vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into python-registry-v2

Browse source
This commit is contained in:
Jake Moshenko 2015-09-17 16:16:27 -04:00
commit 26cea9a07c
96 changed files with 2044 additions and 626 deletions
Download patch file Download diff file Expand all files Collapse all files

32
endpoints/api/billing.py
View file

@ -12,12 +12,42 @@ from auth.permissions import AdministerOrganizationPermission
from auth.auth_context import get_authenticated_user
from auth import scopes
from data import model
from data.billing import PLANS
from data.billing import PLANS, get_plan
import features
import uuid
import json
def lookup_allowed_private_repos(namespace):
""" Returns false if the given namespace has used its allotment of private repositories. """
# Lookup the namespace and verify it has a subscription.
namespace_user = model.user.get_namespace_user(namespace)
if namespace_user is None:
return False
if not namespace_user.stripe_id:
return False
# Ask Stripe for the subscribed plan.
# TODO: Can we cache this or make it faster somehow?
try:
cus = billing.Customer.retrieve(namespace_user.stripe_id)
except stripe.APIConnectionError:
abort(503, message='Cannot contact Stripe')
if not cus.subscription:
return False
# Find the number of private repositories used by the namespace and compare it to the
# plan subscribed.
private_repos = model.user.get_private_repo_count(namespace)
current_plan = get_plan(cus.subscription.plan.id)
if current_plan is None:
return False
return private_repos < current_plan['privateRepos']
def carderror_response(e):
return {'carderror': e.message}, 402

24
endpoints/api/repository.py
View file

@ -2,6 +2,7 @@
import logging
import datetime
import features
from datetime import timedelta
@ -15,7 +16,8 @@ from endpoints.api import (truthy_bool, format_date, nickname, log_action, valid
require_repo_read, require_repo_write, require_repo_admin,
RepositoryParamResource, resource, query_param, parse_args, ApiResource,
request_error, require_scope, Unauthorized, NotFound, InvalidRequest,
path_param)
path_param, ExceedsLicenseException)
from endpoints.api.billing import lookup_allowed_private_repos
from auth.permissions import (ModifyRepositoryPermission, AdministerRepositoryPermission,
CreateRepositoryPermission)
@ -26,6 +28,18 @@ from auth import scopes
logger = logging.getLogger(__name__)
def check_allowed_private_repos(namespace):
""" Checks to see if the given namespace has reached its private repository limit. If so,
raises a ExceedsLicenseException.
"""
# Not enabled if billing is disabled.
if not features.BILLING:
return
if not lookup_allowed_private_repos(namespace):
raise ExceedsLicenseException()
@resource('/v1/repository')
class RepositoryList(ApiResource):
"""Operations for creating and listing repositories."""
@ -87,6 +101,8 @@ class RepositoryList(ApiResource):
raise request_error(message='Repository already exists')
visibility = req['visibility']
if visibility == 'private':
check_allowed_private_repos(namespace_name)
repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility)
repo.description = req['description']
@ -339,7 +355,11 @@ class RepositoryVisibility(RepositoryParamResource):
repo = model.repository.get_repository(namespace, repository)
if repo:
values = request.get_json()
model.repository.set_repository_visibility(repo, values['visibility'])
visibility = values['visibility']
if visibility == 'private':
check_allowed_private_repos(namespace)
model.repository.set_repository_visibility(repo, visibility)
log_action('change_repo_visibility', namespace,
{'repo': repository, 'visibility': values['visibility']},
repo=repo)

36
endpoints/api/suconfig.py
View file

@ -9,7 +9,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_
require_fresh_login, request, validate_json_request, verify_not_prod)
from endpoints.common import common_login
from app import app, CONFIG_PROVIDER, superusers
from app import app, config_provider, superusers
from data import model
from data.database import configure
from auth.permissions import SuperUserPermission
@ -56,13 +56,13 @@ class SuperUserRegistryStatus(ApiResource):
""" Returns the status of the registry. """
# If there is no conf/stack volume, then report that status.
if not CONFIG_PROVIDER.volume_exists():
if not config_provider.volume_exists():
return {
'status': 'missing-config-dir'
}
# If there is no config file, we need to setup the database.
if not CONFIG_PROVIDER.yaml_exists():
if not config_provider.config_exists():
return {
'status': 'config-db'
}
@ -76,7 +76,7 @@ class SuperUserRegistryStatus(ApiResource):
# If we have SETUP_COMPLETE, then we're ready to go!
if app.config.get('SETUP_COMPLETE', False):
return {
'requires_restart': CONFIG_PROVIDER.requires_restart(app.config),
'requires_restart': config_provider.requires_restart(app.config),
'status': 'ready'
}
@ -107,10 +107,10 @@ class SuperUserSetupDatabase(ApiResource):
""" Invokes the alembic upgrade process. """
# Note: This method is called after the database configured is saved, but before the
# database has any tables. Therefore, we only allow it to be run in that unique case.
if CONFIG_PROVIDER.yaml_exists() and not database_is_valid():
if config_provider.config_exists() and not database_is_valid():
# Note: We need to reconfigure the database here as the config has changed.
combined = dict(**app.config)
combined.update(CONFIG_PROVIDER.get_yaml())
combined.update(config_provider.get_config())
configure(combined)
app.config['DB_URI'] = combined['DB_URI']
@ -185,7 +185,7 @@ class SuperUserConfig(ApiResource):
def get(self):
""" Returns the currently defined configuration, if any. """
if SuperUserPermission().can():
config_object = CONFIG_PROVIDER.get_yaml()
config_object = config_provider.get_config()
return {
'config': config_object
}
@ -196,18 +196,18 @@ class SuperUserConfig(ApiResource):
@verify_not_prod
@validate_json_request('UpdateConfig')
def put(self):
""" Updates the config.yaml file. """
""" Updates the config override file. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
if not config_provider.config_exists() or SuperUserPermission().can():
config_object = request.get_json()['config']
hostname = request.get_json()['hostname']
# Add any enterprise defaults missing from the config.
add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
# Write the configuration changes to the YAML file.
CONFIG_PROVIDER.save_yaml(config_object)
# Write the configuration changes to the config override file.
config_provider.save_config(config_object)
# If the authentication system is not the database, link the superuser account to the
# the authentication system chosen.
@ -238,7 +238,7 @@ class SuperUserConfigFile(ApiResource):
if SuperUserPermission().can():
return {
'exists': CONFIG_PROVIDER.volume_file_exists(filename)
'exists': config_provider.volume_file_exists(filename)
}
abort(403)
@ -252,12 +252,12 @@ class SuperUserConfigFile(ApiResource):
# Note: This method can be called before the configuration exists
# to upload the database SSL cert.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
if not config_provider.config_exists() or SuperUserPermission().can():
uploaded_file = request.files['file']
if not uploaded_file:
abort(400)
CONFIG_PROVIDER.save_volume_file(filename, uploaded_file)
config_provider.save_volume_file(filename, uploaded_file)
return {
'status': True
}
@ -309,7 +309,7 @@ class SuperUserCreateInitialSuperUser(ApiResource):
#
# We do this special security check because at the point this method is called, the database
# is clean but does not (yet) have any super users for our permissions code to check against.
if CONFIG_PROVIDER.yaml_exists() and not database_has_users():
if config_provider.config_exists() and not database_has_users():
data = request.get_json()
username = data['username']
password = data['password']
@ -319,9 +319,9 @@ class SuperUserCreateInitialSuperUser(ApiResource):
superuser = model.user.create_user(username, password, email, auto_verify=True)
# Add the user to the config.
config_object = CONFIG_PROVIDER.get_yaml()
config_object = config_provider.get_config()
config_object['SUPER_USERS'] = [username]
CONFIG_PROVIDER.save_yaml(config_object)
config_provider.save_config(config_object)
# Update the in-memory config for the new superuser.
superusers.register_superuser(username)
@ -369,7 +369,7 @@ class SuperUserConfigValidate(ApiResource):
# Note: This method is called to validate the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup. Note that
# this is also safe since this method does not access any information not given in the request.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
if not config_provider.config_exists() or SuperUserPermission().can():
config = request.get_json()['config']
return validate_service_for_config(service, config, request.get_json().get('password', ''))

30
endpoints/api/superuser.py
View file

@ -13,7 +13,7 @@ from app import app, avatar, superusers, authentication
from endpoints.api import (ApiResource, nickname, resource, validate_json_request,
internal_only, require_scope, show_if, parse_args,
query_param, abort, require_fresh_login, path_param, verify_not_prod)
from endpoints.api.logs import get_logs
from endpoints.api.logs import get_logs, get_aggregate_logs
from data import model
from auth.permissions import SuperUserPermission
from auth import scopes
@ -83,6 +83,26 @@ class SuperUserSystemLogServices(ApiResource):
abort(403)
@resource('/v1/superuser/aggregatelogs')
@internal_only
class SuperUserAggregateLogs(ApiResource):
""" Resource for fetching aggregated logs for the current user. """
@require_fresh_login
@verify_not_prod
@nickname('listAllAggregateLogs')
@parse_args
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
def get(self, args):
""" Returns the aggregated logs for the current system. """
if SuperUserPermission().can():
start_time = args['starttime']
end_time = args['endtime']
return get_aggregate_logs(start_time, end_time)
abort(403)
@resource('/v1/superuser/logs')
@internal_only
@ -93,9 +113,9 @@ class SuperUserLogs(ApiResource):
@verify_not_prod
@nickname('listAllLogs')
@parse_args
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('performer', 'Username for which to filter logs.', type=str)
@query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
@query_param('page', 'The page number for the logs', type=int, default=1)
@require_scope(scopes.SUPERUSER)
def get(self, args):
""" List the usage logs for the current system. """
@ -103,7 +123,7 @@ class SuperUserLogs(ApiResource):
start_time = args['starttime']
end_time = args['endtime']
return get_logs(start_time, end_time)
return get_logs(start_time, end_time, page=args['page'])
abort(403)

2
endpoints/api/tag.py
View file

@ -43,7 +43,7 @@ class ListRepositoryTags(RepositoryParamResource):
specific_tag = args.get('specificTag') or None
page = min(1, args.get('start', 1))
page = max(1, args.get('page', 1))
limit = min(100, max(1, args.get('limit', 50)))
# Note: We ask for limit+1 here, so we can check to see if there are

3
endpoints/api/user.py
View file

@ -306,6 +306,7 @@ class User(ApiResource):
return user_view(user)
@show_if(features.USER_CREATION)
@show_if(features.DIRECT_LOGIN)
@nickname('createNewUser')
@internal_only
@validate_json_request('NewUser')
@ -496,6 +497,7 @@ class ConvertToOrganization(ApiResource):
@resource('/v1/signin')
@show_if(features.DIRECT_LOGIN)
@internal_only
class Signin(ApiResource):
""" Operations for signing in the user. """
@ -595,6 +597,7 @@ class Signout(ApiResource):
@resource('/v1/detachexternal/<servicename>')
@show_if(features.DIRECT_LOGIN)
@internal_only
class DetachExternal(ApiResource):
""" Resource for detaching an external login. """

2
endpoints/decorated.py
View file

@ -4,7 +4,7 @@ import json
from flask import make_response
from app import app
from util.useremails import CannotSendEmailException
from util.config.provider import CannotWriteConfigException
from util.config.provider.baseprovider import CannotWriteConfigException
from data import model
logger = logging.getLogger(__name__)

91
endpoints/oauthlogin.py
View file

@ -5,7 +5,7 @@ from flask import request, redirect, url_for, Blueprint
from flask.ext.login import current_user
from endpoints.common import render_page_template, common_login, route_show_if
from app import app, analytics, get_app_url, github_login, google_login
from app import app, analytics, get_app_url, github_login, google_login, dex_login
from data import model
from util.names import parse_repository_name
from util.validation import generate_valid_usernames
@ -14,6 +14,7 @@ from auth.auth import require_session_login
from peewee import IntegrityError
import features
from util.security.strictjwt import decode, InvalidTokenError
logger = logging.getLogger(__name__)
client = app.config['HTTPCLIENT']
@ -24,7 +25,7 @@ def render_ologin_error(service_name,
return render_page_template('ologinerror.html', service_name=service_name,
error_message=error_message,
service_url=get_app_url(),
user_creation=features.USER_CREATION)
user_creation=features.USER_CREATION and features.DIRECT_LOGIN)
def get_user(service, token):
@ -86,7 +87,7 @@ def conduct_oauth_login(service, user_id, username, email, metadata={}):
return render_ologin_error(service_name)
def get_google_username(user_data):
def get_email_username(user_data):
username = user_data['email']
at = username.find('@')
if at > 0:
@ -108,7 +109,7 @@ def google_oauth_callback():
if not user_data or not user_data.get('id', None) or not user_data.get('email', None):
return render_ologin_error('Google')
username = get_google_username(user_data)
username = get_email_username(user_data)
metadata = {
'service_username': user_data['email']
}
@ -194,7 +195,7 @@ def google_oauth_attach():
google_id = user_data['id']
user_obj = current_user.db_user()
username = get_google_username(user_data)
username = get_email_username(user_data)
metadata = {
'service_username': user_data['email']
}
@ -236,3 +237,83 @@ def github_oauth_attach():
return render_ologin_error('GitHub', err)
return redirect(url_for('web.user'))
def decode_user_jwt(token, oidc_provider):
try:
return decode(token, oidc_provider.get_public_key(), algorithms=['RS256'],
audience=oidc_provider.client_id(),
issuer=oidc_provider.issuer)
except InvalidTokenError:
# Public key may have expired. Try to retrieve an updated public key and use it to decode.
return decode(token, oidc_provider.get_public_key(force_refresh=True), algorithms=['RS256'],
audience=oidc_provider.client_id(),
issuer=oidc_provider.issuer)
@oauthlogin.route('/dex/callback', methods=['GET', 'POST'])
@route_show_if(features.DEX_LOGIN)
def dex_oauth_callback():
error = request.values.get('error', None)
if error:
return render_ologin_error(dex_login.public_title, error)
code = request.values.get('code')
if not code:
return render_ologin_error(dex_login.public_title, 'Missing OAuth code')
token = dex_login.exchange_code_for_token(app.config, client, code, client_auth=True,
form_encode=True)
try:
payload = decode_user_jwt(token, dex_login)
except InvalidTokenError:
logger.exception('Exception when decoding returned JWT')
return render_ologin_error(dex_login.public_title,
'Could not decode response. Please contact your system administrator about this error.')
username = get_email_username(payload)
metadata = {}
dex_id = payload['sub']
email_address = payload['email']
if not payload.get('email_verified', False):
return render_ologin_error(dex_login.public_title,
'A verified e-mail address is required for login. Please verify your ' +
'e-mail address in %s and try again.' % dex_login.public_title)
return conduct_oauth_login(dex_login, dex_id, username, email_address,
metadata=metadata)
@oauthlogin.route('/dex/callback/attach', methods=['GET', 'POST'])
@route_show_if(features.DEX_LOGIN)
@require_session_login
def dex_oauth_attach():
code = request.args.get('code')
token = dex_login.exchange_code_for_token(app.config, client, code, redirect_suffix='/attach',
client_auth=True, form_encode=True)
if not token:
return render_ologin_error(dex_login.public_title)
try:
payload = decode_user_jwt(token, dex_login)
except jwt.InvalidTokenError:
logger.exception('Exception when decoding returned JWT')
return render_ologin_error(dex_login.public_title,
'Could not decode response. Please contact your system administrator about this error.')
user_obj = current_user.db_user()
dex_id = payload['sub']
metadata = {}
try:
model.user.attach_federated_login(user_obj, 'dex', dex_id, metadata=metadata)
except IntegrityError:
err = '%s account is already attached to a %s account' % (dex_login.public_title,
app.config['REGISTRY_TITLE_SHORT'])
return render_ologin_error(dex_login.public_title, err)
return redirect(url_for('web.user'))

10
endpoints/trackhelper.py
View file

@ -1,4 +1,5 @@
import logging
import random
from app import analytics, app, userevents
from data import model
@ -7,7 +8,7 @@ from auth.auth_context import get_authenticated_user, get_validated_token, get_v
logger = logging.getLogger(__name__)
def track_and_log(event_name, repo, **kwargs):
def track_and_log(event_name, repo, analytics_name=None, analytics_sample=1, **kwargs):
repository = repo.name
namespace = repo.namespace_user.username
metadata = {
@ -62,8 +63,11 @@ def track_and_log(event_name, repo, **kwargs):
event.publish_event_data('docker-cli', user_event_data)
# Save the action to mixpanel.
logger.debug('Logging the %s to Mixpanel', event_name)
analytics.track(analytics_id, event_name, extra_params)
if random.random() < analytics_sample:
if analytics_name is None:
analytics_name = event_name
logger.debug('Logging the %s to Mixpanel', analytics_name)
analytics.track(analytics_id, analytics_name, extra_params)
# Log the action to the database.
logger.debug('Logging the %s to logs system', event_name)

2
endpoints/v1/index.py
View file

@ -270,7 +270,7 @@ def get_repository_images(namespace, repository):
resp = make_response(json.dumps([]), 200)
resp.mimetype = 'application/json'
track_and_log('pull_repo', repo)
track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
return resp
abort(403)

6
endpoints/web.py
View file

@ -9,7 +9,7 @@ from health.healthcheck import get_healthchecker
from data import model
from data.database import db
from app import app, billing as stripe, build_logs, avatar, signer, log_archive
from app import app, billing as stripe, build_logs, avatar, signer, log_archive, config_provider
from auth.auth import require_session_login, process_oauth
from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission,
SuperUserPermission, AdministerRepositoryPermission,
@ -209,7 +209,7 @@ def v1():
@web.route('/health/instance', methods=['GET'])
@no_cache
def instance_health():
checker = get_healthchecker(app)
checker = get_healthchecker(app, config_provider)
(data, status_code) = checker.check_instance()
response = jsonify(dict(data=data, status_code=status_code))
response.status_code = status_code
@ -221,7 +221,7 @@ def instance_health():
@web.route('/health/endtoend', methods=['GET'])
@no_cache
def endtoend_health():
checker = get_healthchecker(app)
checker = get_healthchecker(app, config_provider)
(data, status_code) = checker.check_endtoend()
response = jsonify(dict(data=data, status_code=status_code))
response.status_code = status_code