vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into star

Browse source
This commit is contained in:
Jimmy Zelinskie 2015-02-18 17:36:58 -05:00
commit 917dd6b674
229 changed files with 10807 additions and 3003 deletions
Download patch file Download diff file Expand all files Collapse all files

25
endpoints/api/__init__.py
View file

@ -280,6 +280,23 @@ require_user_read = require_user_permission(UserReadPermission, scopes.READ_USER
require_user_admin = require_user_permission(UserAdminPermission, None)
require_fresh_user_admin = require_user_permission(UserAdminPermission, None)
def verify_not_prod(func):
@add_method_metadata('enterprise_only', True)
@wraps(func)
def wrapped(*args, **kwargs):
# Verify that we are not running on a production (i.e. hosted) stack. If so, we fail.
# This should never happen (because of the feature-flag on SUPER_USERS), but we want to be
# absolutely sure.
if app.config['SERVER_HOSTNAME'].find('quay.io') >= 0:
logger.error('!!! Super user method called IN PRODUCTION !!!')
raise NotFound()
return func(*args, **kwargs)
return wrapped
def require_fresh_login(func):
@add_method_metadata('requires_fresh_login', True)
@wraps(func)
@ -317,7 +334,11 @@ def validate_json_request(schema_name):
def wrapped(self, *args, **kwargs):
schema = self.schemas[schema_name]
try:
validate(request.get_json(), schema)
json_data = request.get_json()
if json_data is None:
raise InvalidRequest('Missing JSON body')
validate(json_data, schema)
return func(self, *args, **kwargs)
except ValidationError as ex:
raise InvalidRequest(ex.message)
@ -385,8 +406,10 @@ import endpoints.api.repoemail
import endpoints.api.repotoken
import endpoints.api.robot
import endpoints.api.search
import endpoints.api.suconfig
import endpoints.api.superuser
import endpoints.api.tag
import endpoints.api.team
import endpoints.api.trigger
import endpoints.api.user

44
endpoints/api/build.py
View file

@ -9,7 +9,7 @@ from app import app, userfiles as user_files, build_logs, log_archive
from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
require_repo_read, require_repo_write, validate_json_request,
ApiResource, internal_only, format_date, api, Unauthorized, NotFound,
path_param)
path_param, InvalidRequest, require_repo_admin)
from endpoints.common import start_build
from endpoints.trigger import BuildTrigger
from data import model, database
@ -70,10 +70,17 @@ def build_status_view(build_obj, can_write=False):
# If the status contains a heartbeat, then check to see if has been written in the last few
# minutes. If not, then the build timed out.
if status is not None and 'heartbeat' in status and status['heartbeat']:
heartbeat = datetime.datetime.fromtimestamp(status['heartbeat'])
if datetime.datetime.now() - heartbeat > datetime.timedelta(minutes=1):
phase = database.BUILD_PHASE.INTERNAL_ERROR
if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR:
if status is not None and 'heartbeat' in status and status['heartbeat']:
heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat'])
if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1):
phase = database.BUILD_PHASE.INTERNAL_ERROR
# If the phase is internal error, return 'error' instead of the number if retries
# on the queue item is 0.
if phase == database.BUILD_PHASE.INTERNAL_ERROR:
if build_obj.queue_item is None or build_obj.queue_item.retries_remaining == 0:
phase = database.BUILD_PHASE.ERROR
logger.debug('Can write: %s job_config: %s', can_write, build_obj.job_config)
resp = {
@ -86,7 +93,7 @@ def build_status_view(build_obj, can_write=False):
'is_writer': can_write,
'trigger': trigger_view(build_obj.trigger),
'resource_key': build_obj.resource_key,
'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None,
'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None
}
if can_write:
@ -200,6 +207,31 @@ class RepositoryBuildList(RepositoryParamResource):
return resp, 201, headers
@resource('/v1/repository/<repopath:repository>/build/<build_uuid>')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')
class RepositoryBuildResource(RepositoryParamResource):
""" Resource for dealing with repository builds. """
@require_repo_admin
@nickname('cancelRepoBuild')
def delete(self, namespace, repository, build_uuid):
""" Cancels a repository build if it has not yet been picked up by a build worker. """
try:
build = model.get_repository_build(build_uuid)
except model.InvalidRepositoryBuildException:
raise NotFound()
if build.repository.name != repository or build.repository.namespace_user.username != namespace:
raise NotFound()
if model.cancel_repository_build(build):
return 'Okay', 201
else:
raise InvalidRequest('Build is currently running or has finished')
@resource('/v1/repository/<repopath:repository>/build/<build_uuid>/status')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')

9
endpoints/api/organization.py
View file

@ -116,6 +116,11 @@ class Organization(ApiResource):
'type': 'boolean',
'description': 'Whether the organization desires to receive emails for invoices',
},
'tag_expiration': {
'type': 'integer',
'maximum': 2592000,
'minimum': 0,
},
},
},
}
@ -161,6 +166,10 @@ class Organization(ApiResource):
logger.debug('Changing email address for organization: %s', org.username)
model.update_email(org, new_email)
if 'tag_expiration' in org_data:
logger.debug('Changing organization tag expiration to: %ss', org_data['tag_expiration'])
model.change_user_tag_expiration(org, org_data['tag_expiration'])
teams = model.get_teams_within_org(org)
return org_view(org, teams)
raise Unauthorized()

362
endpoints/api/suconfig.py Normal file
View file

@ -0,0 +1,362 @@
import logging
import os
import json
import signal
from flask import abort, Response
from endpoints.api import (ApiResource, nickname, resource, internal_only, show_if,
require_fresh_login, request, validate_json_request, verify_not_prod)
from endpoints.common import common_login
from app import app, CONFIG_PROVIDER, superusers
from data import model
from data.database import configure
from auth.permissions import SuperUserPermission
from auth.auth_context import get_authenticated_user
from data.database import User
from util.config.configutil import add_enterprise_config_defaults
from util.config.validator import validate_service_for_config, SSL_FILENAMES
from data.runmigration import run_alembic_migration
import features
logger = logging.getLogger(__name__)
def database_is_valid():
""" Returns whether the database, as configured, is valid. """
if app.config['TESTING']:
return False
try:
list(User.select().limit(1))
return True
except:
return False
def database_has_users():
""" Returns whether the database has any users defined. """
return bool(list(User.select().limit(1)))
@resource('/v1/superuser/registrystatus')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserRegistryStatus(ApiResource):
""" Resource for determining the status of the registry, such as if config exists,
if a database is configured, and if it has any defined users.
"""
@nickname('scRegistryStatus')
@verify_not_prod
def get(self):
""" Returns the status of the registry. """
# If there is no conf/stack volume, then report that status.
if not CONFIG_PROVIDER.volume_exists():
return {
'status': 'missing-config-dir'
}
# If there is no config file, we need to setup the database.
if not CONFIG_PROVIDER.yaml_exists():
return {
'status': 'config-db'
}
# If the database isn't yet valid, then we need to set it up.
if not database_is_valid():
return {
'status': 'setup-db'
}
# If we have SETUP_COMPLETE, then we're ready to go!
if app.config.get('SETUP_COMPLETE', False):
return {
'requires_restart': CONFIG_PROVIDER.requires_restart(app.config),
'status': 'ready'
}
return {
'status': 'create-superuser' if not database_has_users() else 'config'
}
class _AlembicLogHandler(logging.Handler):
def __init__(self):
super(_AlembicLogHandler, self).__init__()
self.records = []
def emit(self, record):
self.records.append({
'level': record.levelname,
'message': record.getMessage()
})
@resource('/v1/superuser/setupdb')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserSetupDatabase(ApiResource):
""" Resource for invoking alembic to setup the database. """
@verify_not_prod
@nickname('scSetupDatabase')
def get(self):
""" Invokes the alembic upgrade process. """
# Note: This method is called after the database configured is saved, but before the
# database has any tables. Therefore, we only allow it to be run in that unique case.
if CONFIG_PROVIDER.yaml_exists() and not database_is_valid():
# Note: We need to reconfigure the database here as the config has changed.
combined = dict(**app.config)
combined.update(CONFIG_PROVIDER.get_yaml())
configure(combined)
app.config['DB_URI'] = combined['DB_URI']
log_handler = _AlembicLogHandler()
try:
run_alembic_migration(log_handler)
except Exception as ex:
return {
'error': str(ex)
}
return {
'logs': log_handler.records
}
abort(403)
@resource('/v1/superuser/shutdown')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserShutdown(ApiResource):
""" Resource for sending a shutdown signal to the container. """
@verify_not_prod
@nickname('scShutdownContainer')
def post(self):
""" Sends a signal to the phusion init system to shut down the container. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
if app.config['TESTING'] or not database_has_users() or SuperUserPermission().can():
# Note: We skip if debugging locally.
if app.config.get('DEBUGGING') == True:
return {}
os.kill(1, signal.SIGINT)
return {}
abort(403)
@resource('/v1/superuser/config')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserConfig(ApiResource):
""" Resource for fetching and updating the current configuration, if any. """
schemas = {
'UpdateConfig': {
'id': 'UpdateConfig',
'type': 'object',
'description': 'Updates the YAML config file',
'required': [
'config',
'hostname'
],
'properties': {
'config': {
'type': 'object'
},
'hostname': {
'type': 'string'
}
},
},
}
@require_fresh_login
@verify_not_prod
@nickname('scGetConfig')
def get(self):
""" Returns the currently defined configuration, if any. """
if SuperUserPermission().can():
config_object = CONFIG_PROVIDER.get_yaml()
return {
'config': config_object
}
abort(403)
@nickname('scUpdateConfig')
@verify_not_prod
@validate_json_request('UpdateConfig')
def put(self):
""" Updates the config.yaml file. """
# Note: This method is called to set the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
config_object = request.get_json()['config']
hostname = request.get_json()['hostname']
# Add any enterprise defaults missing from the config.
add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname)
# Write the configuration changes to the YAML file.
CONFIG_PROVIDER.save_yaml(config_object)
return {
'exists': True,
'config': config_object
}
abort(403)
@resource('/v1/superuser/config/file/<filename>')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserConfigFile(ApiResource):
""" Resource for fetching the status of config files and overriding them. """
@nickname('scConfigFileExists')
@verify_not_prod
def get(self, filename):
""" Returns whether the configuration file with the given name exists. """
if not filename in SSL_FILENAMES:
abort(404)
if SuperUserPermission().can():
return {
'exists': CONFIG_PROVIDER.volume_file_exists(filename)
}
abort(403)
@nickname('scUpdateConfigFile')
@verify_not_prod
def post(self, filename):
""" Updates the configuration file with the given name. """
if not filename in SSL_FILENAMES:
abort(404)
if SuperUserPermission().can():
uploaded_file = request.files['file']
if not uploaded_file:
abort(400)
CONFIG_PROVIDER.save_volume_file(filename, uploaded_file)
return {
'status': True
}
abort(403)
@resource('/v1/superuser/config/createsuperuser')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserCreateInitialSuperUser(ApiResource):
""" Resource for creating the initial super user. """
schemas = {
'CreateSuperUser': {
'id': 'CreateSuperUser',
'type': 'object',
'description': 'Information for creating the initial super user',
'required': [
'username',
'password',
'email'
],
'properties': {
'username': {
'type': 'string',
'description': 'The username for the superuser'
},
'password': {
'type': 'string',
'description': 'The password for the superuser'
},
'email': {
'type': 'string',
'description': 'The e-mail address for the superuser'
},
},
},
}
@nickname('scCreateInitialSuperuser')
@verify_not_prod
@validate_json_request('CreateSuperUser')
def post(self):
""" Creates the initial super user, updates the underlying configuration and
sets the current session to have that super user. """
# Special security check: This method is only accessible when:
# - There is a valid config YAML file.
# - There are currently no users in the database (clean install)
#
# We do this special security check because at the point this method is called, the database
# is clean but does not (yet) have any super users for our permissions code to check against.
if CONFIG_PROVIDER.yaml_exists() and not database_has_users():
data = request.get_json()
username = data['username']
password = data['password']
email = data['email']
# Create the user in the database.
superuser = model.create_user(username, password, email, auto_verify=True)
# Add the user to the config.
config_object = CONFIG_PROVIDER.get_yaml()
config_object['SUPER_USERS'] = [username]
CONFIG_PROVIDER.save_yaml(config_object)
# Update the in-memory config for the new superuser.
superusers.register_superuser(username)
# Conduct login with that user.
common_login(superuser)
return {
'status': True
}
abort(403)
@resource('/v1/superuser/config/validate/<service>')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserConfigValidate(ApiResource):
""" Resource for validating a block of configuration against an external service. """
schemas = {
'ValidateConfig': {
'id': 'ValidateConfig',
'type': 'object',
'description': 'Validates configuration',
'required': [
'config'
],
'properties': {
'config': {
'type': 'object'
}
},
},
}
@nickname('scValidateConfig')
@verify_not_prod
@validate_json_request('ValidateConfig')
def post(self, service):
""" Validates the given config for the given service. """
# Note: This method is called to validate the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup. Note that
# this is also safe since this method does not access any information not given in the request.
if not CONFIG_PROVIDER.yaml_exists() or SuperUserPermission().can():
config = request.get_json()['config']
return validate_service_for_config(service, config)
abort(403)

81
endpoints/api/superuser.py
View file

@ -1,15 +1,16 @@
import string
import logging
import json
import os
from random import SystemRandom
from app import app
from app import app, avatar, superusers
from flask import request
from endpoints.api import (ApiResource, nickname, resource, validate_json_request, request_error,
log_action, internal_only, NotFound, require_user_admin, format_date,
InvalidToken, require_scope, format_date, hide_if, show_if, parse_args,
query_param, abort, require_fresh_login, path_param)
query_param, abort, require_fresh_login, path_param, verify_not_prod)
from endpoints.api.logs import get_logs
@ -22,18 +23,76 @@ import features
logger = logging.getLogger(__name__)
def get_immediate_subdirectories(directory):
return [name for name in os.listdir(directory) if os.path.isdir(os.path.join(directory, name))]
def get_services():
services = set(get_immediate_subdirectories(app.config['SYSTEM_SERVICES_PATH']))
services = services - set(app.config['SYSTEM_SERVICE_BLACKLIST'])
return services
@resource('/v1/superuser/systemlogs/<service>')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserGetLogsForService(ApiResource):
""" Resource for fetching the kinds of system logs in the system. """
@require_fresh_login
@verify_not_prod
@nickname('getSystemLogs')
def get(self, service):
""" Returns the logs for the specific service. """
if SuperUserPermission().can():
if not service in get_services():
abort(404)
try:
with open(app.config['SYSTEM_SERVICE_LOGS_PATH'] % service, 'r') as f:
logs = f.read()
except Exception as ex:
logger.exception('Cannot read logs')
abort(400)
return {
'logs': logs
}
abort(403)
@resource('/v1/superuser/systemlogs/')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserSystemLogServices(ApiResource):
""" Resource for fetching the kinds of system logs in the system. """
@require_fresh_login
@verify_not_prod
@nickname('listSystemLogServices')
def get(self):
""" List the system logs for the current system. """
if SuperUserPermission().can():
return {
'services': list(get_services())
}
abort(403)
@resource('/v1/superuser/logs')
@internal_only
@show_if(features.SUPER_USERS)
class SuperUserLogs(ApiResource):
""" Resource for fetching all logs in the system. """
@require_fresh_login
@verify_not_prod
@nickname('listAllLogs')
@parse_args
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('performer', 'Username for which to filter logs.', type=str)
def get(self, args):
""" List the logs for the current system. """
""" List the usage logs for the current system. """
if SuperUserPermission().can():
performer_name = args['performer']
start_time = args['starttime']
@ -49,7 +108,8 @@ def user_view(user):
'username': user.username,
'email': user.email,
'verified': user.verified,
'super_user': user.username in app.config['SUPER_USERS']
'avatar': avatar.compute_hash(user.email, name=user.username),
'super_user': superusers.is_superuser(user.username)
}
@resource('/v1/superuser/usage/')
@ -58,6 +118,7 @@ def user_view(user):
class UsageInformation(ApiResource):
""" Resource for returning the usage information for enterprise customers. """
@require_fresh_login
@verify_not_prod
@nickname('getSystemUsage')
def get(self):
""" Returns the number of repository handles currently held. """
@ -96,6 +157,7 @@ class SuperUserList(ApiResource):
}
@require_fresh_login
@verify_not_prod
@nickname('listAllUsers')
def get(self):
""" Returns a list of all users in the system. """
@ -109,6 +171,7 @@ class SuperUserList(ApiResource):
@require_fresh_login
@verify_not_prod
@nickname('createInstallUser')
@validate_json_request('CreateInstallUser')
def post(self):
@ -146,6 +209,7 @@ class SuperUserList(ApiResource):
class SuperUserSendRecoveryEmail(ApiResource):
""" Resource for sending a recovery user on behalf of a user. """
@require_fresh_login
@verify_not_prod
@nickname('sendInstallUserRecoveryEmail')
def post(self, username):
if SuperUserPermission().can():
@ -153,7 +217,7 @@ class SuperUserSendRecoveryEmail(ApiResource):
if not user or user.organization or user.robot:
abort(404)
if username in app.config['SUPER_USERS']:
if superusers.is_superuser(username):
abort(403)
code = model.create_reset_password_email_code(user.email)
@ -190,6 +254,7 @@ class SuperUserManagement(ApiResource):
}
@require_fresh_login
@verify_not_prod
@nickname('getInstallUser')
def get(self, username):
""" Returns information about the specified user. """
@ -203,6 +268,7 @@ class SuperUserManagement(ApiResource):
abort(403)
@require_fresh_login
@verify_not_prod
@nickname('deleteInstallUser')
def delete(self, username):
""" Deletes the specified user. """
@ -211,7 +277,7 @@ class SuperUserManagement(ApiResource):
if not user or user.organization or user.robot:
abort(404)
if username in app.config['SUPER_USERS']:
if superusers.is_superuser(username):
abort(403)
model.delete_user(user)
@ -220,6 +286,7 @@ class SuperUserManagement(ApiResource):
abort(403)
@require_fresh_login
@verify_not_prod
@nickname('changeInstallUser')
@validate_json_request('UpdateUser')
def put(self, username):
@ -229,7 +296,7 @@ class SuperUserManagement(ApiResource):
if not user or user.organization or user.robot:
abort(404)
if username in app.config['SUPER_USERS']:
if superusers.is_superuser(username):
abort(403)
user_data = request.get_json()

4
endpoints/api/tag.py
View file

@ -54,8 +54,8 @@ class RepositoryTag(RepositoryParamResource):
username = get_authenticated_user().username
log_action('move_tag' if original_image_id else 'create_tag', namespace,
{ 'username': username, 'repo': repository, 'tag': tag,
'image': image_id, 'original_image': original_image_id },
{'username': username, 'repo': repository, 'tag': tag,
'image': image_id, 'original_image': original_image_id},
repo=model.get_repository(namespace, repository))
return 'Updated', 201

4
endpoints/api/trigger.py
View file

@ -415,13 +415,13 @@ class ActivateBuildTrigger(RepositoryParamResource):
try:
run_parameters = request.get_json()
specs = handler.manual_start(trigger.auth_token, config_dict, run_parameters=run_parameters)
dockerfile_id, tags, name, subdir = specs
dockerfile_id, tags, name, subdir, metadata = specs
repo = model.get_repository(namespace, repository)
pull_robot_name = model.get_pull_robot_name(trigger)
build_request = start_build(repo, dockerfile_id, tags, name, subdir, True,
pull_robot_name=pull_robot_name)
pull_robot_name=pull_robot_name, trigger_metadata=metadata)
except TriggerStartException as tse:
raise InvalidRequest(tse.message)

13
endpoints/api/user.py
View file

@ -73,6 +73,7 @@ def user_view(user):
'can_create_repo': True,
'invoice_email': user.invoice_email,
'preferred_namespace': not (user.stripe_id is None),
'tag_expiration': user.removed_tag_expiration_s,
})
if features.SUPER_USERS:
@ -144,6 +145,11 @@ class User(ApiResource):
'type': 'string',
'description': 'The user\'s email address',
},
'tag_expiration': {
'type': 'integer',
'maximum': 2592000,
'minimum': 0,
},
'username': {
'type': 'string',
'description': 'The user\'s username',
@ -227,6 +233,10 @@ class User(ApiResource):
logger.debug('Changing invoice_email for user: %s', user.username)
model.change_invoice_email(user, user_data['invoice_email'])
if 'tag_expiration' in user_data:
logger.debug('Changing user tag expiration to: %ss', user_data['tag_expiration'])
model.change_user_tag_expiration(user, user_data['tag_expiration'])
if 'email' in user_data and user_data['email'] != user.email:
new_email = user_data['email']
if model.find_user_by_email(new_email):
@ -248,7 +258,8 @@ class User(ApiResource):
# Username already used
raise request_error(message='Username is already in use')
model.change_username(user, new_username)
model.change_username(user.id, new_username)
except model.InvalidPasswordException, ex:
raise request_error(exception=ex)