import logging import json import time import datetime from flask import request, redirect from app import app, userfiles as user_files, build_logs, log_archive, dockerfile_build_queue from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource, require_repo_read, require_repo_write, validate_json_request, ApiResource, internal_only, format_date, api, Unauthorized, NotFound, path_param, InvalidRequest, require_repo_admin) from endpoints.building import start_build, PreparedBuild from endpoints.trigger import BuildTriggerHandler from data import model, database from auth.auth_context import get_authenticated_user from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, AdministerRepositoryPermission, AdministerOrganizationPermission) from data.buildlogs import BuildStatusRetrievalError from util.names import parse_robot_username logger = logging.getLogger(__name__) def get_trigger_config(trigger): try: return json.loads(trigger.config) except: return {} def get_job_config(build_obj): try: return json.loads(build_obj.job_config) except: return {} def user_view(user): return { 'name': user.username, 'kind': 'user', 'is_robot': user.robot, } def trigger_view(trigger, can_read=False, can_admin=False, for_build=False): if trigger and trigger.uuid: build_trigger = BuildTriggerHandler.get_handler(trigger) build_source = build_trigger.config.get('build_source') repo_url = build_trigger.get_repository_url() if build_source else None if can_admin: can_read = True is_connected_user = False if (can_admin and get_authenticated_user() and trigger.connected_user_id == get_authenticated_user().id): is_connected_user = True trigger_data = { 'id': trigger.uuid, 'service': trigger.service.name, 'is_active': build_trigger.is_active(), 'build_source': build_source if can_read else None, 'repository_url': repo_url if can_read else None, 'config': build_trigger.config if can_admin else {}, 'is_connected_user': is_connected_user, } if not for_build and can_admin and trigger.pull_robot: trigger_data['pull_robot'] = user_view(trigger.pull_robot) return trigger_data return None def build_status_view(build_obj): phase = build_obj.phase try: status = build_logs.get_status(build_obj.uuid) except BuildStatusRetrievalError: status = {} phase = 'cannot_load' # If the status contains a heartbeat, then check to see if has been written in the last few # minutes. If not, then the build timed out. if phase != database.BUILD_PHASE.COMPLETE and phase != database.BUILD_PHASE.ERROR: if status is not None and 'heartbeat' in status and status['heartbeat']: heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat']) if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1): phase = database.BUILD_PHASE.INTERNAL_ERROR # If the phase is internal error, return 'error' instead if the number of retries # on the queue item is 0. if phase == database.BUILD_PHASE.INTERNAL_ERROR: retry = build_obj.queue_id and dockerfile_build_queue.has_retries_remaining(build_obj.queue_id) if not retry: phase = database.BUILD_PHASE.ERROR repo_namespace = build_obj.repository.namespace_user.username repo_name = build_obj.repository.name can_read = ReadRepositoryPermission(repo_namespace, repo_name).can() can_write = ModifyRepositoryPermission(repo_namespace, repo_name).can() can_admin = AdministerRepositoryPermission(repo_namespace, repo_name).can() job_config = get_job_config(build_obj) resp = { 'id': build_obj.uuid, 'phase': phase, 'started': format_date(build_obj.started), 'display_name': build_obj.display_name, 'status': status or {}, 'subdirectory': job_config.get('build_subdir', ''), 'tags': job_config.get('docker_tags', []), 'manual_user': job_config.get('manual_user', None), 'is_writer': can_write, 'trigger': trigger_view(build_obj.trigger, can_read, can_admin, for_build=True), 'trigger_metadata': job_config.get('trigger_metadata', None) if can_read else None, 'resource_key': build_obj.resource_key, 'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None, 'repository': { 'namespace': repo_namespace, 'name': repo_name } } if can_write and build_obj.resource_key is not None: resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True) return resp @resource('/v1/repository//build/') @path_param('repository', 'The full path of the repository. e.g. namespace/name') class RepositoryBuildList(RepositoryParamResource): """ Resource related to creating and listing repository builds. """ schemas = { 'RepositoryBuildRequest': { 'id': 'RepositoryBuildRequest', 'type': 'object', 'description': 'Description of a new repository build.', 'required': [ 'file_id', ], 'properties': { 'file_id': { 'type': 'string', 'description': 'The file id that was generated when the build spec was uploaded', }, 'subdirectory': { 'type': 'string', 'description': 'Subdirectory in which the Dockerfile can be found', }, 'pull_robot': { 'type': 'string', 'description': 'Username of a Quay robot account to use as pull credentials', }, 'docker_tags': { 'type': 'array', 'description': 'The tags to which the built images will be pushed', 'items': { 'type': 'string' }, 'minItems': 1, 'uniqueItems': True } }, }, } @require_repo_read @parse_args @query_param('limit', 'The maximum number of builds to return', type=int, default=5) @query_param('since', 'Returns all builds since the given unix timecode', type=int, default=None) @nickname('getRepoBuilds') def get(self, args, namespace, repository): """ Get the list of repository builds. """ limit = args.get('limit', 5) since = args.get('since', None) if since is not None: since = datetime.datetime.utcfromtimestamp(since) builds = model.list_repository_builds(namespace, repository, limit, since=since) return { 'builds': [build_status_view(build) for build in builds] } @require_repo_write @nickname('requestRepoBuild') @validate_json_request('RepositoryBuildRequest') def post(self, namespace, repository): """ Request that a repository be built and pushed from the specified input. """ logger.debug('User requested repository initialization.') request_json = request.get_json() dockerfile_id = request_json['file_id'] subdir = request_json['subdirectory'] if 'subdirectory' in request_json else '' tags = request_json.get('docker_tags', ['latest']) pull_robot_name = request_json.get('pull_robot', None) # Verify the security behind the pull robot. if pull_robot_name: result = parse_robot_username(pull_robot_name) if result: pull_robot = model.lookup_robot(pull_robot_name) if not pull_robot: raise NotFound() # Make sure the user has administer permissions for the robot's namespace. (robot_namespace, shortname) = result if not AdministerOrganizationPermission(robot_namespace).can(): raise Unauthorized() else: raise Unauthorized() # Check if the dockerfile resource has already been used. If so, then it # can only be reused if the user has access to the repository in which the # dockerfile was previously built. associated_repository = model.get_repository_for_resource(dockerfile_id) if associated_repository: if not ModifyRepositoryPermission(associated_repository.namespace_user.username, associated_repository.name): raise Unauthorized() # Start the build. repo = model.get_repository(namespace, repository) prepared = PreparedBuild() prepared.build_name = user_files.get_file_checksum(dockerfile_id) prepared.dockerfile_id = dockerfile_id prepared.tags = tags prepared.subdirectory = subdir prepared.is_manual = True prepared.metadata = {} build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name) resp = build_status_view(build_request) repo_string = '%s/%s' % (namespace, repository) headers = { 'Location': api.url_for(RepositoryBuildStatus, repository=repo_string, build_uuid=build_request.uuid), } return resp, 201, headers @resource('/v1/repository//build/') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @path_param('build_uuid', 'The UUID of the build') class RepositoryBuildResource(RepositoryParamResource): """ Resource for dealing with repository builds. """ @require_repo_read @nickname('getRepoBuild') def get(self, namespace, repository, build_uuid): """ Returns information about a build. """ try: build = model.get_repository_build(build_uuid) except model.InvalidRepositoryBuildException: raise NotFound() return build_status_view(build) @require_repo_admin @nickname('cancelRepoBuild') def delete(self, namespace, repository, build_uuid): """ Cancels a repository build if it has not yet been picked up by a build worker. """ try: build = model.get_repository_build(build_uuid) except model.InvalidRepositoryBuildException: raise NotFound() if build.repository.name != repository or build.repository.namespace_user.username != namespace: raise NotFound() if model.cancel_repository_build(build, dockerfile_build_queue): return 'Okay', 201 else: raise InvalidRequest('Build is currently running or has finished') @resource('/v1/repository//build//status') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @path_param('build_uuid', 'The UUID of the build') class RepositoryBuildStatus(RepositoryParamResource): """ Resource for dealing with repository build status. """ @require_repo_read @nickname('getRepoBuildStatus') def get(self, namespace, repository, build_uuid): """ Return the status for the builds specified by the build uuids. """ build = model.get_repository_build(build_uuid) if (not build or build.repository.name != repository or build.repository.namespace_user.username != namespace): raise NotFound() return build_status_view(build) @resource('/v1/repository//build//logs') @path_param('repository', 'The full path of the repository. e.g. namespace/name') @path_param('build_uuid', 'The UUID of the build') class RepositoryBuildLogs(RepositoryParamResource): """ Resource for loading repository build logs. """ @require_repo_write @nickname('getRepoBuildLogs') def get(self, namespace, repository, build_uuid): """ Return the build logs for the build specified by the build uuid. """ response_obj = {} build = model.get_repository_build(build_uuid) if (not build or build.repository.name != repository or build.repository.namespace_user.username != namespace): raise NotFound() # If the logs have been archived, just return a URL of the completed archive if build.logs_archived: return { 'logs_url': log_archive.get_file_url(build.uuid) } start = int(request.args.get('start', 0)) try: count, logs = build_logs.get_log_entries(build.uuid, start) except BuildStatusRetrievalError: count, logs = (0, []) response_obj.update({ 'start': start, 'total': count, 'logs': [log for log in logs], }) return response_obj @resource('/v1/filedrop/') @internal_only class FileDropResource(ApiResource): """ Custom verb for setting up a client side file transfer. """ schemas = { 'FileDropRequest': { 'id': 'FileDropRequest', 'type': 'object', 'description': 'Description of the file that the user wishes to upload.', 'required': [ 'mimeType', ], 'properties': { 'mimeType': { 'type': 'string', 'description': 'Type of the file which is about to be uploaded', }, }, }, } @nickname('getFiledropUrl') @validate_json_request('FileDropRequest') def post(self): """ Request a URL to which a file may be uploaded. """ mime_type = request.get_json()['mimeType'] (url, file_id) = user_files.prepare_for_drop(mime_type, requires_cors=True) return { 'url': url, 'file_id': str(file_id), }