quay/endpoints/api/build.py

""" Create, list, cancel and get status/logs of repository builds. """
import os
from urlparse import urlparse

import logging
import json
import datetime
import hashlib

from flask import request

from app import userfiles as user_files, build_logs, log_archive, dockerfile_build_queue
from buildtrigger.basehandler import BuildTriggerHandler
from endpoints.api import (RepositoryParamResource, parse_args, query_param, nickname, resource,
                           require_repo_read, require_repo_write, validate_json_request,
                           ApiResource, internal_only, format_date, api, path_param,
                           require_repo_admin, abort, disallow_for_app_repositories)
from endpoints.exception import Unauthorized, NotFound, InvalidRequest
from endpoints.building import start_build, PreparedBuild, MaximumBuildsQueuedException
from data import database
from data import model
from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
                              AdministerRepositoryPermission, AdministerOrganizationPermission,
                              SuperUserPermission)

from data.buildlogs import BuildStatusRetrievalError
from util.names import parse_robot_username


logger = logging.getLogger(__name__)


def get_trigger_config(trigger):
  try:
    return json.loads(trigger.config)
  except:
    return {}


def get_job_config(build_obj):
  try:
    return json.loads(build_obj.job_config)
  except:
    return {}


def user_view(user):
  return {
    'name': user.username,
    'kind': 'user',
    'is_robot': user.robot,
  }


def trigger_view(trigger, can_read=False, can_admin=False, for_build=False):
  if trigger and trigger.uuid:
    build_trigger = BuildTriggerHandler.get_handler(trigger)
    build_source = build_trigger.config.get('build_source')

    repo_url = build_trigger.get_repository_url() if build_source else None
    can_read = can_read or can_admin

    trigger_data = {
      'id': trigger.uuid,
      'service': trigger.service.name,
      'is_active': build_trigger.is_active(),

      'build_source': build_source if can_read else None,
      'repository_url': repo_url if can_read else None,

      'config': build_trigger.config if can_admin else {},
      'can_invoke': can_admin,
    }

    if not for_build and can_admin and trigger.pull_robot:
      trigger_data['pull_robot'] = user_view(trigger.pull_robot)

    return trigger_data

  return None


def _get_build_status(build_obj):
  """ Returns the updated build phase, status and (if any) error for the build object. """
  phase = build_obj.phase
  status = {}
  error = None

  # If the build is currently running, then load its "real-time" status from Redis.
  if not database.BUILD_PHASE.is_terminal_phase(phase):
    try:
      status = build_logs.get_status(build_obj.uuid)
    except BuildStatusRetrievalError as bsre:
      phase = 'cannot_load'
      if SuperUserPermission().can():
        error = str(bsre)
      else:
        error = 'Redis may be down. Please contact support.'

    if phase != 'cannot_load':
      # If the status contains a heartbeat, then check to see if has been written in the last few
      # minutes. If not, then the build timed out.
      if status is not None and 'heartbeat' in status and status['heartbeat']:
        heartbeat = datetime.datetime.utcfromtimestamp(status['heartbeat'])
        if datetime.datetime.utcnow() - heartbeat > datetime.timedelta(minutes=1):
          phase = database.BUILD_PHASE.INTERNAL_ERROR

  # If the phase is internal error, return 'expired' instead if the number of retries
  # on the queue item is 0.
  if phase == database.BUILD_PHASE.INTERNAL_ERROR:
    retry = (build_obj.queue_id and
             dockerfile_build_queue.has_retries_remaining(build_obj.queue_id))
    if not retry:
      phase = 'expired'

  return (phase, status, error)


def build_status_view(build_obj):
  phase, status, error = _get_build_status(build_obj)
  repo_namespace = build_obj.repository.namespace_user.username
  repo_name = build_obj.repository.name

  can_read = ReadRepositoryPermission(repo_namespace, repo_name).can()
  can_write = ModifyRepositoryPermission(repo_namespace, repo_name).can()
  can_admin = AdministerRepositoryPermission(repo_namespace, repo_name).can()

  job_config = get_job_config(build_obj)

  resp = {
    'id': build_obj.uuid,
    'phase': phase,
    'started': format_date(build_obj.started),
    'display_name': build_obj.display_name,
    'status': status or {},
    'subdirectory': job_config.get('build_subdir', ''),
    'dockerfile_path': job_config.get('build_subdir', ''),
    'context': job_config.get('context', ''),
    'tags': job_config.get('docker_tags', []),
    'manual_user': job_config.get('manual_user', None),
    'is_writer': can_write,
    'trigger': trigger_view(build_obj.trigger, can_read, can_admin, for_build=True),
    'trigger_metadata': job_config.get('trigger_metadata', None) if can_read else None,
    'resource_key': build_obj.resource_key,
    'pull_robot': user_view(build_obj.pull_robot) if build_obj.pull_robot else None,
    'repository': {
      'namespace': repo_namespace,
      'name': repo_name
    },
    'error': error,
  }

  if can_write:
    if build_obj.resource_key is not None:
      resp['archive_url'] = user_files.get_file_url(build_obj.resource_key, requires_cors=True)
    elif job_config.get('archive_url', None):
      resp['archive_url'] = job_config['archive_url']

  return resp


@resource('/v1/repository/<apirepopath:repository>/build/')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
class RepositoryBuildList(RepositoryParamResource):
  """ Resource related to creating and listing repository builds. """
  schemas = {
    'RepositoryBuildRequest': {
      'type': 'object',
      'description': 'Description of a new repository build.',
      'properties': {
        'file_id': {
          'type': 'string',
          'description': 'The file id that was generated when the build spec was uploaded',
        },
        'archive_url': {
          'type': 'string',
          'description': 'The URL of the .tar.gz to build. Must start with "http" or "https".',
        },
        'subdirectory': {
          'type': 'string',
          'description': 'Subdirectory in which the Dockerfile can be found',
        },
        'pull_robot': {
          'type': 'string',
          'description': 'Username of a Quay robot account to use as pull credentials',
        },
        'docker_tags': {
          'type': 'array',
          'description': 'The tags to which the built images will be pushed. ' +
                         'If none specified, "latest" is used.',
          'items': {
            'type': 'string'
          },
          'minItems': 1,
          'uniqueItems': True
        }
      },
    },
  }

  @require_repo_read
  @parse_args()
  @query_param('limit', 'The maximum number of builds to return', type=int, default=5)
  @query_param('since', 'Returns all builds since the given unix timecode', type=int, default=None)
  @nickname('getRepoBuilds')
  @disallow_for_app_repositories
  def get(self, namespace, repository, parsed_args):
    """ Get the list of repository builds. """
    limit = parsed_args.get('limit', 5)
    since = parsed_args.get('since', None)

    if since is not None:
      since = datetime.datetime.utcfromtimestamp(since)

    builds = model.build.list_repository_builds(namespace, repository, limit, since=since)
    return {
      'builds': [build_status_view(build) for build in builds]
    }

  @require_repo_write
  @nickname('requestRepoBuild')
  @disallow_for_app_repositories
  @validate_json_request('RepositoryBuildRequest')
  def post(self, namespace, repository):
    """ Request that a repository be built and pushed from the specified input. """
    logger.debug('User requested repository initialization.')
    request_json = request.get_json()

    dockerfile_id = request_json.get('file_id', None)
    archive_url = request_json.get('archive_url', None)

    if not dockerfile_id and not archive_url:
      raise InvalidRequest('file_id or archive_url required')

    if archive_url:
      archive_match = None
      try:
        archive_match = urlparse(archive_url)
      except ValueError:
        pass

      if not archive_match:
        raise InvalidRequest('Invalid Archive URL: Must be a valid URI')

      scheme = archive_match.scheme
      if scheme != 'http' and scheme != 'https':
        raise InvalidRequest('Invalid Archive URL: Must be http or https')

    subdir = request_json['subdirectory'] if 'subdirectory' in request_json else ''
    context = request_json['context'] if 'context' in request_json else os.path.dirname(subdir)
    tags = request_json.get('docker_tags', ['latest'])
    pull_robot_name = request_json.get('pull_robot', None)

    # Verify the security behind the pull robot.
    if pull_robot_name:
      result = parse_robot_username(pull_robot_name)
      if result:
        try:
          model.user.lookup_robot(pull_robot_name)
        except model.InvalidRobotException:
          raise NotFound()

        # Make sure the user has administer permissions for the robot's namespace.
        (robot_namespace, _) = result
        if not AdministerOrganizationPermission(robot_namespace).can():
          raise Unauthorized()
      else:
        raise Unauthorized()

    # Check if the dockerfile resource has already been used. If so, then it
    # can only be reused if the user has access to the repository in which the
    # dockerfile was previously built.
    if dockerfile_id:
      associated_repository = model.build.get_repository_for_resource(dockerfile_id)
      if associated_repository:
        if not ModifyRepositoryPermission(associated_repository.namespace_user.username,
                                          associated_repository.name):
          raise Unauthorized()

    # Start the build.
    repo = model.repository.get_repository(namespace, repository)
    if repo is None:
      raise NotFound()

    try:
      build_name = (user_files.get_file_checksum(dockerfile_id)
                    if dockerfile_id
                    else hashlib.sha224(archive_url).hexdigest()[0:7])
    except IOError:
      raise InvalidRequest('File %s could not be found or is invalid' %  dockerfile_id)

    prepared = PreparedBuild()
    prepared.build_name = build_name
    prepared.dockerfile_id = dockerfile_id
    prepared.archive_url = archive_url
    prepared.tags = tags
    prepared.subdirectory = subdir
    prepared.context = context
    prepared.is_manual = True
    prepared.metadata = {}
    try:
      build_request = start_build(repo, prepared, pull_robot_name=pull_robot_name)
    except MaximumBuildsQueuedException:
      abort(429, message='Maximum queued build rate exceeded.')

    resp = build_status_view(build_request)
    repo_string = '%s/%s' % (namespace, repository)
    headers = {
      'Location': api.url_for(RepositoryBuildStatus, repository=repo_string,
                              build_uuid=build_request.uuid),
    }
    return resp, 201, headers




@resource('/v1/repository/<apirepopath:repository>/build/<build_uuid>')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')
class RepositoryBuildResource(RepositoryParamResource):
  """ Resource for dealing with repository builds. """
  @require_repo_read
  @nickname('getRepoBuild')
  @disallow_for_app_repositories
  def get(self, namespace, repository, build_uuid):
    """ Returns information about a build. """
    try:
      build = model.build.get_repository_build(build_uuid)
    except model.build.InvalidRepositoryBuildException:
      raise NotFound()

    if build.repository.name != repository or build.repository.namespace_user.username != namespace:
      raise NotFound()

    return build_status_view(build)

  @require_repo_admin
  @nickname('cancelRepoBuild')
  @disallow_for_app_repositories
  def delete(self, namespace, repository, build_uuid):
    """ Cancels a repository build. """
    try:
      build = model.build.get_repository_build(build_uuid)
    except model.build.InvalidRepositoryBuildException:
      raise NotFound()

    if build.repository.name != repository or build.repository.namespace_user.username != namespace:
      raise NotFound()

    if model.build.cancel_repository_build(build, dockerfile_build_queue):
      return 'Okay', 201
    else:
      raise InvalidRequest('Build is currently running or has finished')


@resource('/v1/repository/<apirepopath:repository>/build/<build_uuid>/status')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')
class RepositoryBuildStatus(RepositoryParamResource):
  """ Resource for dealing with repository build status. """
  @require_repo_read
  @nickname('getRepoBuildStatus')
  @disallow_for_app_repositories
  def get(self, namespace, repository, build_uuid):
    """ Return the status for the builds specified by the build uuids. """
    build = model.build.get_repository_build(build_uuid)
    if (not build or build.repository.name != repository or
        build.repository.namespace_user.username != namespace):
      raise NotFound()

    return build_status_view(build)


def get_logs_or_log_url(build):
  # If the logs have been archived, just return a URL of the completed archive
  if build.logs_archived:
    return {
      'logs_url': log_archive.get_file_url(build.uuid, requires_cors=True)
    }
  start = int(request.args.get('start', 0))

  try:
    count, logs = build_logs.get_log_entries(build.uuid, start)
  except BuildStatusRetrievalError:
    count, logs = (0, [])

  response_obj = {}
  response_obj.update({
    'start': start,
    'total': count,
    'logs': [log for log in logs],
  })

  return response_obj


@resource('/v1/repository/<apirepopath:repository>/build/<build_uuid>/logs')
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
@path_param('build_uuid', 'The UUID of the build')
class RepositoryBuildLogs(RepositoryParamResource):
  """ Resource for loading repository build logs. """
  @require_repo_write
  @nickname('getRepoBuildLogs')
  @disallow_for_app_repositories
  def get(self, namespace, repository, build_uuid):
    """ Return the build logs for the build specified by the build uuid. """
    build = model.build.get_repository_build(build_uuid)
    if (not build or build.repository.name != repository or
        build.repository.namespace_user.username != namespace):
      raise NotFound()

    return get_logs_or_log_url(build)


@resource('/v1/filedrop/')
@internal_only
class FileDropResource(ApiResource):
  """ Custom verb for setting up a client side file transfer. """
  schemas = {
    'FileDropRequest': {
      'type': 'object',
      'description': 'Description of the file that the user wishes to upload.',
      'required': [
        'mimeType',
      ],
      'properties': {
        'mimeType': {
          'type': 'string',
          'description': 'Type of the file which is about to be uploaded',
        },
      },
    },
  }

  @nickname('getFiledropUrl')
  @validate_json_request('FileDropRequest')
  def post(self):
    """ Request a URL to which a file may be uploaded. """
    mime_type = request.get_json()['mimeType']
    (url, file_id) = user_files.prepare_for_drop(mime_type, requires_cors=True)
    return {
      'url': url,
      'file_id': str(file_id),
    }