import logging import json from datetime import datetime, timedelta from flask import request from app import app, dockerfile_build_queue, metric_queue from data import model from data.database import db from auth.auth_context import get_authenticated_user from notifications import spawn_notification from util.names import escape_tag from util.morecollections import AttrDict logger = logging.getLogger(__name__) MAX_BUILD_QUEUE_RATE_ITEMS = app.config.get('MAX_BUILD_QUEUE_RATE_ITEMS', -1) MAX_BUILD_QUEUE_RATE_SECS = app.config.get('MAX_BUILD_QUEUE_RATE_SECS', -1) class MaximumBuildsQueuedException(Exception): """ This exception is raised when a build is requested, but the incoming build would exceed the configured maximum build rate. """ pass def start_build(repository, prepared_build, pull_robot_name=None): if repository.kind.name != 'image': raise Exception('Attempt to start a build for application repository %s' % repository.id) if MAX_BUILD_QUEUE_RATE_ITEMS > 0 and MAX_BUILD_QUEUE_RATE_SECS > 0: queue_item_canonical_name = [repository.namespace_user.username, repository.name] now = datetime.utcnow() available_min = now - timedelta(seconds=MAX_BUILD_QUEUE_RATE_SECS) available_builds = dockerfile_build_queue.num_available_jobs_between(available_min, now, queue_item_canonical_name) if available_builds >= MAX_BUILD_QUEUE_RATE_ITEMS: raise MaximumBuildsQueuedException() host = app.config['SERVER_HOSTNAME'] repo_path = '%s/%s/%s' % (host, repository.namespace_user.username, repository.name) new_token = model.token.create_access_token(repository, 'write', kind='build-worker', friendly_name='Repository Build Token') logger.debug('Creating build %s with repo %s tags %s', prepared_build.build_name, repo_path, prepared_build.tags) job_config = { 'docker_tags': prepared_build.tags, 'registry': host, 'build_subdir': prepared_build.subdirectory, 'context': prepared_build.context, 'trigger_metadata': prepared_build.metadata or {}, 'is_manual': prepared_build.is_manual, 'manual_user': get_authenticated_user().username if get_authenticated_user() else None, 'archive_url': prepared_build.archive_url } with app.config['DB_TRANSACTION_FACTORY'](db): build_request = model.build.create_repository_build(repository, new_token, job_config, prepared_build.dockerfile_id, prepared_build.build_name, prepared_build.trigger, pull_robot_name=pull_robot_name) pull_creds = model.user.get_pull_credentials(pull_robot_name) if pull_robot_name else None json_data = json.dumps({ 'build_uuid': build_request.uuid, 'pull_credentials': pull_creds }) queue_id = dockerfile_build_queue.put([repository.namespace_user.username, repository.name], json_data, retries_remaining=3) build_request.queue_id = queue_id build_request.save() # Add the queueing of the build to the metrics queue. metric_queue.repository_build_queued.Inc(labelvalues=[repository.namespace_user.username, repository.name]) # Add the build to the repo's log and spawn the build_queued notification. event_log_metadata = { 'build_id': build_request.uuid, 'docker_tags': prepared_build.tags, 'repo': repository.name, 'namespace': repository.namespace_user.username, 'is_manual': prepared_build.is_manual, 'manual_user': get_authenticated_user().username if get_authenticated_user() else None } if prepared_build.trigger: event_log_metadata['trigger_id'] = prepared_build.trigger.uuid event_log_metadata['trigger_kind'] = prepared_build.trigger.service.name event_log_metadata['trigger_metadata'] = prepared_build.metadata or {} model.log.log_action('build_dockerfile', repository.namespace_user.username, ip=request.remote_addr, metadata=event_log_metadata, repository=repository) # TODO(jzelinskie): remove when more endpoints have been converted to using interfaces repo = AttrDict({ 'namespace_name': repository.namespace_user.username, 'name': repository.name, }) spawn_notification(repo, 'build_queued', event_log_metadata, subpage='build/%s' % build_request.uuid, pathargs=['build', build_request.uuid]) return build_request class PreparedBuild(object): """ Class which holds all the information about a prepared build. The build queuing service will use this result to actually invoke the build. """ def __init__(self, trigger=None): self._dockerfile_id = None self._archive_url = None self._tags = None self._build_name = None self._subdirectory = None self._context = None self._metadata = None self._trigger = trigger self._is_manual = None @staticmethod def get_display_name(sha): return sha[0:7] def tags_from_ref(self, ref, default_branch=None): branch = ref.split('/', 2)[-1] tags = {branch} if branch == default_branch: tags.add('latest') self.tags = tags def name_from_sha(self, sha): self.build_name = PreparedBuild.get_display_name(sha) @property def is_manual(self): if self._is_manual is None: raise Exception('Property is_manual not set') return self._is_manual @is_manual.setter def is_manual(self, value): if self._is_manual is not None: raise Exception('Property is_manual already set') self._is_manual = value @property def trigger(self): return self._trigger @property def archive_url(self): return self._archive_url @archive_url.setter def archive_url(self, value): if self._archive_url: raise Exception('Property archive_url already set') self._archive_url = value @property def dockerfile_id(self): return self._dockerfile_id @dockerfile_id.setter def dockerfile_id(self, value): if self._dockerfile_id: raise Exception('Property dockerfile_id already set') self._dockerfile_id = value @property def tags(self): if not self._tags: raise Exception('Missing property tags') return self._tags @tags.setter def tags(self, value): if self._tags: raise Exception('Property tags already set') self._tags = [escape_tag(tag, default='latest') for tag in value] @property def build_name(self): if not self._build_name: raise Exception('Missing property build_name') return self._build_name @build_name.setter def build_name(self, value): if self._build_name: raise Exception('Property build_name already set') self._build_name = value @property def subdirectory(self): if self._subdirectory is None: raise Exception('Missing property subdirectory') return self._subdirectory @subdirectory.setter def subdirectory(self, value): if self._subdirectory: raise Exception('Property subdirectory already set') self._subdirectory = value @property def context(self): if self._context is None: raise Exception('Missing property context') return self._context @context.setter def context(self, value): if self._context: raise Exception('Property context already set') self._context = value @property def metadata(self): if self._metadata is None: raise Exception('Missing property metadata') return self._metadata @metadata.setter def metadata(self, value): if self._metadata: raise Exception('Property metadata already set') self._metadata = value