Merge branch 'rustedbuilds' of https://bitbucket.org/yackob03/quay into rustedbuilds

data/database.py

@@ -237,8 +237,7 @@ class RepositoryBuild(BaseModel):
   uuid = CharField(default=uuid_generator, index=True)
   repository = ForeignKeyField(Repository, index=True)
   access_token = ForeignKeyField(AccessToken)
-  resource_key = CharField()
+  job_config = TextField()
-  tag = CharField()
   phase = CharField(default='waiting')
   started = DateTimeField(default=datetime.now)
   display_name = CharField()

data/model.py

@@ -1409,10 +1409,10 @@ def list_repository_builds(namespace_name, repository_name,
   return query
 
 
-def create_repository_build(repo, access_token, resource_key, tag,
+def create_repository_build(repo, access_token, job_config_obj,
                             display_name, trigger=None):
   return RepositoryBuild.create(repository=repo, access_token=access_token,
-                                resource_key=resource_key, tag=tag,
+                                job_config=json.dumps(job_config_obj),
                                 display_name=display_name, trigger=trigger)
 
 
@@ -1508,16 +1508,6 @@ def list_build_triggers(namespace_name, repository_name):
            Repository.name == repository_name))
 
 
-def delete_build_trigger(namespace_name, repository_name, trigger_uuid):
-  trigger = get_build_trigger(namespace_name, repository_name, trigger_uuid)
-
-  # Delete the access token created for this trigger, and the trigger itself
-  if trigger.write_token and trigger.write_token.code:
-    trigger.write_token.delete_instance()
-
-  trigger.delete_instance()
-
-
 def list_trigger_builds(namespace_name, repository_name, trigger_uuid,
                         limit=None):
   query = (list_repository_builds(namespace_name, repository_name)

data/queue.py

@@ -68,5 +68,5 @@ class WorkQueue(object):
 
 
 image_diff_queue = WorkQueue('imagediff')
-dockerfile_build_queue = WorkQueue('dockerfilebuild2')
+dockerfile_build_queue = WorkQueue('dockerfilebuild3')
 webhook_queue = WorkQueue('webhook')

endpoints/api.py

@@ -29,7 +29,9 @@ from auth.permissions import (ReadRepositoryPermission,
                               ViewTeamPermission,
                               UserPermission)
 from endpoints.common import common_login, get_route_data, truthy_param
-from endpoints.trigger import BuildTrigger, TriggerActivationException, EmptyRepositoryException
+from endpoints.trigger import (BuildTrigger, TriggerActivationException,
+                               TriggerDeactivationException, EmptyRepositoryException)
+
 from util.cache import cache_control
 from datetime import datetime, timedelta
 
@@ -1144,7 +1146,7 @@ def build_status_view(build_obj, can_write=False):
     'started': build_obj.started,
     'display_name': build_obj.display_name,
     'status': status,
-    'resource_key': build_obj.resource_key if can_write else None,
+    'job_config': json.loads(build_obj.job_config) if can_write else None,
     'is_writer': can_write,
     'trigger': trigger_view(build_obj.trigger),
   }
@@ -1234,11 +1236,13 @@ def request_repo_build(namespace, repository):
     logger.debug('User requested repository initialization.')
     dockerfile_id = request.get_json()['file_id']
 
-    # Check if the dockerfile resource has already been used. If so, then it can only be reused if the
+    # Check if the dockerfile resource has already been used. If so, then it
-    # user has access to the repository for which it was used.
+    # can only be reused if the user has access to the repository for which it
+    # was used.
     associated_repository = model.get_repository_for_resource(dockerfile_id)
     if associated_repository:
-      if not ModifyRepositoryPermission(associated_repository.namespace, associated_repository.name):
+      if not ModifyRepositoryPermission(associated_repository.namespace,
+                                        associated_repository.name):
         abort(403)
 
     # Start the build.
@@ -1248,9 +1252,15 @@ def request_repo_build(namespace, repository):
     logger.debug('**********Md5: %s' % display_name)
 
     host = urlparse.urlparse(request.url).netloc
-    tag = '%s/%s/%s' % (host, repo.namespace, repo.name)
+    repo = '%s/%s/%s' % (host, repo.namespace, repo.name)
-    build_request = model.create_repository_build(repo, token, dockerfile_id,
+    job_config = {
-                                                  tag, display_name)
+      'docker_tags': ['latest'],
+      'build_subdir': '',
+      'repository': repo,
+      'resource_key': dockerfile_id,
+    }
+    build_request = model.create_repository_build(repo, token, job_config,
+                                                  display_name)
     dockerfile_build_queue.put(json.dumps({
       'build_uuid': build_request.uuid,
       'namespace': namespace,
@@ -1438,15 +1448,15 @@ def activate_build_trigger(namespace, repository, trigger_uuid):
                                           token.code, app.config['URL_HOST'],
                                           path)
 
-        handler.activate(trigger.uuid, authed_url, trigger.auth_token,
+        final_config = handler.activate(trigger.uuid, authed_url,
-                         new_config_dict)
+                                        trigger.auth_token, new_config_dict)
       except TriggerActivationException as e:
         token.delete_instance()
         abort(400, message = e.msg)
         return
 
       # Save the updated config.
-      trigger.config = json.dumps(new_config_dict)
+      trigger.config = json.dumps(final_config)
       trigger.write_token = token
       trigger.save()
 
@@ -1517,7 +1527,22 @@ def list_build_triggers(namespace, repository):
 def delete_build_trigger(namespace, repository, trigger_uuid):
   permission = AdministerRepositoryPermission(namespace, repository)
   if permission.can():
-    model.delete_build_trigger(namespace, repository, trigger_uuid)
+    try:
+      trigger = model.get_build_trigger(namespace, repository, trigger_uuid)
+    except model.InvalidBuildTriggerException:
+      abort(404)
+      return
+
+    handler = BuildTrigger.get_trigger_for_service(trigger.service.name)
+    config_dict = json.loads(trigger.config)
+    if handler.is_active(config_dict):
+      try:
+        handler.deactivate(trigger.auth_token, config_dict)
+      except TriggerDeactivationException as ex:
+        # We are just going to eat this error
+        logger.warning('Trigger deactivation problem.', ex)
+
+    trigger.delete_instance()
     log_action('delete_repo_trigger', namespace, 
                {'repo': repository, 'trigger_id': trigger_uuid},
                repo=model.get_repository(namespace, repository))

endpoints/index.py

@@ -120,6 +120,11 @@ def get_user():
       'username': get_authenticated_user().username,
       'email': get_authenticated_user().email,
     })
+  elif get_validated_token():
+    return jsonify({
+      'username': '$token',
+      'email': None,
+    })
   abort(404)
 
 

endpoints/trigger.py

@@ -28,6 +28,9 @@ class InvalidServiceException(Exception):
 class TriggerActivationException(Exception):
   pass
 
+class TriggerDeactivationException(Exception):
+  pass
+
 class ValidationRequestException(Exception):
   pass
 
@@ -55,7 +58,8 @@ class BuildTrigger(object):
 
   def handle_trigger_request(self, request, auth_token, config):
     """
-    Transform the incoming request data into a set of actions.
+    Transform the incoming request data into a set of actions. Returns a tuple
+    of usefiles resource id, docker tags, build name, and resource subdir.
     """
     raise NotImplementedError
 
@@ -69,6 +73,15 @@ class BuildTrigger(object):
   def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
     """ 
     Activates the trigger for the service, with the given new configuration.
+    Returns new configuration that should be stored if successful.
+    """
+    raise NotImplementedError
+
+  def deactivate(self, auth_token, config):
+    """
+    Deactivates the trigger for the service, removing any hooks installed in
+    the remote service. Returns the new config that should be stored if this
+    trigger is going to be re-activated.
     """
     raise NotImplementedError
 
@@ -103,7 +116,7 @@ class GithubBuildTrigger(BuildTrigger):
     return 'github'
 
   def is_active(self, config):
-    return 'build_source' in config and len(config['build_source']) > 0
+    return 'hook_id' in config
 
   def activate(self, trigger_uuid, standard_webhook_url, auth_token, config):
     new_build_source = config['build_source']
@@ -121,11 +134,30 @@ class GithubBuildTrigger(BuildTrigger):
     }
 
     try:
-      to_add_webhook.create_hook('web', webhook_config)
+      hook = to_add_webhook.create_hook('web', webhook_config)
+      config['hook_id'] = hook.id
     except GithubException:
       msg = 'Unable to create webhook on repository: %s'
       raise TriggerActivationException(msg % new_build_source)
 
+    return config
+
+  def deactivate(self, auth_token, config):
+    gh_client = self._get_client(auth_token)
+
+    try:
+      repo = gh_client.get_repo(config['build_source'])
+      to_delete = repo.get_hook(config['hook_id'])
+      to_delete.delete()
+    except GithubException:
+      msg = 'Unable to remove hook: %s' % config['hook_id']
+      raise TriggerDeactivationException(msg)
+
+    config.pop('hook_id', None)
+
+    return config
+
+
   def list_build_sources(self, auth_token):
     gh_client = self._get_client(auth_token)
     usr = gh_client.get_user()
@@ -163,7 +195,7 @@ class GithubBuildTrigger(BuildTrigger):
 
     try:
       repo = gh_client.get_repo(source)
-      default_commit = repo.get_branch(repo.default_branch).commit
+      default_commit = repo.get_branch(repo.master_branch).commit
       commit_tree = repo.get_git_tree(default_commit.sha, recursive=True)
 
       return [os.path.dirname(elem.path) for elem in commit_tree.tree
@@ -181,7 +213,8 @@ class GithubBuildTrigger(BuildTrigger):
 
     logger.debug('Payload %s', payload)
     ref = payload['ref']
-    commit_id = payload['head_commit']['id'][0:7]
+    commit_sha = payload['head_commit']['id']
+    short_sha = commit_sha[0:7]
 
     gh_client = self._get_client(auth_token)
 
@@ -192,8 +225,7 @@ class GithubBuildTrigger(BuildTrigger):
     logger.debug('Github repo: %s', repo)
 
     # Prepare the download and upload URLs
-    branch_name = ref.split('/')[-1]
+    archive_link = repo.get_archive_link('zipball', short_sha)
-    archive_link = repo.get_archive_link('zipball', branch_name)
     download_archive = client.get(archive_link, stream=True)
 
     with SpooledTemporaryFile(CHUNK_SIZE) as zipball:
@@ -204,4 +236,17 @@ class GithubBuildTrigger(BuildTrigger):
 
     logger.debug('Successfully prepared job')
 
-    return dockerfile_id, branch_name, commit_id
+    # compute the tag(s)
+    pushed_branch = ref.split('/')[-1]
+    tags = {pushed_branch}
+    if pushed_branch == repo.master_branch:
+      tags.add('latest')
+    logger.debug('Pushing to tags: %s' % tags)
+
+    # compute the subdir
+    repo_subdir = config['subdir']
+    zipball_subdir = '%s-%s-%s' % (repo.owner.login, repo.name, short_sha)
+    joined_subdir = os.path.join(zipball_subdir, repo_subdir)
+    logger.debug('Final subdir: %s' % joined_subdir)
+
+    return dockerfile_id, list(tags), short_sha, joined_subdir

endpoints/webhooks.py

@@ -1,10 +1,12 @@
 import logging
 import stripe
 import urlparse
+import json
 
 from flask import request, make_response, Blueprint
 
 from data import model
+from data.queue import dockerfile_build_queue
 from auth.auth import process_auth
 from auth.permissions import ModifyRepositoryPermission
 from util.invoice import renderInvoiceToHtml
@@ -62,22 +64,36 @@ def build_trigger_webhook(namespace, repository, trigger_uuid):
 
     logger.debug('Passing webhook request to handler %s', handler)
     try:
-      df_id, tag, name = handler.handle_trigger_request(request,
+      specs = handler.handle_trigger_request(request, trigger.auth_token,
-                                                        trigger.auth_token,
+                                             json.loads(trigger.config))
-                                                        trigger.config)
+      dockerfile_id, tags, name, subdir = specs
+
     except ValidationRequestException:
-      # This was just a validation request, don't need to build anything
+      # This was just a validation request, we don't need to build anything
       return make_response('Okay')
 
     host = urlparse.urlparse(request.url).netloc
-    full_tag = '%s/%s/%s:%s' % (host, trigger.repository.namespace,
+    repo = '%s/%s/%s' % (host, trigger.repository.namespace,
-                                trigger.repository.name, tag)
+                         trigger.repository.name)
 
     token = model.create_access_token(trigger.repository, 'write')
-    logger.debug('Creating build %s with full_tag %s and dockerfile_id %s',
+    logger.debug('Creating build %s with repo %s tags %s and dockerfile_id %s',
-                 name, full_tag, df_id)
+                 name, repo, tags, dockerfile_id)
-    model.create_repository_build(trigger.repository, token, df_id, full_tag,
+
-                                  name)
+    job_config = {
+      'docker_tags': tags,
+      'repository': repo,
+      'build_subdir': subdir,
+      'resource_key': dockerfile_id,
+    }
+    build_request = model.create_repository_build(trigger.repository, token,
+                                                  job_config, name)
+
+    dockerfile_build_queue.put(json.dumps({
+      'build_uuid': build_request.uuid,
+      'namespace': namespace,
+      'repository': repository,
+    }), retries_remaining=1)
 
     return make_response('Okay')
 

initdb.py

@@ -309,18 +309,24 @@ def populate_database():
                                    False, [], (0, [], None))
 
   token = model.create_access_token(building, 'write')
-  tag = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
 
   trigger = model.create_build_trigger(building, 'github', '123authtoken',
                                        new_user_1)
   trigger.config = json.dumps({
     'build_source': 'jakedt/testconnect',
+    'subdir': '',
   })
   trigger.save()
 
-  build = model.create_repository_build(building, token,
+  repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace, building.name)
-                                        '701dcc3724fb4f2ea6c31400528343cd',
+  job_config = {
-                                        tag, 'build-name', trigger)
+    'repository': repo,
+    'docker_tags': ['latest'],
+    'build_subdir': '',
+    'resource_key': '701dcc3724fb4f2ea6c31400528343cd',
+  }
+  build = model.create_repository_build(building, token, job_config,
+                                        'build-name', trigger)
   build.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef'
   build.save()
 

workers/dockerfilebuild.py

@@ -50,18 +50,23 @@ class StatusWrapper(object):
 
 
 class DockerfileBuildContext(object):
-  def __init__(self, build_context_dir, tag_name, push_token, build_uuid):
+  def __init__(self, build_context_dir, dockerfile_subdir, repo, tag_names,
+               push_token, build_uuid):
     self._build_dir = build_context_dir
-    self._tag_name = tag_name
+    self._dockerfile_subdir = dockerfile_subdir
+    self._repo = repo
+    self._tag_names = tag_names
     self._push_token = push_token
     self._cl = Client(timeout=1200, version='1.7')
     self._status = StatusWrapper(build_uuid)
     self._build_logger = partial(build_logs.append_log_message, build_uuid)
 
-    dockerfile_path = os.path.join(self._build_dir, "Dockerfile")
+    dockerfile_path = os.path.join(self._build_dir, dockerfile_subdir,
+                                   "Dockerfile")
     self._num_steps = DockerfileBuildContext.__count_steps(dockerfile_path)
 
-    logger.debug('Will build and push to tag named: %s' % self._tag_name)
+    logger.debug('Will build and push to repo %s with tags named: %s' %
+                 (self._repo, self._tag_names))
 
   def __enter__(self):
     return self
@@ -94,9 +99,13 @@ class DockerfileBuildContext(object):
     with self._status as status:
       status['total_commands'] = self._num_steps
 
-    logger.debug('Building to tag named: %s' % self._tag_name)
+    logger.debug('Building to tags named: %s' % self._tag_names)
-    build_status = self._cl.build(path=self._build_dir, tag=self._tag_name,
+    context_path = os.path.join(self._build_dir, self._dockerfile_subdir)
-                                  stream=True)
+
+    logger.debug('Final context path: %s exists: %s' %
+                 (context_path, os.path.exists(context_path)))
+
+    build_status = self._cl.build(path=context_path, stream=True)
 
     current_step = 0
     built_image = None
@@ -128,9 +137,9 @@ class DockerfileBuildContext(object):
 
   def push(self, built_image):
     # Login to the registry
-    host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._tag_name)
+    host = re.match(r'([a-z0-9.:]+)/.+/.+$', self._repo)
     if not host:
-      raise RuntimeError('Invalid tag name: %s' % self._tag_name)
+      raise RuntimeError('Invalid repo name: %s' % self._repo)
 
     for protocol in ['https', 'http']:
       registry_endpoint = '%s://%s/v1/' % (protocol, host.group(1))
@@ -142,13 +151,18 @@ class DockerfileBuildContext(object):
       except APIError:
         pass  # Probably the wrong protocol
 
+    for tag in self._tag_names:
+      logger.debug('Tagging image %s as %s:%s' %
+                   (built_image, self._repo, tag))
+      self._cl.tag(built_image, self._repo, tag)
+
     history = json.loads(self._cl.history(built_image))
     num_images = len(history)
     with self._status as status:
       status['total_images'] = num_images
 
-    logger.debug('Pushing to tag name: %s' % self._tag_name)
+    logger.debug('Pushing to repo %s' % self._repo)
-    resp = self._cl.push(self._tag_name, stream=True)
+    resp = self._cl.push(self._repo, stream=True)
 
     for status_str in resp:
       status = json.loads(status_str)
@@ -258,8 +272,13 @@ class DockerfileBuildWorker(Worker):
                                                   job_details['repository'],
                                                   job_details['build_uuid'])
 
-    resource_url = user_files.get_file_url(repository_build.resource_key)
+    job_config = json.loads(repository_build.job_config)
-    tag_name = repository_build.tag
+
+    resource_url = user_files.get_file_url(job_config['resource_key'])
+    tag_names = job_config['docker_tags']
+    build_subdir = job_config['build_subdir']
+    repo = job_config['repository']
+
     access_token = repository_build.access_token.code
 
     log_appender = partial(build_logs.append_log_message,
@@ -267,16 +286,15 @@ class DockerfileBuildWorker(Worker):
 
     log_appender('initializing', build_logs.PHASE)
 
-    start_msg = ('Starting job with resource url: %s tag: %s' % (resource_url,
+    start_msg = ('Starting job with resource url: %s repo: %s' % (resource_url,
-                                                                 tag_name))
+                                                                  repo))
-    logger.debug(start_msg)
     log_appender(start_msg)
 
     docker_resource = requests.get(resource_url)
     c_type = docker_resource.headers['content-type']
 
-    filetype_msg = ('Request to build file of type: %s with tag: %s' %
+    filetype_msg = ('Request to build type: %s with repo: %s and tags: %s' %
-                    (c_type, tag_name))
+                    (c_type, repo, tag_names))
     logger.info(filetype_msg)
     log_appender(filetype_msg)
 
@@ -288,7 +306,8 @@ class DockerfileBuildWorker(Worker):
     repository_build.phase = 'building'
     repository_build.save()
 
-    with DockerfileBuildContext(build_dir, tag_name, access_token,
+    with DockerfileBuildContext(build_dir, build_subdir, repo, tag_names,
+                                access_token,
                                 repository_build.uuid) as build_ctxt:
       try:
         built_image = build_ctxt.build()
@@ -298,7 +317,7 @@ class DockerfileBuildWorker(Worker):
           repository_build.phase = 'error'
           repository_build.save()
           log_appender('Unable to build dockerfile.', build_logs.ERROR)
-          return False
+          return True
 
         log_appender('pushing', build_logs.PHASE)
         repository_build.phase = 'pushing'
@@ -316,7 +335,7 @@ class DockerfileBuildWorker(Worker):
         repository_build.phase = 'error'
         repository_build.save()
         log_appender(str(exc), build_logs.ERROR)
-        return False
+        return True
 
     return True