Merge branch 'master' into ackbar

buildman/component/buildcomponent.py

@@ -10,7 +10,6 @@ from autobahn.wamp.exception import ApplicationError
 from buildman.server import BuildJobResult
 from buildman.component.basecomponent import BaseComponent
 from buildman.jobutil.buildjob import BuildJobLoadException
-from buildman.jobutil.buildpack import BuildPackage, BuildPackageException
 from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.workererror import WorkerError
 
@@ -20,7 +19,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
 INITIAL_TIMEOUT = 25
 
-SUPPORTED_WORKER_VERSIONS = ['0.1-beta', '0.2']
+SUPPORTED_WORKER_VERSIONS = ['0.3']
 
 logger = logging.getLogger(__name__)
 
@@ -56,7 +55,10 @@ class BuildComponent(BaseComponent):
   def onJoin(self, details):
     logger.debug('Registering methods and listeners for component %s', self.builder_realm)
     yield trollius.From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
+    yield trollius.From(self.register(self._determine_cache_tag,
+                                      u'io.quay.buildworker.determinecachetag'))
     yield trollius.From(self.register(self._ping, u'io.quay.buildworker.ping'))
+
     yield trollius.From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
     yield trollius.From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
 
@@ -91,46 +93,6 @@ class BuildComponent(BaseComponent):
     buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key,
                                                  requires_cors=False)
 
-    # TODO(jschorr): Remove as soon as the fleet has been transitioned to 0.2.
-    if self._worker_version == '0.1-beta':
-      # Retrieve the job's buildpack.
-      logger.debug('Retrieving build package: %s', buildpack_url)
-      buildpack = None
-      try:
-        buildpack = BuildPackage.from_url(buildpack_url)
-      except BuildPackageException as bpe:
-        self._build_failure('Could not retrieve build package', bpe)
-        raise trollius.Return()
-
-      # Extract the base image information from the Dockerfile.
-      parsed_dockerfile = None
-      logger.debug('Parsing dockerfile')
-
-      try:
-        parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
-      except BuildPackageException as bpe:
-        self._build_failure('Could not find Dockerfile in build package', bpe)
-        raise trollius.Return()
-
-      image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
-      if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
-        self._build_failure('Missing FROM line in Dockerfile')
-        raise trollius.Return()
-
-      base_image_information = {
-          'repository': image_and_tag_tuple[0],
-          'tag': image_and_tag_tuple[1]
-      }
-
-      # Extract the number of steps from the Dockerfile.
-      with self._build_status as status_dict:
-        status_dict['total_commands'] = len(parsed_dockerfile.commands)
-    else:
-      # TODO(jschorr): This is a HACK to make sure the progress bar (sort of) continues working
-      # until such time as we have the caching code in place.
-      with self._build_status as status_dict:
-        status_dict['total_commands'] = 25
-
     # Add the pull robot information, if any.
     if build_job.pull_credentials:
       base_image_information['username'] = build_job.pull_credentials.get('username', '')
@@ -161,8 +123,7 @@ class BuildComponent(BaseComponent):
         'pull_token': build_job.repo_build.access_token.code,
         'push_token': build_job.repo_build.access_token.code,
         'tag_names': build_config.get('docker_tags', ['latest']),
-        'base_image': base_image_information,
-        'cached_tag': build_job.determine_cached_tag() or ''
+        'base_image': base_image_information
     }
 
     # Invoke the build.
@@ -256,6 +217,16 @@ class BuildComponent(BaseComponent):
     elif phase == BUILD_PHASE.BUILDING:
       self._build_status.append_log(current_status_string)
 
+  @trollius.coroutine
+  def _determine_cache_tag(self, command_comments, base_image_name, base_image_tag, base_image_id):
+    with self._build_status as status_dict:
+      status_dict['total_commands'] = len(command_comments) + 1
+
+    logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm,
+                 base_image_name, base_image_tag, base_image_id)
+
+    tag_found = self._current_job.determine_cached_tag(base_image_id, command_comments)
+    raise trollius.Return(tag_found or '')
 
   def _build_failure(self, error_message, exception=None):
     """ Handles and logs a failed build. """

buildman/jobutil/buildjob.py

@@ -1,8 +1,12 @@
 import json
+import logging
 
 from cachetools import lru_cache
 from endpoints.notificationhelper import spawn_notification
 from data import model
+from util.imagetree import ImageTree
+
+logger = logging.getLogger(__name__)
 
 
 class BuildJobLoadException(Exception):
@@ -65,14 +69,65 @@ class BuildJob(object):
           'Could not parse repository build job config with ID %s' % self.job_details['build_uuid']
       )
 
-  def determine_cached_tag(self):
+  def determine_cached_tag(self, base_image_id=None, cache_comments=None):
     """ Returns the tag to pull to prime the cache or None if none. """
-    # TODO(jschorr): Change this to use the more complicated caching rules, once we have caching
-    # be a pull of things besides the constructed tags.
-    tags = self.build_config.get('docker_tags', ['latest'])
-    existing_tags = model.list_repository_tags(self.repo_build.repository.namespace_user.username,
-                                               self.repo_build.repository.name)
+    cached_tag = None
+    if base_image_id and cache_comments:
+      cached_tag = self._determine_cached_tag_by_comments(base_image_id, cache_comments)
 
+    if not cached_tag:
+      cached_tag = self._determine_cached_tag_by_tag()
+
+    logger.debug('Determined cached tag %s for %s: %s', cached_tag, base_image_id, cache_comments)
+
+    return cached_tag
+
+  def _determine_cached_tag_by_comments(self, base_image_id, cache_commands):
+    """ Determines the tag to use for priming the cache for this build job, by matching commands
+        starting at the given base_image_id. This mimics the Docker cache checking, so it should,
+        in theory, provide "perfect" caching.
+    """
+    # Lookup the base image in the repository. If it doesn't exist, nothing more to do.
+    repo_build = self.repo_build
+    repo_namespace = repo_build.repository.namespace_user.username
+    repo_name = repo_build.repository.name
+
+    base_image = model.get_image(repo_build.repository, base_image_id)
+    if base_image is None:
+      return None
+
+    # Build an in-memory tree of the full heirarchy of images in the repository.
+    all_images = model.get_repository_images(repo_namespace, repo_name)
+    all_tags = model.list_repository_tags(repo_namespace, repo_name)
+    tree = ImageTree(all_images, all_tags, base_filter=base_image.id)
+
+    # Find a path in the tree, starting at the base image, that matches the cache comments
+    # or some subset thereof.
+    def checker(step, image):
+      if step >= len(cache_commands):
+        return False
+
+      full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
+      logger.debug('Checking step #%s: %s, %s == %s', step, image.id,
+                   image.storage.command, full_command)
+
+      return image.storage.command == full_command
+
+    path = tree.find_longest_path(base_image.id, checker)
+    if not path:
+      return None
+
+    # Find any tag associated with the last image in the path.
+    return tree.tag_containing_image(path[-1])
+
+
+  def _determine_cached_tag_by_tag(self):
+    """ Determines the cached tag by looking for one of the tags being built, and seeing if it
+        exists in the repository. This is a fallback for when no comment information is available.
+    """
+    tags = self.build_config.get('docker_tags', ['latest'])
+    repository = self.repo_build.repository
+    existing_tags = model.list_repository_tags(repository.namespace_user.username, repository.name)
     cached_tags = set(tags) & set([tag.name for tag in existing_tags])
     if cached_tags:
       return list(cached_tags)[0]

buildman/jobutil/buildpack.py

@@ -1,88 +0,0 @@
-import tarfile
-import requests
-import os
-
-from tempfile import TemporaryFile, mkdtemp
-from zipfile import ZipFile
-from util.dockerfileparse import parse_dockerfile
-from util.safetar import safe_extractall
-
-class BuildPackageException(Exception):
-  """ Exception raised when retrieving or parsing a build package. """
-  pass
-
-
-class BuildPackage(object):
-  """ Helper class for easy reading and updating of a Dockerfile build pack. """
-
-  def __init__(self, requests_file):
-    self._mime_processors = {
-        'application/zip': BuildPackage._prepare_zip,
-        'application/x-zip-compressed': BuildPackage._prepare_zip,
-        'text/plain': BuildPackage._prepare_dockerfile,
-        'application/octet-stream': BuildPackage._prepare_dockerfile,
-        'application/x-tar': BuildPackage._prepare_tarball,
-        'application/gzip': BuildPackage._prepare_tarball,
-        'application/x-gzip': BuildPackage._prepare_tarball,
-    }
-
-    c_type = requests_file.headers['content-type']
-    c_type = c_type.split(';')[0] if ';' in c_type else c_type
-
-    if c_type not in self._mime_processors:
-      raise BuildPackageException('Unknown build package mime type: %s' % c_type)
-
-    self._package_directory = None
-    try:
-      self._package_directory = self._mime_processors[c_type](requests_file)
-    except Exception as ex:
-      raise BuildPackageException(ex.message)
-
-  def parse_dockerfile(self, subdirectory):
-    dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
-    if not os.path.exists(dockerfile_path):
-      if subdirectory:
-        message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory
-      else:
-        message = 'Build package did not contain a Dockerfile at the root directory.'
-
-      raise BuildPackageException(message)
-
-    with open(dockerfile_path, 'r') as dockerfileobj:
-      return parse_dockerfile(dockerfileobj.read())
-
-  @staticmethod
-  def from_url(url):
-    buildpack_resource = requests.get(url, stream=True)
-    return BuildPackage(buildpack_resource)
-
-  @staticmethod
-  def _prepare_zip(request_file):
-    build_dir = mkdtemp(prefix='docker-build-')
-
-    # Save the zip file to temp somewhere
-    with TemporaryFile() as zip_file:
-      zip_file.write(request_file.content)
-      to_extract = ZipFile(zip_file)
-      to_extract.extractall(build_dir)
-
-    return build_dir
-
-  @staticmethod
-  def _prepare_dockerfile(request_file):
-    build_dir = mkdtemp(prefix='docker-build-')
-    dockerfile_path = os.path.join(build_dir, "Dockerfile")
-    with open(dockerfile_path, 'w') as dockerfile:
-      dockerfile.write(request_file.content)
-
-    return build_dir
-
-  @staticmethod
-  def _prepare_tarball(request_file):
-    build_dir = mkdtemp(prefix='docker-build-')
-
-    # Save the zip file to temp somewhere
-    with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
-      safe_extractall(tar_stream, build_dir)
-
-    return build_dir

buildman/jobutil/buildstatus.py

@@ -7,11 +7,12 @@ class StatusHandler(object):
 
   def __init__(self, build_logs, repository_build_uuid):
     self._current_phase = None
+    self._current_command = None
     self._uuid = repository_build_uuid
     self._build_logs = build_logs
 
     self._status = {
-        'total_commands': None,
+        'total_commands': 0,
         'current_command': None,
         'push_completion': 0.0,
         'pull_completion': 0.0,
@@ -26,9 +27,16 @@ class StatusHandler(object):
     self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data)
 
   def append_log(self, log_message, extra_data=None):
+    if log_message is None:
+      return
+
     self._append_log_message(log_message, log_data=extra_data)
 
   def set_command(self, command, extra_data=None):
+    if self._current_command == command:
+      return
+
+    self._current_command = command
     self._append_log_message(command, self._build_logs.COMMAND, extra_data)
 
   def set_error(self, error_message, extra_data=None, internal_error=False):

buildman/jobutil/workererror.py

@@ -63,6 +63,11 @@ class WorkerError(object):
         'io.quay.builder.missingorinvalidargument': {
             'message': 'Missing required arguments for builder',
             'is_internal': True
+        },
+
+        'io.quay.builder.cachelookupissue': {
+            'message': 'Error checking for a cached tag',
+            'is_internal': True
         }
     }
 

buildman/manager/executor.py

@@ -11,6 +11,7 @@ from trollius import coroutine, From, Return, get_event_loop
 from functools import partial
 
 from buildman.asyncutil import AsyncWrapper
+from container_cloud_config import CloudConfigContext
 
 
 logger = logging.getLogger(__name__)
@@ -20,7 +21,7 @@ ONE_HOUR = 60*60
 
 ENV = Environment(loader=FileSystemLoader('buildman/templates'))
 TEMPLATE = ENV.get_template('cloudconfig.yaml')
-
+CloudConfigContext().populate_jinja_environment(ENV)
 
 class ExecutorException(Exception):
   """ Exception raised when there is a problem starting or stopping a builder.

buildman/templates/cloudconfig.yaml

@@ -19,18 +19,13 @@ coreos:
     group: {{ coreos_channel }}
 
   units:
-    - name: quay-builder.service
-      command: start
-      content: |
-        [Unit]
-        Description=Quay builder container
-        Author=Jake Moshenko
-        After=docker.service
-
-        [Service]
-        TimeoutStartSec=600
-        TimeoutStopSec=2000
-        ExecStartPre=/usr/bin/docker login -u {{ quay_username }} -p {{ quay_password }} -e unused quay.io
-        ExecStart=/usr/bin/docker run --rm --net=host --name quay-builder --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs quay.io/coreos/registry-build-worker:{{ worker_tag }}
-        ExecStop=/usr/bin/docker stop quay-builder
-        ExecStopPost=/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"
+    {{ dockersystemd('quay-builder',
+                     'quay.io/coreos/registry-build-worker',
+                     quay_username,
+                     quay_password,
+                     worker_tag,
+                     extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /usr/share/ca-certificates:/etc/ssl/certs',
+                     exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
+                     flattened=True,
+                     restart_policy='no'
+                    ) | indent(4) }}