Add back in the cache checking code and remove the old 0.1 build pack code

buildman/component/buildcomponent.py

@@ -10,7 +10,6 @@ from autobahn.wamp.exception import ApplicationError
 from buildman.server import BuildJobResult
 from buildman.component.basecomponent import BaseComponent
 from buildman.jobutil.buildjob import BuildJobLoadException
-from buildman.jobutil.buildpack import BuildPackage, BuildPackageException
 from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.workererror import WorkerError
 
@@ -20,7 +19,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
 INITIAL_TIMEOUT = 25
 
-SUPPORTED_WORKER_VERSIONS = ['0.1-beta', '0.2']
+SUPPORTED_WORKER_VERSIONS = ['0.2']
 
 logger = logging.getLogger(__name__)
 
@@ -56,7 +55,10 @@ class BuildComponent(BaseComponent):
   def onJoin(self, details):
     logger.debug('Registering methods and listeners for component %s', self.builder_realm)
     yield trollius.From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
+    yield trollius.From(self.register(self._determine_cache_tag,
+                                      u'io.quay.buildworker.determinecachetag'))
     yield trollius.From(self.register(self._ping, u'io.quay.buildworker.ping'))
+
     yield trollius.From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
     yield trollius.From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
 
@@ -91,46 +93,6 @@ class BuildComponent(BaseComponent):
     buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key,
                                                  requires_cors=False)
 
-    # TODO(jschorr): Remove as soon as the fleet has been transitioned to 0.2.
-    if self._worker_version == '0.1-beta':
-      # Retrieve the job's buildpack.
-      logger.debug('Retrieving build package: %s', buildpack_url)
-      buildpack = None
-      try:
-        buildpack = BuildPackage.from_url(buildpack_url)
-      except BuildPackageException as bpe:
-        self._build_failure('Could not retrieve build package', bpe)
-        raise trollius.Return()
-
-      # Extract the base image information from the Dockerfile.
-      parsed_dockerfile = None
-      logger.debug('Parsing dockerfile')
-
-      try:
-        parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
-      except BuildPackageException as bpe:
-        self._build_failure('Could not find Dockerfile in build package', bpe)
-        raise trollius.Return()
-
-      image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
-      if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
-        self._build_failure('Missing FROM line in Dockerfile')
-        raise trollius.Return()
-
-      base_image_information = {
-          'repository': image_and_tag_tuple[0],
-          'tag': image_and_tag_tuple[1]
-      }
-
-      # Extract the number of steps from the Dockerfile.
-      with self._build_status as status_dict:
-        status_dict['total_commands'] = len(parsed_dockerfile.commands)
-    else:
-      # TODO(jschorr): This is a HACK to make sure the progress bar (sort of) continues working
-      # until such time as we have the caching code in place.
-      with self._build_status as status_dict:
-        status_dict['total_commands'] = 25
-
     # Add the pull robot information, if any.
     if build_job.pull_credentials:
       base_image_information['username'] = build_job.pull_credentials.get('username', '')
@@ -161,8 +123,7 @@ class BuildComponent(BaseComponent):
         'pull_token': build_job.repo_build.access_token.code,
         'push_token': build_job.repo_build.access_token.code,
         'tag_names': build_config.get('docker_tags', ['latest']),
-        'base_image': base_image_information,
-        'cached_tag': build_job.determine_cached_tag() or ''
+        'base_image': base_image_information
     }
 
     # Invoke the build.
@@ -256,6 +217,14 @@ class BuildComponent(BaseComponent):
     elif phase == BUILD_PHASE.BUILDING:
       self._build_status.append_log(current_status_string)
 
+  def _determine_cache_tag(self, cache_commands, base_image_name, base_image_tag, base_image_id):
+    with self._build_status as status_dict:
+      status_dict['total_commands'] = len(cache_commands) + 1
+
+    logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm,
+                 base_image_name, base_image_tag, base_image_id)
+
+    return self._current_job.determine_cached_tag(base_image_id, cache_commands) or ''
 
   def _build_failure(self, error_message, exception=None):
     """ Handles and logs a failed build. """

buildman/jobutil/buildpack.py

@@ -1,88 +0,0 @@
-import tarfile
-import requests
-import os
-
-from tempfile import TemporaryFile, mkdtemp
-from zipfile import ZipFile
-from util.dockerfileparse import parse_dockerfile
-from util.safetar import safe_extractall
-
-class BuildPackageException(Exception):
-  """ Exception raised when retrieving or parsing a build package. """
-  pass
-
-
-class BuildPackage(object):
-  """ Helper class for easy reading and updating of a Dockerfile build pack. """
-
-  def __init__(self, requests_file):
-    self._mime_processors = {
-        'application/zip': BuildPackage._prepare_zip,
-        'application/x-zip-compressed': BuildPackage._prepare_zip,
-        'text/plain': BuildPackage._prepare_dockerfile,
-        'application/octet-stream': BuildPackage._prepare_dockerfile,
-        'application/x-tar': BuildPackage._prepare_tarball,
-        'application/gzip': BuildPackage._prepare_tarball,
-        'application/x-gzip': BuildPackage._prepare_tarball,
-    }
-
-    c_type = requests_file.headers['content-type']
-    c_type = c_type.split(';')[0] if ';' in c_type else c_type
-
-    if c_type not in self._mime_processors:
-      raise BuildPackageException('Unknown build package mime type: %s' % c_type)
-
-    self._package_directory = None
-    try:
-      self._package_directory = self._mime_processors[c_type](requests_file)
-    except Exception as ex:
-      raise BuildPackageException(ex.message)
-
-  def parse_dockerfile(self, subdirectory):
-    dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
-    if not os.path.exists(dockerfile_path):
-      if subdirectory:
-        message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory
-      else:
-        message = 'Build package did not contain a Dockerfile at the root directory.'
-
-      raise BuildPackageException(message)
-
-    with open(dockerfile_path, 'r') as dockerfileobj:
-      return parse_dockerfile(dockerfileobj.read())
-
-  @staticmethod
-  def from_url(url):
-    buildpack_resource = requests.get(url, stream=True)
-    return BuildPackage(buildpack_resource)
-
-  @staticmethod
-  def _prepare_zip(request_file):
-    build_dir = mkdtemp(prefix='docker-build-')
-
-    # Save the zip file to temp somewhere
-    with TemporaryFile() as zip_file:
-      zip_file.write(request_file.content)
-      to_extract = ZipFile(zip_file)
-      to_extract.extractall(build_dir)
-
-    return build_dir
-
-  @staticmethod
-  def _prepare_dockerfile(request_file):
-    build_dir = mkdtemp(prefix='docker-build-')
-    dockerfile_path = os.path.join(build_dir, "Dockerfile")
-    with open(dockerfile_path, 'w') as dockerfile:
-      dockerfile.write(request_file.content)
-
-    return build_dir
-
-  @staticmethod
-  def _prepare_tarball(request_file):
-    build_dir = mkdtemp(prefix='docker-build-')
-
-    # Save the zip file to temp somewhere
-    with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
-      safe_extractall(tar_stream, build_dir)
-
-    return build_dir

buildman/jobutil/buildstatus.py

@@ -11,7 +11,7 @@ class StatusHandler(object):
     self._build_logs = build_logs
 
     self._status = {
-        'total_commands': None,
+        'total_commands': 0,
         'current_command': None,
         'push_completion': 0.0,
         'pull_completion': 0.0,