Add support for v2 of the build worker, which performs the Dockerfile parsing on its own. Note that this version is backwards compatible with v1-beta of the build worker, so it should be pushed first. Also note that this version is temporary until such time as we get the caching branches merged.

buildman/component/buildcomponent.py

@@ -20,7 +20,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30)
 HEARTBEAT_TIMEOUT = 10
 INITIAL_TIMEOUT = 25
 
-SUPPORTED_WORKER_VERSIONS = ['0.1-beta']
+SUPPORTED_WORKER_VERSIONS = ['0.1-beta', '0.2']
 
 logger = logging.getLogger(__name__)
 
@@ -46,6 +46,7 @@ class BuildComponent(BaseComponent):
     self._current_job = None
     self._build_status = None
     self._image_info = None
+    self._worker_version = None
 
     BaseComponent.__init__(self, config, **kwargs)
 
@@ -68,6 +69,9 @@ class BuildComponent(BaseComponent):
   @trollius.coroutine
   def start_build(self, build_job):
     """ Starts a build. """
+    logger.debug('Starting build for component %s (worker version: %s)',
+                 self.builder_realm, self._worker_version)
+
     self._current_job = build_job
     self._build_status = StatusHandler(self.build_logs, build_job.repo_build.uuid)
     self._image_info = {}
@@ -77,46 +81,55 @@ class BuildComponent(BaseComponent):
     # Send the notification that the build has started.
     build_job.send_notification('build_start')
 
-    # Retrieve the job's buildpack.
-    buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key,
-                                                 requires_cors=False)
-
-    logger.debug('Retrieving build package: %s', buildpack_url)
-    buildpack = None
-    try:
-      buildpack = BuildPackage.from_url(buildpack_url)
-    except BuildPackageException as bpe:
-      self._build_failure('Could not retrieve build package', bpe)
-      raise trollius.Return()
-
-    # Extract the base image information from the Dockerfile.
-    parsed_dockerfile = None
-    logger.debug('Parsing dockerfile')
-
+    # Parse the build configuration.
     try:
       build_config = build_job.build_config
     except BuildJobLoadException as irbe:
       self._build_failure('Could not load build job information', irbe)
 
-    try:
-      parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
-    except BuildPackageException as bpe:
-      self._build_failure('Could not find Dockerfile in build package', bpe)
-      raise trollius.Return()
+    base_image_information = {}
+    buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key,
+                                                 requires_cors=False)
 
-    image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
-    if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
-      self._build_failure('Missing FROM line in Dockerfile')
-      raise trollius.Return()
+    # TODO(jschorr): Remove as soon as the fleet has been transitioned to 0.2.
+    if self._worker_version == '0.1-beta':
+      # Retrieve the job's buildpack.
+      logger.debug('Retrieving build package: %s', buildpack_url)
+      buildpack = None
+      try:
+        buildpack = BuildPackage.from_url(buildpack_url)
+      except BuildPackageException as bpe:
+        self._build_failure('Could not retrieve build package', bpe)
+        raise trollius.Return()
 
-    base_image_information = {
-        'repository': image_and_tag_tuple[0],
-        'tag': image_and_tag_tuple[1]
-    }
+      # Extract the base image information from the Dockerfile.
+      parsed_dockerfile = None
+      logger.debug('Parsing dockerfile')
 
-    # Extract the number of steps from the Dockerfile.
-    with self._build_status as status_dict:
-      status_dict['total_commands'] = len(parsed_dockerfile.commands)
+      try:
+        parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
+      except BuildPackageException as bpe:
+        self._build_failure('Could not find Dockerfile in build package', bpe)
+        raise trollius.Return()
+
+      image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
+      if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
+        self._build_failure('Missing FROM line in Dockerfile')
+        raise trollius.Return()
+
+      base_image_information = {
+          'repository': image_and_tag_tuple[0],
+          'tag': image_and_tag_tuple[1]
+      }
+
+      # Extract the number of steps from the Dockerfile.
+      with self._build_status as status_dict:
+        status_dict['total_commands'] = len(parsed_dockerfile.commands)
+    else:
+      # TODO(jschorr): This is a HACK to make sure the progress bar (sort of) continues working
+      # until such time as we have the caching code in place.
+      with self._build_status as status_dict:
+        status_dict['total_commands'] = 25
 
     # Add the pull robot information, if any.
     if build_job.pull_credentials:
@@ -136,8 +149,8 @@ class BuildComponent(BaseComponent):
     #  push_token: The token to use to push the built image.
     #  tag_names: The name(s) of the tag(s) for the newly built image.
     #  base_image: The image name and credentials to use to conduct the base image pull.
-    #   repository: The repository to pull.
-    #   tag: The tag to pull.
+    #   repository: The repository to pull (DEPRECATED 0.2)
+    #   tag: The tag to pull (DEPRECATED in 0.2)
     #   username: The username for pulling the base image (if any).
     #   password: The password for pulling the base image (if any).
     build_arguments = {
@@ -299,6 +312,8 @@ class BuildComponent(BaseComponent):
 
   @trollius.coroutine
   def _on_ready(self, token, version):
+    self._worker_version = version
+
     if not version in SUPPORTED_WORKER_VERSIONS:
       logger.warning('Build component (token "%s") is running an out-of-date version: %s', token,
                      version)

buildman/jobutil/workererror.py

@@ -19,6 +19,11 @@ class WorkerError(object):
             'is_internal': True
         },
 
+        'io.quay.builder.dockerfileissue': {
+            'message': 'Could not find or parse Dockerfile',
+            'show_base_error': True
+        },
+
         'io.quay.builder.cannotpullbaseimage': {
             'message': 'Could not pull base image',
             'show_base_error': True