Better organize the source file structure of the build manager and change it to choose a lifecycle manager based on the config

buildman/jobutil/buildjob.py

@@ -0,0 +1,60 @@
+from data import model
+
+import json
+
+class BuildJobLoadException(Exception):
+  """ Exception raised if a build job could not be instantiated for some reason. """
+  pass
+
+class BuildJob(object):
+  """ Represents a single in-progress build job. """
+  def __init__(self, job_item):
+    self._job_item = job_item
+
+    try:
+      self._job_details = json.loads(job_item.body)
+    except ValueError:
+      raise BuildJobLoadException(
+          'Could not parse build queue item config with ID %s' % self._job_details['build_uuid']
+      )
+
+    try:
+      self._repo_build = model.get_repository_build(self._job_details['namespace'],
+                                                    self._job_details['repository'],
+                                                    self._job_details['build_uuid'])
+    except model.InvalidRepositoryBuildException:
+      raise BuildJobLoadException(
+          'Could not load repository build with ID %s' % self._job_details['build_uuid'])
+
+    try:
+      self._build_config = json.loads(self._repo_build.job_config)
+    except ValueError:
+      raise BuildJobLoadException(
+          'Could not parse repository build job config with ID %s' % self._job_details['build_uuid']
+      )
+
+  def determine_cached_tag(self):
+    """ Returns the tag to pull to prime the cache or None if none. """
+    # TODO(jschorr): Change this to use the more complicated caching rules, once we have caching
+    # be a pull of things besides the constructed tags.
+    tags = self._build_config.get('docker_tags', ['latest'])
+    existing_tags = model.list_repository_tags(self._job_details['namespace'],
+                                               self._job_details['repository'])
+
+    cached_tags = set(tags) & set([tag.name for tag in existing_tags])
+    if cached_tags:
+      return list(cached_tags)[0]
+
+    return None
+
+  def job_item(self):
+    """ Returns the job's queue item. """
+    return self._job_item
+
+  def repo_build(self):
+    """ Returns the repository build DB row for the job. """
+    return self._repo_build
+
+  def build_config(self):
+    """ Returns the parsed repository build config for the job. """
+    return self._build_config

buildman/jobutil/buildpack.py

@@ -0,0 +1,88 @@
+import tarfile
+import requests
+import os
+
+from tempfile import TemporaryFile, mkdtemp
+from zipfile import ZipFile
+from util.dockerfileparse import parse_dockerfile
+from util.safetar import safe_extractall
+
+class BuildPackageException(Exception):
+  """ Exception raised when retrieving or parsing a build package. """
+  pass
+
+
+class BuildPackage(object):
+  """ Helper class for easy reading and updating of a Dockerfile build pack. """
+
+  def __init__(self, requests_file):
+    self._mime_processors = {
+        'application/zip': BuildPackage._prepare_zip,
+        'application/x-zip-compressed': BuildPackage._prepare_zip,
+        'text/plain': BuildPackage._prepare_dockerfile,
+        'application/octet-stream': BuildPackage._prepare_dockerfile,
+        'application/x-tar': BuildPackage._prepare_tarball,
+        'application/gzip': BuildPackage._prepare_tarball,
+        'application/x-gzip': BuildPackage._prepare_tarball,
+    }
+
+    c_type = requests_file.headers['content-type']
+    c_type = c_type.split(';')[0] if ';' in c_type else c_type
+
+    if c_type not in self._mime_processors:
+      raise BuildPackageException('Unknown build package mime type: %s' % c_type)
+
+    self._package_directory = None
+    try:
+      self._package_directory = self._mime_processors[c_type](requests_file)
+    except Exception as ex:
+      raise BuildPackageException(ex.message)
+
+  def parse_dockerfile(self, subdirectory):
+    dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile')
+    if not os.path.exists(dockerfile_path):
+      if subdirectory:
+        message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory
+      else:
+        message = 'Build package did not contain a Dockerfile at the root directory.'
+
+      raise BuildPackageException(message)
+
+    with open(dockerfile_path, 'r') as dockerfileobj:
+      return parse_dockerfile(dockerfileobj.read())
+
+  @staticmethod
+  def from_url(url):
+    buildpack_resource = requests.get(url, stream=True)
+    return BuildPackage(buildpack_resource)
+
+  @staticmethod
+  def _prepare_zip(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+
+    # Save the zip file to temp somewhere
+    with TemporaryFile() as zip_file:
+      zip_file.write(request_file.content)
+      to_extract = ZipFile(zip_file)
+      to_extract.extractall(build_dir)
+
+    return build_dir
+
+  @staticmethod
+  def _prepare_dockerfile(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+    dockerfile_path = os.path.join(build_dir, "Dockerfile")
+    with open(dockerfile_path, 'w') as dockerfile:
+      dockerfile.write(request_file.content)
+
+    return build_dir
+
+  @staticmethod
+  def _prepare_tarball(request_file):
+    build_dir = mkdtemp(prefix='docker-build-')
+
+    # Save the zip file to temp somewhere
+    with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream:
+      safe_extractall(tar_stream, build_dir)
+
+    return build_dir

buildman/jobutil/buildstatus.py

@@ -0,0 +1,52 @@
+from data.database import BUILD_PHASE
+
+class StatusHandler(object):
+  """ Context wrapper for writing status to build logs. """
+
+  def __init__(self, build_logs, repository_build):
+    self._current_phase = None
+    self._repository_build = repository_build
+    self._uuid = repository_build.uuid
+    self._build_logs = build_logs
+
+    self._status = {
+        'total_commands': None,
+        'current_command': None,
+        'push_completion': 0.0,
+        'pull_completion': 0.0,
+    }
+
+    # Write the initial status.
+    self.__exit__(None, None, None)
+
+  def _append_log_message(self, log_message, log_type=None, log_data=None):
+    self._build_logs.append_log_message(self._uuid, log_message, log_type, log_data)
+
+  def append_log(self, log_message, extra_data=None):
+    self._append_log_message(log_message, log_data=extra_data)
+
+  def set_command(self, command, extra_data=None):
+    self._append_log_message(command, self._build_logs.COMMAND, extra_data)
+
+  def set_error(self, error_message, extra_data=None, internal_error=False):
+    self.set_phase(BUILD_PHASE.INTERNAL_ERROR if internal_error else BUILD_PHASE.ERROR)
+
+    extra_data = extra_data or {}
+    extra_data['internal_error'] = internal_error
+    self._append_log_message(error_message, self._build_logs.ERROR, extra_data)
+
+  def set_phase(self, phase, extra_data=None):
+    if phase == self._current_phase:
+      return False
+
+    self._current_phase = phase
+    self._append_log_message(phase, self._build_logs.PHASE, extra_data)
+    self._repository_build.phase = phase
+    self._repository_build.save()
+    return True
+
+  def __enter__(self):
+    return self._status
+
+  def __exit__(self, exc_type, value, traceback):
+    self._build_logs.set_status(self._uuid, self._status)

buildman/jobutil/workererror.py

@@ -0,0 +1,84 @@
+class WorkerError(object):
+  """ Helper class which represents errors raised by a build worker. """
+  def __init__(self, error_code, base_message=None):
+    self._error_code = error_code
+    self._base_message = base_message
+
+    self._error_handlers = {
+        'io.quay.builder.buildpackissue': {
+            'message': 'Could not load build package',
+            'is_internal': True
+        },
+
+        'io.quay.builder.cannotextractbuildpack': {
+            'message': 'Could not extract the contents of the build package'
+        },
+
+        'io.quay.builder.cannotpullforcache': {
+            'message': 'Could not pull cached image',
+            'is_internal': True
+        },
+
+        'io.quay.builder.cannotpullbaseimage': {
+            'message': 'Could not pull base image',
+            'show_base_error': True
+        },
+
+        'io.quay.builder.internalerror': {
+            'message': 'An internal error occurred while building. Please submit a ticket.'
+        },
+
+        'io.quay.builder.buildrunerror': {
+            'message': 'Could not start the build process',
+            'is_internal': True
+        },
+
+        'io.quay.builder.builderror': {
+            'message': 'A build step failed',
+            'show_base_error': True
+        },
+
+        'io.quay.builder.tagissue': {
+            'message': 'Could not tag built image',
+            'is_internal': True
+        },
+
+        'io.quay.builder.pushissue': {
+            'message': 'Could not push built image',
+            'show_base_error': True,
+            'is_internal': True
+        },
+
+        'io.quay.builder.dockerconnecterror': {
+            'message': 'Could not connect to Docker daemon',
+            'is_internal': True
+        },
+
+        'io.quay.builder.missingorinvalidargument': {
+            'message': 'Missing required arguments for builder',
+            'is_internal': True
+        }
+    }
+
+  def is_internal_error(self):
+    handler = self._error_handlers.get(self._error_code)
+    return handler.get('is_internal', False) if handler else True
+
+  def public_message(self):
+    handler = self._error_handlers.get(self._error_code)
+    if not handler:
+      return 'An unknown error occurred'
+
+    message = handler['message']
+    if handler.get('show_base_error', False) and self._base_message:
+      message = message + ': ' + self._base_message
+
+    return message
+
+  def extra_data(self):
+    if self._base_message:
+      return {
+          'base_error': self._base_message
+      }
+
+    return {}