WIP: Start implementation of the build manager/controller. This code is not yet working completely.

buildman/buildcomponent.py

@@ -0,0 +1,212 @@
+import datetime
+import logging
+import json
+import trollius
+
+from trollius.coroutines import From
+from buildman.basecomponent import BaseComponent
+
+HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
+
+logger = logging.getLogger(__name__)
+
+class BuildComponent(BaseComponent):
+  """ An application session component which conducts one (or more) builds. """
+
+  server_hostname = None
+  expected_token = None
+  builder_realm = None
+  last_heartbeat = None
+
+  current_phase = 'joining'
+  current_job = None
+
+  def __init__(self, config, realm=None, token=None, **kwargs):
+    self.expected_token = token
+    self.builder_realm = realm
+
+    BaseComponent.__init__(self, config, **kwargs)
+
+  def onConnect(self):    
+    self.join(self.builder_realm)
+
+  def onJoin(self, details):
+    logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
+    yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
+    yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
+    yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
+
+    self._set_phase('waiting')
+
+  def is_ready(self):
+    return self.current_phase == 'running'
+
+  def start_build(self, job_item):
+    if not self.is_ready():
+      return False
+
+    self.job_item = job_item
+    self._set_phase('building')
+
+    # Parse the build job's config.
+    logger.debug('Parsing job JSON configuration block')
+    try:
+      job_config = json.loads(job_item.body)
+    except ValueError:
+      self._build_failure('Could not parse build job configuration')
+      return False
+
+    # Retrieve the job's buildpack.
+    buildpack_url = self.user_files.get_file_url(job_item.resource_key, requires_cors=False)
+    logger.debug('Retreiving build package: %s' % buildpack_url)
+
+    buildpack = None
+    try:
+      buildpack = BuildPack.from_url(buildpack_url)
+    except BuildPackageException as bpe:
+      self._build_failure('Could not retrieve build package', bpe)
+      return False
+
+    # Extract the base image information from the Dockerfile.
+    parsed_dockerfile = None
+    logger.debug('Parsing dockerfile')
+
+    try:
+      parsed_dockerfile = buildpack.parse_dockerfile(job_config.get('build_subdir'))
+    except BuildPackageException as bpe:
+      self._build_failure('Could not find Dockerfile in build package', bpe)
+      return False
+
+    image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
+    if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
+      self._build_failure('Missing FROM line Dockerfile')
+      return False      
+
+    base_image_information = {
+      'repository': image_and_tag_tuple[0],
+      'tag': image_and_tag_tuple[1]
+    }
+
+    # Add the pull robot information, if any.
+    if job_config.get('pull_credentials') is not None:
+      base_image_information['username'] = job_config['pull_credentials'].get('username', '')
+      base_image_information['password'] = job_config['pull_credentials'].get('password', '')
+
+    # Retrieve the repository's full name.
+    repo = job_config.repository
+    repository_name = repo.namespace_user.username + '/' + repo.name
+
+    # Parse the build queue item into build arguments.
+    #   build_package: URL to the build package to download and untar/unzip.
+    #   sub_directory: The location within the build package of the Dockerfile and the build context.
+    #   repository: The repository for which this build is occurring.
+    #   registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
+    #   pull_token: The token to use when pulling the cache for building.
+    #   push_token: The token to use to push the built image.
+    #   tag_names: The name(s) of the tag(s) for the newly built image.
+    #   base_image: The image name and credentials to use to conduct the base image pull.
+    #     repository: The repository to pull.
+    #     tag: The tag to pull.
+    #     username: The username for pulling the base image (if any).
+    #     password: The password for pulling the base image (if any).
+    build_arguments = {
+      'build_package': buildpack_url,
+      'sub_directory': job_config.get('build_subdir', ''),
+      'repository': repository_name,
+      'registry': self.server_hostname,
+      'pull_token': job_item.access_token.code,
+      'push_token': job_item.access_token.code,
+      'tag_names': job_config.get('docker_tags', ['latest']),
+      'base_image': base_image_information
+    }
+
+    # Invoke the build.    
+    logger.debug('Invoking build: %s', token)
+    logger.debug('With Arguments: %s', build_arguments)
+
+    (self.call("io.quay.builder.build", **build_arguments)
+         .add_done_callback(self._build_complete))
+
+    return True
+
+  def _build_failure(self, error_message, exception=None):
+    # TODO: log this message
+    print error_kind
+    self._set_phase('running')
+
+  def _build_complete(self, result):
+    try:
+      status = result.result()
+      # TODO: log the success
+      print status
+    except ApplicationError as ae:
+      error_kind = ae.error
+      # TODO: log the error
+      print error_kind
+    finally:
+      self._set_phase('running')
+
+  def _on_ready(self, token):
+    if self.current_phase != 'waiting':
+      logger.warning('Build component with token %s is already connected', self.expected_token)
+      return
+
+    if token != self.expected_token:
+      logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
+      return
+
+    self._set_phase('running')
+
+    # Start the heartbeat check.
+    loop = trollius.get_event_loop()
+    loop.create_task(self._check_heartbeat(loop))
+    logger.debug('Build worker %s is connected and ready' % self.builder_realm)
+    return True
+
+  def _on_log_message(self, status, json):
+    # TODO: log the message
+    print json
+
+  def _set_phase(self, phase):
+    self.current_phase = phase
+
+  def _on_heartbeat(self):
+    self.last_heartbeat = datetime.datetime.now()
+
+  def _start_heartbeat_check(self, loop):
+    trollius.set_event_loop(loop)
+    loop.run_until_complete(self._check_heartbeat())
+
+  @trollius.coroutine
+  def _check_heartbeat(self, loop):
+    while True:
+      if self.current_phase != 'running' or self.current_phase != 'building':
+        return
+
+      logger.debug('Checking heartbeat on realm %s and build %s',
+        self.builder_realm, self.expected_token)
+
+      if not self.last_heartbeat:
+        self._timeout()
+        return
+
+      if self.last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
+        self._timeout()
+        return      
+
+      yield From(trollius.sleep(5))
+
+  def _timeout(self):
+    self._set_phase('timeout')
+    logger.warning('Build component %s timed out', self.expected_token)
+    self._dispose(timed_out=True)
+
+  def _dispose(self, timed_out=False):
+    # If we still have a running job, then it has not completed and we need to tell the parent
+    # manager.
+    if self.job_item is not None:
+      self.parent_manager.job_completed(job_item, 'incomplete', self)
+      self.job_item = None
+
+    # Unregister the current component so that it cannot be invoked again.
+    self.parent_manager.build_component_disposed(self, timed_out)