quay/buildman/buildcomponent.py

import datetime
import logging
import json
import trollius

from autobahn.wamp.exception import ApplicationError
from trollius.coroutines import From
from buildman.basecomponent import BaseComponent
from buildman.buildpack import BuildPackage, BuildPackageException

HEARTBEAT_DELTA = datetime.timedelta(seconds=15)

logger = logging.getLogger(__name__)

class BuildComponent(BaseComponent):
  """ An application session component which conducts one (or more) builds. """

  server_hostname = None
  expected_token = None
  builder_realm = None
  last_heartbeat = None

  current_phase = 'joining'
  current_job = None

  def __init__(self, config, realm=None, token=None, **kwargs):
    self.expected_token = token
    self.builder_realm = realm

    BaseComponent.__init__(self, config, **kwargs)

  def onConnect(self):    
    self.join(self.builder_realm)

  def onJoin(self, details):
    logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
    yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
    yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
    yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))

    self._set_phase('waiting')

  def is_ready(self):
    return self.current_phase == 'running'

  def start_build(self, build_job):    
    if not self.is_ready():
      return False

    self.current_job = build_job
    self._set_phase('building')

    # Retrieve the job's buildpack.
    buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key,
                                                 requires_cors=False)

    logger.debug('Retreiving build package: %s' % buildpack_url)
    buildpack = None
    try:
      buildpack = BuildPackage.from_url(buildpack_url)
    except BuildPackageException as bpe:
      self._build_failure('Could not retrieve build package', bpe)
      return False

    # Extract the base image information from the Dockerfile.
    parsed_dockerfile = None
    logger.debug('Parsing dockerfile')

    build_config = build_job.build_config()
    try:
      parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir'))
    except BuildPackageException as bpe:
      self._build_failure('Could not find Dockerfile in build package', bpe)
      return False

    image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
    if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
      self._build_failure('Missing FROM line Dockerfile')
      return False      

    base_image_information = {
      'repository': image_and_tag_tuple[0],
      'tag': image_and_tag_tuple[1]
    }

    # Add the pull robot information, if any.
    if build_config.get('pull_credentials') is not None:
      base_image_information['username'] = build_config['pull_credentials'].get('username', '')
      base_image_information['password'] = build_config['pull_credentials'].get('password', '')

    # Retrieve the repository's full name.
    repo = build_job.repo_build().repository
    repository_name = repo.namespace_user.username + '/' + repo.name

    # Parse the build queue item into build arguments.
    #   build_package: URL to the build package to download and untar/unzip.
    #   sub_directory: The location within the build package of the Dockerfile and the build context.
    #   repository: The repository for which this build is occurring.
    #   registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
    #   pull_token: The token to use when pulling the cache for building.
    #   push_token: The token to use to push the built image.
    #   tag_names: The name(s) of the tag(s) for the newly built image.
    #   base_image: The image name and credentials to use to conduct the base image pull.
    #     repository: The repository to pull.
    #     tag: The tag to pull.
    #     username: The username for pulling the base image (if any).
    #     password: The password for pulling the base image (if any).
    build_arguments = {
      'build_package': buildpack_url,
      'sub_directory': build_config.get('build_subdir', ''),
      'repository': repository_name,
      'registry': self.server_hostname,
      'pull_token': build_job.repo_build().access_token.code,
      'push_token': build_job.repo_build().access_token.code,
      'tag_names': build_config.get('docker_tags', ['latest']),
      'base_image': base_image_information,
      'cached_tag': build_job.determine_cached_tag() or ''
    }

    # Invoke the build.
    logger.debug('Invoking build: %s', self.builder_realm)
    logger.debug('With Arguments: %s', build_arguments)

    (self.call("io.quay.builder.build", **build_arguments)
         .add_done_callback(self._build_complete))

    return True

  def _build_failure(self, error_message, exception=None):
    # TODO: log this message
    print error_message
    print exception
    self._set_phase('running')

  def _build_complete(self, result):
    try:
      status = result.result()
      # TODO: log the success
      print status
    except ApplicationError as ae:
      error_kind = ae.error
      # TODO: log the error
      print error_kind
    finally:
      self._set_phase('running')

  def _on_ready(self, token):
    if self.current_phase != 'waiting':
      logger.warning('Build component with token %s is already connected', self.expected_token)
      return

    if token != self.expected_token:
      logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
      return

    self._set_phase('running')

    # Start the heartbeat check.
    loop = trollius.get_event_loop()
    loop.create_task(self._check_heartbeat(loop))
    logger.debug('Build worker %s is connected and ready' % self.builder_realm)
    return True

  def _on_log_message(self, status, json):
    # TODO: log the message
    print json

  def _set_phase(self, phase):
    self.current_phase = phase

  def _on_heartbeat(self):
    self.last_heartbeat = datetime.datetime.now()

  def _start_heartbeat_check(self, loop):
    trollius.set_event_loop(loop)
    loop.run_until_complete(self._check_heartbeat())

  @trollius.coroutine
  def _check_heartbeat(self, loop):
    while True:
      if self.current_phase != 'running' or self.current_phase != 'building':
        return

      logger.debug('Checking heartbeat on realm %s and build %s',
        self.builder_realm, self.expected_token)

      if not self.last_heartbeat:
        self._timeout()
        return

      if self.last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
        self._timeout()
        return      

      yield From(trollius.sleep(5))

  def _timeout(self):
    self._set_phase('timeout')
    logger.warning('Build component %s timed out', self.expected_token)
    self._dispose(timed_out=True)

  def _dispose(self, timed_out=False):
    # If we still have a running job, then it has not completed and we need to tell the parent
    # manager.
    if self.current_job is not None:
      self.parent_manager.job_completed(self.current_job, 'incomplete', self)
      self.current_job = None

    # Unregister the current component so that it cannot be invoked again.
    self.parent_manager.build_component_disposed(self, timed_out)