WIP: Start implementation of the build manager/controller. This code is not yet working completely.

This commit is contained in:
Joseph Schorr 2014-11-11 18:23:15 -05:00
parent 2ccbea95a5
commit eacf3f01d2
9 changed files with 576 additions and 0 deletions

212
buildman/buildcomponent.py Normal file
View file

@ -0,0 +1,212 @@
import datetime
import logging
import json
import trollius
from trollius.coroutines import From
from buildman.basecomponent import BaseComponent
HEARTBEAT_DELTA = datetime.timedelta(seconds=15)
logger = logging.getLogger(__name__)
class BuildComponent(BaseComponent):
""" An application session component which conducts one (or more) builds. """
server_hostname = None
expected_token = None
builder_realm = None
last_heartbeat = None
current_phase = 'joining'
current_job = None
def __init__(self, config, realm=None, token=None, **kwargs):
self.expected_token = token
self.builder_realm = realm
BaseComponent.__init__(self, config, **kwargs)
def onConnect(self):
self.join(self.builder_realm)
def onJoin(self, details):
logger.debug('Registering methods and listeners for component %s' % self.builder_realm)
yield From(self.register(self._on_ready, u'io.quay.buildworker.ready'))
yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat'))
yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage'))
self._set_phase('waiting')
def is_ready(self):
return self.current_phase == 'running'
def start_build(self, job_item):
if not self.is_ready():
return False
self.job_item = job_item
self._set_phase('building')
# Parse the build job's config.
logger.debug('Parsing job JSON configuration block')
try:
job_config = json.loads(job_item.body)
except ValueError:
self._build_failure('Could not parse build job configuration')
return False
# Retrieve the job's buildpack.
buildpack_url = self.user_files.get_file_url(job_item.resource_key, requires_cors=False)
logger.debug('Retreiving build package: %s' % buildpack_url)
buildpack = None
try:
buildpack = BuildPack.from_url(buildpack_url)
except BuildPackageException as bpe:
self._build_failure('Could not retrieve build package', bpe)
return False
# Extract the base image information from the Dockerfile.
parsed_dockerfile = None
logger.debug('Parsing dockerfile')
try:
parsed_dockerfile = buildpack.parse_dockerfile(job_config.get('build_subdir'))
except BuildPackageException as bpe:
self._build_failure('Could not find Dockerfile in build package', bpe)
return False
image_and_tag_tuple = parsed_dockerfile.get_image_and_tag()
if image_and_tag_tuple is None or image_and_tag_tuple[0] is None:
self._build_failure('Missing FROM line Dockerfile')
return False
base_image_information = {
'repository': image_and_tag_tuple[0],
'tag': image_and_tag_tuple[1]
}
# Add the pull robot information, if any.
if job_config.get('pull_credentials') is not None:
base_image_information['username'] = job_config['pull_credentials'].get('username', '')
base_image_information['password'] = job_config['pull_credentials'].get('password', '')
# Retrieve the repository's full name.
repo = job_config.repository
repository_name = repo.namespace_user.username + '/' + repo.name
# Parse the build queue item into build arguments.
# build_package: URL to the build package to download and untar/unzip.
# sub_directory: The location within the build package of the Dockerfile and the build context.
# repository: The repository for which this build is occurring.
# registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io'
# pull_token: The token to use when pulling the cache for building.
# push_token: The token to use to push the built image.
# tag_names: The name(s) of the tag(s) for the newly built image.
# base_image: The image name and credentials to use to conduct the base image pull.
# repository: The repository to pull.
# tag: The tag to pull.
# username: The username for pulling the base image (if any).
# password: The password for pulling the base image (if any).
build_arguments = {
'build_package': buildpack_url,
'sub_directory': job_config.get('build_subdir', ''),
'repository': repository_name,
'registry': self.server_hostname,
'pull_token': job_item.access_token.code,
'push_token': job_item.access_token.code,
'tag_names': job_config.get('docker_tags', ['latest']),
'base_image': base_image_information
}
# Invoke the build.
logger.debug('Invoking build: %s', token)
logger.debug('With Arguments: %s', build_arguments)
(self.call("io.quay.builder.build", **build_arguments)
.add_done_callback(self._build_complete))
return True
def _build_failure(self, error_message, exception=None):
# TODO: log this message
print error_kind
self._set_phase('running')
def _build_complete(self, result):
try:
status = result.result()
# TODO: log the success
print status
except ApplicationError as ae:
error_kind = ae.error
# TODO: log the error
print error_kind
finally:
self._set_phase('running')
def _on_ready(self, token):
if self.current_phase != 'waiting':
logger.warning('Build component with token %s is already connected', self.expected_token)
return
if token != self.expected_token:
logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token)
return
self._set_phase('running')
# Start the heartbeat check.
loop = trollius.get_event_loop()
loop.create_task(self._check_heartbeat(loop))
logger.debug('Build worker %s is connected and ready' % self.builder_realm)
return True
def _on_log_message(self, status, json):
# TODO: log the message
print json
def _set_phase(self, phase):
self.current_phase = phase
def _on_heartbeat(self):
self.last_heartbeat = datetime.datetime.now()
def _start_heartbeat_check(self, loop):
trollius.set_event_loop(loop)
loop.run_until_complete(self._check_heartbeat())
@trollius.coroutine
def _check_heartbeat(self, loop):
while True:
if self.current_phase != 'running' or self.current_phase != 'building':
return
logger.debug('Checking heartbeat on realm %s and build %s',
self.builder_realm, self.expected_token)
if not self.last_heartbeat:
self._timeout()
return
if self.last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA:
self._timeout()
return
yield From(trollius.sleep(5))
def _timeout(self):
self._set_phase('timeout')
logger.warning('Build component %s timed out', self.expected_token)
self._dispose(timed_out=True)
def _dispose(self, timed_out=False):
# If we still have a running job, then it has not completed and we need to tell the parent
# manager.
if self.job_item is not None:
self.parent_manager.job_completed(job_item, 'incomplete', self)
self.job_item = None
# Unregister the current component so that it cannot be invoked again.
self.parent_manager.build_component_disposed(self, timed_out)