import datetime import logging import json import trollius from autobahn.wamp.exception import ApplicationError from trollius.coroutines import From from buildman.basecomponent import BaseComponent from buildman.buildpack import BuildPackage, BuildPackageException HEARTBEAT_DELTA = datetime.timedelta(seconds=15) logger = logging.getLogger(__name__) class BuildComponent(BaseComponent): """ An application session component which conducts one (or more) builds. """ server_hostname = None expected_token = None builder_realm = None last_heartbeat = None current_phase = 'joining' current_job = None def __init__(self, config, realm=None, token=None, **kwargs): self.expected_token = token self.builder_realm = realm BaseComponent.__init__(self, config, **kwargs) def onConnect(self): self.join(self.builder_realm) def onJoin(self, details): logger.debug('Registering methods and listeners for component %s' % self.builder_realm) yield From(self.register(self._on_ready, u'io.quay.buildworker.ready')) yield From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat')) yield From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage')) self._set_phase('waiting') def is_ready(self): return self.current_phase == 'running' def start_build(self, build_job): if not self.is_ready(): return False self.current_job = build_job self._set_phase('building') # Retrieve the job's buildpack. buildpack_url = self.user_files.get_file_url(build_job.repo_build().resource_key, requires_cors=False) logger.debug('Retreiving build package: %s' % buildpack_url) buildpack = None try: buildpack = BuildPackage.from_url(buildpack_url) except BuildPackageException as bpe: self._build_failure('Could not retrieve build package', bpe) return False # Extract the base image information from the Dockerfile. parsed_dockerfile = None logger.debug('Parsing dockerfile') build_config = build_job.build_config() try: parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir')) except BuildPackageException as bpe: self._build_failure('Could not find Dockerfile in build package', bpe) return False image_and_tag_tuple = parsed_dockerfile.get_image_and_tag() if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: self._build_failure('Missing FROM line Dockerfile') return False base_image_information = { 'repository': image_and_tag_tuple[0], 'tag': image_and_tag_tuple[1] } # Add the pull robot information, if any. if build_config.get('pull_credentials') is not None: base_image_information['username'] = build_config['pull_credentials'].get('username', '') base_image_information['password'] = build_config['pull_credentials'].get('password', '') # Retrieve the repository's full name. repo = build_job.repo_build().repository repository_name = repo.namespace_user.username + '/' + repo.name # Parse the build queue item into build arguments. # build_package: URL to the build package to download and untar/unzip. # sub_directory: The location within the build package of the Dockerfile and the build context. # repository: The repository for which this build is occurring. # registry: The registry for which this build is occuring. Example: 'quay.io', 'staging.quay.io' # pull_token: The token to use when pulling the cache for building. # push_token: The token to use to push the built image. # tag_names: The name(s) of the tag(s) for the newly built image. # base_image: The image name and credentials to use to conduct the base image pull. # repository: The repository to pull. # tag: The tag to pull. # username: The username for pulling the base image (if any). # password: The password for pulling the base image (if any). build_arguments = { 'build_package': buildpack_url, 'sub_directory': build_config.get('build_subdir', ''), 'repository': repository_name, 'registry': self.server_hostname, 'pull_token': build_job.repo_build().access_token.code, 'push_token': build_job.repo_build().access_token.code, 'tag_names': build_config.get('docker_tags', ['latest']), 'base_image': base_image_information, 'cached_tag': build_job.determine_cached_tag() or '' } # Invoke the build. logger.debug('Invoking build: %s', self.builder_realm) logger.debug('With Arguments: %s', build_arguments) (self.call("io.quay.builder.build", **build_arguments) .add_done_callback(self._build_complete)) return True def _build_failure(self, error_message, exception=None): # TODO: log this message print error_message print exception self._set_phase('running') def _build_complete(self, result): try: status = result.result() # TODO: log the success print status except ApplicationError as ae: error_kind = ae.error # TODO: log the error print error_kind finally: self._set_phase('running') def _on_ready(self, token): if self.current_phase != 'waiting': logger.warning('Build component with token %s is already connected', self.expected_token) return if token != self.expected_token: logger.warning('Builder token mismatch. Expected: %s. Found: %s', self.expected_token, token) return self._set_phase('running') # Start the heartbeat check. loop = trollius.get_event_loop() loop.create_task(self._check_heartbeat(loop)) logger.debug('Build worker %s is connected and ready' % self.builder_realm) return True def _on_log_message(self, status, json): # TODO: log the message print json def _set_phase(self, phase): self.current_phase = phase def _on_heartbeat(self): self.last_heartbeat = datetime.datetime.now() def _start_heartbeat_check(self, loop): trollius.set_event_loop(loop) loop.run_until_complete(self._check_heartbeat()) @trollius.coroutine def _check_heartbeat(self, loop): while True: if self.current_phase != 'running' or self.current_phase != 'building': return logger.debug('Checking heartbeat on realm %s and build %s', self.builder_realm, self.expected_token) if not self.last_heartbeat: self._timeout() return if self.last_heartbeat < datetime.datetime.now() - HEARTBEAT_DELTA: self._timeout() return yield From(trollius.sleep(5)) def _timeout(self): self._set_phase('timeout') logger.warning('Build component %s timed out', self.expected_token) self._dispose(timed_out=True) def _dispose(self, timed_out=False): # If we still have a running job, then it has not completed and we need to tell the parent # manager. if self.current_job is not None: self.parent_manager.job_completed(self.current_job, 'incomplete', self) self.current_job = None # Unregister the current component so that it cannot be invoked again. self.parent_manager.build_component_disposed(self, timed_out)