From 0065ac8503e0cc80125e7d74de02a1aee35853e1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 9 Feb 2015 12:13:40 -0500 Subject: [PATCH] Add back in the cache checking code and remove the old 0.1 build pack code --- buildman/component/buildcomponent.py | 57 ++++-------------- buildman/jobutil/buildpack.py | 88 ---------------------------- buildman/jobutil/buildstatus.py | 2 +- 3 files changed, 14 insertions(+), 133 deletions(-) delete mode 100644 buildman/jobutil/buildpack.py diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index c1fb41a02..be0e17d7f 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -10,7 +10,6 @@ from autobahn.wamp.exception import ApplicationError from buildman.server import BuildJobResult from buildman.component.basecomponent import BaseComponent from buildman.jobutil.buildjob import BuildJobLoadException -from buildman.jobutil.buildpack import BuildPackage, BuildPackageException from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.workererror import WorkerError @@ -20,7 +19,7 @@ HEARTBEAT_DELTA = datetime.timedelta(seconds=30) HEARTBEAT_TIMEOUT = 10 INITIAL_TIMEOUT = 25 -SUPPORTED_WORKER_VERSIONS = ['0.1-beta', '0.2'] +SUPPORTED_WORKER_VERSIONS = ['0.2'] logger = logging.getLogger(__name__) @@ -56,7 +55,10 @@ class BuildComponent(BaseComponent): def onJoin(self, details): logger.debug('Registering methods and listeners for component %s', self.builder_realm) yield trollius.From(self.register(self._on_ready, u'io.quay.buildworker.ready')) + yield trollius.From(self.register(self._determine_cache_tag, + u'io.quay.buildworker.determinecachetag')) yield trollius.From(self.register(self._ping, u'io.quay.buildworker.ping')) + yield trollius.From(self.subscribe(self._on_heartbeat, 'io.quay.builder.heartbeat')) yield trollius.From(self.subscribe(self._on_log_message, 'io.quay.builder.logmessage')) @@ -91,46 +93,6 @@ class BuildComponent(BaseComponent): buildpack_url = self.user_files.get_file_url(build_job.repo_build.resource_key, requires_cors=False) - # TODO(jschorr): Remove as soon as the fleet has been transitioned to 0.2. - if self._worker_version == '0.1-beta': - # Retrieve the job's buildpack. - logger.debug('Retrieving build package: %s', buildpack_url) - buildpack = None - try: - buildpack = BuildPackage.from_url(buildpack_url) - except BuildPackageException as bpe: - self._build_failure('Could not retrieve build package', bpe) - raise trollius.Return() - - # Extract the base image information from the Dockerfile. - parsed_dockerfile = None - logger.debug('Parsing dockerfile') - - try: - parsed_dockerfile = buildpack.parse_dockerfile(build_config.get('build_subdir')) - except BuildPackageException as bpe: - self._build_failure('Could not find Dockerfile in build package', bpe) - raise trollius.Return() - - image_and_tag_tuple = parsed_dockerfile.get_image_and_tag() - if image_and_tag_tuple is None or image_and_tag_tuple[0] is None: - self._build_failure('Missing FROM line in Dockerfile') - raise trollius.Return() - - base_image_information = { - 'repository': image_and_tag_tuple[0], - 'tag': image_and_tag_tuple[1] - } - - # Extract the number of steps from the Dockerfile. - with self._build_status as status_dict: - status_dict['total_commands'] = len(parsed_dockerfile.commands) - else: - # TODO(jschorr): This is a HACK to make sure the progress bar (sort of) continues working - # until such time as we have the caching code in place. - with self._build_status as status_dict: - status_dict['total_commands'] = 25 - # Add the pull robot information, if any. if build_job.pull_credentials: base_image_information['username'] = build_job.pull_credentials.get('username', '') @@ -161,8 +123,7 @@ class BuildComponent(BaseComponent): 'pull_token': build_job.repo_build.access_token.code, 'push_token': build_job.repo_build.access_token.code, 'tag_names': build_config.get('docker_tags', ['latest']), - 'base_image': base_image_information, - 'cached_tag': build_job.determine_cached_tag() or '' + 'base_image': base_image_information } # Invoke the build. @@ -256,6 +217,14 @@ class BuildComponent(BaseComponent): elif phase == BUILD_PHASE.BUILDING: self._build_status.append_log(current_status_string) + def _determine_cache_tag(self, cache_commands, base_image_name, base_image_tag, base_image_id): + with self._build_status as status_dict: + status_dict['total_commands'] = len(cache_commands) + 1 + + logger.debug('Checking cache on realm %s. Base image: %s:%s (%s)', self.builder_realm, + base_image_name, base_image_tag, base_image_id) + + return self._current_job.determine_cached_tag(base_image_id, cache_commands) or '' def _build_failure(self, error_message, exception=None): """ Handles and logs a failed build. """ diff --git a/buildman/jobutil/buildpack.py b/buildman/jobutil/buildpack.py deleted file mode 100644 index 9892c65d3..000000000 --- a/buildman/jobutil/buildpack.py +++ /dev/null @@ -1,88 +0,0 @@ -import tarfile -import requests -import os - -from tempfile import TemporaryFile, mkdtemp -from zipfile import ZipFile -from util.dockerfileparse import parse_dockerfile -from util.safetar import safe_extractall - -class BuildPackageException(Exception): - """ Exception raised when retrieving or parsing a build package. """ - pass - - -class BuildPackage(object): - """ Helper class for easy reading and updating of a Dockerfile build pack. """ - - def __init__(self, requests_file): - self._mime_processors = { - 'application/zip': BuildPackage._prepare_zip, - 'application/x-zip-compressed': BuildPackage._prepare_zip, - 'text/plain': BuildPackage._prepare_dockerfile, - 'application/octet-stream': BuildPackage._prepare_dockerfile, - 'application/x-tar': BuildPackage._prepare_tarball, - 'application/gzip': BuildPackage._prepare_tarball, - 'application/x-gzip': BuildPackage._prepare_tarball, - } - - c_type = requests_file.headers['content-type'] - c_type = c_type.split(';')[0] if ';' in c_type else c_type - - if c_type not in self._mime_processors: - raise BuildPackageException('Unknown build package mime type: %s' % c_type) - - self._package_directory = None - try: - self._package_directory = self._mime_processors[c_type](requests_file) - except Exception as ex: - raise BuildPackageException(ex.message) - - def parse_dockerfile(self, subdirectory): - dockerfile_path = os.path.join(self._package_directory, subdirectory, 'Dockerfile') - if not os.path.exists(dockerfile_path): - if subdirectory: - message = 'Build package did not contain a Dockerfile at sub directory %s.' % subdirectory - else: - message = 'Build package did not contain a Dockerfile at the root directory.' - - raise BuildPackageException(message) - - with open(dockerfile_path, 'r') as dockerfileobj: - return parse_dockerfile(dockerfileobj.read()) - - @staticmethod - def from_url(url): - buildpack_resource = requests.get(url, stream=True) - return BuildPackage(buildpack_resource) - - @staticmethod - def _prepare_zip(request_file): - build_dir = mkdtemp(prefix='docker-build-') - - # Save the zip file to temp somewhere - with TemporaryFile() as zip_file: - zip_file.write(request_file.content) - to_extract = ZipFile(zip_file) - to_extract.extractall(build_dir) - - return build_dir - - @staticmethod - def _prepare_dockerfile(request_file): - build_dir = mkdtemp(prefix='docker-build-') - dockerfile_path = os.path.join(build_dir, "Dockerfile") - with open(dockerfile_path, 'w') as dockerfile: - dockerfile.write(request_file.content) - - return build_dir - - @staticmethod - def _prepare_tarball(request_file): - build_dir = mkdtemp(prefix='docker-build-') - - # Save the zip file to temp somewhere - with tarfile.open(mode='r|*', fileobj=request_file.raw) as tar_stream: - safe_extractall(tar_stream, build_dir) - - return build_dir diff --git a/buildman/jobutil/buildstatus.py b/buildman/jobutil/buildstatus.py index 217e3aa6c..393ecd3b4 100644 --- a/buildman/jobutil/buildstatus.py +++ b/buildman/jobutil/buildstatus.py @@ -11,7 +11,7 @@ class StatusHandler(object): self._build_logs = build_logs self._status = { - 'total_commands': None, + 'total_commands': 0, 'current_command': None, 'push_completion': 0.0, 'pull_completion': 0.0,