diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index e1fa37cf2..8ebbeb92f 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -15,8 +15,9 @@ from buildman.jobutil.buildjob import BuildJobLoadException from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.workererror import WorkerError +from app import app from data import model -from data.database import BUILD_PHASE +from data.database import BUILD_PHASE, UseThenDisconnect from data.model import InvalidRepositoryBuildException from util import slash_join @@ -358,16 +359,17 @@ class BuildComponent(BaseComponent): # Label the pushed manifests with the build metadata. manifest_digests = kwargs.get('digests') or [] for digest in manifest_digests: - try: - manifest = model.tag.load_manifest_by_digest(self._current_job.namespace, - self._current_job.repo_name, digest) - model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID, - build_id, 'internal', 'text/plain') - except model.InvalidManifestException: - logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s', - digest, self._current_job.namespace, self._current_job.repo_name, - build_id) - continue + with UseThenDisconnect(app.config): + try: + manifest = model.tag.load_manifest_by_digest(self._current_job.namespace, + self._current_job.repo_name, digest) + model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID, + build_id, 'internal', 'text/plain') + except model.InvalidManifestException: + logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s', + digest, self._current_job.namespace, self._current_job.repo_name, + build_id) + continue # Send the notification that the build has completed successfully. self._current_job.send_notification('build_success', diff --git a/buildman/jobutil/buildjob.py b/buildman/jobutil/buildjob.py index 18381b0f2..1e33ce9b0 100644 --- a/buildman/jobutil/buildjob.py +++ b/buildman/jobutil/buildjob.py @@ -1,9 +1,11 @@ import json import logging +from app import app from cachetools import lru_cache from notifications import spawn_notification from data import model +from data.database import UseThenDisconnect from util.imagetree import ImageTree from util.morecollections import AttrDict @@ -40,11 +42,12 @@ class BuildJob(object): @lru_cache(maxsize=1) def _load_repo_build(self): - try: - return model.build.get_repository_build(self.build_uuid) - except model.InvalidRepositoryBuildException: - raise BuildJobLoadException( - 'Could not load repository build with ID %s' % self.build_uuid) + with UseThenDisconnect(app.config): + try: + return model.build.get_repository_build(self.build_uuid) + except model.InvalidRepositoryBuildException: + raise BuildJobLoadException( + 'Could not load repository build with ID %s' % self.build_uuid) @property def build_uuid(self): @@ -108,54 +111,56 @@ class BuildJob(object): starting at the given base_image_id. This mimics the Docker cache checking, so it should, in theory, provide "perfect" caching. """ - # Lookup the base image in the repository. If it doesn't exist, nothing more to do. - repo_build = self.repo_build - repo_namespace = repo_build.repository.namespace_user.username - repo_name = repo_build.repository.name + with UseThenDisconnect(app.config): + # Lookup the base image in the repository. If it doesn't exist, nothing more to do. + repo_build = self.repo_build + repo_namespace = repo_build.repository.namespace_user.username + repo_name = repo_build.repository.name - base_image = model.image.get_image(repo_build.repository, base_image_id) - if base_image is None: - return None + base_image = model.image.get_image(repo_build.repository, base_image_id) + if base_image is None: + return None - # Build an in-memory tree of the full heirarchy of images in the repository. - all_images = model.image.get_repository_images_without_placements(repo_build.repository, - with_ancestor=base_image) + # Build an in-memory tree of the full heirarchy of images in the repository. + all_images = model.image.get_repository_images_without_placements(repo_build.repository, + with_ancestor=base_image) - all_tags = model.tag.list_repository_tags(repo_namespace, repo_name) - tree = ImageTree(all_images, all_tags, base_filter=base_image.id) + all_tags = model.tag.list_repository_tags(repo_namespace, repo_name) + tree = ImageTree(all_images, all_tags, base_filter=base_image.id) - # Find a path in the tree, starting at the base image, that matches the cache comments - # or some subset thereof. - def checker(step, image): - if step >= len(cache_commands): - return False + # Find a path in the tree, starting at the base image, that matches the cache comments + # or some subset thereof. + def checker(step, image): + if step >= len(cache_commands): + return False - full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step] - logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command) + full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step] + logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command) - return image.command == full_command + return image.command == full_command - path = tree.find_longest_path(base_image.id, checker) - if not path: - return None + path = tree.find_longest_path(base_image.id, checker) + if not path: + return None - # Find any tag associated with the last image in the path. - return tree.tag_containing_image(path[-1]) + # Find any tag associated with the last image in the path. + return tree.tag_containing_image(path[-1]) def _determine_cached_tag_by_tag(self): """ Determines the cached tag by looking for one of the tags being built, and seeing if it exists in the repository. This is a fallback for when no comment information is available. """ - tags = self.build_config.get('docker_tags', ['latest']) - repository = self.repo_build.repository - existing_tags = model.tag.list_repository_tags(repository.namespace_user.username, - repository.name) - cached_tags = set(tags) & set([tag.name for tag in existing_tags]) - if cached_tags: - return list(cached_tags)[0] + with UseThenDisconnect(app.config): + tags = self.build_config.get('docker_tags', ['latest']) + repository = self.repo_build.repository + existing_tags = model.tag.list_repository_tags(repository.namespace_user.username, + repository.name) + cached_tags = set(tags) & set([tag.name for tag in existing_tags]) + if cached_tags: + return list(cached_tags)[0] - return None + return None class BuildJobNotifier(object): @@ -186,31 +191,32 @@ class BuildJobNotifier(object): ) def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None): - tags = self.build_config.get('docker_tags', ['latest']) - event_data = { - 'build_id': self.repo_build.uuid, - 'build_name': self.repo_build.display_name, - 'docker_tags': tags, - 'trigger_id': self.repo_build.trigger.uuid, - 'trigger_kind': self.repo_build.trigger.service.name, - 'trigger_metadata': self.build_config.get('trigger_metadata', {}) - } + with UseThenDisconnect(app.config): + tags = self.build_config.get('docker_tags', ['latest']) + event_data = { + 'build_id': self.repo_build.uuid, + 'build_name': self.repo_build.display_name, + 'docker_tags': tags, + 'trigger_id': self.repo_build.trigger.uuid, + 'trigger_kind': self.repo_build.trigger.service.name, + 'trigger_metadata': self.build_config.get('trigger_metadata', {}) + } - if image_id is not None: - event_data['image_id'] = image_id + if image_id is not None: + event_data['image_id'] = image_id - if manifest_digests: - event_data['manifest_digests'] = manifest_digests + if manifest_digests: + event_data['manifest_digests'] = manifest_digests - if error_message is not None: - event_data['error_message'] = error_message + if error_message is not None: + event_data['error_message'] = error_message - # TODO(jzelinskie): remove when more endpoints have been converted to using - # interfaces - repo = AttrDict({ - 'namespace_name': self.repo_build.repository.namespace_user.username, - 'name': self.repo_build.repository.name, - }) - spawn_notification(repo, kind, event_data, - subpage='build/%s' % self.repo_build.uuid, - pathargs=['build', self.repo_build.uuid]) \ No newline at end of file + # TODO(jzelinskie): remove when more endpoints have been converted to using + # interfaces + repo = AttrDict({ + 'namespace_name': self.repo_build.repository.namespace_user.username, + 'name': self.repo_build.repository.name, + }) + spawn_notification(repo, kind, event_data, + subpage='build/%s' % self.repo_build.uuid, + pathargs=['build', self.repo_build.uuid]) diff --git a/data/database.py b/data/database.py index 56d62e621..eccbc31f1 100644 --- a/data/database.py +++ b/data/database.py @@ -168,6 +168,7 @@ class CloseForLongOperation(object): self.config_object = config_object def __enter__(self): + # TODO(jschorr): Remove this stupid hack. if self.config_object.get('TESTING') is True: return @@ -185,9 +186,17 @@ class UseThenDisconnect(object): self.config_object = config_object def __enter__(self): + # TODO(jschorr): Remove this stupid hack. + if self.config_object.get('TESTING') is True: + return + configure(self.config_object) def __exit__(self, typ, value, traceback): + # TODO(jschorr): Remove this stupid hack. + if self.config_object.get('TESTING') is True: + return + close_db_filter(None)