vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2824 from coreos-inc/close-build-connection

Browse source 
Close the database connection after operations in buildman
This commit is contained in:
josephschorr 2017-07-26 12:19:04 -04:00 committed by GitHub
commit 543bf21e6b
3 changed files with 90 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
buildman/component/buildcomponent.py
View file

@ -15,8 +15,9 @@ from buildman.jobutil.buildjob import BuildJobLoadException
from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.buildstatus import StatusHandler
from buildman.jobutil.workererror import WorkerError from buildman.jobutil.workererror import WorkerError
from app import app
from data import model from data import model
from data.database import BUILD_PHASE from data.database import BUILD_PHASE, UseThenDisconnect
from data.model import InvalidRepositoryBuildException from data.model import InvalidRepositoryBuildException
from util import slash_join from util import slash_join
@ -358,16 +359,17 @@ class BuildComponent(BaseComponent):
# Label the pushed manifests with the build metadata. # Label the pushed manifests with the build metadata.
manifest_digests = kwargs.get('digests') or [] manifest_digests = kwargs.get('digests') or []
for digest in manifest_digests: for digest in manifest_digests:
try: with UseThenDisconnect(app.config):
manifest = model.tag.load_manifest_by_digest(self._current_job.namespace, try:
self._current_job.repo_name, digest) manifest = model.tag.load_manifest_by_digest(self._current_job.namespace,
model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID, self._current_job.repo_name, digest)
build_id, 'internal', 'text/plain') model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID,
except model.InvalidManifestException: build_id, 'internal', 'text/plain')
logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s', except model.InvalidManifestException:
digest, self._current_job.namespace, self._current_job.repo_name, logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s',
build_id) digest, self._current_job.namespace, self._current_job.repo_name,
continue build_id)
continue
# Send the notification that the build has completed successfully. # Send the notification that the build has completed successfully.
self._current_job.send_notification('build_success', self._current_job.send_notification('build_success',

130
buildman/jobutil/buildjob.py
View file

@ -1,9 +1,11 @@
import json import json
import logging import logging
from app import app
from cachetools import lru_cache from cachetools import lru_cache
from notifications import spawn_notification from notifications import spawn_notification
from data import model from data import model
from data.database import UseThenDisconnect
from util.imagetree import ImageTree from util.imagetree import ImageTree
from util.morecollections import AttrDict from util.morecollections import AttrDict
@ -40,11 +42,12 @@ class BuildJob(object):
@lru_cache(maxsize=1) @lru_cache(maxsize=1)
def _load_repo_build(self): def _load_repo_build(self):
try: with UseThenDisconnect(app.config):
return model.build.get_repository_build(self.build_uuid) try:
except model.InvalidRepositoryBuildException: return model.build.get_repository_build(self.build_uuid)
raise BuildJobLoadException( except model.InvalidRepositoryBuildException:
'Could not load repository build with ID %s' % self.build_uuid) raise BuildJobLoadException(
'Could not load repository build with ID %s' % self.build_uuid)
@property @property
def build_uuid(self): def build_uuid(self):
@ -108,54 +111,56 @@ class BuildJob(object):
starting at the given base_image_id. This mimics the Docker cache checking, so it should, starting at the given base_image_id. This mimics the Docker cache checking, so it should,
in theory, provide "perfect" caching. in theory, provide "perfect" caching.
""" """
# Lookup the base image in the repository. If it doesn't exist, nothing more to do. with UseThenDisconnect(app.config):
repo_build = self.repo_build # Lookup the base image in the repository. If it doesn't exist, nothing more to do.
repo_namespace = repo_build.repository.namespace_user.username repo_build = self.repo_build
repo_name = repo_build.repository.name repo_namespace = repo_build.repository.namespace_user.username
repo_name = repo_build.repository.name
base_image = model.image.get_image(repo_build.repository, base_image_id) base_image = model.image.get_image(repo_build.repository, base_image_id)
if base_image is None: if base_image is None:
return None return None
# Build an in-memory tree of the full heirarchy of images in the repository. # Build an in-memory tree of the full heirarchy of images in the repository.
all_images = model.image.get_repository_images_without_placements(repo_build.repository, all_images = model.image.get_repository_images_without_placements(repo_build.repository,
with_ancestor=base_image) with_ancestor=base_image)
all_tags = model.tag.list_repository_tags(repo_namespace, repo_name) all_tags = model.tag.list_repository_tags(repo_namespace, repo_name)
tree = ImageTree(all_images, all_tags, base_filter=base_image.id) tree = ImageTree(all_images, all_tags, base_filter=base_image.id)
# Find a path in the tree, starting at the base image, that matches the cache comments # Find a path in the tree, starting at the base image, that matches the cache comments
# or some subset thereof. # or some subset thereof.
def checker(step, image): def checker(step, image):
if step >= len(cache_commands): if step >= len(cache_commands):
return False return False
full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step] full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command) logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command)
return image.command == full_command return image.command == full_command
path = tree.find_longest_path(base_image.id, checker) path = tree.find_longest_path(base_image.id, checker)
if not path: if not path:
return None return None
# Find any tag associated with the last image in the path. # Find any tag associated with the last image in the path.
return tree.tag_containing_image(path[-1]) return tree.tag_containing_image(path[-1])
def _determine_cached_tag_by_tag(self): def _determine_cached_tag_by_tag(self):
""" Determines the cached tag by looking for one of the tags being built, and seeing if it """ Determines the cached tag by looking for one of the tags being built, and seeing if it
exists in the repository. This is a fallback for when no comment information is available. exists in the repository. This is a fallback for when no comment information is available.
""" """
tags = self.build_config.get('docker_tags', ['latest']) with UseThenDisconnect(app.config):
repository = self.repo_build.repository tags = self.build_config.get('docker_tags', ['latest'])
existing_tags = model.tag.list_repository_tags(repository.namespace_user.username, repository = self.repo_build.repository
repository.name) existing_tags = model.tag.list_repository_tags(repository.namespace_user.username,
cached_tags = set(tags) & set([tag.name for tag in existing_tags]) repository.name)
if cached_tags: cached_tags = set(tags) & set([tag.name for tag in existing_tags])
return list(cached_tags)[0] if cached_tags:
return list(cached_tags)[0]
return None return None
class BuildJobNotifier(object): class BuildJobNotifier(object):
@ -186,31 +191,32 @@ class BuildJobNotifier(object):
) )
def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None): def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None):
tags = self.build_config.get('docker_tags', ['latest']) with UseThenDisconnect(app.config):
event_data = { tags = self.build_config.get('docker_tags', ['latest'])
'build_id': self.repo_build.uuid, event_data = {
'build_name': self.repo_build.display_name, 'build_id': self.repo_build.uuid,
'docker_tags': tags, 'build_name': self.repo_build.display_name,
'trigger_id': self.repo_build.trigger.uuid, 'docker_tags': tags,
'trigger_kind': self.repo_build.trigger.service.name, 'trigger_id': self.repo_build.trigger.uuid,
'trigger_metadata': self.build_config.get('trigger_metadata', {}) 'trigger_kind': self.repo_build.trigger.service.name,
} 'trigger_metadata': self.build_config.get('trigger_metadata', {})
}
if image_id is not None: if image_id is not None:
event_data['image_id'] = image_id event_data['image_id'] = image_id
if manifest_digests: if manifest_digests:
event_data['manifest_digests'] = manifest_digests event_data['manifest_digests'] = manifest_digests
if error_message is not None: if error_message is not None:
event_data['error_message'] = error_message event_data['error_message'] = error_message
# TODO(jzelinskie): remove when more endpoints have been converted to using # TODO(jzelinskie): remove when more endpoints have been converted to using
# interfaces # interfaces
repo = AttrDict({ repo = AttrDict({
'namespace_name': self.repo_build.repository.namespace_user.username, 'namespace_name': self.repo_build.repository.namespace_user.username,
'name': self.repo_build.repository.name, 'name': self.repo_build.repository.name,
}) })
spawn_notification(repo, kind, event_data, spawn_notification(repo, kind, event_data,
subpage='build/%s' % self.repo_build.uuid, subpage='build/%s' % self.repo_build.uuid,
pathargs=['build', self.repo_build.uuid]) pathargs=['build', self.repo_build.uuid])

9
data/database.py
View file

@ -168,6 +168,7 @@ class CloseForLongOperation(object):
self.config_object = config_object self.config_object = config_object
def __enter__(self): def __enter__(self):
# TODO(jschorr): Remove this stupid hack.
if self.config_object.get('TESTING') is True: if self.config_object.get('TESTING') is True:
return return
@ -185,9 +186,17 @@ class UseThenDisconnect(object):
self.config_object = config_object self.config_object = config_object
def __enter__(self): def __enter__(self):
# TODO(jschorr): Remove this stupid hack.
if self.config_object.get('TESTING') is True:
return
configure(self.config_object) configure(self.config_object)
def __exit__(self, typ, value, traceback): def __exit__(self, typ, value, traceback):
# TODO(jschorr): Remove this stupid hack.
if self.config_object.get('TESTING') is True:
return
close_db_filter(None) close_db_filter(None)