Close the database connection after operations in buildman
Also adds a *temporary* hack to prevent this from breaking tests
This commit is contained in:
Joseph Schorr
parent
c271b1f386
commit
9febb539a7
3 changed files with 90 additions and 73 deletions
|
|
@ -15,8 +15,9 @@ from buildman.jobutil.buildjob import BuildJobLoadException
|
|||
from buildman.jobutil.buildstatus import StatusHandler
|
||||
from buildman.jobutil.workererror import WorkerError
|
||||
|
||||
from app import app
|
||||
from data import model
|
||||
from data.database import BUILD_PHASE
|
||||
from data.database import BUILD_PHASE, UseThenDisconnect
|
||||
from data.model import InvalidRepositoryBuildException
|
||||
from util import slash_join
|
||||
|
||||
|
|
@ -358,16 +359,17 @@ class BuildComponent(BaseComponent):
|
|||
# Label the pushed manifests with the build metadata.
|
||||
manifest_digests = kwargs.get('digests') or []
|
||||
for digest in manifest_digests:
|
||||
try:
|
||||
manifest = model.tag.load_manifest_by_digest(self._current_job.namespace,
|
||||
self._current_job.repo_name, digest)
|
||||
model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID,
|
||||
build_id, 'internal', 'text/plain')
|
||||
except model.InvalidManifestException:
|
||||
logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s',
|
||||
digest, self._current_job.namespace, self._current_job.repo_name,
|
||||
build_id)
|
||||
continue
|
||||
with UseThenDisconnect(app.config):
|
||||
try:
|
||||
manifest = model.tag.load_manifest_by_digest(self._current_job.namespace,
|
||||
self._current_job.repo_name, digest)
|
||||
model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID,
|
||||
build_id, 'internal', 'text/plain')
|
||||
except model.InvalidManifestException:
|
||||
logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s',
|
||||
digest, self._current_job.namespace, self._current_job.repo_name,
|
||||
build_id)
|
||||
continue
|
||||
|
||||
# Send the notification that the build has completed successfully.
|
||||
self._current_job.send_notification('build_success',
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import json
|
||||
import logging
|
||||
|
||||
from app import app
|
||||
from cachetools import lru_cache
|
||||
from notifications import spawn_notification
|
||||
from data import model
|
||||
from data.database import UseThenDisconnect
|
||||
from util.imagetree import ImageTree
|
||||
from util.morecollections import AttrDict
|
||||
|
||||
|
|
@ -40,11 +42,12 @@ class BuildJob(object):
|
|||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_repo_build(self):
|
||||
try:
|
||||
return model.build.get_repository_build(self.build_uuid)
|
||||
except model.InvalidRepositoryBuildException:
|
||||
raise BuildJobLoadException(
|
||||
'Could not load repository build with ID %s' % self.build_uuid)
|
||||
with UseThenDisconnect(app.config):
|
||||
try:
|
||||
return model.build.get_repository_build(self.build_uuid)
|
||||
except model.InvalidRepositoryBuildException:
|
||||
raise BuildJobLoadException(
|
||||
'Could not load repository build with ID %s' % self.build_uuid)
|
||||
|
||||
@property
|
||||
def build_uuid(self):
|
||||
|
|
@ -108,54 +111,56 @@ class BuildJob(object):
|
|||
starting at the given base_image_id. This mimics the Docker cache checking, so it should,
|
||||
in theory, provide "perfect" caching.
|
||||
"""
|
||||
# Lookup the base image in the repository. If it doesn't exist, nothing more to do.
|
||||
repo_build = self.repo_build
|
||||
repo_namespace = repo_build.repository.namespace_user.username
|
||||
repo_name = repo_build.repository.name
|
||||
with UseThenDisconnect(app.config):
|
||||
# Lookup the base image in the repository. If it doesn't exist, nothing more to do.
|
||||
repo_build = self.repo_build
|
||||
repo_namespace = repo_build.repository.namespace_user.username
|
||||
repo_name = repo_build.repository.name
|
||||
|
||||
base_image = model.image.get_image(repo_build.repository, base_image_id)
|
||||
if base_image is None:
|
||||
return None
|
||||
base_image = model.image.get_image(repo_build.repository, base_image_id)
|
||||
if base_image is None:
|
||||
return None
|
||||
|
||||
# Build an in-memory tree of the full heirarchy of images in the repository.
|
||||
all_images = model.image.get_repository_images_without_placements(repo_build.repository,
|
||||
with_ancestor=base_image)
|
||||
# Build an in-memory tree of the full heirarchy of images in the repository.
|
||||
all_images = model.image.get_repository_images_without_placements(repo_build.repository,
|
||||
with_ancestor=base_image)
|
||||
|
||||
all_tags = model.tag.list_repository_tags(repo_namespace, repo_name)
|
||||
tree = ImageTree(all_images, all_tags, base_filter=base_image.id)
|
||||
all_tags = model.tag.list_repository_tags(repo_namespace, repo_name)
|
||||
tree = ImageTree(all_images, all_tags, base_filter=base_image.id)
|
||||
|
||||
# Find a path in the tree, starting at the base image, that matches the cache comments
|
||||
# or some subset thereof.
|
||||
def checker(step, image):
|
||||
if step >= len(cache_commands):
|
||||
return False
|
||||
# Find a path in the tree, starting at the base image, that matches the cache comments
|
||||
# or some subset thereof.
|
||||
def checker(step, image):
|
||||
if step >= len(cache_commands):
|
||||
return False
|
||||
|
||||
full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
|
||||
logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command)
|
||||
full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
|
||||
logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command)
|
||||
|
||||
return image.command == full_command
|
||||
return image.command == full_command
|
||||
|
||||
path = tree.find_longest_path(base_image.id, checker)
|
||||
if not path:
|
||||
return None
|
||||
path = tree.find_longest_path(base_image.id, checker)
|
||||
if not path:
|
||||
return None
|
||||
|
||||
# Find any tag associated with the last image in the path.
|
||||
return tree.tag_containing_image(path[-1])
|
||||
# Find any tag associated with the last image in the path.
|
||||
return tree.tag_containing_image(path[-1])
|
||||
|
||||
|
||||
def _determine_cached_tag_by_tag(self):
|
||||
""" Determines the cached tag by looking for one of the tags being built, and seeing if it
|
||||
exists in the repository. This is a fallback for when no comment information is available.
|
||||
"""
|
||||
tags = self.build_config.get('docker_tags', ['latest'])
|
||||
repository = self.repo_build.repository
|
||||
existing_tags = model.tag.list_repository_tags(repository.namespace_user.username,
|
||||
repository.name)
|
||||
cached_tags = set(tags) & set([tag.name for tag in existing_tags])
|
||||
if cached_tags:
|
||||
return list(cached_tags)[0]
|
||||
with UseThenDisconnect(app.config):
|
||||
tags = self.build_config.get('docker_tags', ['latest'])
|
||||
repository = self.repo_build.repository
|
||||
existing_tags = model.tag.list_repository_tags(repository.namespace_user.username,
|
||||
repository.name)
|
||||
cached_tags = set(tags) & set([tag.name for tag in existing_tags])
|
||||
if cached_tags:
|
||||
return list(cached_tags)[0]
|
||||
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
class BuildJobNotifier(object):
|
||||
|
|
@ -186,31 +191,32 @@ class BuildJobNotifier(object):
|
|||
)
|
||||
|
||||
def send_notification(self, kind, error_message=None, image_id=None, manifest_digests=None):
|
||||
tags = self.build_config.get('docker_tags', ['latest'])
|
||||
event_data = {
|
||||
'build_id': self.repo_build.uuid,
|
||||
'build_name': self.repo_build.display_name,
|
||||
'docker_tags': tags,
|
||||
'trigger_id': self.repo_build.trigger.uuid,
|
||||
'trigger_kind': self.repo_build.trigger.service.name,
|
||||
'trigger_metadata': self.build_config.get('trigger_metadata', {})
|
||||
}
|
||||
with UseThenDisconnect(app.config):
|
||||
tags = self.build_config.get('docker_tags', ['latest'])
|
||||
event_data = {
|
||||
'build_id': self.repo_build.uuid,
|
||||
'build_name': self.repo_build.display_name,
|
||||
'docker_tags': tags,
|
||||
'trigger_id': self.repo_build.trigger.uuid,
|
||||
'trigger_kind': self.repo_build.trigger.service.name,
|
||||
'trigger_metadata': self.build_config.get('trigger_metadata', {})
|
||||
}
|
||||
|
||||
if image_id is not None:
|
||||
event_data['image_id'] = image_id
|
||||
if image_id is not None:
|
||||
event_data['image_id'] = image_id
|
||||
|
||||
if manifest_digests:
|
||||
event_data['manifest_digests'] = manifest_digests
|
||||
if manifest_digests:
|
||||
event_data['manifest_digests'] = manifest_digests
|
||||
|
||||
if error_message is not None:
|
||||
event_data['error_message'] = error_message
|
||||
if error_message is not None:
|
||||
event_data['error_message'] = error_message
|
||||
|
||||
# TODO(jzelinskie): remove when more endpoints have been converted to using
|
||||
# interfaces
|
||||
repo = AttrDict({
|
||||
'namespace_name': self.repo_build.repository.namespace_user.username,
|
||||
'name': self.repo_build.repository.name,
|
||||
})
|
||||
spawn_notification(repo, kind, event_data,
|
||||
subpage='build/%s' % self.repo_build.uuid,
|
||||
pathargs=['build', self.repo_build.uuid])
|
||||
# TODO(jzelinskie): remove when more endpoints have been converted to using
|
||||
# interfaces
|
||||
repo = AttrDict({
|
||||
'namespace_name': self.repo_build.repository.namespace_user.username,
|
||||
'name': self.repo_build.repository.name,
|
||||
})
|
||||
spawn_notification(repo, kind, event_data,
|
||||
subpage='build/%s' % self.repo_build.uuid,
|
||||
pathargs=['build', self.repo_build.uuid])
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ class CloseForLongOperation(object):
|
|||
self.config_object = config_object
|
||||
|
||||
def __enter__(self):
|
||||
# TODO(jschorr): Remove this stupid hack.
|
||||
if self.config_object.get('TESTING') is True:
|
||||
return
|
||||
|
||||
|
|
@ -185,9 +186,17 @@ class UseThenDisconnect(object):
|
|||
self.config_object = config_object
|
||||
|
||||
def __enter__(self):
|
||||
# TODO(jschorr): Remove this stupid hack.
|
||||
if self.config_object.get('TESTING') is True:
|
||||
return
|
||||
|
||||
configure(self.config_object)
|
||||
|
||||
def __exit__(self, typ, value, traceback):
|
||||
# TODO(jschorr): Remove this stupid hack.
|
||||
if self.config_object.get('TESTING') is True:
|
||||
return
|
||||
|
||||
close_db_filter(None)
|
||||
|
||||
|
||||
|
|
|
|||
Reference in a new issue