vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Start on a basic registry_model interface and change a single module to use it. This will allow us to completely abstract out how we deal with registry-related tables and ensure that transitioning to the new OCI-like model will be easier to do.

Browse source
This commit is contained in:
Joseph Schorr 2018-08-13 18:09:05 -04:00
parent 73e585df49
commit c30214c7a8
7 changed files with 150 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

68
buildman/jobutil/buildjob.py
View file

@ -5,6 +5,8 @@ from app import app
from cachetools import lru_cache
from notifications import spawn_notification
from data import model
from data.registry_model import registry_model
from data.registry_model.datatypes import RepositoryReference
from data.database import UseThenDisconnect
from util.imagetree import ImageTree
from util.morecollections import AttrDict
@ -27,7 +29,7 @@ class BuildJob(object):
self.build_notifier = BuildJobNotifier(self.build_uuid)
except ValueError:
raise BuildJobLoadException(
'Could not parse build queue item config with ID %s' % self.job_details['build_uuid']
'Could not parse build queue item config with ID %s' % self.job_details['build_uuid']
)
@property
@ -95,70 +97,24 @@ class BuildJob(object):
def determine_cached_tag(self, base_image_id=None, cache_comments=None):
""" Returns the tag to pull to prime the cache or None if none. """
cached_tag = None
if base_image_id and cache_comments:
cached_tag = self._determine_cached_tag_by_comments(base_image_id, cache_comments)
if not cached_tag:
cached_tag = self._determine_cached_tag_by_tag()
cached_tag = self._determine_cached_tag_by_tag()
logger.debug('Determined cached tag %s for %s: %s', cached_tag, base_image_id, cache_comments)
return cached_tag
def _determine_cached_tag_by_comments(self, base_image_id, cache_commands):
""" Determines the tag to use for priming the cache for this build job, by matching commands
starting at the given base_image_id. This mimics the Docker cache checking, so it should,
in theory, provide "perfect" caching.
"""
with UseThenDisconnect(app.config):
# Lookup the base image in the repository. If it doesn't exist, nothing more to do.
repo_build = self.repo_build
repo_namespace = repo_build.repository.namespace_user.username
repo_name = repo_build.repository.name
base_image = model.image.get_image(repo_build.repository, base_image_id)
if base_image is None:
return None
# Build an in-memory tree of the full heirarchy of images in the repository.
all_images = model.image.get_repository_images_without_placements(repo_build.repository,
with_ancestor=base_image)
all_tags = model.tag.list_repository_tags(repo_namespace, repo_name)
tree = ImageTree(all_images, all_tags, base_filter=base_image.id)
# Find a path in the tree, starting at the base image, that matches the cache comments
# or some subset thereof.
def checker(step, image):
if step >= len(cache_commands):
return False
full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command)
return image.command == full_command
path = tree.find_longest_path(base_image.id, checker)
if not path:
return None
# Find any tag associated with the last image in the path.
return tree.tag_containing_image(path[-1])
def _determine_cached_tag_by_tag(self):
""" Determines the cached tag by looking for one of the tags being built, and seeing if it
exists in the repository. This is a fallback for when no comment information is available.
"""
with UseThenDisconnect(app.config):
tags = self.build_config.get('docker_tags', ['latest'])
repository = self.repo_build.repository
existing_tags = model.tag.list_repository_tags(repository.namespace_user.username,
repository.name)
cached_tags = set(tags) & set([tag.name for tag in existing_tags])
if cached_tags:
return list(cached_tags)[0]
repository = RepositoryReference.for_repo_obj(self.repo_build.repository)
matching_tag = registry_model.find_matching_tag(repository, tags)
if matching_tag is not None:
return matching_tag.name
most_recent_tag = registry_model.get_most_recent_tag(repository)
if most_recent_tag is not None:
return most_recent_tag.name
return None