diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index 4b25c4636..d0e9edfae 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -17,9 +17,10 @@ from buildman.jobutil.buildstatus import StatusHandler from buildman.jobutil.workererror import WorkerError from app import app -from data import model from data.database import BUILD_PHASE, UseThenDisconnect from data.model import InvalidRepositoryBuildException +from data.registry_model import registry_model +from data.registry_model.datatypes import RepositoryReference from util import slash_join HEARTBEAT_DELTA = datetime.timedelta(seconds=60) @@ -29,6 +30,9 @@ INITIAL_TIMEOUT = 25 SUPPORTED_WORKER_VERSIONS = ['0.3'] +# Label which marks a manifest with its source build ID. +INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid' + logger = logging.getLogger(__name__) class ComponentStatus(object): @@ -357,18 +361,17 @@ class BuildComponent(BaseComponent): # Label the pushed manifests with the build metadata. manifest_digests = kwargs.get('digests') or [] - for digest in manifest_digests: - with UseThenDisconnect(app.config): - try: - manifest = model.tag.load_manifest_by_digest(self._current_job.namespace, - self._current_job.repo_name, digest) - model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID, - build_id, 'internal', 'text/plain') - except model.InvalidManifestException: - logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s', - digest, self._current_job.namespace, self._current_job.repo_name, - build_id) - continue + repository = registry_model.lookup_repository(self._current_job.namespace, + self._current_job.repo_name) + if repository is not None: + for digest in manifest_digests: + with UseThenDisconnect(app.config): + manifest = registry_model.lookup_manifest_by_digest(repository, digest) + if manifest is None: + continue + + registry_model.create_manifest_label(manifest, INTERNAL_LABEL_BUILD_UUID, + build_id, 'internal', 'text/plain') # Send the notification that the build has completed successfully. self._current_job.send_notification('build_success', diff --git a/data/model/label.py b/data/model/label.py index 1592efaa6..3e461e2d7 100644 --- a/data/model/label.py +++ b/data/model/label.py @@ -11,9 +11,6 @@ from util.validation import is_json logger = logging.getLogger(__name__) -# Label which marks a manifest with its source build ID. -INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid' - @lru_cache(maxsize=1) def get_label_source_types(): diff --git a/data/registry_model/datatypes.py b/data/registry_model/datatypes.py index 3f2cae187..4cf2a0015 100644 --- a/data/registry_model/datatypes.py +++ b/data/registry_model/datatypes.py @@ -7,6 +7,9 @@ class RepositoryReference(object): @classmethod def for_repo_obj(cls, repo_obj): + if repo_obj is None: + return None + return RepositoryReference(repo_obj.id) @@ -18,3 +21,13 @@ class Tag(namedtuple('Tag', ['id', 'name'])): return None return Tag(id=repository_tag.id, name=repository_tag.name) + + +class Manifest(namedtuple('Manifest', ['id', 'digest'])): + """ Manifest represents a manifest in a repository. """ + @classmethod + def for_tag_manifest(cls, tag_manifest): + if tag_manifest is None: + return None + + return Manifest(id=tag_manifest.id, digest=tag_manifest.digest) diff --git a/data/registry_model/interface.py b/data/registry_model/interface.py index e67366733..d92ccdea9 100644 --- a/data/registry_model/interface.py +++ b/data/registry_model/interface.py @@ -19,3 +19,21 @@ class RegistryDataInterface(object): """ Returns the most recently pushed alive tag in the repository, if any. If none, returns None. """ + + @abstractmethod + def lookup_repository(self, namespace_name, repo_name, kind_filter=None): + """ Looks up and returns a reference to the repository with the given namespace and name, + or None if none. """ + + @abstractmethod + def get_manifest_for_tag(self, tag): + """ Returns the manifest associated with the given tag. """ + + @abstractmethod + def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False): + """ Looks up the manifest with the given digest under the given repository and returns it + or None if none. """ + + @abstractmethod + def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None): + """ Creates a label on the manifest with the given key and value. """ diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index 5cd6ed631..69443ec71 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -1,6 +1,7 @@ +from data import database from data import model from data.registry_model.interface import RegistryDataInterface -from data.registry_model.datatypes import Tag +from data.registry_model.datatypes import Tag, RepositoryReference, Manifest class PreOCIModel(RegistryDataInterface): @@ -23,5 +24,41 @@ class PreOCIModel(RegistryDataInterface): found_tag = model.tag.get_most_recent_tag(repository_ref.repo_id) return Tag.for_repository_tag(found_tag) + def lookup_repository(self, namespace_name, repo_name, kind_filter=None): + """ Looks up and returns a reference to the repository with the given namespace and name, + or None if none. """ + repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter) + return RepositoryReference.for_repo_obj(repo) + + def get_manifest_for_tag(self, tag): + """ Returns the manifest associated with the given tag. """ + try: + tag_manifest = database.TagManifest.get(tag_id=tag.id) + except database.TagManifest.DoesNotExist: + return + + return Manifest.for_tag_manifest(tag_manifest) + + def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False): + """ Looks up the manifest with the given digest under the given repository and returns it + or None if none. """ + repo = model.repository.lookup_repository(repository_ref.repo_id) + if repo is None: + return None + + tag_manifest = model.tag.load_manifest_by_digest(repo.namespace_user.username, + repo.name, + manifest_digest, allow_dead=allow_dead) + return Manifest.for_tag_manifest(tag_manifest) + + def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None): + """ Creates a label on the manifest with the given key and value. """ + try: + tag_manifest = database.TagManifest.get(id=manifest.id) + except database.TagManifest.DoesNotExist: + return + + model.label.create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name) + pre_oci_model = PreOCIModel() diff --git a/data/registry_model/test/test_pre_oci_model.py b/data/registry_model/test/test_pre_oci_model.py index 6e81c3fff..7e46b8cc6 100644 --- a/data/registry_model/test/test_pre_oci_model.py +++ b/data/registry_model/test/test_pre_oci_model.py @@ -39,3 +39,39 @@ def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model) assert found is None else: assert found.name in expected + + +@pytest.mark.parametrize('repo_namespace, repo_name, expected', [ + ('devtable', 'simple', True), + ('buynlarge', 'orgrepo', True), + ('buynlarge', 'unknownrepo', False), +]) +def test_lookup_repository(repo_namespace, repo_name, expected, pre_oci_model): + repo_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) + if expected: + assert repo_ref + else: + assert repo_ref is None + + +@pytest.mark.parametrize('repo_namespace, repo_name', [ + ('devtable', 'simple'), + ('buynlarge', 'orgrepo'), +]) +def test_lookup_manifests(repo_namespace, repo_name, pre_oci_model): + repo = model.repository.get_repository(repo_namespace, repo_name) + repository_ref = RepositoryReference.for_repo_obj(repo) + found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) + found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) + found = pre_oci_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest) + assert found.id == found_manifest.id + assert found.digest == found_manifest.digest + + +def test_create_manifest_label(pre_oci_model): + repo = model.repository.get_repository('devtable', 'simple') + repository_ref = RepositoryReference.for_repo_obj(repo) + found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) + found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) + + pre_oci_model.create_manifest_label(found_manifest, 'foo', 'bar', 'internal')