Change build component labeling to use new registry interface

buildman/component/buildcomponent.py

@@ -17,9 +17,10 @@ from buildman.jobutil.buildstatus import StatusHandler
 from buildman.jobutil.workererror import WorkerError
 
 from app import app
-from data import model
 from data.database import BUILD_PHASE, UseThenDisconnect
 from data.model import InvalidRepositoryBuildException
+from data.registry_model import registry_model
+from data.registry_model.datatypes import RepositoryReference
 from util import slash_join
 
 HEARTBEAT_DELTA = datetime.timedelta(seconds=60)
@@ -29,6 +30,9 @@ INITIAL_TIMEOUT = 25
 
 SUPPORTED_WORKER_VERSIONS = ['0.3']
 
+# Label which marks a manifest with its source build ID.
+INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid'
+
 logger = logging.getLogger(__name__)
 
 class ComponentStatus(object):
@@ -357,18 +361,17 @@ class BuildComponent(BaseComponent):
 
       # Label the pushed manifests with the build metadata.
       manifest_digests = kwargs.get('digests') or []
-      for digest in manifest_digests:
-        with UseThenDisconnect(app.config):
-          try:
-            manifest = model.tag.load_manifest_by_digest(self._current_job.namespace,
-                                                         self._current_job.repo_name, digest)
-            model.label.create_manifest_label(manifest, model.label.INTERNAL_LABEL_BUILD_UUID,
-                                              build_id, 'internal', 'text/plain')
-          except model.InvalidManifestException:
-            logger.debug('Could not find built manifest with digest %s under repo %s/%s for build %s',
-                         digest, self._current_job.namespace, self._current_job.repo_name,
-                         build_id)
-            continue
+      repository = registry_model.lookup_repository(self._current_job.namespace,
+                                                    self._current_job.repo_name)
+      if repository is not None:
+        for digest in manifest_digests:
+          with UseThenDisconnect(app.config):
+            manifest = registry_model.lookup_manifest_by_digest(repository, digest)
+            if manifest is None:
+              continue
+            
+            registry_model.create_manifest_label(manifest, INTERNAL_LABEL_BUILD_UUID,
+                                                 build_id, 'internal', 'text/plain')
 
       # Send the notification that the build has completed successfully.
       self._current_job.send_notification('build_success',

data/model/label.py

@@ -11,9 +11,6 @@ from util.validation import is_json
 
 logger = logging.getLogger(__name__)
 
-# Label which marks a manifest with its source build ID.
-INTERNAL_LABEL_BUILD_UUID = 'quay.build.uuid'
-
 
 @lru_cache(maxsize=1)
 def get_label_source_types():

data/registry_model/datatypes.py

@@ -7,6 +7,9 @@ class RepositoryReference(object):
 
   @classmethod
   def for_repo_obj(cls, repo_obj):
+    if repo_obj is None:
+      return None
+
     return RepositoryReference(repo_obj.id)
 
 
@@ -18,3 +21,13 @@ class Tag(namedtuple('Tag', ['id', 'name'])):
       return None
 
     return Tag(id=repository_tag.id, name=repository_tag.name)
+
+
+class Manifest(namedtuple('Manifest', ['id', 'digest'])):
+  """ Manifest represents a manifest in a repository. """
+  @classmethod
+  def for_tag_manifest(cls, tag_manifest):
+    if tag_manifest is None:
+      return None
+
+    return Manifest(id=tag_manifest.id, digest=tag_manifest.digest)

data/registry_model/interface.py

@@ -19,3 +19,21 @@ class RegistryDataInterface(object):
     """ Returns the most recently pushed alive tag in the repository, if any. If none, returns
         None.
     """
+
+  @abstractmethod
+  def lookup_repository(self, namespace_name, repo_name, kind_filter=None):
+    """ Looks up and returns a reference to the repository with the given namespace and name,
+        or None if none. """
+
+  @abstractmethod
+  def get_manifest_for_tag(self, tag):
+    """ Returns the manifest associated with the given tag. """
+
+  @abstractmethod
+  def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False):
+    """ Looks up the manifest with the given digest under the given repository and returns it
+        or None if none. """
+
+  @abstractmethod
+  def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
+    """ Creates a label on the manifest with the given key and value. """

data/registry_model/registry_pre_oci_model.py

@@ -1,6 +1,7 @@
+from data import database
 from data import model
 from data.registry_model.interface import RegistryDataInterface
-from data.registry_model.datatypes import Tag
+from data.registry_model.datatypes import Tag, RepositoryReference, Manifest
 
 
 class PreOCIModel(RegistryDataInterface):
@@ -23,5 +24,41 @@ class PreOCIModel(RegistryDataInterface):
     found_tag = model.tag.get_most_recent_tag(repository_ref.repo_id)
     return Tag.for_repository_tag(found_tag)
 
+  def lookup_repository(self, namespace_name, repo_name, kind_filter=None):
+    """ Looks up and returns a reference to the repository with the given namespace and name,
+        or None if none. """
+    repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter)
+    return RepositoryReference.for_repo_obj(repo)
+
+  def get_manifest_for_tag(self, tag):
+    """ Returns the manifest associated with the given tag. """
+    try:
+      tag_manifest = database.TagManifest.get(tag_id=tag.id)
+    except database.TagManifest.DoesNotExist:
+      return
+
+    return Manifest.for_tag_manifest(tag_manifest)
+
+  def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False):
+    """ Looks up the manifest with the given digest under the given repository and returns it
+        or None if none. """
+    repo = model.repository.lookup_repository(repository_ref.repo_id)
+    if repo is None:
+      return None
+
+    tag_manifest = model.tag.load_manifest_by_digest(repo.namespace_user.username,
+                                                     repo.name,
+                                                     manifest_digest, allow_dead=allow_dead)
+    return Manifest.for_tag_manifest(tag_manifest)
+
+  def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
+    """ Creates a label on the manifest with the given key and value. """
+    try:
+      tag_manifest = database.TagManifest.get(id=manifest.id)
+    except database.TagManifest.DoesNotExist:
+      return
+
+    model.label.create_manifest_label(tag_manifest, key, value, source_type_name, media_type_name)
+
 
 pre_oci_model = PreOCIModel()

data/registry_model/test/test_pre_oci_model.py

@@ -39,3 +39,39 @@ def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model)
     assert found is None
   else:
     assert found.name in expected
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name, expected', [
+  ('devtable', 'simple', True),
+  ('buynlarge', 'orgrepo', True),
+  ('buynlarge', 'unknownrepo', False),
+])
+def test_lookup_repository(repo_namespace, repo_name, expected, pre_oci_model):
+  repo_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
+  if expected:
+    assert repo_ref
+  else:
+    assert repo_ref is None
+
+
+@pytest.mark.parametrize('repo_namespace, repo_name', [
+  ('devtable', 'simple'),
+  ('buynlarge', 'orgrepo'),
+])
+def test_lookup_manifests(repo_namespace, repo_name, pre_oci_model):
+  repo = model.repository.get_repository(repo_namespace, repo_name)
+  repository_ref = RepositoryReference.for_repo_obj(repo)
+  found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest'])
+  found_manifest = pre_oci_model.get_manifest_for_tag(found_tag)
+  found = pre_oci_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest)
+  assert found.id == found_manifest.id
+  assert found.digest == found_manifest.digest
+
+
+def test_create_manifest_label(pre_oci_model):
+  repo = model.repository.get_repository('devtable', 'simple')
+  repository_ref = RepositoryReference.for_repo_obj(repo)
+  found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest'])
+  found_manifest = pre_oci_model.get_manifest_for_tag(found_tag)
+
+  pre_oci_model.create_manifest_label(found_manifest, 'foo', 'bar', 'internal')