Merge pull request #3234 from quay/joseph.schorr/QUAY-1030/interfacing-part-8

Change verbs endpoint to use the new registry data model interface

data/registry_model/registry_pre_oci_model.py

@@ -1,4 +1,5 @@
 # pylint: disable=protected-access
+import logging
 
 from collections import defaultdict
 
@@ -8,8 +9,12 @@ from data import database
 from data import model
 from data.registry_model.interface import RegistryDataInterface
 from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label,
-                                           SecurityScanStatus)
-from image.docker.schema1 import DockerSchema1ManifestBuilder
+                                           SecurityScanStatus, ManifestLayer, Blob, DerivedImage,
+                                           TorrentInfo)
+from image.docker.schema1 import DockerSchema1ManifestBuilder, ManifestException
+
+
+logger = logging.getLogger(__name__)
 
 
 class PreOCIModel(RegistryDataInterface):
@@ -40,11 +45,14 @@ class PreOCIModel(RegistryDataInterface):
     repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter)
     return RepositoryReference.for_repo_obj(repo)
 
-  def get_manifest_for_tag(self, tag):
+  def get_manifest_for_tag(self, tag, backfill_if_necessary=False):
     """ Returns the manifest associated with the given tag. """
     try:
       tag_manifest = database.TagManifest.get(tag_id=tag._db_id)
     except database.TagManifest.DoesNotExist:
+      if backfill_if_necessary:
+        return self.backfill_manifest_for_tag(tag)
+
       return
 
     return Manifest.for_tag_manifest(tag_manifest)
@@ -173,6 +181,7 @@ class PreOCIModel(RegistryDataInterface):
     Returns the latest, *active* tag found in the repository, with the matching name
     or None if none.
     """
+    assert isinstance(tag_name, basestring)
     tag = model.tag.get_active_tag_for_repo(repository_ref._db_id, tag_name)
     if tag is None:
       return None
@@ -359,5 +368,204 @@ class PreOCIModel(RegistryDataInterface):
 
     return Manifest.for_tag_manifest(tag_manifest)
 
+  def is_namespace_enabled(self, namespace_name):
+    """ Returns whether the given namespace exists and is enabled. """
+    namespace = model.user.get_namespace_user(namespace_name)
+    return namespace is not None and namespace.enabled
+
+  def list_manifest_layers(self, manifest, include_placements=False):
+    """ Returns an *ordered list* of the layers found in the manifest, starting at the base and
+        working towards the leaf, including the associated Blob and its placements (if specified).
+        Returns None if the manifest could not be parsed and validated.
+    """
+    try:
+      parsed = manifest.get_parsed_manifest()
+    except ManifestException:
+      logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
+      return None
+
+    try:
+      tag_manifest = database.TagManifest.get(id=manifest._db_id)
+    except database.TagManifest.DoesNotExist:
+      logger.exception('Could not find tag manifest for manifest `%s`', manifest._db_id)
+      return None
+
+    repo = tag_manifest.tag.repository
+    blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo, parsed.checksums)
+    storage_map = {blob.content_checksum: blob for blob in blob_query}
+
+    manifest_layers = []
+    for layer in parsed.layers:
+      digest_str = str(layer.digest)
+      if digest_str not in storage_map:
+        logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
+        return None
+
+      image_storage = storage_map[digest_str]
+      assert image_storage.cas_path is not None
+
+      placements = None
+      if include_placements:
+        placements = list(model.storage.get_storage_locations(image_storage.uuid))
+
+      blob = Blob.for_image_storage(image_storage,
+                                    storage_path=model.storage.get_layer_path(image_storage),
+                                    placements=placements)
+      manifest_layers.append(ManifestLayer(layer, blob))
+
+    return manifest_layers
+
+  def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
+    """
+    Looks up the derived image for the given manifest, verb and optional varying metadata and
+    returns it or None if none.
+    """
+    try:
+      tag_manifest = database.TagManifest.get(id=manifest._db_id)
+    except database.TagManifest.DoesNotExist:
+      logger.exception('Could not find tag manifest for manifest `%s`', manifest._db_id)
+      return None
+
+    repo_image = tag_manifest.tag.image
+    derived = model.image.find_derived_storage_for_image(repo_image, verb, varying_metadata)
+    return self._build_derived(derived, verb, varying_metadata, include_placements)
+
+  def lookup_or_create_derived_image(self, manifest, verb, storage_location, varying_metadata=None,
+                                     include_placements=False):
+    """
+    Looks up the derived image for the given maniest, verb and optional varying metadata
+    and returns it. If none exists, a new derived image is created.
+    """
+    try:
+      tag_manifest = database.TagManifest.get(id=manifest._db_id)
+    except database.TagManifest.DoesNotExist:
+      logger.exception('Could not find tag manifest for manifest `%s`', manifest._db_id)
+      return None
+
+    repo_image = tag_manifest.tag.image
+    derived = model.image.find_or_create_derived_storage(repo_image, verb, storage_location,
+                                                         varying_metadata)
+    return self._build_derived(derived, verb, varying_metadata, include_placements)
+
+  def _build_derived(self, derived, verb, varying_metadata, include_placements):
+    if derived is None:
+      return None
+
+    derived_storage = derived.derivative
+    placements = None
+    if include_placements:
+      placements = list(model.storage.get_storage_locations(derived_storage.uuid))
+
+    blob = Blob.for_image_storage(derived_storage,
+                                  storage_path=model.storage.get_layer_path(derived_storage),
+                                  placements=placements)
+
+    return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
+
+  def get_derived_image_signature(self, derived_image, signer_name):
+    """
+    Returns the signature associated with the derived image and a specific signer or None if none.
+    """
+    try:
+      derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
+    except database.DerivedStorageForImage.DoesNotExist:
+      return None
+
+    storage = derived_storage.derivative
+    signature_entry = model.storage.lookup_storage_signature(storage, signer_name)
+    if signature_entry is None:
+      return None
+
+    return signature_entry.signature
+
+  def set_derived_image_signature(self, derived_image, signer_name, signature):
+    """
+    Sets the calculated signature for the given derived image and signer to that specified.
+    """
+    try:
+      derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
+    except database.DerivedStorageForImage.DoesNotExist:
+      return None
+
+    storage = derived_storage.derivative
+    signature_entry = model.storage.find_or_create_storage_signature(storage, signer_name)
+    signature_entry.signature = signature
+    signature_entry.uploading = False
+    signature_entry.save()
+
+  def delete_derived_image(self, derived_image):
+    """
+    Deletes a derived image and all of its storage.
+    """
+    try:
+      derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
+    except database.DerivedStorageForImage.DoesNotExist:
+      return None
+
+    model.image.delete_derived_storage(derived_storage)
+
+  def set_derived_image_size(self, derived_image, compressed_size):
+    """
+    Sets the compressed size on the given derived image.
+    """
+    try:
+      derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
+    except database.DerivedStorageForImage.DoesNotExist:
+      return None
+
+    storage_entry = derived_storage.derivative
+    storage_entry.image_size = compressed_size
+    storage_entry.uploading = False
+    storage_entry.save()
+
+  def get_torrent_info(self, blob):
+    """
+    Returns the torrent information associated with the given blob or None if none.
+    """
+    try:
+      image_storage = database.ImageStorage.get(id=blob._db_id)
+    except database.ImageStorage.DoesNotExist:
+      return None
+
+    try:
+      torrent_info = model.storage.get_torrent_info(image_storage)
+    except model.TorrentInfoDoesNotExist:
+      return None
+
+    return TorrentInfo.for_torrent_info(torrent_info)
+
+  def set_torrent_info(self, blob, piece_length, pieces):
+    """
+    Sets the torrent infomation associated with the given blob to that specified.
+    """
+    try:
+      image_storage = database.ImageStorage.get(id=blob._db_id)
+    except database.ImageStorage.DoesNotExist:
+      return None
+
+    torrent_info = model.storage.save_torrent_info(image_storage, piece_length, pieces)
+    return TorrentInfo.for_torrent_info(torrent_info)
+
+  def get_repo_blob_by_digest(self, repo_ref, blob_digest, include_placements=False):
+    """
+    Returns the blob in the repository with the given digest, if any or None if none. Note that
+    there may be multiple records in the same repository for the same blob digest, so the return
+    value of this function may change.
+    """
+    try:
+      image_storage = model.blob.get_repository_blob_by_digest(repo_ref._db_id, blob_digest)
+    except model.BlobDoesNotExist:
+      return None
+
+    assert image_storage.cas_path is not None
+
+    placements = None
+    if include_placements:
+      placements = list(model.storage.get_storage_locations(image_storage.uuid))
+
+    return Blob.for_image_storage(image_storage,
+                                  storage_path=model.storage.get_layer_path(image_storage),
+                                  placements=placements)
+
 
 pre_oci_model = PreOCIModel()