Implement the new OCI-based registry data model

Note that this change does *not* enable the new data model by default, but does allow it to be used when a special environment variable is specified.

data/registry_model/shared.py

@@ -1,11 +1,15 @@
 # pylint: disable=protected-access
 import logging
 
+from collections import defaultdict
+
 from data import database
 from data import model
 from data.cache import cache_key
 from data.registry_model.datatype import FromDictionaryException
-from data.registry_model.datatypes import RepositoryReference, Blob, TorrentInfo, BlobUpload
+from data.registry_model.datatypes import (RepositoryReference, Blob, TorrentInfo, BlobUpload,
+                                           LegacyImage, ManifestLayer, DerivedImage)
+from image.docker.schema1 import ManifestException, DockerSchema1ManifestBuilder
 
 logger = logging.getLogger(__name__)
 
@@ -254,3 +258,130 @@ class SharedModel:
     storage = model.blob.temp_link_blob(namespace_name, repo_name, blob.digest,
                                         expiration_sec)
     return bool(storage)
+
+  def get_legacy_images(self, repository_ref):
+    """
+    Returns an iterator of all the LegacyImage's defined in the matching repository.
+    """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    all_images = model.image.get_repository_images_without_placements(repo)
+    all_images_map = {image.id: image for image in all_images}
+
+    all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name)
+    tags_by_image_id = defaultdict(list)
+    for tag in all_tags:
+      tags_by_image_id[tag.image_id].append(tag)
+
+    return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
+            for image in all_images]
+
+  def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False,
+                       include_blob=False):
+    """
+    Returns the matching LegacyImages under the matching repository, if any. If none,
+    returns None.
+    """
+    repo = model.repository.lookup_repository(repository_ref._db_id)
+    if repo is None:
+      return None
+
+    image = model.image.get_image(repository_ref._db_id, docker_image_id)
+    if image is None:
+      return None
+
+    parent_images_map = None
+    if include_parents:
+      parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image)
+      parent_images_map = {image.id: image for image in parent_images}
+
+    blob = None
+    if include_blob:
+      placements = list(model.storage.get_storage_locations(image.storage.uuid))
+      blob = Blob.for_image_storage(image.storage,
+                                    storage_path=model.storage.get_layer_path(image.storage),
+                                    placements=placements)
+
+    return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
+
+  def _list_manifest_layers(self, manifest, repo_id, include_placements=False):
+    """ Returns an *ordered list* of the layers found in the manifest, starting at the base and
+        working towards the leaf, including the associated Blob and its placements (if specified).
+        Returns None if the manifest could not be parsed and validated.
+    """
+    try:
+      parsed = manifest.get_parsed_manifest()
+    except ManifestException:
+      logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
+      return None
+
+    blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id, parsed.checksums)
+    storage_map = {blob.content_checksum: blob for blob in blob_query}
+
+    manifest_layers = []
+    for layer in parsed.layers:
+      digest_str = str(layer.digest)
+      if digest_str not in storage_map:
+        logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
+        return None
+
+      image_storage = storage_map[digest_str]
+      assert image_storage.cas_path is not None
+
+      placements = None
+      if include_placements:
+        placements = list(model.storage.get_storage_locations(image_storage.uuid))
+
+      blob = Blob.for_image_storage(image_storage,
+                                    storage_path=model.storage.get_layer_path(image_storage),
+                                    placements=placements)
+      manifest_layers.append(ManifestLayer(layer, blob))
+
+    return manifest_layers
+
+  def _build_derived(self, derived, verb, varying_metadata, include_placements):
+    if derived is None:
+      return None
+
+    derived_storage = derived.derivative
+    placements = None
+    if include_placements:
+      placements = list(model.storage.get_storage_locations(derived_storage.uuid))
+
+    blob = Blob.for_image_storage(derived_storage,
+                                  storage_path=model.storage.get_layer_path(derived_storage),
+                                  placements=placements)
+
+    return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
+
+  def _build_manifest_for_legacy_image(self, tag_name, legacy_image_row):
+    import features
+
+    from app import app, docker_v2_signing_key
+
+    repo = legacy_image_row.repository
+    namespace_name = repo.namespace_user.username
+    repo_name = repo.name
+
+    # Find the v1 metadata for this image and its parents.
+    parents = model.image.get_parent_images(namespace_name, repo_name, legacy_image_row)
+
+    # If the manifest is being generated under the library namespace, then we make its namespace
+    # empty.
+    manifest_namespace = namespace_name
+    if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
+      manifest_namespace = ''
+
+    # Create and populate the manifest builder
+    builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)
+
+    # Add the leaf layer
+    builder.add_layer(legacy_image_row.storage.content_checksum, legacy_image_row.v1_json_metadata)
+
+    for parent_image in parents:
+      builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
+
+    # Sign the manifest with our signing key.
+    return builder.build(docker_v2_signing_key)