Add manifest creation to new registry data model interface

data/registry_model/registry_pre_oci_model.py

@@ -8,11 +8,15 @@ from peewee import IntegrityError
 
 from data import database
 from data import model
+from data.database import db_transaction
 from data.registry_model.interface import RegistryDataInterface
 from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label,
                                            SecurityScanStatus, ManifestLayer, Blob, DerivedImage,
                                            TorrentInfo, BlobUpload)
-from image.docker.schema1 import DockerSchema1ManifestBuilder, ManifestException
+from data.registry_model.label_handlers import apply_label_to_manifest
+from image.docker.schema1 import (DockerSchema1ManifestBuilder, ManifestException,
+                                  DockerSchema1Manifest)
+from util.validation import is_json
 
 
 logger = logging.getLogger(__name__)
@@ -81,6 +85,75 @@ class PreOCIModel(RegistryDataInterface):
 
     return Manifest.for_tag_manifest(tag_manifest, legacy_image)
 
+  def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name):
+    """ Creates a manifest in a repository, adding all of the necessary data in the model.
+
+        The `manifest_interface_instance` parameter must be an instance of the manifest
+        interface as returned by the image/docker package.
+
+        Note that all blobs referenced by the manifest must exist under the repository or this
+        method will fail and return None.
+
+        Returns a reference to the (created manifest, tag) or (None, None) on error.
+    """
+    # NOTE: Only Schema1 is supported by the pre_oci_model.
+    assert isinstance(manifest_interface_instance, DockerSchema1Manifest)
+    if not manifest_interface_instance.layers:
+      return None, None
+
+    # Ensure all the blobs in the manifest exist.
+    digests = manifest_interface_instance.checksums
+    query = model.storage.lookup_repo_storages_by_content_checksum(repository_ref._db_id, digests)
+    blob_map = {s.content_checksum: s.id for s in query}
+    for layer in manifest_interface_instance.layers:
+      digest_str = str(layer.digest)
+      if digest_str not in blob_map:
+        return None, None
+
+    # Lookup all the images and their parent images (if any) inside the manifest.
+    # This will let us know which v1 images we need to synthesize and which ones are invalid.
+    docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
+    images_query = model.image.lookup_repository_images(repository_ref._db_id, docker_image_ids)
+    images_map = {i.docker_image_id: i.storage for i in images_query}
+
+    # Rewrite any v1 image IDs that do not match the checksum in the database.
+    try:
+      rewritten_images = list(manifest_interface_instance.rewrite_invalid_image_ids(images_map))
+      for rewritten_image in rewritten_images:
+        if not rewritten_image.image_id in images_map:
+          model.image.synthesize_v1_image(
+            repository_ref._db_id,
+            blob_map[rewritten_image.content_checksum],
+            rewritten_image.image_id,
+            rewritten_image.created,
+            rewritten_image.comment,
+            rewritten_image.command,
+            rewritten_image.compat_json,
+            rewritten_image.parent_image_id,
+          )
+    except ManifestException:
+      logger.exception("exception when rewriting v1 metadata")
+      return None, None
+
+    # Store the manifest pointing to the tag.
+    leaf_layer_id = rewritten_images[-1].image_id
+    tag_manifest, newly_created = model.tag.store_tag_manifest_for_repo(repository_ref._db_id,
+                                                                        tag_name,
+                                                                        manifest_interface_instance,
+                                                                        leaf_layer_id,
+                                                                        blob_map)
+
+    manifest = Manifest.for_tag_manifest(tag_manifest)
+
+    # Save the labels on the manifest.
+    if newly_created:
+      with self.batch_create_manifest_labels(manifest) as add_label:
+        for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
+          media_type = 'application/json' if is_json(value) else 'text/plain'
+          add_label(key, value, 'manifest', media_type)
+
+    return manifest, Tag.for_repository_tag(tag_manifest.tag)
+
   def get_legacy_images(self, repository_ref):
     """
     Returns an iterator of all the LegacyImage's defined in the matching repository.
@@ -135,8 +208,17 @@ class PreOCIModel(RegistryDataInterface):
     except database.TagManifest.DoesNotExist:
       return None
 
-    label = model.label.create_manifest_label(tag_manifest, key, value, source_type_name,
-                                              media_type_name)
+    label_data = dict(key=key, value=value, source_type_name=source_type_name,
+                      media_type_name=media_type_name)
+
+    with db_transaction():
+      # Create the label itself.
+      label = model.label.create_manifest_label(tag_manifest, key, value, source_type_name,
+                                                media_type_name)
+
+      # Apply any changes to the manifest that the label prescribes.
+      apply_label_to_manifest(label_data, manifest, self)
+
     return Label.for_label(label)
 
   @contextmanager
@@ -164,7 +246,12 @@ class PreOCIModel(RegistryDataInterface):
     # TODO: make this truly batch once we've fully transitioned to V2_2 and no longer need
     # the mapping tables.
     for label in labels_to_add:
-      model.label.create_manifest_label(tag_manifest, **label)
+      with db_transaction():
+        # Create the label itself.
+        model.label.create_manifest_label(tag_manifest, **label)
+
+        # Apply any changes to the manifest that the label prescribes.
+        apply_label_to_manifest(label, manifest, self)
 
   def list_manifest_labels(self, manifest, key_prefix=None):
     """ Returns all labels found on the manifest. If specified, the key_prefix will filter the
@@ -708,4 +795,15 @@ class PreOCIModel(RegistryDataInterface):
                                         expiration_sec)
     return bool(storage)
 
+  def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
+    """
+    Sets the expiration on all tags that point to the given manifest to that specified.
+    """
+    try:
+      tag_manifest = database.TagManifest.get(id=manifest._db_id)
+    except database.TagManifest.DoesNotExist:
+      return None
+
+    model.tag.set_tag_expiration_for_manifest(tag_manifest, expiration_sec)
+
 pre_oci_model = PreOCIModel()