Add support for direct pushing of schema 2 manifests without tags

This is required for manifest lists

data/model/oci/tag.py

@@ -1,3 +1,4 @@
+import uuid
 import logging
 
 from calendar import timegm
@@ -155,6 +156,47 @@ def get_expired_tag(repository_id, tag_name):
     return None
 
 
+def create_temporary_tag(manifest, expiration_sec):
+  """ Creates a temporary tag pointing to the given manifest, with the given expiration in seconds.
+  """
+  tag_name = '$temp-%s' % str(uuid.uuid4())
+  now_ms = get_epoch_timestamp_ms()
+  end_ms = now_ms + (expiration_sec * 1000)
+
+  legacy_image = get_legacy_image_for_manifest(manifest)
+
+  with db_transaction():
+    created_tag = Tag.create(name=tag_name,
+                             repository=manifest.repository_id,
+                             lifetime_start_ms=now_ms,
+                             lifetime_end_ms=end_ms,
+                             reversion=False,
+                             hidden=True,
+                             manifest=manifest,
+                             tag_kind=Tag.tag_kind.get_id('tag'))
+
+    # TODO(jschorr): Remove the linkage code once RepositoryTag is gone.
+    # If this is a schema 1 manifest, then add a TagManifest linkage to it. Otherwise, it will only
+    # be pullable via the new OCI model.
+    if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES and legacy_image is not None:
+      now_ts = int(now_ms / 1000)
+      end_ts = int(end_ms / 1000)
+
+      old_style_tag = RepositoryTag.create(repository=manifest.repository_id, image=legacy_image,
+                                           name=tag_name, lifetime_start_ts=now_ts,
+                                           lifetime_end_ts=end_ts,
+                                           reversion=False, hidden=True)
+      TagToRepositoryTag.create(tag=created_tag, repository_tag=old_style_tag,
+                                repository=manifest.repository_id)
+
+      tag_manifest = TagManifest.create(tag=old_style_tag, digest=manifest.digest,
+                                        json_data=manifest.manifest_bytes)
+      TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest,
+                                   repository=manifest.repository_id)
+
+    return created_tag
+
+
 def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None):
   """ Creates or updates a tag with the specified name to point to the given manifest under
       its repository. If this action is a reversion to a previous manifest, is_reversion

data/model/oci/test/test_oci_tag.py

@@ -10,7 +10,8 @@ from data.model.oci.tag import (find_matching_tag, get_most_recent_tag, list_ali
                                 filter_to_visible_tags, list_repository_tag_history,
                                 get_expired_tag, get_tag, delete_tag,
                                 delete_tags_for_manifest, change_tag_expiration,
-                                set_tag_expiration_for_manifest, retarget_tag)
+                                set_tag_expiration_for_manifest, retarget_tag,
+                                create_temporary_tag)
 from data.model.repository import get_repository, create_repository
 
 from test.fixtures import *
@@ -207,6 +208,31 @@ def test_set_tag_expiration_for_manifest(initialized_db):
   assert updated_tag.lifetime_end_ms is not None
 
 
+def test_create_temporary_tag(initialized_db):
+  tag = Tag.get()
+  manifest = tag.manifest
+  assert manifest is not None
+
+  created = create_temporary_tag(manifest, 30)
+  assert created is not None
+
+  assert created.hidden
+  assert created.name.startswith('$temp-')
+  assert created.manifest == manifest
+  assert created.lifetime_end_ms is not None
+  assert created.lifetime_end_ms == (created.lifetime_start_ms + 30000)
+
+  # Verify old-style tables.
+  repository_tag = TagToRepositoryTag.get(tag=created).repository_tag
+  assert repository_tag.lifetime_start_ts == int(created.lifetime_start_ms / 1000)
+  assert repository_tag.lifetime_end_ts == int(created.lifetime_end_ms / 1000)
+  assert repository_tag.name == created.name
+  assert repository_tag.hidden
+
+  tag_manifest = TagManifest.get(tag=repository_tag)
+  assert TagManifestToManifest.get(tag_manifest=tag_manifest).manifest == manifest
+
+
 def test_retarget_tag(initialized_db):
   repo = get_repository('devtable', 'history')
   results, _ = list_repository_tag_history(repo, 1, 100, specific_tag_name='latest')

data/registry_model/interface.py

@@ -303,3 +303,10 @@ class RegistryDataInterface(object):
   @abstractmethod
   def get_schema1_parsed_manifest(self, manifest, namespace_name, repo_name, tag_name, storage):
     """ Returns the schema 1 version of this manifest, or None if none. """
+
+  @abstractmethod
+  def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance,
+                                    expiration_sec, storage):
+    """ Creates a manifest under the repository and sets a temporary tag to point to it.
+        Returns the manifest object created or None on error.
+    """

data/registry_model/registry_oci_model.py

@@ -219,17 +219,6 @@ class OCIModel(SharedModel, RegistryDataInterface):
 
         Returns a reference to the (created manifest, tag) or (None, None) on error.
     """
-    def _retrieve_repo_blob(digest):
-      blob_found = self.get_repo_blob_by_digest(repository_ref, digest, include_placements=True)
-      if blob_found is None:
-        return None
-
-      try:
-        return storage.get_content(blob_found.placements, blob_found.storage_path)
-      except IOError:
-        logger.exception('Could not retrieve configuration blob `%s`', digest)
-        return None
-
     # Get or create the manifest itself.
     created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id,
                                                            manifest_interface_instance,
@@ -460,4 +449,26 @@ class OCIModel(SharedModel, RegistryDataInterface):
     retriever = RepositoryContentRetriever(manifest_row.repository_id, storage)
     return parsed.get_v1_compatible_manifest(namespace_name, repo_name, tag_name, retriever)
 
+  def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance,
+                                    expiration_sec, storage):
+    """ Creates a manifest under the repository and sets a temporary tag to point to it.
+        Returns the manifest object created or None on error.
+    """
+    # Get or create the manifest itself.
+    created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id,
+                                                           manifest_interface_instance,
+                                                           storage)
+    if created_manifest is None:
+      return None
+
+    # Point a temporary tag to the manifest.
+    tag = oci.tag.create_temporary_tag(created_manifest.manifest, expiration_sec)
+    if tag is None:
+      return None
+
+    legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
+    li = LegacyImage.for_image(legacy_image)
+    return Manifest.for_manifest(created_manifest.manifest, li)
+
+
 oci_model = OCIModel()

data/registry_model/registry_pre_oci_model.py

@@ -541,5 +541,12 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
     except ManifestException:
       return None
 
+  def create_manifest_with_temp_tag(self, repository_ref, manifest_interface_instance,
+                                    expiration_sec, storage):
+    """ Creates a manifest under the repository and sets a temporary tag to point to it.
+        Returns the manifest object created or None on error.
+    """
+    raise NotImplementedError('Unsupported in pre OCI model')
+
 
 pre_oci_model = PreOCIModel()

endpoints/v2/manifest.py

@@ -62,7 +62,8 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
     logger.exception('Got exception when trying to parse manifest `%s`', manifest_ref)
     raise ManifestInvalid()
 
-  manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, manifest_ref, parsed)
+  manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, manifest_ref, manifest,
+                                              parsed)
   if manifest is None:
     raise ManifestUnknown()
 
@@ -100,7 +101,8 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
     logger.exception('Got exception when trying to parse manifest `%s`', manifest_ref)
     raise ManifestInvalid()
 
-  manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, '$digest', parsed)
+  manifest = _rewrite_to_schema1_if_necessary(namespace_name, repo_name, '$digest', manifest,
+                                              parsed)
   if manifest is None:
     raise ManifestUnknown()
 
@@ -113,24 +115,17 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
   })
 
 
-def _rewrite_to_schema1_if_necessary(namespace_name, repo_name, tag_name, manifest):
+def _rewrite_to_schema1_if_necessary(namespace_name, repo_name, tag_name, manifest, parsed):
   # As per the Docker protocol, if the manifest is not schema version 1 and the manifest's
   # media type is not in the Accept header, we return a schema 1 version of the manifest for
   # the amd64+linux platform, if any, or None if none.
   # See: https://docs.docker.com/registry/spec/manifest-v2-2
   mimetypes = [mimetype for mimetype, _ in request.accept_mimetypes]
-  if manifest.media_type in mimetypes:
+  if parsed.media_type in mimetypes:
-    return manifest
+    return parsed
 
-  def lookup_fn(config_or_manifest_digest):
+  return registry_model.get_schema1_parsed_manifest(manifest, namespace_name, repo_name, tag_name,
-    blob = registry_model.get_cached_repo_blob(model_cache, namespace_name, repo_name,
+                                                    storage)
-                                               config_or_manifest_digest)
-    if blob is None:
-      return None
-
-    return storage.get_content(blob.placements, blob.storage_path)
-
-  return manifest.get_v1_compatible_manifest(namespace_name, repo_name, tag_name, lookup_fn)
 
 
 def _reject_manifest2_schema2(func):
@@ -162,19 +157,8 @@ def _doesnt_accept_schema_v1():
 @require_repo_write
 @anon_protect
 def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
-  content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
+  parsed = _parse_manifest()
-
+  return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, parsed)
-  if content_type == 'application/json':
-    # For back-compat.
-    content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
-
-  try:
-    manifest = parse_manifest_from_bytes(request.data, content_type)
-  except ManifestException as me:
-    logger.exception("failed to parse manifest when writing by tagname")
-    raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
-
-  return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, manifest)
 
 
 @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
@@ -184,16 +168,50 @@ def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
 @require_repo_write
 @anon_protect
 def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
-  try:
+  parsed = _parse_manifest()
-    manifest = DockerSchema1Manifest(request.data)
+  if parsed.digest != manifest_ref:
-  except ManifestException as me:
-    logger.exception("failed to parse manifest when writing by digest")
-    raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
-
-  if manifest.digest != manifest_ref:
     raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})
 
-  return _write_manifest_and_log(namespace_name, repo_name, manifest.tag, manifest)
+  if parsed.schema_version != 2:
+    return _write_manifest_and_log(namespace_name, repo_name, parsed.tag, parsed)
+
+  # If the manifest is schema version 2, then this cannot be a normal tag-based push, as the
+  # manifest does not contain the tag and this call was not given a tag name. Instead, we write the
+  # manifest with a temporary tag, as it is being pushed as part of a call for a manifest list.
+  repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
+  if repository_ref is None:
+    raise NameUnknown()
+
+  expiration_sec = app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']
+  manifest = registry_model.create_manifest_with_temp_tag(repository_ref, parsed, expiration_sec,
+                                                          storage)
+  if manifest is None:
+    raise ManifestInvalid()
+
+  return Response(
+    'OK',
+    status=202,
+    headers={
+      'Docker-Content-Digest': manifest.digest,
+      'Location':
+        url_for('v2.fetch_manifest_by_digest',
+                repository='%s/%s' % (namespace_name, repo_name),
+                manifest_ref=manifest.digest),
+    },
+  )
+
+
+def _parse_manifest():
+  content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
+  if content_type == 'application/json':
+    # For back-compat.
+    content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
+
+  try:
+    return parse_manifest_from_bytes(request.data, content_type)
+  except ManifestException as me:
+    logger.exception("failed to parse manifest when writing by tagname")
+    raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
 
 
 @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])