Fix lookup of manifests referenced solely by a manifest list

We need to ensure we can find them if there is an active tag pointing to the parent list

data/model/oci/manifest.py

@@ -2,7 +2,7 @@ import logging
 
 from collections import namedtuple
 
-from peewee import IntegrityError
+from peewee import IntegrityError, JOIN
 
 from data.database import (Tag, Manifest, ManifestBlob, ManifestLegacyImage, ManifestChild,
                            db_transaction)
@@ -32,8 +32,24 @@ def lookup_manifest(repository_id, manifest_digest, allow_dead=False):
            .where(Manifest.repository == repository_id)
            .where(Manifest.digest == manifest_digest))
 
-  if not allow_dead:
-    query = filter_to_alive_tags(query.join(Tag)).group_by(Manifest.id)
+  if allow_dead:
+    try:
+      return query.get()
+    except Manifest.DoesNotExist:
+      return None
+
+  # Try first to filter to those manifests referenced by an alive tag,
+  try:
+    return filter_to_alive_tags(query.join(Tag)).get()
+  except Manifest.DoesNotExist:
+    pass
+
+  # Try referenced as the child of a manifest that has an alive tag.
+  query = (query
+           .join(ManifestChild, on=(ManifestChild.child_manifest == Manifest.id))
+           .join(Tag, on=(Tag.manifest == ManifestChild.manifest)))
+
+  query = filter_to_alive_tags(query)
 
   try:
     return query.get()

data/model/oci/tag.py

@@ -321,7 +321,7 @@ def filter_to_visible_tags(query):
   return query.where(Tag.hidden == False)
 
 
-def filter_to_alive_tags(query, now_ms=None):
+def filter_to_alive_tags(query, now_ms=None, model=Tag):
   """ Adjusts the specified Tag query to only return those tags alive. If now_ms is specified,
       the given timestamp (in MS) is used in place of the current timestamp for determining wherther
       a tag is alive.
@@ -329,7 +329,7 @@ def filter_to_alive_tags(query, now_ms=None):
   if now_ms is None:
     now_ms = get_epoch_timestamp_ms()
 
-  return query.where((Tag.lifetime_end_ms >> None) | (Tag.lifetime_end_ms > now_ms))
+  return query.where((model.lifetime_end_ms >> None) | (model.lifetime_end_ms > now_ms))
 
 
 def set_tag_expiration_sec_for_manifest(manifest_id, expiration_seconds):

data/model/oci/test/test_oci_manifest.py

@@ -5,9 +5,10 @@ from playhouse.test_utils import assert_query_count
 from app import docker_v2_signing_key, storage
 
 from digest.digest_tools import sha256_digest
-from data.database import Tag, ManifestBlob, ImageStorageLocation, ManifestChild, get_epoch_timestamp_ms
+from data.database import (Tag, ManifestBlob, ImageStorageLocation, ManifestChild,
+                           get_epoch_timestamp_ms)
 from data.model.oci.manifest import lookup_manifest, get_or_create_manifest
-from data.model.oci.tag import filter_to_alive_tags, get_tag
+from data.model.oci.tag import filter_to_alive_tags, get_tag, create_temporary_tag
 from data.model.oci.shared import get_legacy_image_for_manifest
 from data.model.oci.label import list_manifest_labels
 from data.model.repository import get_repository, create_repository
@@ -47,6 +48,53 @@ def test_lookup_manifest_dead_tag(initialized_db):
           dead_tag.manifest)
 
 
+def test_lookup_manifest_child_tag(initialized_db):
+  repository = create_repository('devtable', 'newrepo', None)
+
+  # Populate a manifest.
+  layer_json = json.dumps({
+    'config': {},
+    "rootfs": {
+      "type": "layers",
+      "diff_ids": []
+    },
+    "history": [],
+  })
+
+  # Add a blob containing the config.
+  _, config_digest = _populate_blob(layer_json)
+
+  remote_digest = sha256_digest('something')
+  builder = DockerSchema2ManifestBuilder()
+  builder.set_config_digest(config_digest, len(layer_json))
+  builder.add_layer(remote_digest, 1234, urls=['http://hello/world'])
+  manifest = builder.build()
+
+  assert get_or_create_manifest(repository, manifest, storage)
+
+  # Ensure the manifest cannot currently be looked up, as it is pointed to by an alive tag.
+  assert lookup_manifest(repository, manifest.digest) is None
+  assert lookup_manifest(repository, manifest.digest, allow_dead=True) is not None
+
+  # Populate a manifest list.
+  list_builder = DockerSchema2ManifestListBuilder()
+  list_builder.add_manifest(manifest, 'amd64', 'linux')
+  manifest_list = list_builder.build()
+
+  # Write the manifest list, which should also write the manifests themselves.
+  created_tuple = get_or_create_manifest(repository, manifest_list, storage)
+  assert created_tuple is not None
+
+  assert lookup_manifest(repository, manifest.digest) is None
+  assert lookup_manifest(repository, manifest_list.digest) is None
+
+  # Point a tag at the manifest list. This should make it and its child manifest visible.
+  create_temporary_tag(created_tuple.manifest, 1000)
+
+  assert lookup_manifest(repository, manifest.digest) is not None
+  assert lookup_manifest(repository, manifest_list.digest) is not None
+
+
 def _populate_blob(content):
   digest = str(sha256_digest(content))
   location = ImageStorageLocation.get(name='local_us')