Merge pull request #3356 from quay/tag-backfill-skip-improvements

Tag backfill improvements

workers/tagbackfillworker.py

@@ -156,7 +156,7 @@ class TagBackfillWorker(Worker):
 
       for candidate, abt, _ in iterator:
         if not backfill_tag(candidate):
-          logger.info('Another worker pre-empted us for label: %s', candidate.id)
+          logger.info('Another worker pre-empted us for tag: %s', candidate.id)
           abt.set()
 
 
@@ -171,14 +171,14 @@ def lookup_map_row(repositorytag):
 def backfill_tag(repositorytag):
   logger.info('Backfilling tag %s', repositorytag.id)
 
-  # Ensure that a mapping row doesn't already exist. If it does, we've been preempted.
+  # Ensure that a mapping row doesn't already exist. If it does, nothing more to do.
   if lookup_map_row(repositorytag):
     return False
 
   # Grab the manifest for the RepositoryTag, backfilling as necessary.
   manifest_id = _get_manifest_id(repositorytag)
   if manifest_id is None:
-    return False
+    return True
 
   lifetime_start_ms = (repositorytag.lifetime_start_ts * 1000
                        if repositorytag.lifetime_start_ts is not None else None)
@@ -225,15 +225,19 @@ def _get_manifest_id(repositorytag):
     manifest_datatype = pre_oci_model.get_manifest_for_tag(repository_tag_datatype,
                                                            backfill_if_necessary=True)
     if manifest_datatype is None:
-      logger.error('Missing manifest for tag `%s`', repositorytag.id)
-      return None
+      logger.error('Could not load or backfill manifest for tag `%s`', repositorytag.id)
 
-  # Retrieve the new-style Manifest for the TagManifest, if any.
-  try:
-    tag_manifest = TagManifest.get(id=manifest_datatype._db_id)
-  except TagManifest.DoesNotExist:
-    logger.exception('Could not find tag manifest')
-    return None
+      # Create a broken manifest for the tag.
+      tag_manifest = TagManifest.create(tag=repositorytag,
+                                        digest='BROKEN-%s' % repositorytag.id,
+                                        json_data='{}')
+    else:
+      # Retrieve the new-style Manifest for the TagManifest, if any.
+      try:
+        tag_manifest = TagManifest.get(id=manifest_datatype._db_id)
+      except TagManifest.DoesNotExist:
+        logger.exception('Could not find tag manifest')
+        return None
 
   try:
     found = TagManifestToManifest.get(tag_manifest=tag_manifest).manifest

workers/test/test_tagbackfillworker.py

@@ -2,7 +2,8 @@ from app import docker_v2_signing_key
 from data import model
 from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
                            ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
-                           TagManifestLabel, Tag, TagToRepositoryTag, Repository)
+                           TagManifestLabel, Tag, TagToRepositoryTag, Repository,
+                           ImageStorage)
 from image.docker.schema1 import DockerSchema1ManifestBuilder
 from workers.tagbackfillworker import backfill_tag, _backfill_manifest
 
@@ -251,3 +252,30 @@ def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db):
   map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2)
 
   assert map_row1.manifest == map_row2.manifest
+
+
+def test_manifest_backfill_broken_tag(clear_rows, initialized_db):
+  """ Tests backfilling a broken tag. """
+  # Delete existing tag manifest so we can reuse the tag.
+  TagManifestLabel.delete().execute()
+  TagManifest.delete().execute()
+
+  # Create a tag with an image referenced missing parent images.
+  repo = model.repository.get_repository('devtable', 'gargantuan')
+  broken_image = Image.create(docker_image_id='foo', repository=repo, ancestors='/348723847234/',
+                              storage=ImageStorage.get())
+  broken_image_tag = RepositoryTag.create(repository=repo, image=broken_image, name='broken')
+
+  # Backfill the tag.
+  assert backfill_tag(broken_image_tag)
+
+  # Ensure we backfilled, even though we reference a broken manifest.
+  tag_manifest = TagManifest.get(tag=broken_image_tag)
+
+  map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
+  manifest = map_row.manifest
+  assert manifest.manifest_bytes == tag_manifest.json_data
+
+  tag = TagToRepositoryTag.get(repository_tag=broken_image_tag).tag
+  assert tag.name == 'broken'
+  assert tag.manifest == manifest