quay/workers/test/test_manifestbackfillworker.py

from app import docker_v2_signing_key
from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
                           ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
                           TagManifestLabel, Tag, TagToRepositoryTag)
from image.docker.schema1 import DockerSchema1ManifestBuilder
from workers.manifestbackfillworker import backfill_manifest
from workers.labelbackfillworker import backfill_label

from test.fixtures import *

@pytest.fixture()
def clear_rows(initialized_db):
  # Remove all new-style rows so we can backfill.
  TagToRepositoryTag.delete().execute()
  Tag.delete().execute()
  TagManifestLabelMap.delete().execute()
  ManifestLabel.delete().execute()
  ManifestBlob.delete().execute()
  ManifestLegacyImage.delete().execute()
  TagManifestToManifest.delete().execute()
  Manifest.delete().execute()


def test_manifestbackfillworker(clear_rows, initialized_db):
  for tag_manifest in TagManifest.select():
    # Backfill the manifest.
    assert backfill_manifest(tag_manifest)

    # Ensure if we try again, the backfill is skipped.
    assert not backfill_manifest(tag_manifest)

    # Ensure that we now have the expected manifest rows.
    map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
    assert not map_row.broken

    manifest_row = map_row.manifest
    assert manifest_row.manifest_bytes == tag_manifest.json_data
    assert manifest_row.digest == tag_manifest.digest
    assert manifest_row.repository == tag_manifest.tag.repository

    legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
    assert tag_manifest.tag.image == legacy_image

    expected_storages = {tag_manifest.tag.image.storage.id}
    for parent_image_id in tag_manifest.tag.image.ancestor_id_list():
      expected_storages.add(Image.get(id=parent_image_id).storage_id)

    found_storages = {manifest_blob.blob_id for manifest_blob
                      in ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)}
    assert expected_storages == found_storages

    # Ensure that backfilling labels now works.
    for tml in TagManifestLabel.select().where(TagManifestLabel.annotated == tag_manifest):
      assert backfill_label(tml)

      label_map = TagManifestLabelMap.get(tag_manifest_label=tml)
      assert label_map.tag_manifest == tag_manifest
      assert label_map.manifest == manifest_row
      assert label_map.manifest_label.label == label_map.tag_manifest_label.label
      assert label_map.label == tml.label


def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db):
  # Delete existing tag manifest so we can reuse the tag.
  TagManifestLabel.delete().execute()
  TagManifest.delete().execute()

  # Add a broken manifest.
  broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar',
                                       tag=RepositoryTag.get())
  
  # Ensure the backfill works.
  assert backfill_manifest(broken_manifest)

  # Ensure the mapping is marked as broken.
  map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
  assert map_row.broken

  manifest_row = map_row.manifest
  assert manifest_row.manifest_bytes == broken_manifest.json_data
  assert manifest_row.digest == broken_manifest.digest
  assert manifest_row.repository == broken_manifest.tag.repository

  legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
  assert broken_manifest.tag.image == legacy_image


def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db):
  """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """
  # Delete existing tag manifest so we can reuse the tag.
  TagManifestLabel.delete().execute()
  TagManifest.delete().execute()

  repo = model.repository.get_repository('devtable', 'complex')
  tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
  tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')

  # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image
  # be the v5.0 image.
  builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
  builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}')
  manifest = builder.build(docker_v2_signing_key)

  mislinked_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
                                          tag=tag_v50)

  # Backfill the manifest and ensure its proper content checksum was linked.
  assert backfill_manifest(mislinked_manifest)

  map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest)
  assert not map_row.broken

  manifest_row = map_row.manifest
  legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
  assert legacy_image == tag_v50.image

  manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
  assert len(manifest_blobs) == 1
  assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum


def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db):
  """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked
      properly. """
  # Delete existing tag manifest so we can reuse the tag.
  TagManifestLabel.delete().execute()
  TagManifest.delete().execute()

  repo = model.repository.get_repository('devtable', 'complex')
  tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')

  # Add a mislinked manifest, by having its layer point to an invalid blob but its image
  # be the v5.0 image.
  builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
  builder.add_layer('sha256:deadbeef', '{"id": "foo"}')
  manifest = builder.build(docker_v2_signing_key)

  broken_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
                                       tag=tag_v50)

  # Backfill the manifest and ensure it is marked as broken.
  assert backfill_manifest(broken_manifest)

  map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
  assert map_row.broken

  manifest_row = map_row.manifest
  legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
  assert legacy_image == tag_v50.image

  manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
  assert len(manifest_blobs) == 0


def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db):
  """ Tests that a manifest with a shared digest will be properly linked. """
  # Delete existing tag manifest so we can reuse the tag.
  TagManifestLabel.delete().execute()
  TagManifest.delete().execute()

  repo = model.repository.get_repository('devtable', 'gargantuan')
  tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
  tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')

  # Build a manifest and assign it to both tags (this is allowed in the old model).
  builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
  builder.add_layer('sha256:deadbeef', '{"id": "foo"}')
  manifest = builder.build(docker_v2_signing_key)

  manifest_1 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
                                  tag=tag_v30)
  manifest_2 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
                                  tag=tag_v50)

  # Backfill "both" manifests and ensure both are pointed to by a single resulting row.
  assert backfill_manifest(manifest_1)
  assert backfill_manifest(manifest_2)

  map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1)
  map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2)

  assert map_row1.manifest == map_row2.manifest