e30b746aef
TagManifests can (apparently, in very rare scenarios) share manifests with the exact same digests, so we need to support that case in the backfill worker. We also need to remove a unique constraint on the manifest column in the mapping table to support this case.
170 lines
7.1 KiB
Python
170 lines
7.1 KiB
Python
from app import docker_v2_signing_key
|
|
from data import model
|
|
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
|
|
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
|
|
TagManifestLabel)
|
|
from image.docker.schema1 import DockerSchema1ManifestBuilder
|
|
from workers.manifestbackfillworker import backfill_manifest
|
|
|
|
from test.fixtures import *
|
|
|
|
@pytest.fixture()
|
|
def clear_rows(initialized_db):
|
|
# Remove all new-style rows so we can backfill.
|
|
TagManifestLabelMap.delete().execute()
|
|
ManifestLabel.delete().execute()
|
|
ManifestBlob.delete().execute()
|
|
ManifestLegacyImage.delete().execute()
|
|
TagManifestToManifest.delete().execute()
|
|
Manifest.delete().execute()
|
|
|
|
|
|
def test_manifestbackfillworker(clear_rows, initialized_db):
|
|
for tag_manifest in TagManifest.select():
|
|
# Backfill the manifest.
|
|
assert backfill_manifest(tag_manifest)
|
|
|
|
# Ensure if we try again, the backfill is skipped.
|
|
assert not backfill_manifest(tag_manifest)
|
|
|
|
# Ensure that we now have the expected manifest rows.
|
|
map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
|
|
assert not map_row.broken
|
|
|
|
manifest_row = map_row.manifest
|
|
assert manifest_row.manifest_bytes == tag_manifest.json_data
|
|
assert manifest_row.digest == tag_manifest.digest
|
|
assert manifest_row.repository == tag_manifest.tag.repository
|
|
|
|
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
|
assert tag_manifest.tag.image == legacy_image
|
|
|
|
expected_storages = {tag_manifest.tag.image.storage.id}
|
|
for parent_image_id in tag_manifest.tag.image.ancestor_id_list():
|
|
expected_storages.add(Image.get(id=parent_image_id).storage_id)
|
|
|
|
found_storages = {manifest_blob.blob_id for manifest_blob
|
|
in ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)}
|
|
assert expected_storages == found_storages
|
|
|
|
|
|
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db):
|
|
# Delete existing tag manifest so we can reuse the tag.
|
|
TagManifestLabel.delete().execute()
|
|
TagManifest.delete().execute()
|
|
|
|
# Add a broken manifest.
|
|
broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar',
|
|
tag=RepositoryTag.get())
|
|
|
|
# Ensure the backfill works.
|
|
assert backfill_manifest(broken_manifest)
|
|
|
|
# Ensure the mapping is marked as broken.
|
|
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
|
|
assert map_row.broken
|
|
|
|
manifest_row = map_row.manifest
|
|
assert manifest_row.manifest_bytes == broken_manifest.json_data
|
|
assert manifest_row.digest == broken_manifest.digest
|
|
assert manifest_row.repository == broken_manifest.tag.repository
|
|
|
|
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
|
assert broken_manifest.tag.image == legacy_image
|
|
|
|
|
|
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db):
|
|
""" Tests that a manifest whose image is mislinked will have its storages relinked properly. """
|
|
# Delete existing tag manifest so we can reuse the tag.
|
|
TagManifestLabel.delete().execute()
|
|
TagManifest.delete().execute()
|
|
|
|
repo = model.repository.get_repository('devtable', 'complex')
|
|
tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
|
|
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
|
|
|
|
# Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image
|
|
# be the v5.0 image.
|
|
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
|
|
builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}')
|
|
manifest = builder.build(docker_v2_signing_key)
|
|
|
|
mislinked_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
|
tag=tag_v50)
|
|
|
|
# Backfill the manifest and ensure its proper content checksum was linked.
|
|
assert backfill_manifest(mislinked_manifest)
|
|
|
|
map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest)
|
|
assert not map_row.broken
|
|
|
|
manifest_row = map_row.manifest
|
|
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
|
assert legacy_image == tag_v50.image
|
|
|
|
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
|
|
assert len(manifest_blobs) == 1
|
|
assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
|
|
|
|
|
|
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db):
|
|
""" Tests that a manifest whose image is mislinked will attempt to have its storages relinked
|
|
properly. """
|
|
# Delete existing tag manifest so we can reuse the tag.
|
|
TagManifestLabel.delete().execute()
|
|
TagManifest.delete().execute()
|
|
|
|
repo = model.repository.get_repository('devtable', 'complex')
|
|
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
|
|
|
|
# Add a mislinked manifest, by having its layer point to an invalid blob but its image
|
|
# be the v5.0 image.
|
|
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
|
|
builder.add_layer('sha256:deadbeef', '{"id": "foo"}')
|
|
manifest = builder.build(docker_v2_signing_key)
|
|
|
|
broken_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
|
tag=tag_v50)
|
|
|
|
# Backfill the manifest and ensure it is marked as broken.
|
|
assert backfill_manifest(broken_manifest)
|
|
|
|
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
|
|
assert map_row.broken
|
|
|
|
manifest_row = map_row.manifest
|
|
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
|
assert legacy_image == tag_v50.image
|
|
|
|
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
|
|
assert len(manifest_blobs) == 0
|
|
|
|
|
|
def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db):
|
|
""" Tests that a manifest with a shared digest will be properly linked. """
|
|
# Delete existing tag manifest so we can reuse the tag.
|
|
TagManifestLabel.delete().execute()
|
|
TagManifest.delete().execute()
|
|
|
|
repo = model.repository.get_repository('devtable', 'gargantuan')
|
|
tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
|
|
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
|
|
|
|
# Build a manifest and assign it to both tags (this is allowed in the old model).
|
|
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
|
|
builder.add_layer('sha256:deadbeef', '{"id": "foo"}')
|
|
manifest = builder.build(docker_v2_signing_key)
|
|
|
|
manifest_1 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
|
tag=tag_v30)
|
|
manifest_2 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
|
tag=tag_v50)
|
|
|
|
# Backfill "both" manifests and ensure both are pointed to by a single resulting row.
|
|
assert backfill_manifest(manifest_1)
|
|
assert backfill_manifest(manifest_2)
|
|
|
|
map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1)
|
|
map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2)
|
|
|
|
assert map_row1.manifest == map_row2.manifest
|