Remove manifest backfill worker
This commit is contained in:
Joseph Schorr
parent
cb7369c2ca
commit
57e93a82c9
3 changed files with 0 additions and 196 deletions
|
|
@ -1,4 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Start the logger
|
||||
exec logger -i -t manifestbackfillworker
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
#! /bin/bash
|
||||
|
||||
echo 'Starting manifest backfill worker'
|
||||
|
||||
QUAYPATH=${QUAYPATH:-"."}
|
||||
cd ${QUAYDIR:-"/"}
|
||||
PYTHONPATH=$QUAYPATH venv/bin/python -m workers.manifestbackfillworker 2>&1
|
||||
|
||||
echo 'Repository manifest backfill exited'
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
from app import docker_v2_signing_key
|
||||
from data import model
|
||||
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
|
||||
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
|
||||
TagManifestLabel, Tag, TagToRepositoryTag)
|
||||
from image.docker.schema1 import DockerSchema1ManifestBuilder
|
||||
from workers.manifestbackfillworker import backfill_manifest
|
||||
from workers.labelbackfillworker import backfill_label
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.fixture()
|
||||
def clear_rows(initialized_db):
|
||||
# Remove all new-style rows so we can backfill.
|
||||
TagToRepositoryTag.delete().execute()
|
||||
Tag.delete().execute()
|
||||
TagManifestLabelMap.delete().execute()
|
||||
ManifestLabel.delete().execute()
|
||||
ManifestBlob.delete().execute()
|
||||
ManifestLegacyImage.delete().execute()
|
||||
TagManifestToManifest.delete().execute()
|
||||
Manifest.delete().execute()
|
||||
|
||||
|
||||
def test_manifestbackfillworker(clear_rows, initialized_db):
|
||||
for tag_manifest in TagManifest.select():
|
||||
# Backfill the manifest.
|
||||
assert backfill_manifest(tag_manifest)
|
||||
|
||||
# Ensure if we try again, the backfill is skipped.
|
||||
assert not backfill_manifest(tag_manifest)
|
||||
|
||||
# Ensure that we now have the expected manifest rows.
|
||||
map_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
|
||||
assert not map_row.broken
|
||||
|
||||
manifest_row = map_row.manifest
|
||||
assert manifest_row.manifest_bytes == tag_manifest.json_data
|
||||
assert manifest_row.digest == tag_manifest.digest
|
||||
assert manifest_row.repository == tag_manifest.tag.repository
|
||||
|
||||
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
||||
assert tag_manifest.tag.image == legacy_image
|
||||
|
||||
expected_storages = {tag_manifest.tag.image.storage.id}
|
||||
for parent_image_id in tag_manifest.tag.image.ancestor_id_list():
|
||||
expected_storages.add(Image.get(id=parent_image_id).storage_id)
|
||||
|
||||
found_storages = {manifest_blob.blob_id for manifest_blob
|
||||
in ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)}
|
||||
assert expected_storages == found_storages
|
||||
|
||||
# Ensure that backfilling labels now works.
|
||||
for tml in TagManifestLabel.select().where(TagManifestLabel.annotated == tag_manifest):
|
||||
assert backfill_label(tml)
|
||||
|
||||
label_map = TagManifestLabelMap.get(tag_manifest_label=tml)
|
||||
assert label_map.tag_manifest == tag_manifest
|
||||
assert label_map.manifest == manifest_row
|
||||
assert label_map.manifest_label.label == label_map.tag_manifest_label.label
|
||||
assert label_map.label == tml.label
|
||||
|
||||
|
||||
def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db):
|
||||
# Delete existing tag manifest so we can reuse the tag.
|
||||
TagManifestLabel.delete().execute()
|
||||
TagManifest.delete().execute()
|
||||
|
||||
# Add a broken manifest.
|
||||
broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar',
|
||||
tag=RepositoryTag.get())
|
||||
|
||||
# Ensure the backfill works.
|
||||
assert backfill_manifest(broken_manifest)
|
||||
|
||||
# Ensure the mapping is marked as broken.
|
||||
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
|
||||
assert map_row.broken
|
||||
|
||||
manifest_row = map_row.manifest
|
||||
assert manifest_row.manifest_bytes == broken_manifest.json_data
|
||||
assert manifest_row.digest == broken_manifest.digest
|
||||
assert manifest_row.repository == broken_manifest.tag.repository
|
||||
|
||||
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
||||
assert broken_manifest.tag.image == legacy_image
|
||||
|
||||
|
||||
def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db):
|
||||
""" Tests that a manifest whose image is mislinked will have its storages relinked properly. """
|
||||
# Delete existing tag manifest so we can reuse the tag.
|
||||
TagManifestLabel.delete().execute()
|
||||
TagManifest.delete().execute()
|
||||
|
||||
repo = model.repository.get_repository('devtable', 'complex')
|
||||
tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
|
||||
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
|
||||
|
||||
# Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image
|
||||
# be the v5.0 image.
|
||||
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
|
||||
builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}')
|
||||
manifest = builder.build(docker_v2_signing_key)
|
||||
|
||||
mislinked_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
||||
tag=tag_v50)
|
||||
|
||||
# Backfill the manifest and ensure its proper content checksum was linked.
|
||||
assert backfill_manifest(mislinked_manifest)
|
||||
|
||||
map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest)
|
||||
assert not map_row.broken
|
||||
|
||||
manifest_row = map_row.manifest
|
||||
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
||||
assert legacy_image == tag_v50.image
|
||||
|
||||
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
|
||||
assert len(manifest_blobs) == 1
|
||||
assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum
|
||||
|
||||
|
||||
def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db):
|
||||
""" Tests that a manifest whose image is mislinked will attempt to have its storages relinked
|
||||
properly. """
|
||||
# Delete existing tag manifest so we can reuse the tag.
|
||||
TagManifestLabel.delete().execute()
|
||||
TagManifest.delete().execute()
|
||||
|
||||
repo = model.repository.get_repository('devtable', 'complex')
|
||||
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
|
||||
|
||||
# Add a mislinked manifest, by having its layer point to an invalid blob but its image
|
||||
# be the v5.0 image.
|
||||
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
|
||||
builder.add_layer('sha256:deadbeef', '{"id": "foo"}')
|
||||
manifest = builder.build(docker_v2_signing_key)
|
||||
|
||||
broken_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
||||
tag=tag_v50)
|
||||
|
||||
# Backfill the manifest and ensure it is marked as broken.
|
||||
assert backfill_manifest(broken_manifest)
|
||||
|
||||
map_row = TagManifestToManifest.get(tag_manifest=broken_manifest)
|
||||
assert map_row.broken
|
||||
|
||||
manifest_row = map_row.manifest
|
||||
legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image
|
||||
assert legacy_image == tag_v50.image
|
||||
|
||||
manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row))
|
||||
assert len(manifest_blobs) == 0
|
||||
|
||||
|
||||
def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db):
|
||||
""" Tests that a manifest with a shared digest will be properly linked. """
|
||||
# Delete existing tag manifest so we can reuse the tag.
|
||||
TagManifestLabel.delete().execute()
|
||||
TagManifest.delete().execute()
|
||||
|
||||
repo = model.repository.get_repository('devtable', 'gargantuan')
|
||||
tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0')
|
||||
tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0')
|
||||
|
||||
# Build a manifest and assign it to both tags (this is allowed in the old model).
|
||||
builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag')
|
||||
builder.add_layer('sha256:deadbeef', '{"id": "foo"}')
|
||||
manifest = builder.build(docker_v2_signing_key)
|
||||
|
||||
manifest_1 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
||||
tag=tag_v30)
|
||||
manifest_2 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest,
|
||||
tag=tag_v50)
|
||||
|
||||
# Backfill "both" manifests and ensure both are pointed to by a single resulting row.
|
||||
assert backfill_manifest(manifest_1)
|
||||
assert backfill_manifest(manifest_2)
|
||||
|
||||
map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1)
|
||||
map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2)
|
||||
|
||||
assert map_row1.manifest == map_row2.manifest
|
||||
Reference in a new issue