From 57e93a82c9b7b4395cbbd08e1a4be3e6063894a6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 6 Dec 2018 15:05:11 -0500 Subject: [PATCH] Remove manifest backfill worker --- .../batch/manifestbackfillworker/log/run | 4 - .../service/batch/manifestbackfillworker/run | 9 - workers/test/test_manifestbackfillworker.py | 183 ------------------ 3 files changed, 196 deletions(-) delete mode 100755 conf/init/service/batch/manifestbackfillworker/log/run delete mode 100755 conf/init/service/batch/manifestbackfillworker/run delete mode 100644 workers/test/test_manifestbackfillworker.py diff --git a/conf/init/service/batch/manifestbackfillworker/log/run b/conf/init/service/batch/manifestbackfillworker/log/run deleted file mode 100755 index de0b8e251..000000000 --- a/conf/init/service/batch/manifestbackfillworker/log/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# Start the logger -exec logger -i -t manifestbackfillworker \ No newline at end of file diff --git a/conf/init/service/batch/manifestbackfillworker/run b/conf/init/service/batch/manifestbackfillworker/run deleted file mode 100755 index 4dc77e7f6..000000000 --- a/conf/init/service/batch/manifestbackfillworker/run +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/bash - -echo 'Starting manifest backfill worker' - -QUAYPATH=${QUAYPATH:-"."} -cd ${QUAYDIR:-"/"} -PYTHONPATH=$QUAYPATH venv/bin/python -m workers.manifestbackfillworker 2>&1 - -echo 'Repository manifest backfill exited' diff --git a/workers/test/test_manifestbackfillworker.py b/workers/test/test_manifestbackfillworker.py deleted file mode 100644 index c1c31d6bf..000000000 --- a/workers/test/test_manifestbackfillworker.py +++ /dev/null @@ -1,183 +0,0 @@ -from app import docker_v2_signing_key -from data import model -from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob, - ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image, - TagManifestLabel, Tag, TagToRepositoryTag) -from image.docker.schema1 import DockerSchema1ManifestBuilder -from workers.manifestbackfillworker import backfill_manifest -from workers.labelbackfillworker import backfill_label - -from test.fixtures import * - -@pytest.fixture() -def clear_rows(initialized_db): - # Remove all new-style rows so we can backfill. - TagToRepositoryTag.delete().execute() - Tag.delete().execute() - TagManifestLabelMap.delete().execute() - ManifestLabel.delete().execute() - ManifestBlob.delete().execute() - ManifestLegacyImage.delete().execute() - TagManifestToManifest.delete().execute() - Manifest.delete().execute() - - -def test_manifestbackfillworker(clear_rows, initialized_db): - for tag_manifest in TagManifest.select(): - # Backfill the manifest. - assert backfill_manifest(tag_manifest) - - # Ensure if we try again, the backfill is skipped. - assert not backfill_manifest(tag_manifest) - - # Ensure that we now have the expected manifest rows. - map_row = TagManifestToManifest.get(tag_manifest=tag_manifest) - assert not map_row.broken - - manifest_row = map_row.manifest - assert manifest_row.manifest_bytes == tag_manifest.json_data - assert manifest_row.digest == tag_manifest.digest - assert manifest_row.repository == tag_manifest.tag.repository - - legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image - assert tag_manifest.tag.image == legacy_image - - expected_storages = {tag_manifest.tag.image.storage.id} - for parent_image_id in tag_manifest.tag.image.ancestor_id_list(): - expected_storages.add(Image.get(id=parent_image_id).storage_id) - - found_storages = {manifest_blob.blob_id for manifest_blob - in ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)} - assert expected_storages == found_storages - - # Ensure that backfilling labels now works. - for tml in TagManifestLabel.select().where(TagManifestLabel.annotated == tag_manifest): - assert backfill_label(tml) - - label_map = TagManifestLabelMap.get(tag_manifest_label=tml) - assert label_map.tag_manifest == tag_manifest - assert label_map.manifest == manifest_row - assert label_map.manifest_label.label == label_map.tag_manifest_label.label - assert label_map.label == tml.label - - -def test_manifestbackfillworker_broken_manifest(clear_rows, initialized_db): - # Delete existing tag manifest so we can reuse the tag. - TagManifestLabel.delete().execute() - TagManifest.delete().execute() - - # Add a broken manifest. - broken_manifest = TagManifest.create(json_data='wat?', digest='sha256:foobar', - tag=RepositoryTag.get()) - - # Ensure the backfill works. - assert backfill_manifest(broken_manifest) - - # Ensure the mapping is marked as broken. - map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) - assert map_row.broken - - manifest_row = map_row.manifest - assert manifest_row.manifest_bytes == broken_manifest.json_data - assert manifest_row.digest == broken_manifest.digest - assert manifest_row.repository == broken_manifest.tag.repository - - legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image - assert broken_manifest.tag.image == legacy_image - - -def test_manifestbackfillworker_mislinked_manifest(clear_rows, initialized_db): - """ Tests that a manifest whose image is mislinked will have its storages relinked properly. """ - # Delete existing tag manifest so we can reuse the tag. - TagManifestLabel.delete().execute() - TagManifest.delete().execute() - - repo = model.repository.get_repository('devtable', 'complex') - tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') - tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') - - # Add a mislinked manifest, by having its layer point to a blob in v3.0 but its image - # be the v5.0 image. - builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') - builder.add_layer(tag_v30.image.storage.content_checksum, '{"id": "foo"}') - manifest = builder.build(docker_v2_signing_key) - - mislinked_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest, - tag=tag_v50) - - # Backfill the manifest and ensure its proper content checksum was linked. - assert backfill_manifest(mislinked_manifest) - - map_row = TagManifestToManifest.get(tag_manifest=mislinked_manifest) - assert not map_row.broken - - manifest_row = map_row.manifest - legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image - assert legacy_image == tag_v50.image - - manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) - assert len(manifest_blobs) == 1 - assert manifest_blobs[0].blob.content_checksum == tag_v30.image.storage.content_checksum - - -def test_manifestbackfillworker_mislinked_invalid_manifest(clear_rows, initialized_db): - """ Tests that a manifest whose image is mislinked will attempt to have its storages relinked - properly. """ - # Delete existing tag manifest so we can reuse the tag. - TagManifestLabel.delete().execute() - TagManifest.delete().execute() - - repo = model.repository.get_repository('devtable', 'complex') - tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') - - # Add a mislinked manifest, by having its layer point to an invalid blob but its image - # be the v5.0 image. - builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') - builder.add_layer('sha256:deadbeef', '{"id": "foo"}') - manifest = builder.build(docker_v2_signing_key) - - broken_manifest = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest, - tag=tag_v50) - - # Backfill the manifest and ensure it is marked as broken. - assert backfill_manifest(broken_manifest) - - map_row = TagManifestToManifest.get(tag_manifest=broken_manifest) - assert map_row.broken - - manifest_row = map_row.manifest - legacy_image = ManifestLegacyImage.get(manifest=manifest_row).image - assert legacy_image == tag_v50.image - - manifest_blobs = list(ManifestBlob.select().where(ManifestBlob.manifest == manifest_row)) - assert len(manifest_blobs) == 0 - - -def test_manifestbackfillworker_repeat_digest(clear_rows, initialized_db): - """ Tests that a manifest with a shared digest will be properly linked. """ - # Delete existing tag manifest so we can reuse the tag. - TagManifestLabel.delete().execute() - TagManifest.delete().execute() - - repo = model.repository.get_repository('devtable', 'gargantuan') - tag_v30 = model.tag.get_active_tag('devtable', 'gargantuan', 'v3.0') - tag_v50 = model.tag.get_active_tag('devtable', 'gargantuan', 'v5.0') - - # Build a manifest and assign it to both tags (this is allowed in the old model). - builder = DockerSchema1ManifestBuilder('devtable', 'gargantuan', 'sometag') - builder.add_layer('sha256:deadbeef', '{"id": "foo"}') - manifest = builder.build(docker_v2_signing_key) - - manifest_1 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest, - tag=tag_v30) - manifest_2 = TagManifest.create(json_data=manifest.bytes, digest=manifest.digest, - tag=tag_v50) - - # Backfill "both" manifests and ensure both are pointed to by a single resulting row. - assert backfill_manifest(manifest_1) - assert backfill_manifest(manifest_2) - - map_row1 = TagManifestToManifest.get(tag_manifest=manifest_1) - map_row2 = TagManifestToManifest.get(tag_manifest=manifest_2) - - assert map_row1.manifest == map_row2.manifest