From 114e2c3bf232548b8271d27a008219d52604f30c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 30 Oct 2018 11:35:31 -0400 Subject: [PATCH] Have all tag code add, modify and delete both old and new style tags This preps us for being able to use the new data model with existing repositories --- data/database.py | 9 +- ...f0abd172ae_add_tagtorepositorytag_table.py | 44 +++++ data/model/tag.py | 166 +++++++++++++++--- data/model/test/test_gc.py | 7 +- data/model/test/test_tag.py | 85 ++++++--- data/registry_model/registry_pre_oci_model.py | 1 + data/registry_model/test/test_interface.py | 4 +- endpoints/v2/test/test_manifest.py | 4 +- initdb.py | 4 +- workers/test/test_manifestbackfillworker.py | 4 +- 10 files changed, 274 insertions(+), 54 deletions(-) create mode 100644 data/migrations/versions/67f0abd172ae_add_tagtorepositorytag_table.py diff --git a/data/database.py b/data/database.py index e25c28cac..3601fd1a3 100644 --- a/data/database.py +++ b/data/database.py @@ -1519,12 +1519,19 @@ class TagManifestLabelMap(BaseModel): broken_manifest = BooleanField(index=True, default=False) +class TagToRepositoryTag(BaseModel): + """ NOTE: Only used for the duration of the migrations. """ + repository = ForeignKeyField(Repository, index=True) + tag = ForeignKeyField(Tag, index=True, unique=True) + repository_tag = ForeignKeyField(RepositoryTag, index=True, unique=True) + + appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, ApprBlobPlacement]) v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage, TagKind, ManifestChild, Tag]) -transition_classes = set([TagManifestToManifest, TagManifestLabelMap]) +transition_classes = set([TagManifestToManifest, TagManifestLabelMap, TagToRepositoryTag]) is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] diff --git a/data/migrations/versions/67f0abd172ae_add_tagtorepositorytag_table.py b/data/migrations/versions/67f0abd172ae_add_tagtorepositorytag_table.py new file mode 100644 index 000000000..241ac693a --- /dev/null +++ b/data/migrations/versions/67f0abd172ae_add_tagtorepositorytag_table.py @@ -0,0 +1,44 @@ +"""Add TagToRepositoryTag table + +Revision ID: 67f0abd172ae +Revises: 10f45ee2310b +Create Date: 2018-10-30 11:31:06.615488 + +""" + +# revision identifiers, used by Alembic. +revision = '67f0abd172ae' +down_revision = '10f45ee2310b' + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +def upgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('tagtorepositorytag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('repository_tag_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tagtorepositorytag_repository_id_repository')), + sa.ForeignKeyConstraint(['repository_tag_id'], ['repositorytag.id'], name=op.f('fk_tagtorepositorytag_repository_tag_id_repositorytag')), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tagtorepositorytag_tag_id_tag')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tagtorepositorytag')) + ) + op.create_index('tagtorepositorytag_repository_id', 'tagtorepositorytag', ['repository_id'], unique=False) + op.create_index('tagtorepositorytag_repository_tag_id', 'tagtorepositorytag', ['repository_tag_id'], unique=True) + op.create_index('tagtorepositorytag_tag_id', 'tagtorepositorytag', ['tag_id'], unique=True) + # ### end Alembic commands ### + + tester.populate_table('tagtorepositorytag', [ + ('repository_id', tester.TestDataType.Foreign('repository')), + ('tag_id', tester.TestDataType.Foreign('tag')), + ('repository_tag_id', tester.TestDataType.Foreign('repositorytag')), + ]) + + +def downgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('tagtorepositorytag') + # ### end Alembic commands ### diff --git a/data/model/tag.py b/data/model/tag.py index 7a92fda6e..d817fb15d 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -12,7 +12,7 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp, db_for_update, Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage, TagManifestToManifest, - TagManifestLabelMap) + TagManifestLabelMap, TagToRepositoryTag, Tag, get_epoch_timestamp_ms) from util.timedeltastring import convert_to_timedelta @@ -259,8 +259,10 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_i return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion) -def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False): - now_ts = get_epoch_timestamp() +def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False, + oci_manifest=None): + now_ms = get_epoch_timestamp_ms() + now_ts = int(now_ms / 1000) with db_transaction(): try: @@ -270,6 +272,17 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, RepositoryTag.name == tag_name), now_ts)).get() tag.lifetime_end_ts = now_ts tag.save() + + # Check for an OCI tag. + try: + oci_tag = db_for_update(Tag + .select() + .join(TagToRepositoryTag) + .where(TagToRepositoryTag.repository_tag == tag)).get() + oci_tag.lifetime_end_ms = now_ms + oci_tag.save() + except Tag.DoesNotExist: + pass except RepositoryTag.DoesNotExist: pass except IntegrityError: @@ -283,8 +296,16 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) try: - return RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name, - lifetime_start_ts=now_ts, reversion=reversion) + created = RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name, + lifetime_start_ts=now_ts, reversion=reversion) + if oci_manifest: + # Create the OCI tag as well. + oci_tag = Tag.create(repository=repository_id, manifest=oci_manifest, name=tag_name, + lifetime_start_ms=now_ms, reversion=reversion, + tag_kind=Tag.tag_kind.get_id('tag')) + TagToRepositoryTag.create(tag=oci_tag, repository_tag=created, repository=repository_id) + + return created except IntegrityError: msg = 'Tag with name %s and lifetime start %s already exists' raise TagAlreadyCreatedException(msg % (tag_name, now_ts)) @@ -302,7 +323,9 @@ def create_temporary_hidden_tag(repo, image_obj, expiration_s): def delete_tag(namespace_name, repository_name, tag_name): - now_ts = get_epoch_timestamp() + now_ms = get_epoch_timestamp_ms() + now_ts = int(now_ms / 1000) + with db_transaction(): try: query = _tag_alive(RepositoryTag @@ -320,6 +343,15 @@ def delete_tag(namespace_name, repository_name, tag_name): found.lifetime_end_ts = now_ts found.save() + + try: + oci_tag_query = TagToRepositoryTag.select().where(TagToRepositoryTag.repository_tag == found) + oci_tag = db_for_update(oci_tag_query).get().tag + oci_tag.lifetime_end_ms = now_ms + oci_tag.save() + except TagToRepositoryTag.DoesNotExist: + pass + return found @@ -362,6 +394,24 @@ def _delete_tags(repo, query_modifier=None): return set() with db_transaction(): + # Delete any associated new-style OCI tags. + # NOTE: This will *not* work once we have tags pointing to other tags (e.g. for channels), + # but this should all be changed over to new-style-only before we make *that* change. + oci_tags_to_delete = list(Tag + .select() + .join(TagToRepositoryTag) + .where(TagToRepositoryTag.repository == repo, + TagToRepositoryTag.repository_tag << tags_to_delete)) + + if oci_tags_to_delete: + # Delete the mapping entries. + (TagToRepositoryTag.delete().where(TagToRepositoryTag.repository == repo, + TagToRepositoryTag.repository_tag << tags_to_delete) + .execute()) + + # Delete the tags themselves. + Tag.delete().where(Tag.id << oci_tags_to_delete).execute() + # TODO(jschorr): Update to not use TagManifest once that table has been deprecated. tag_manifests_to_delete = list(TagManifest .select() @@ -548,10 +598,16 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest): except DataModelException: existing_image = None - # Change the tag manifest to point to the updated image. docker_image_id = tag_manifest.tag.image.docker_image_id + oci_manifest = None + try: + oci_manifest = Manifest.get(repository=repo_obj, digest=manifest_digest) + except Manifest.DoesNotExist: + pass + + # Change the tag and tag manifest to point to the updated image. updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, - reversion=True) + reversion=True, oci_manifest=oci_manifest) tag_manifest.tag = updated_tag tag_manifest.save() return existing_image @@ -601,9 +657,13 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id """ Stores a tag manifest for a specific tag name in the database. Returns the TagManifest object, as well as a boolean indicating whether the TagManifest was created. """ + # Create the new-style OCI manifest and its blobs. + oci_manifest = _populate_manifest_and_blobs(repository_id, manifest, storage_id_map, + leaf_layer_id=leaf_layer_id) + # Create the tag for the tag manifest. tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id, - reversion=reversion) + reversion=reversion, oci_manifest=oci_manifest) # Add a tag manifest pointing to that tag. try: @@ -612,7 +672,8 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id manifest.save() return manifest, False except TagManifest.DoesNotExist: - return _create_manifest(tag, manifest, storage_id_map), True + created = _associate_manifest(tag, oci_manifest) + return created, True def get_active_tag(namespace, repo_name, tag_name): @@ -661,10 +722,42 @@ def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest, s manifest.save() return manifest, False except TagManifest.DoesNotExist: - return _create_manifest(tag, manifest, storage_id_map), True + oci_manifest = _populate_manifest_and_blobs(tag.repository, manifest, storage_id_map) + + with db_transaction(): + try: + (Tag + .select() + .join(TagToRepositoryTag) + .where(TagToRepositoryTag.repository_tag == tag)).get() + except Tag.DoesNotExist: + oci_tag = Tag.create(repository=tag.repository, manifest=oci_manifest, name=tag_name, + reversion=tag.reversion, + lifetime_start_ms=tag.lifetime_start_ts * 1000, + lifetime_end_ms=(tag.lifetime_end_ts * 1000 + if tag.lifetime_end_ts else None), + tag_kind=Tag.tag_kind.get_id('tag')) + TagToRepositoryTag.create(tag=oci_tag, repository_tag=tag, repository=tag.repository) + + return _associate_manifest(tag, oci_manifest), True -def _create_manifest(tag, manifest, storage_id_map): +def _associate_manifest(tag, oci_manifest): + with db_transaction(): + tag_manifest = TagManifest.create(tag=tag, digest=oci_manifest.digest, + json_data=oci_manifest.manifest_bytes) + TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=oci_manifest) + return tag_manifest + + +def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_layer_id=None): + leaf_layer_id = leaf_layer_id or manifest.leaf_layer_v1_image_id + try: + legacy_image = Image.get(Image.docker_image_id == leaf_layer_id, + Image.repository == repository) + except Image.DoesNotExist: + raise DataModelException('Invalid image with id: %s' % leaf_layer_id) + storage_ids = set() for blob_digest in manifest.blob_digests: image_storage_id = storage_id_map.get(blob_digest) @@ -677,12 +770,7 @@ def _create_manifest(tag, manifest, storage_id_map): storage_ids.add(image_storage_id) - manifest_row = populate_manifest(tag.repository, manifest, tag.image, storage_ids) - - with db_transaction(): - tag_manifest = TagManifest.create(tag=tag, digest=manifest.digest, json_data=manifest.bytes) - TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row) - return tag_manifest + return populate_manifest(repository, manifest, legacy_image, storage_ids) def populate_manifest(repository, manifest, legacy_image, storage_ids): @@ -800,15 +888,41 @@ def change_tag_expiration(tag, expiration_date): if end_ts == tag.lifetime_end_ts: return (None, True) - # Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has - # not changed while we were updatings it expiration. - result = (RepositoryTag - .update(lifetime_end_ts=end_ts) - .where(RepositoryTag.id == tag.id, - RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts) - .execute()) + return set_tag_end_ts(tag, end_ts) - return (tag.lifetime_end_ts, result > 0) + +def set_tag_end_ts(tag, end_ts): + """ Sets the end timestamp for a tag. Should only be called by change_tag_expiration + or tests. + """ + end_ms = end_ts * 1000 + + with db_transaction(): + # Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has + # not changed while we were updating it expiration. + result = (RepositoryTag + .update(lifetime_end_ts=end_ts) + .where(RepositoryTag.id == tag.id, + RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts) + .execute()) + + # Check for a mapping to an OCI tag. + try: + oci_tag = (Tag + .select() + .join(TagToRepositoryTag) + .where(TagToRepositoryTag.repository_tag == tag) + .get()) + + (Tag + .update(lifetime_end_ms=end_ms) + .where(Tag.id == oci_tag.id, + Tag.lifetime_end_ms == oci_tag.lifetime_end_ms) + .execute()) + except Tag.DoesNotExist: + pass + + return (tag.lifetime_end_ts, result > 0) def find_matching_tag(repo_id, tag_names): diff --git a/data/model/test/test_gc.py b/data/model/test/test_gc.py index f40154c3a..7af75430b 100644 --- a/data/model/test/test_gc.py +++ b/data/model/test/test_gc.py @@ -12,7 +12,7 @@ from playhouse.test_utils import assert_query_count from data import model, database from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, ApprBlob, Manifest, TagManifest, TagManifestToManifest, ManifestLabel, - TagManifestLabelMap, ManifestBlob) + TagManifestLabelMap, ManifestBlob, Tag, TagToRepositoryTag) from image.docker.schema1 import DockerSchema1ManifestBuilder from test.fixtures import * @@ -223,6 +223,11 @@ def assert_gc_integrity(expect_storage_removed=True): for blob_row in ApprBlob.select(): storage.get_content({preferred}, storage.blob_path(blob_row.digest)) + # Ensure there are no danglings OCI tags. + oci_tags = {t.id for t in Tag.select()} + referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()} + assert not (oci_tags - referenced_oci_tags) + def test_has_garbage(default_tag_policy, initialized_db): """ Remove all existing repositories, then add one without garbage, check, then add one with diff --git a/data/model/test/test_tag.py b/data/model/test/test_tag.py index 5a7f69d61..baa164d5b 100644 --- a/data/model/test/test_tag.py +++ b/data/model/test/test_tag.py @@ -9,12 +9,13 @@ from mock import patch from app import docker_v2_signing_key from data.database import (Image, RepositoryTag, ImageStorage, Repository, Manifest, ManifestBlob, - ManifestLegacyImage, TagManifestToManifest) + ManifestLegacyImage, TagManifestToManifest, Tag, TagToRepositoryTag) from data.model.repository import create_repository from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag, get_matching_tags, _tag_alive, get_matching_tags_for_images, change_tag_expiration, get_active_tag, store_tag_manifest_for_testing, - get_most_recent_tag, get_active_tag_for_repo) + get_most_recent_tag, get_active_tag_for_repo, + create_or_update_tag_for_repo, set_tag_end_ts) from data.model.image import find_create_or_link_image from image.docker.schema1 import DockerSchema1ManifestBuilder from util.timedeltastring import convert_to_timedelta @@ -24,12 +25,13 @@ from test.fixtures import * def _get_expected_tags(image): expected_query = (RepositoryTag - .select() - .join(Image) - .where(RepositoryTag.hidden == False) - .where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id)))) + .select() + .join(Image) + .where(RepositoryTag.hidden == False) + .where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id)))) return set([tag.id for tag in _tag_alive(expected_query)]) + @pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [ (1, 1), (10, 10), @@ -45,6 +47,12 @@ def test_get_matching_tags(max_subqueries, max_image_lookup_count, initialized_d expected_tags = _get_expected_tags(image) assert matching_tags == expected_tags, "mismatch for image %s" % image.id + oci_tags = list(Tag + .select() + .join(TagToRepositoryTag) + .where(TagToRepositoryTag.repository_tag << expected_tags)) + assert len(oci_tags) == len(expected_tags) + @pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [ (1, 1), @@ -116,6 +124,13 @@ def test_get_matching_tag_ids_images_filtered(initialized_db): assert matching_tags_ids == expected_tag_ids +def _get_oci_tag(tag): + return (Tag + .select() + .join(TagToRepositoryTag) + .where(TagToRepositoryTag.repository_tag == tag)).get() + + def assert_tags(repository, *args): tags = list(list_active_repo_tags(repository)) assert len(tags) == len(args) @@ -128,12 +143,23 @@ def assert_tags(repository, *args): tags_dict[tag.name] = tag + oci_tag = _get_oci_tag(tag) + assert oci_tag.name == tag.name + assert not oci_tag.hidden + + if tag.lifetime_end_ts: + assert oci_tag.lifetime_end_ms == (tag.lifetime_end_ts * 1000) + else: + assert oci_tag.lifetime_end_ms is None + for expected in args: assert expected in tags_dict + def test_list_active_tags(initialized_db): # Create a new repository. repository = create_repository('devtable', 'somenewrepo', None) + manifest = Manifest.get() # Create some images. image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') @@ -143,50 +169,62 @@ def test_list_active_tags(initialized_db): assert_tags(repository) # Add some new tags. - footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) - bartag = create_or_update_tag('devtable', 'somenewrepo', 'bar', image1.docker_image_id) + footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, + oci_manifest=manifest) + bartag = create_or_update_tag_for_repo(repository, 'bar', image1.docker_image_id, + oci_manifest=manifest) - # Since timestamps are stored on a second-granuality, we need to make the tags "start" + # Since timestamps are stored on a second-granularity, we need to make the tags "start" # before "now", so when we recreate them below, they don't conflict. footag.lifetime_start_ts -= 5 - bartag.lifetime_start_ts -= 5 - footag.save() + + bartag.lifetime_start_ts -= 5 bartag.save() + footag_oci = _get_oci_tag(footag) + footag_oci.lifetime_start_ms -= 5000 + footag_oci.save() + + bartag_oci = _get_oci_tag(bartag) + bartag_oci.lifetime_start_ms -= 5000 + bartag_oci.save() + # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # Mark as a tag as expiring in the far future, and make sure it is still returned. - footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000 - footag.save() + set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000) # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # Delete a tag and make sure it isn't returned. footag = delete_tag('devtable', 'somenewrepo', 'foo') - footag.lifetime_end_ts -= 4 - footag.save() + set_tag_end_ts(footag, footag.lifetime_end_ts - 4) assert_tags(repository, 'bar') # Add a new foo again. - footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) + footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, + oci_manifest=manifest) footag.lifetime_start_ts -= 3 footag.save() + footag_oci = _get_oci_tag(footag) + footag_oci.lifetime_start_ms -= 3000 + footag_oci.save() + assert_tags(repository, 'foo', 'bar') # Mark as a tag as expiring in the far future, and make sure it is still returned. - footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000 - footag.save() + set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000) # Make sure they are returned. assert_tags(repository, 'foo', 'bar') # "Move" foo by updating it and make sure we don't get duplicates. - create_or_update_tag('devtable', 'somenewrepo', 'foo', image2.docker_image_id) + create_or_update_tag_for_repo(repository, 'foo', image2.docker_image_id, oci_manifest=manifest) assert_tags(repository, 'foo', 'bar') @@ -201,7 +239,10 @@ def test_list_active_tags(initialized_db): def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db): repository = create_repository('devtable', 'somenewrepo', None) image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') - footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) + + manifest = Manifest.get() + footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id, + oci_manifest=manifest) expiration_date = None if expiration_offset is not None: @@ -211,15 +252,19 @@ def test_change_tag_expiration(expiration_offset, expected_offset, initialized_d # Lookup the tag again. footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo') + oci_tag = _get_oci_tag(footag_updated) if expected_offset is None: assert footag_updated.lifetime_end_ts is None + assert oci_tag.lifetime_end_ms is None else: start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts) end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts) expected_end_date = start_date + convert_to_timedelta(expected_offset) assert (expected_end_date - end_date).total_seconds() < 5 # variance in test + assert oci_tag.lifetime_end_ms == (footag_updated.lifetime_end_ts * 1000) + def random_storages(): return list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(10)) diff --git a/data/registry_model/registry_pre_oci_model.py b/data/registry_model/registry_pre_oci_model.py index 9748cf74d..1e0b4065a 100644 --- a/data/registry_model/registry_pre_oci_model.py +++ b/data/registry_model/registry_pre_oci_model.py @@ -522,6 +522,7 @@ class PreOCIModel(SharedModel, RegistryDataInterface): try: tag_manifest, _ = model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag.name, manifest, storage_map) + assert tag_manifest except IntegrityError: tag_manifest = model.tag.get_tag_manifest(tag_obj) diff --git a/data/registry_model/test/test_interface.py b/data/registry_model/test/test_interface.py index 695ecc3ec..1e5a8d510 100644 --- a/data/registry_model/test/test_interface.py +++ b/data/registry_model/test/test_interface.py @@ -13,7 +13,7 @@ from data import model from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob, ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image, TagManifestLabel, TagManifest, TagManifestLabel, DerivedStorageForImage, - TorrentInfo, close_db_filter) + TorrentInfo, Tag, TagToRepositoryTag, close_db_filter) from data.cache.impl import InMemoryDataModelCache from data.registry_model.registry_pre_oci_model import PreOCIModel from data.registry_model.datatypes import RepositoryReference @@ -381,6 +381,8 @@ def test_get_security_status(registry_model): @pytest.fixture() def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. + TagToRepositoryTag.delete().execute() + Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute() diff --git a/endpoints/v2/test/test_manifest.py b/endpoints/v2/test/test_manifest.py index 673163bb9..b16278e01 100644 --- a/endpoints/v2/test/test_manifest.py +++ b/endpoints/v2/test/test_manifest.py @@ -46,10 +46,10 @@ def test_e2e_query_count_manifest_norewrite(client, app): return timecode + 10 with patch('time.time', get_time): - # Necessary in order to have the tag updates not occurr in the same second, which is the + # Necessary in order to have the tag updates not occur in the same second, which is the # granularity supported currently. with count_queries() as counter: conduct_call(client, 'v2.write_manifest_by_digest', url_for, 'PUT', params, expected_code=202, headers=headers, raw_body=tag_manifest.json_data) - assert counter.count <= 16 + assert counter.count <= 25 diff --git a/initdb.py b/initdb.py index 0b4d9994c..2294bbfb4 100644 --- a/initdb.py +++ b/initdb.py @@ -20,7 +20,7 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind, RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes, - ApprTagKind, ApprBlobPlacementLocation, Repository, Tag, TagKind, + ApprTagKind, ApprBlobPlacementLocation, Repository, TagKind, ManifestChild) from data import model from data.queue import WorkQueue @@ -913,7 +913,7 @@ def populate_database(minimal=False, with_storage=False): model.repositoryactioncount.update_repository_score(to_count) -WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'Tag', 'ManifestChild'] +WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'ManifestChild'] def find_models_missing_data(): # As a sanity check we are going to make sure that all db tables have some data, unless explicitly diff --git a/workers/test/test_manifestbackfillworker.py b/workers/test/test_manifestbackfillworker.py index 58a918524..c1c31d6bf 100644 --- a/workers/test/test_manifestbackfillworker.py +++ b/workers/test/test_manifestbackfillworker.py @@ -2,7 +2,7 @@ from app import docker_v2_signing_key from data import model from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob, ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image, - TagManifestLabel) + TagManifestLabel, Tag, TagToRepositoryTag) from image.docker.schema1 import DockerSchema1ManifestBuilder from workers.manifestbackfillworker import backfill_manifest from workers.labelbackfillworker import backfill_label @@ -12,6 +12,8 @@ from test.fixtures import * @pytest.fixture() def clear_rows(initialized_db): # Remove all new-style rows so we can backfill. + TagToRepositoryTag.delete().execute() + Tag.delete().execute() TagManifestLabelMap.delete().execute() ManifestLabel.delete().execute() ManifestBlob.delete().execute()