vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Have all tag code add, modify and delete both old and new style tags

Browse source 
This preps us for being able to use the new data model with existing repositories
This commit is contained in:
Joseph Schorr 2018-10-30 11:35:31 -04:00
parent 36db133b86
commit 114e2c3bf2
10 changed files with 274 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
data/database.py
View file

@ -1519,12 +1519,19 @@ class TagManifestLabelMap(BaseModel):
broken_manifest = BooleanField(index=True, default=False)
class TagToRepositoryTag(BaseModel):
""" NOTE: Only used for the duration of the migrations. """
repository = ForeignKeyField(Repository, index=True)
tag = ForeignKeyField(Tag, index=True, unique=True)
repository_tag = ForeignKeyField(RepositoryTag, index=True, unique=True)
appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement])
v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage, TagKind,
ManifestChild, Tag])
transition_classes = set([TagManifestToManifest, TagManifestLabelMap])
transition_classes = set([TagManifestToManifest, TagManifestLabelMap, TagToRepositoryTag])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

44
data/migrations/versions/67f0abd172ae_add_tagtorepositorytag_table.py Normal file
View file

@ -0,0 +1,44 @@
"""Add TagToRepositoryTag table
Revision ID: 67f0abd172ae
Revises: 10f45ee2310b
Create Date: 2018-10-30 11:31:06.615488
"""
# revision identifiers, used by Alembic.
revision = '67f0abd172ae'
down_revision = '10f45ee2310b'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('tagtorepositorytag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('tag_id', sa.Integer(), nullable=False),
sa.Column('repository_tag_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tagtorepositorytag_repository_id_repository')),
sa.ForeignKeyConstraint(['repository_tag_id'], ['repositorytag.id'], name=op.f('fk_tagtorepositorytag_repository_tag_id_repositorytag')),
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tagtorepositorytag_tag_id_tag')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_tagtorepositorytag'))
)
op.create_index('tagtorepositorytag_repository_id', 'tagtorepositorytag', ['repository_id'], unique=False)
op.create_index('tagtorepositorytag_repository_tag_id', 'tagtorepositorytag', ['repository_tag_id'], unique=True)
op.create_index('tagtorepositorytag_tag_id', 'tagtorepositorytag', ['tag_id'], unique=True)
# ### end Alembic commands ###
tester.populate_table('tagtorepositorytag', [
('repository_id', tester.TestDataType.Foreign('repository')),
('tag_id', tester.TestDataType.Foreign('tag')),
('repository_tag_id', tester.TestDataType.Foreign('repositorytag')),
])
def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('tagtorepositorytag')
# ### end Alembic commands ###

166
data/model/tag.py
View file

@ -12,7 +12,7 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names
RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp,
db_for_update, Manifest, ManifestLabel, ManifestBlob,
ManifestLegacyImage, TagManifestToManifest,
TagManifestLabelMap)
TagManifestLabelMap, TagToRepositoryTag, Tag, get_epoch_timestamp_ms)
from util.timedeltastring import convert_to_timedelta
@ -259,8 +259,10 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_i
return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion)
def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False):
now_ts = get_epoch_timestamp()
def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False,
oci_manifest=None):
now_ms = get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
@ -270,6 +272,17 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id,
RepositoryTag.name == tag_name), now_ts)).get()
tag.lifetime_end_ts = now_ts
tag.save()
# Check for an OCI tag.
try:
oci_tag = db_for_update(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)).get()
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except Tag.DoesNotExist:
pass
except RepositoryTag.DoesNotExist:
pass
except IntegrityError:
@ -283,8 +296,16 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id,
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
try:
return RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name,
lifetime_start_ts=now_ts, reversion=reversion)
created = RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name,
lifetime_start_ts=now_ts, reversion=reversion)
if oci_manifest:
# Create the OCI tag as well.
oci_tag = Tag.create(repository=repository_id, manifest=oci_manifest, name=tag_name,
lifetime_start_ms=now_ms, reversion=reversion,
tag_kind=Tag.tag_kind.get_id('tag'))
TagToRepositoryTag.create(tag=oci_tag, repository_tag=created, repository=repository_id)
return created
except IntegrityError:
msg = 'Tag with name %s and lifetime start %s already exists'
raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
@ -302,7 +323,9 @@ def create_temporary_hidden_tag(repo, image_obj, expiration_s):
def delete_tag(namespace_name, repository_name, tag_name):
now_ts = get_epoch_timestamp()
now_ms = get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction():
try:
query = _tag_alive(RepositoryTag
@ -320,6 +343,15 @@ def delete_tag(namespace_name, repository_name, tag_name):
found.lifetime_end_ts = now_ts
found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found
@ -362,6 +394,24 @@ def _delete_tags(repo, query_modifier=None):
return set()
with db_transaction():
# Delete any associated new-style OCI tags.
# NOTE: This will *not* work once we have tags pointing to other tags (e.g. for channels),
# but this should all be changed over to new-style-only before we make *that* change.
oci_tags_to_delete = list(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository == repo,
TagToRepositoryTag.repository_tag << tags_to_delete))
if oci_tags_to_delete:
# Delete the mapping entries.
(TagToRepositoryTag.delete().where(TagToRepositoryTag.repository == repo,
TagToRepositoryTag.repository_tag << tags_to_delete)
.execute())
# Delete the tags themselves.
Tag.delete().where(Tag.id << oci_tags_to_delete).execute()
# TODO(jschorr): Update to not use TagManifest once that table has been deprecated.
tag_manifests_to_delete = list(TagManifest
.select()
@ -548,10 +598,16 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
except DataModelException:
existing_image = None
# Change the tag manifest to point to the updated image.
docker_image_id = tag_manifest.tag.image.docker_image_id
oci_manifest = None
try:
oci_manifest = Manifest.get(repository=repo_obj, digest=manifest_digest)
except Manifest.DoesNotExist:
pass
# Change the tag and tag manifest to point to the updated image.
updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id,
reversion=True)
reversion=True, oci_manifest=oci_manifest)
tag_manifest.tag = updated_tag
tag_manifest.save()
return existing_image
@ -601,9 +657,13 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
object, as well as a boolean indicating whether the TagManifest was created.
"""
# Create the new-style OCI manifest and its blobs.
oci_manifest = _populate_manifest_and_blobs(repository_id, manifest, storage_id_map,
leaf_layer_id=leaf_layer_id)
# Create the tag for the tag manifest.
tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id,
reversion=reversion)
reversion=reversion, oci_manifest=oci_manifest)
# Add a tag manifest pointing to that tag.
try:
@ -612,7 +672,8 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
manifest.save()
return manifest, False
except TagManifest.DoesNotExist:
return _create_manifest(tag, manifest, storage_id_map), True
created = _associate_manifest(tag, oci_manifest)
return created, True
def get_active_tag(namespace, repo_name, tag_name):
@ -661,10 +722,42 @@ def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest, s
manifest.save()
return manifest, False
except TagManifest.DoesNotExist:
return _create_manifest(tag, manifest, storage_id_map), True
oci_manifest = _populate_manifest_and_blobs(tag.repository, manifest, storage_id_map)
with db_transaction():
try:
(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)).get()
except Tag.DoesNotExist:
oci_tag = Tag.create(repository=tag.repository, manifest=oci_manifest, name=tag_name,
reversion=tag.reversion,
lifetime_start_ms=tag.lifetime_start_ts * 1000,
lifetime_end_ms=(tag.lifetime_end_ts * 1000
if tag.lifetime_end_ts else None),
tag_kind=Tag.tag_kind.get_id('tag'))
TagToRepositoryTag.create(tag=oci_tag, repository_tag=tag, repository=tag.repository)
return _associate_manifest(tag, oci_manifest), True
def _create_manifest(tag, manifest, storage_id_map):
def _associate_manifest(tag, oci_manifest):
with db_transaction():
tag_manifest = TagManifest.create(tag=tag, digest=oci_manifest.digest,
json_data=oci_manifest.manifest_bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=oci_manifest)
return tag_manifest
def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_layer_id=None):
leaf_layer_id = leaf_layer_id or manifest.leaf_layer_v1_image_id
try:
legacy_image = Image.get(Image.docker_image_id == leaf_layer_id,
Image.repository == repository)
except Image.DoesNotExist:
raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
storage_ids = set()
for blob_digest in manifest.blob_digests:
image_storage_id = storage_id_map.get(blob_digest)
@ -677,12 +770,7 @@ def _create_manifest(tag, manifest, storage_id_map):
storage_ids.add(image_storage_id)
manifest_row = populate_manifest(tag.repository, manifest, tag.image, storage_ids)
with db_transaction():
tag_manifest = TagManifest.create(tag=tag, digest=manifest.digest, json_data=manifest.bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row)
return tag_manifest
return populate_manifest(repository, manifest, legacy_image, storage_ids)
def populate_manifest(repository, manifest, legacy_image, storage_ids):
@ -800,15 +888,41 @@ def change_tag_expiration(tag, expiration_date):
if end_ts == tag.lifetime_end_ts:
return (None, True)
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
# not changed while we were updatings it expiration.
result = (RepositoryTag
.update(lifetime_end_ts=end_ts)
.where(RepositoryTag.id == tag.id,
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts)
.execute())
return set_tag_end_ts(tag, end_ts)
return (tag.lifetime_end_ts, result > 0)
def set_tag_end_ts(tag, end_ts):
""" Sets the end timestamp for a tag. Should only be called by change_tag_expiration
or tests.
"""
end_ms = end_ts * 1000
with db_transaction():
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
# not changed while we were updating it expiration.
result = (RepositoryTag
.update(lifetime_end_ts=end_ts)
.where(RepositoryTag.id == tag.id,
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts)
.execute())
# Check for a mapping to an OCI tag.
try:
oci_tag = (Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)
.get())
(Tag
.update(lifetime_end_ms=end_ms)
.where(Tag.id == oci_tag.id,
Tag.lifetime_end_ms == oci_tag.lifetime_end_ms)
.execute())
except Tag.DoesNotExist:
pass
return (tag.lifetime_end_ts, result > 0)
def find_matching_tag(repo_id, tag_names):

7
data/model/test/test_gc.py
View file

@ -12,7 +12,7 @@ from playhouse.test_utils import assert_query_count
from data import model, database
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
ApprBlob, Manifest, TagManifest, TagManifestToManifest, ManifestLabel,
TagManifestLabelMap, ManifestBlob)
TagManifestLabelMap, ManifestBlob, Tag, TagToRepositoryTag)
from image.docker.schema1 import DockerSchema1ManifestBuilder
from test.fixtures import *
@ -223,6 +223,11 @@ def assert_gc_integrity(expect_storage_removed=True):
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure there are no danglings OCI tags.
oci_tags = {t.id for t in Tag.select()}
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
assert not (oci_tags - referenced_oci_tags)
def test_has_garbage(default_tag_policy, initialized_db):
""" Remove all existing repositories, then add one without garbage, check, then add one with

85
data/model/test/test_tag.py
View file

@ -9,12 +9,13 @@ from mock import patch
from app import docker_v2_signing_key
from data.database import (Image, RepositoryTag, ImageStorage, Repository, Manifest, ManifestBlob,
ManifestLegacyImage, TagManifestToManifest)
ManifestLegacyImage, TagManifestToManifest, Tag, TagToRepositoryTag)
from data.model.repository import create_repository
from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag,
get_matching_tags, _tag_alive, get_matching_tags_for_images,
change_tag_expiration, get_active_tag, store_tag_manifest_for_testing,
get_most_recent_tag, get_active_tag_for_repo)
get_most_recent_tag, get_active_tag_for_repo,
create_or_update_tag_for_repo, set_tag_end_ts)
from data.model.image import find_create_or_link_image
from image.docker.schema1 import DockerSchema1ManifestBuilder
from util.timedeltastring import convert_to_timedelta
@ -24,12 +25,13 @@ from test.fixtures import *
def _get_expected_tags(image):
expected_query = (RepositoryTag
.select()
.join(Image)
.where(RepositoryTag.hidden == False)
.where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
.select()
.join(Image)
.where(RepositoryTag.hidden == False)
.where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
return set([tag.id for tag in _tag_alive(expected_query)])
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1),
(10, 10),
@ -45,6 +47,12 @@ def test_get_matching_tags(max_subqueries, max_image_lookup_count, initialized_d
expected_tags = _get_expected_tags(image)
assert matching_tags == expected_tags, "mismatch for image %s" % image.id
oci_tags = list(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag << expected_tags))
assert len(oci_tags) == len(expected_tags)
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1),
@ -116,6 +124,13 @@ def test_get_matching_tag_ids_images_filtered(initialized_db):
assert matching_tags_ids == expected_tag_ids
def _get_oci_tag(tag):
return (Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)).get()
def assert_tags(repository, *args):
tags = list(list_active_repo_tags(repository))
assert len(tags) == len(args)
@ -128,12 +143,23 @@ def assert_tags(repository, *args):
tags_dict[tag.name] = tag
oci_tag = _get_oci_tag(tag)
assert oci_tag.name == tag.name
assert not oci_tag.hidden
if tag.lifetime_end_ts:
assert oci_tag.lifetime_end_ms == (tag.lifetime_end_ts * 1000)
else:
assert oci_tag.lifetime_end_ms is None
for expected in args:
assert expected in tags_dict
def test_list_active_tags(initialized_db):
# Create a new repository.
repository = create_repository('devtable', 'somenewrepo', None)
manifest = Manifest.get()
# Create some images.
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
@ -143,50 +169,62 @@ def test_list_active_tags(initialized_db):
assert_tags(repository)
# Add some new tags.
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
bartag = create_or_update_tag('devtable', 'somenewrepo', 'bar', image1.docker_image_id)
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
oci_manifest=manifest)
bartag = create_or_update_tag_for_repo(repository, 'bar', image1.docker_image_id,
oci_manifest=manifest)
# Since timestamps are stored on a second-granuality, we need to make the tags "start"
# Since timestamps are stored on a second-granularity, we need to make the tags "start"
# before "now", so when we recreate them below, they don't conflict.
footag.lifetime_start_ts -= 5
bartag.lifetime_start_ts -= 5
footag.save()
bartag.lifetime_start_ts -= 5
bartag.save()
footag_oci = _get_oci_tag(footag)
footag_oci.lifetime_start_ms -= 5000
footag_oci.save()
bartag_oci = _get_oci_tag(bartag)
bartag_oci.lifetime_start_ms -= 5000
bartag_oci.save()
# Make sure they are returned.
assert_tags(repository, 'foo', 'bar')
# Mark as a tag as expiring in the far future, and make sure it is still returned.
footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000
footag.save()
set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
# Make sure they are returned.
assert_tags(repository, 'foo', 'bar')
# Delete a tag and make sure it isn't returned.
footag = delete_tag('devtable', 'somenewrepo', 'foo')
footag.lifetime_end_ts -= 4
footag.save()
set_tag_end_ts(footag, footag.lifetime_end_ts - 4)
assert_tags(repository, 'bar')
# Add a new foo again.
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
oci_manifest=manifest)
footag.lifetime_start_ts -= 3
footag.save()
footag_oci = _get_oci_tag(footag)
footag_oci.lifetime_start_ms -= 3000
footag_oci.save()
assert_tags(repository, 'foo', 'bar')
# Mark as a tag as expiring in the far future, and make sure it is still returned.
footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000
footag.save()
set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
# Make sure they are returned.
assert_tags(repository, 'foo', 'bar')
# "Move" foo by updating it and make sure we don't get duplicates.
create_or_update_tag('devtable', 'somenewrepo', 'foo', image2.docker_image_id)
create_or_update_tag_for_repo(repository, 'foo', image2.docker_image_id, oci_manifest=manifest)
assert_tags(repository, 'foo', 'bar')
@ -201,7 +239,10 @@ def test_list_active_tags(initialized_db):
def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db):
repository = create_repository('devtable', 'somenewrepo', None)
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
manifest = Manifest.get()
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
oci_manifest=manifest)
expiration_date = None
if expiration_offset is not None:
@ -211,15 +252,19 @@ def test_change_tag_expiration(expiration_offset, expected_offset, initialized_d
# Lookup the tag again.
footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo')
oci_tag = _get_oci_tag(footag_updated)
if expected_offset is None:
assert footag_updated.lifetime_end_ts is None
assert oci_tag.lifetime_end_ms is None
else:
start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts)
end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts)
expected_end_date = start_date + convert_to_timedelta(expected_offset)
assert (expected_end_date - end_date).total_seconds() < 5 # variance in test
assert oci_tag.lifetime_end_ms == (footag_updated.lifetime_end_ts * 1000)
def random_storages():
return list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(10))

1
data/registry_model/registry_pre_oci_model.py
View file

@ -522,6 +522,7 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
try:
tag_manifest, _ = model.tag.associate_generated_tag_manifest(namespace_name, repo_name,
tag.name, manifest, storage_map)
assert tag_manifest
except IntegrityError:
tag_manifest = model.tag.get_tag_manifest(tag_obj)

4
data/registry_model/test/test_interface.py
View file

@ -13,7 +13,7 @@ from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
TagManifestLabel, TagManifest, TagManifestLabel, DerivedStorageForImage,
TorrentInfo, close_db_filter)
TorrentInfo, Tag, TagToRepositoryTag, close_db_filter)
from data.cache.impl import InMemoryDataModelCache
from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.datatypes import RepositoryReference
@ -381,6 +381,8 @@ def test_get_security_status(registry_model):
@pytest.fixture()
def clear_rows(initialized_db):
# Remove all new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()

4
endpoints/v2/test/test_manifest.py
View file

@ -46,10 +46,10 @@ def test_e2e_query_count_manifest_norewrite(client, app):
return timecode + 10
with patch('time.time', get_time):
# Necessary in order to have the tag updates not occurr in the same second, which is the
# Necessary in order to have the tag updates not occur in the same second, which is the
# granularity supported currently.
with count_queries() as counter:
conduct_call(client, 'v2.write_manifest_by_digest', url_for, 'PUT', params, expected_code=202,
headers=headers, raw_body=tag_manifest.json_data)
assert counter.count <= 16
assert counter.count <= 25

4
initdb.py
View file

@ -20,7 +20,7 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes,
ApprTagKind, ApprBlobPlacementLocation, Repository, Tag, TagKind,
ApprTagKind, ApprBlobPlacementLocation, Repository, TagKind,
ManifestChild)
from data import model
from data.queue import WorkQueue
@ -913,7 +913,7 @@ def populate_database(minimal=False, with_storage=False):
model.repositoryactioncount.update_repository_score(to_count)
WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'Tag', 'ManifestChild']
WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'ManifestChild']
def find_models_missing_data():
# As a sanity check we are going to make sure that all db tables have some data, unless explicitly

4
workers/test/test_manifestbackfillworker.py
View file

@ -2,7 +2,7 @@ from app import docker_v2_signing_key
from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
TagManifestLabel)
TagManifestLabel, Tag, TagToRepositoryTag)
from image.docker.schema1 import DockerSchema1ManifestBuilder
from workers.manifestbackfillworker import backfill_manifest
from workers.labelbackfillworker import backfill_label
@ -12,6 +12,8 @@ from test.fixtures import *
@pytest.fixture()
def clear_rows(initialized_db):
# Remove all new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute()
ManifestBlob.delete().execute()