vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3278 from quay/joseph.schorr/QUAY-1124/new-data-model

Browse source 
Begin work on the new registry data model implementation
This commit is contained in:
Joseph Schorr 2018-11-02 14:04:14 -04:00 • committed by GitHub
commit 876ebc9b2f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 886 additions and 482 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
data/database.py
View file

@ -1385,6 +1385,58 @@ class Manifest(BaseModel):
) )
class TagKind(BaseModel):
""" TagKind describes the various kinds of tags that can be found in the registry.
"""
name = CharField(index=True, unique=True)
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a Manifest or as an alias to another tag.
"""
name = CharField()
repository = ForeignKeyField(Repository)
manifest = ForeignKeyField(Manifest, null=True)
lifetime_start_ms = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end_ms = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reversion = BooleanField(default=False)
tag_kind = EnumField(TagKind)
linked_tag = ForeignKeyField('self', null=True, backref='tag_parents')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'name'), False),
(('repository', 'name', 'hidden'), False),
(('repository', 'name', 'tag_kind'), False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end_ms'), True),
)
class ManifestChild(BaseModel):
""" ManifestChild represents a relationship between a manifest and its child manifest(s).
Multiple manifests can share the same children. Note that since Manifests are stored
per-repository, the repository here is a bit redundant, but we do so to make cleanup easier.
"""
repository = ForeignKeyField(Repository)
manifest = ForeignKeyField(Manifest)
child_manifest = ForeignKeyField(Manifest)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'manifest'), False),
(('repository', 'child_manifest'), False),
(('repository', 'manifest', 'child_manifest'), False),
(('manifest', 'child_manifest'), True),
)
class ManifestLabel(BaseModel): class ManifestLabel(BaseModel):
""" ManifestLabel represents a label applied to a Manifest, within a repository. """ ManifestLabel represents a label applied to a Manifest, within a repository.
Note that since Manifests are stored per-repository, the repository here is Note that since Manifests are stored per-repository, the repository here is
@ -1467,11 +1519,19 @@ class TagManifestLabelMap(BaseModel):
broken_manifest = BooleanField(index=True, default=False) broken_manifest = BooleanField(index=True, default=False)
class TagToRepositoryTag(BaseModel):
""" NOTE: Only used for the duration of the migrations. """
repository = ForeignKeyField(Repository, index=True)
tag = ForeignKeyField(Tag, index=True, unique=True)
repository_tag = ForeignKeyField(RepositoryTag, index=True, unique=True)
appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement]) ApprBlobPlacement])
v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage]) v22_classes = set([Manifest, ManifestLabel, ManifestBlob, ManifestLegacyImage, TagKind,
transition_classes = set([TagManifestToManifest, TagManifestLabelMap]) ManifestChild, Tag])
transition_classes = set([TagManifestToManifest, TagManifestLabelMap, TagToRepositoryTag])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

97
data/migrations/versions/10f45ee2310b_add_tag_tagkind_and_manifestchild_tables.py Normal file
View file

@ -0,0 +1,97 @@
"""Add Tag, TagKind and ManifestChild tables
Revision ID: 10f45ee2310b
Revises: 13411de1c0ff
Create Date: 2018-10-29 15:22:53.552216
"""
# revision identifiers, used by Alembic.
revision = '10f45ee2310b'
down_revision = '13411de1c0ff'
from alembic import op
import sqlalchemy as sa
from util.migrate import UTF8CharField
def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('tagkind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind'))
)
op.create_index('tagkind_name', 'tagkind', ['name'], unique=True)
op.create_table('manifestchild',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('child_manifest_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['child_manifest_id'], ['manifest.id'], name=op.f('fk_manifestchild_child_manifest_id_manifest')),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestchild_manifest_id_manifest')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_manifestchild_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestchild'))
)
op.create_index('manifestchild_child_manifest_id', 'manifestchild', ['child_manifest_id'], unique=False)
op.create_index('manifestchild_manifest_id', 'manifestchild', ['manifest_id'], unique=False)
op.create_index('manifestchild_manifest_id_child_manifest_id', 'manifestchild', ['manifest_id', 'child_manifest_id'], unique=True)
op.create_index('manifestchild_repository_id', 'manifestchild', ['repository_id'], unique=False)
op.create_index('manifestchild_repository_id_child_manifest_id', 'manifestchild', ['repository_id', 'child_manifest_id'], unique=False)
op.create_index('manifestchild_repository_id_manifest_id', 'manifestchild', ['repository_id', 'manifest_id'], unique=False)
op.create_index('manifestchild_repository_id_manifest_id_child_manifest_id', 'manifestchild', ['repository_id', 'manifest_id', 'child_manifest_id'], unique=False)
op.create_table('tag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', UTF8CharField(length=255), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=True),
sa.Column('lifetime_start_ms', sa.BigInteger(), nullable=False),
sa.Column('lifetime_end_ms', sa.BigInteger(), nullable=True),
sa.Column('hidden', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()),
sa.Column('reversion', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()),
sa.Column('tag_kind_id', sa.Integer(), nullable=False),
sa.Column('linked_tag_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_tag_manifest_id_manifest')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')),
sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
)
op.create_index('tag_lifetime_end_ms', 'tag', ['lifetime_end_ms'], unique=False)
op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False)
op.create_index('tag_manifest_id', 'tag', ['manifest_id'], unique=False)
op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False)
op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False)
op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False)
op.create_index('tag_repository_id_name_lifetime_end_ms', 'tag', ['repository_id', 'name', 'lifetime_end_ms'], unique=True)
op.create_index('tag_repository_id_name_tag_kind_id', 'tag', ['repository_id', 'name', 'tag_kind_id'], unique=False)
op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False)
# ### end Alembic commands ###
op.bulk_insert(tables.tagkind,
[
{'name': 'tag'},
])
# ### population of test data ### #
tester.populate_table('tag', [
('repository_id', tester.TestDataType.Foreign('repository')),
('tag_kind_id', tester.TestDataType.Foreign('tagkind')),
('name', tester.TestDataType.String),
('manifest_id', tester.TestDataType.Foreign('manifest')),
('lifetime_start_ms', tester.TestDataType.BigInteger),
])
tester.populate_table('manifestchild', [
('repository_id', tester.TestDataType.Foreign('repository')),
('manifest_id', tester.TestDataType.Foreign('manifest')),
('child_manifest_id', tester.TestDataType.Foreign('manifest')),
])
# ### end population of test data ### #
def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('tag')
op.drop_table('manifestchild')
op.drop_table('tagkind')
# ### end Alembic commands ###

44
data/migrations/versions/67f0abd172ae_add_tagtorepositorytag_table.py Normal file
View file

@ -0,0 +1,44 @@
"""Add TagToRepositoryTag table
Revision ID: 67f0abd172ae
Revises: 10f45ee2310b
Create Date: 2018-10-30 11:31:06.615488
"""
# revision identifiers, used by Alembic.
revision = '67f0abd172ae'
down_revision = '10f45ee2310b'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('tagtorepositorytag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('tag_id', sa.Integer(), nullable=False),
sa.Column('repository_tag_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tagtorepositorytag_repository_id_repository')),
sa.ForeignKeyConstraint(['repository_tag_id'], ['repositorytag.id'], name=op.f('fk_tagtorepositorytag_repository_tag_id_repositorytag')),
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tagtorepositorytag_tag_id_tag')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_tagtorepositorytag'))
)
op.create_index('tagtorepositorytag_repository_id', 'tagtorepositorytag', ['repository_id'], unique=False)
op.create_index('tagtorepositorytag_repository_tag_id', 'tagtorepositorytag', ['repository_tag_id'], unique=True)
op.create_index('tagtorepositorytag_tag_id', 'tagtorepositorytag', ['tag_id'], unique=True)
# ### end Alembic commands ###
tester.populate_table('tagtorepositorytag', [
('repository_id', tester.TestDataType.Foreign('repository')),
('tag_id', tester.TestDataType.Foreign('tag')),
('repository_tag_id', tester.TestDataType.Foreign('repositorytag')),
])
def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('tagtorepositorytag')
# ### end Alembic commands ###

166
data/model/tag.py
View file

@ -12,7 +12,7 @@ from data.database import (RepositoryTag, Repository, Image, ImageStorage, Names
RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp, RepositoryNotification, Label, TagManifestLabel, get_epoch_timestamp,
db_for_update, Manifest, ManifestLabel, ManifestBlob, db_for_update, Manifest, ManifestLabel, ManifestBlob,
ManifestLegacyImage, TagManifestToManifest, ManifestLegacyImage, TagManifestToManifest,
TagManifestLabelMap) TagManifestLabelMap, TagToRepositoryTag, Tag, get_epoch_timestamp_ms)
from util.timedeltastring import convert_to_timedelta from util.timedeltastring import convert_to_timedelta
@ -259,8 +259,10 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_i
return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion) return create_or_update_tag_for_repo(repo.id, tag_name, tag_docker_image_id, reversion=reversion)
def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False): def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id, reversion=False,
now_ts = get_epoch_timestamp() oci_manifest=None):
now_ms = get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction(): with db_transaction():
try: try:
@ -270,6 +272,17 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id,
RepositoryTag.name == tag_name), now_ts)).get() RepositoryTag.name == tag_name), now_ts)).get()
tag.lifetime_end_ts = now_ts tag.lifetime_end_ts = now_ts
tag.save() tag.save()
# Check for an OCI tag.
try:
oci_tag = db_for_update(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)).get()
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except Tag.DoesNotExist:
pass
except RepositoryTag.DoesNotExist: except RepositoryTag.DoesNotExist:
pass pass
except IntegrityError: except IntegrityError:
@ -283,8 +296,16 @@ def create_or_update_tag_for_repo(repository_id, tag_name, tag_docker_image_id,
raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
try: try:
return RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name, created = RepositoryTag.create(repository=repository_id, image=image_obj, name=tag_name,
lifetime_start_ts=now_ts, reversion=reversion) lifetime_start_ts=now_ts, reversion=reversion)
if oci_manifest:
# Create the OCI tag as well.
oci_tag = Tag.create(repository=repository_id, manifest=oci_manifest, name=tag_name,
lifetime_start_ms=now_ms, reversion=reversion,
tag_kind=Tag.tag_kind.get_id('tag'))
TagToRepositoryTag.create(tag=oci_tag, repository_tag=created, repository=repository_id)
return created
except IntegrityError: except IntegrityError:
msg = 'Tag with name %s and lifetime start %s already exists' msg = 'Tag with name %s and lifetime start %s already exists'
raise TagAlreadyCreatedException(msg % (tag_name, now_ts)) raise TagAlreadyCreatedException(msg % (tag_name, now_ts))
@ -302,7 +323,9 @@ def create_temporary_hidden_tag(repo, image_obj, expiration_s):
def delete_tag(namespace_name, repository_name, tag_name): def delete_tag(namespace_name, repository_name, tag_name):
now_ts = get_epoch_timestamp() now_ms = get_epoch_timestamp_ms()
now_ts = int(now_ms / 1000)
with db_transaction(): with db_transaction():
try: try:
query = _tag_alive(RepositoryTag query = _tag_alive(RepositoryTag
@ -320,6 +343,15 @@ def delete_tag(namespace_name, repository_name, tag_name):
found.lifetime_end_ts = now_ts found.lifetime_end_ts = now_ts
found.save() found.save()
try:
oci_tag_query = TagToRepositoryTag.select().where(TagToRepositoryTag.repository_tag == found)
oci_tag = db_for_update(oci_tag_query).get().tag
oci_tag.lifetime_end_ms = now_ms
oci_tag.save()
except TagToRepositoryTag.DoesNotExist:
pass
return found return found
@ -362,6 +394,24 @@ def _delete_tags(repo, query_modifier=None):
return set() return set()
with db_transaction(): with db_transaction():
# Delete any associated new-style OCI tags.
# NOTE: This will *not* work once we have tags pointing to other tags (e.g. for channels),
# but this should all be changed over to new-style-only before we make *that* change.
oci_tags_to_delete = list(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository == repo,
TagToRepositoryTag.repository_tag << tags_to_delete))
if oci_tags_to_delete:
# Delete the mapping entries.
(TagToRepositoryTag.delete().where(TagToRepositoryTag.repository == repo,
TagToRepositoryTag.repository_tag << tags_to_delete)
.execute())
# Delete the tags themselves.
Tag.delete().where(Tag.id << oci_tags_to_delete).execute()
# TODO(jschorr): Update to not use TagManifest once that table has been deprecated. # TODO(jschorr): Update to not use TagManifest once that table has been deprecated.
tag_manifests_to_delete = list(TagManifest tag_manifests_to_delete = list(TagManifest
.select() .select()
@ -548,10 +598,16 @@ def restore_tag_to_manifest(repo_obj, tag_name, manifest_digest):
except DataModelException: except DataModelException:
existing_image = None existing_image = None
# Change the tag manifest to point to the updated image.
docker_image_id = tag_manifest.tag.image.docker_image_id docker_image_id = tag_manifest.tag.image.docker_image_id
oci_manifest = None
try:
oci_manifest = Manifest.get(repository=repo_obj, digest=manifest_digest)
except Manifest.DoesNotExist:
pass
# Change the tag and tag manifest to point to the updated image.
updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id, updated_tag = create_or_update_tag_for_repo(repo_obj, tag_name, docker_image_id,
reversion=True) reversion=True, oci_manifest=oci_manifest)
tag_manifest.tag = updated_tag tag_manifest.tag = updated_tag
tag_manifest.save() tag_manifest.save()
return existing_image return existing_image
@ -601,9 +657,13 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest """ Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
object, as well as a boolean indicating whether the TagManifest was created. object, as well as a boolean indicating whether the TagManifest was created.
""" """
# Create the new-style OCI manifest and its blobs.
oci_manifest = _populate_manifest_and_blobs(repository_id, manifest, storage_id_map,
leaf_layer_id=leaf_layer_id)
# Create the tag for the tag manifest. # Create the tag for the tag manifest.
tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id, tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id,
reversion=reversion) reversion=reversion, oci_manifest=oci_manifest)
# Add a tag manifest pointing to that tag. # Add a tag manifest pointing to that tag.
try: try:
@ -612,7 +672,8 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
manifest.save() manifest.save()
return manifest, False return manifest, False
except TagManifest.DoesNotExist: except TagManifest.DoesNotExist:
return _create_manifest(tag, manifest, storage_id_map), True created = _associate_manifest(tag, oci_manifest)
return created, True
def get_active_tag(namespace, repo_name, tag_name): def get_active_tag(namespace, repo_name, tag_name):
@ -661,10 +722,42 @@ def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest, s
manifest.save() manifest.save()
return manifest, False return manifest, False
except TagManifest.DoesNotExist: except TagManifest.DoesNotExist:
return _create_manifest(tag, manifest, storage_id_map), True oci_manifest = _populate_manifest_and_blobs(tag.repository, manifest, storage_id_map)
with db_transaction():
try:
(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)).get()
except Tag.DoesNotExist:
oci_tag = Tag.create(repository=tag.repository, manifest=oci_manifest, name=tag_name,
reversion=tag.reversion,
lifetime_start_ms=tag.lifetime_start_ts * 1000,
lifetime_end_ms=(tag.lifetime_end_ts * 1000
if tag.lifetime_end_ts else None),
tag_kind=Tag.tag_kind.get_id('tag'))
TagToRepositoryTag.create(tag=oci_tag, repository_tag=tag, repository=tag.repository)
return _associate_manifest(tag, oci_manifest), True
def _create_manifest(tag, manifest, storage_id_map): def _associate_manifest(tag, oci_manifest):
with db_transaction():
tag_manifest = TagManifest.create(tag=tag, digest=oci_manifest.digest,
json_data=oci_manifest.manifest_bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=oci_manifest)
return tag_manifest
def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_layer_id=None):
leaf_layer_id = leaf_layer_id or manifest.leaf_layer_v1_image_id
try:
legacy_image = Image.get(Image.docker_image_id == leaf_layer_id,
Image.repository == repository)
except Image.DoesNotExist:
raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
storage_ids = set() storage_ids = set()
for blob_digest in manifest.blob_digests: for blob_digest in manifest.blob_digests:
image_storage_id = storage_id_map.get(blob_digest) image_storage_id = storage_id_map.get(blob_digest)
@ -677,12 +770,7 @@ def _create_manifest(tag, manifest, storage_id_map):
storage_ids.add(image_storage_id) storage_ids.add(image_storage_id)
manifest_row = populate_manifest(tag.repository, manifest, tag.image, storage_ids) return populate_manifest(repository, manifest, legacy_image, storage_ids)
with db_transaction():
tag_manifest = TagManifest.create(tag=tag, digest=manifest.digest, json_data=manifest.bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row)
return tag_manifest
def populate_manifest(repository, manifest, legacy_image, storage_ids): def populate_manifest(repository, manifest, legacy_image, storage_ids):
@ -800,15 +888,41 @@ def change_tag_expiration(tag, expiration_date):
if end_ts == tag.lifetime_end_ts: if end_ts == tag.lifetime_end_ts:
return (None, True) return (None, True)
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has return set_tag_end_ts(tag, end_ts)
# not changed while we were updatings it expiration.
result = (RepositoryTag
.update(lifetime_end_ts=end_ts)
.where(RepositoryTag.id == tag.id,
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts)
.execute())
return (tag.lifetime_end_ts, result > 0)
def set_tag_end_ts(tag, end_ts):
""" Sets the end timestamp for a tag. Should only be called by change_tag_expiration
or tests.
"""
end_ms = end_ts * 1000
with db_transaction():
# Note: We check not just the ID of the tag but also its lifetime_end_ts, to ensure that it has
# not changed while we were updating it expiration.
result = (RepositoryTag
.update(lifetime_end_ts=end_ts)
.where(RepositoryTag.id == tag.id,
RepositoryTag.lifetime_end_ts == tag.lifetime_end_ts)
.execute())
# Check for a mapping to an OCI tag.
try:
oci_tag = (Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)
.get())
(Tag
.update(lifetime_end_ms=end_ms)
.where(Tag.id == oci_tag.id,
Tag.lifetime_end_ms == oci_tag.lifetime_end_ms)
.execute())
except Tag.DoesNotExist:
pass
return (tag.lifetime_end_ts, result > 0)
def find_matching_tag(repo_id, tag_names): def find_matching_tag(repo_id, tag_names):

7
data/model/test/test_gc.py
View file

@ -12,7 +12,7 @@ from playhouse.test_utils import assert_query_count
from data import model, database from data import model, database
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
ApprBlob, Manifest, TagManifest, TagManifestToManifest, ManifestLabel, ApprBlob, Manifest, TagManifest, TagManifestToManifest, ManifestLabel,
TagManifestLabelMap, ManifestBlob) TagManifestLabelMap, ManifestBlob, Tag, TagToRepositoryTag)
from image.docker.schema1 import DockerSchema1ManifestBuilder from image.docker.schema1 import DockerSchema1ManifestBuilder
from test.fixtures import * from test.fixtures import *
@ -223,6 +223,11 @@ def assert_gc_integrity(expect_storage_removed=True):
for blob_row in ApprBlob.select(): for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest)) storage.get_content({preferred}, storage.blob_path(blob_row.digest))
# Ensure there are no danglings OCI tags.
oci_tags = {t.id for t in Tag.select()}
referenced_oci_tags = {t.tag_id for t in TagToRepositoryTag.select()}
assert not (oci_tags - referenced_oci_tags)
def test_has_garbage(default_tag_policy, initialized_db): def test_has_garbage(default_tag_policy, initialized_db):
""" Remove all existing repositories, then add one without garbage, check, then add one with """ Remove all existing repositories, then add one without garbage, check, then add one with

101
data/model/test/test_tag.py
View file

@ -9,12 +9,13 @@ from mock import patch
from app import docker_v2_signing_key from app import docker_v2_signing_key
from data.database import (Image, RepositoryTag, ImageStorage, Repository, Manifest, ManifestBlob, from data.database import (Image, RepositoryTag, ImageStorage, Repository, Manifest, ManifestBlob,
ManifestLegacyImage, TagManifestToManifest) ManifestLegacyImage, TagManifestToManifest, Tag, TagToRepositoryTag)
from data.model.repository import create_repository from data.model.repository import create_repository
from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag, from data.model.tag import (list_active_repo_tags, create_or_update_tag, delete_tag,
get_matching_tags, _tag_alive, get_matching_tags_for_images, get_matching_tags, _tag_alive, get_matching_tags_for_images,
change_tag_expiration, get_active_tag, store_tag_manifest_for_testing, change_tag_expiration, get_active_tag, store_tag_manifest_for_testing,
get_most_recent_tag, get_active_tag_for_repo) get_most_recent_tag, get_active_tag_for_repo,
create_or_update_tag_for_repo, set_tag_end_ts)
from data.model.image import find_create_or_link_image from data.model.image import find_create_or_link_image
from image.docker.schema1 import DockerSchema1ManifestBuilder from image.docker.schema1 import DockerSchema1ManifestBuilder
from util.timedeltastring import convert_to_timedelta from util.timedeltastring import convert_to_timedelta
@ -24,12 +25,13 @@ from test.fixtures import *
def _get_expected_tags(image): def _get_expected_tags(image):
expected_query = (RepositoryTag expected_query = (RepositoryTag
.select() .select()
.join(Image) .join(Image)
.where(RepositoryTag.hidden == False) .where(RepositoryTag.hidden == False)
.where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id)))) .where((Image.id == image.id) | (Image.ancestors ** ('%%/%s/%%' % image.id))))
return set([tag.id for tag in _tag_alive(expected_query)]) return set([tag.id for tag in _tag_alive(expected_query)])
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [ @pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1), (1, 1),
(10, 10), (10, 10),
@ -45,6 +47,12 @@ def test_get_matching_tags(max_subqueries, max_image_lookup_count, initialized_d
expected_tags = _get_expected_tags(image) expected_tags = _get_expected_tags(image)
assert matching_tags == expected_tags, "mismatch for image %s" % image.id assert matching_tags == expected_tags, "mismatch for image %s" % image.id
oci_tags = list(Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag << expected_tags))
assert len(oci_tags) == len(expected_tags)
@pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [ @pytest.mark.parametrize('max_subqueries,max_image_lookup_count', [
(1, 1), (1, 1),
@ -116,6 +124,13 @@ def test_get_matching_tag_ids_images_filtered(initialized_db):
assert matching_tags_ids == expected_tag_ids assert matching_tags_ids == expected_tag_ids
def _get_oci_tag(tag):
return (Tag
.select()
.join(TagToRepositoryTag)
.where(TagToRepositoryTag.repository_tag == tag)).get()
def assert_tags(repository, *args): def assert_tags(repository, *args):
tags = list(list_active_repo_tags(repository)) tags = list(list_active_repo_tags(repository))
assert len(tags) == len(args) assert len(tags) == len(args)
@ -128,12 +143,39 @@ def assert_tags(repository, *args):
tags_dict[tag.name] = tag tags_dict[tag.name] = tag
oci_tag = _get_oci_tag(tag)
assert oci_tag.name == tag.name
assert not oci_tag.hidden
assert oci_tag.reversion == tag.reversion
if tag.lifetime_end_ts:
assert oci_tag.lifetime_end_ms == (tag.lifetime_end_ts * 1000)
else:
assert oci_tag.lifetime_end_ms is None
for expected in args: for expected in args:
assert expected in tags_dict assert expected in tags_dict
def test_create_reversion_tag(initialized_db):
repository = create_repository('devtable', 'somenewrepo', None)
manifest = Manifest.get()
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
oci_manifest=manifest, reversion=True)
assert footag.reversion
oci_tag = _get_oci_tag(footag)
assert oci_tag.name == footag.name
assert not oci_tag.hidden
assert oci_tag.reversion == footag.reversion
def test_list_active_tags(initialized_db): def test_list_active_tags(initialized_db):
# Create a new repository. # Create a new repository.
repository = create_repository('devtable', 'somenewrepo', None) repository = create_repository('devtable', 'somenewrepo', None)
manifest = Manifest.get()
# Create some images. # Create some images.
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
@ -143,50 +185,62 @@ def test_list_active_tags(initialized_db):
assert_tags(repository) assert_tags(repository)
# Add some new tags. # Add some new tags.
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
bartag = create_or_update_tag('devtable', 'somenewrepo', 'bar', image1.docker_image_id) oci_manifest=manifest)
bartag = create_or_update_tag_for_repo(repository, 'bar', image1.docker_image_id,
oci_manifest=manifest)
# Since timestamps are stored on a second-granuality, we need to make the tags "start" # Since timestamps are stored on a second-granularity, we need to make the tags "start"
# before "now", so when we recreate them below, they don't conflict. # before "now", so when we recreate them below, they don't conflict.
footag.lifetime_start_ts -= 5 footag.lifetime_start_ts -= 5
bartag.lifetime_start_ts -= 5
footag.save() footag.save()
bartag.lifetime_start_ts -= 5
bartag.save() bartag.save()
footag_oci = _get_oci_tag(footag)
footag_oci.lifetime_start_ms -= 5000
footag_oci.save()
bartag_oci = _get_oci_tag(bartag)
bartag_oci.lifetime_start_ms -= 5000
bartag_oci.save()
# Make sure they are returned. # Make sure they are returned.
assert_tags(repository, 'foo', 'bar') assert_tags(repository, 'foo', 'bar')
# Mark as a tag as expiring in the far future, and make sure it is still returned. # Mark as a tag as expiring in the far future, and make sure it is still returned.
footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000 set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
footag.save()
# Make sure they are returned. # Make sure they are returned.
assert_tags(repository, 'foo', 'bar') assert_tags(repository, 'foo', 'bar')
# Delete a tag and make sure it isn't returned. # Delete a tag and make sure it isn't returned.
footag = delete_tag('devtable', 'somenewrepo', 'foo') footag = delete_tag('devtable', 'somenewrepo', 'foo')
footag.lifetime_end_ts -= 4 set_tag_end_ts(footag, footag.lifetime_end_ts - 4)
footag.save()
assert_tags(repository, 'bar') assert_tags(repository, 'bar')
# Add a new foo again. # Add a new foo again.
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id) footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
oci_manifest=manifest)
footag.lifetime_start_ts -= 3 footag.lifetime_start_ts -= 3
footag.save() footag.save()
footag_oci = _get_oci_tag(footag)
footag_oci.lifetime_start_ms -= 3000
footag_oci.save()
assert_tags(repository, 'foo', 'bar') assert_tags(repository, 'foo', 'bar')
# Mark as a tag as expiring in the far future, and make sure it is still returned. # Mark as a tag as expiring in the far future, and make sure it is still returned.
footag.lifetime_end_ts = footag.lifetime_start_ts + 10000000 set_tag_end_ts(footag, footag.lifetime_start_ts + 10000000)
footag.save()
# Make sure they are returned. # Make sure they are returned.
assert_tags(repository, 'foo', 'bar') assert_tags(repository, 'foo', 'bar')
# "Move" foo by updating it and make sure we don't get duplicates. # "Move" foo by updating it and make sure we don't get duplicates.
create_or_update_tag('devtable', 'somenewrepo', 'foo', image2.docker_image_id) create_or_update_tag_for_repo(repository, 'foo', image2.docker_image_id, oci_manifest=manifest)
assert_tags(repository, 'foo', 'bar') assert_tags(repository, 'foo', 'bar')
@ -201,7 +255,10 @@ def test_list_active_tags(initialized_db):
def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db): def test_change_tag_expiration(expiration_offset, expected_offset, initialized_db):
repository = create_repository('devtable', 'somenewrepo', None) repository = create_repository('devtable', 'somenewrepo', None)
image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us') image1 = find_create_or_link_image('foobarimage1', repository, None, {}, 'local_us')
footag = create_or_update_tag('devtable', 'somenewrepo', 'foo', image1.docker_image_id)
manifest = Manifest.get()
footag = create_or_update_tag_for_repo(repository, 'foo', image1.docker_image_id,
oci_manifest=manifest)
expiration_date = None expiration_date = None
if expiration_offset is not None: if expiration_offset is not None:
@ -211,15 +268,19 @@ def test_change_tag_expiration(expiration_offset, expected_offset, initialized_d
# Lookup the tag again. # Lookup the tag again.
footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo') footag_updated = get_active_tag('devtable', 'somenewrepo', 'foo')
oci_tag = _get_oci_tag(footag_updated)
if expected_offset is None: if expected_offset is None:
assert footag_updated.lifetime_end_ts is None assert footag_updated.lifetime_end_ts is None
assert oci_tag.lifetime_end_ms is None
else: else:
start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts) start_date = datetime.utcfromtimestamp(footag_updated.lifetime_start_ts)
end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts) end_date = datetime.utcfromtimestamp(footag_updated.lifetime_end_ts)
expected_end_date = start_date + convert_to_timedelta(expected_offset) expected_end_date = start_date + convert_to_timedelta(expected_offset)
assert (expected_end_date - end_date).total_seconds() < 5 # variance in test assert (expected_end_date - end_date).total_seconds() < 5 # variance in test
assert oci_tag.lifetime_end_ms == (footag_updated.lifetime_end_ts * 1000)
def random_storages(): def random_storages():
return list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(10)) return list(ImageStorage.select().where(~(ImageStorage.content_checksum >> None)).limit(10))

251
data/registry_model/registry_pre_oci_model.py
View file

@ -8,13 +8,11 @@ from peewee import IntegrityError
from data import database from data import database
from data import model from data import model
from data.cache import cache_key
from data.database import db_transaction from data.database import db_transaction
from data.registry_model.interface import RegistryDataInterface from data.registry_model.interface import RegistryDataInterface
from data.registry_model.datatype import FromDictionaryException from data.registry_model.datatypes import (Tag, Manifest, LegacyImage, Label,
from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label, SecurityScanStatus, ManifestLayer, Blob, DerivedImage)
SecurityScanStatus, ManifestLayer, Blob, DerivedImage, from data.registry_model.shared import SharedModel
TorrentInfo, BlobUpload)
from data.registry_model.label_handlers import apply_label_to_manifest from data.registry_model.label_handlers import apply_label_to_manifest
from image.docker.schema1 import (DockerSchema1ManifestBuilder, ManifestException, from image.docker.schema1 import (DockerSchema1ManifestBuilder, ManifestException,
DockerSchema1Manifest) DockerSchema1Manifest)
@ -24,7 +22,7 @@ from util.validation import is_json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class PreOCIModel(RegistryDataInterface): class PreOCIModel(SharedModel, RegistryDataInterface):
""" """
PreOCIModel implements the data model for the registry API using a database schema PreOCIModel implements the data model for the registry API using a database schema
before it was changed to support the OCI specification. before it was changed to support the OCI specification.
@ -46,12 +44,6 @@ class PreOCIModel(RegistryDataInterface):
assert found_tag is None or not found_tag.hidden assert found_tag is None or not found_tag.hidden
return Tag.for_repository_tag(found_tag) return Tag.for_repository_tag(found_tag)
def lookup_repository(self, namespace_name, repo_name, kind_filter=None):
""" Looks up and returns a reference to the repository with the given namespace and name,
or None if none. """
repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter)
return RepositoryReference.for_repo_obj(repo)
def get_manifest_for_tag(self, tag, backfill_if_necessary=False): def get_manifest_for_tag(self, tag, backfill_if_necessary=False):
""" Returns the manifest associated with the given tag. """ """ Returns the manifest associated with the given tag. """
try: try:
@ -530,21 +522,12 @@ class PreOCIModel(RegistryDataInterface):
try: try:
tag_manifest, _ = model.tag.associate_generated_tag_manifest(namespace_name, repo_name, tag_manifest, _ = model.tag.associate_generated_tag_manifest(namespace_name, repo_name,
tag.name, manifest, storage_map) tag.name, manifest, storage_map)
assert tag_manifest
except IntegrityError: except IntegrityError:
tag_manifest = model.tag.get_tag_manifest(tag_obj) tag_manifest = model.tag.get_tag_manifest(tag_obj)
return Manifest.for_tag_manifest(tag_manifest) return Manifest.for_tag_manifest(tag_manifest)
def is_existing_disabled_namespace(self, namespace_name):
""" Returns whether the given namespace exists and is disabled. """
namespace = model.user.get_namespace_user(namespace_name)
return namespace is not None and not namespace.enabled
def is_namespace_enabled(self, namespace_name):
""" Returns whether the given namespace exists and is enabled. """
namespace = model.user.get_namespace_user(namespace_name)
return namespace is not None and namespace.enabled
def list_manifest_layers(self, manifest, include_placements=False): def list_manifest_layers(self, manifest, include_placements=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and """ Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified). working towards the leaf, including the associated Blob and its placements (if specified).
@ -634,230 +617,6 @@ class PreOCIModel(RegistryDataInterface):
return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob) return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
def get_derived_image_signature(self, derived_image, signer_name):
"""
Returns the signature associated with the derived image and a specific signer or None if none.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage = derived_storage.derivative
signature_entry = model.storage.lookup_storage_signature(storage, signer_name)
if signature_entry is None:
return None
return signature_entry.signature
def set_derived_image_signature(self, derived_image, signer_name, signature):
"""
Sets the calculated signature for the given derived image and signer to that specified.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage = derived_storage.derivative
signature_entry = model.storage.find_or_create_storage_signature(storage, signer_name)
signature_entry.signature = signature
signature_entry.uploading = False
signature_entry.save()
def delete_derived_image(self, derived_image):
"""
Deletes a derived image and all of its storage.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
model.image.delete_derived_storage(derived_storage)
def set_derived_image_size(self, derived_image, compressed_size):
"""
Sets the compressed size on the given derived image.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage_entry = derived_storage.derivative
storage_entry.image_size = compressed_size
storage_entry.uploading = False
storage_entry.save()
def get_torrent_info(self, blob):
"""
Returns the torrent information associated with the given blob or None if none.
"""
try:
image_storage = database.ImageStorage.get(id=blob._db_id)
except database.ImageStorage.DoesNotExist:
return None
try:
torrent_info = model.storage.get_torrent_info(image_storage)
except model.TorrentInfoDoesNotExist:
return None
return TorrentInfo.for_torrent_info(torrent_info)
def set_torrent_info(self, blob, piece_length, pieces):
"""
Sets the torrent infomation associated with the given blob to that specified.
"""
try:
image_storage = database.ImageStorage.get(id=blob._db_id)
except database.ImageStorage.DoesNotExist:
return None
torrent_info = model.storage.save_torrent_info(image_storage, piece_length, pieces)
return TorrentInfo.for_torrent_info(torrent_info)
def get_cached_repo_blob(self, model_cache, namespace_name, repo_name, blob_digest):
"""
Returns the blob in the repository with the given digest if any or None if none.
Caches the result in the caching system.
"""
def load_blob():
repository_ref = self.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
return None
blob_found = self.get_repo_blob_by_digest(repository_ref, blob_digest,
include_placements=True)
if blob_found is None:
return None
return blob_found.asdict()
blob_cache_key = cache_key.for_repository_blob(namespace_name, repo_name, blob_digest, 2)
blob_dict = model_cache.retrieve(blob_cache_key, load_blob)
try:
return Blob.from_dict(blob_dict) if blob_dict is not None else None
except FromDictionaryException:
# The data was stale in some way. Simply reload.
repository_ref = self.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
return None
return self.get_repo_blob_by_digest(repository_ref, blob_digest, include_placements=True)
def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
"""
Returns the blob in the repository with the given digest, if any or None if none. Note that
there may be multiple records in the same repository for the same blob digest, so the return
value of this function may change.
"""
try:
image_storage = model.blob.get_repository_blob_by_digest(repository_ref._db_id, blob_digest)
except model.BlobDoesNotExist:
return None
assert image_storage.cas_path is not None
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(image_storage.uuid))
return Blob.for_image_storage(image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
def create_blob_upload(self, repository_ref, new_upload_id, location_name, storage_metadata):
""" Creates a new blob upload and returns a reference. If the blob upload could not be
created, returns None. """
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
try:
upload_record = model.blob.initiate_upload(repo.namespace_user.username, repo.name,
new_upload_id, location_name, storage_metadata)
return BlobUpload.for_upload(upload_record)
except database.Repository.DoesNotExist:
return None
def lookup_blob_upload(self, repository_ref, blob_upload_id):
""" Looks up the blob upload withn the given ID under the specified repository and returns it
or None if none.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload_id)
if upload_record is None:
return None
return BlobUpload.for_upload(upload_record)
def update_blob_upload(self, blob_upload, uncompressed_byte_count, piece_hashes, piece_sha_state,
storage_metadata, byte_count, chunk_count, sha_state):
""" Updates the fields of the blob upload to match those given. Returns the updated blob upload
or None if the record does not exists.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is None:
return None
upload_record.uncompressed_byte_count = uncompressed_byte_count
upload_record.piece_hashes = piece_hashes
upload_record.piece_sha_state = piece_sha_state
upload_record.storage_metadata = storage_metadata
upload_record.byte_count = byte_count
upload_record.chunk_count = chunk_count
upload_record.sha_state = sha_state
upload_record.save()
return BlobUpload.for_upload(upload_record)
def delete_blob_upload(self, blob_upload):
""" Deletes a blob upload record. """
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is not None:
upload_record.delete_instance()
def commit_blob_upload(self, blob_upload, blob_digest_str, blob_expiration_seconds):
""" Commits the blob upload into a blob and sets an expiration before that blob will be GCed.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is None:
return None
repository = upload_record.repository
namespace_name = repository.namespace_user.username
repo_name = repository.name
# Create the blob and temporarily tag it.
location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
blob_record = model.blob.store_blob_record_and_temp_link(
namespace_name, repo_name, blob_digest_str, location_obj.id, blob_upload.byte_count,
blob_expiration_seconds, blob_upload.uncompressed_byte_count)
# Delete the blob upload.
upload_record.delete_instance()
return Blob.for_image_storage(blob_record,
storage_path=model.storage.get_layer_path(blob_record))
def mount_blob_into_repository(self, blob, target_repository_ref, expiration_sec):
"""
Mounts the blob from another repository into the specified target repository, and adds an
expiration before that blob is automatically GCed. This function is useful during push
operations if an existing blob from another repository is being pushed. Returns False if
the mounting fails.
"""
repo = model.repository.lookup_repository(target_repository_ref._db_id)
if repo is None:
return False
namespace_name = repo.namespace_user.username
repo_name = repo.name
storage = model.blob.temp_link_blob(namespace_name, repo_name, blob.digest,
expiration_sec)
return bool(storage)
def set_tags_expiration_for_manifest(self, manifest, expiration_sec): def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
""" """
Sets the expiration on all tags that point to the given manifest to that specified. Sets the expiration on all tags that point to the given manifest to that specified.

256
data/registry_model/shared.py Normal file
View file

@ -0,0 +1,256 @@
# pylint: disable=protected-access
import logging
from data import database
from data import model
from data.cache import cache_key
from data.registry_model.datatype import FromDictionaryException
from data.registry_model.datatypes import RepositoryReference, Blob, TorrentInfo, BlobUpload
logger = logging.getLogger(__name__)
class SharedModel:
"""
SharedModel implements those data model operations for the registry API that are unchanged
between the old and new data models.
"""
def lookup_repository(self, namespace_name, repo_name, kind_filter=None):
""" Looks up and returns a reference to the repository with the given namespace and name,
or None if none. """
repo = model.repository.get_repository(namespace_name, repo_name, kind_filter=kind_filter)
return RepositoryReference.for_repo_obj(repo)
def is_existing_disabled_namespace(self, namespace_name):
""" Returns whether the given namespace exists and is disabled. """
namespace = model.user.get_namespace_user(namespace_name)
return namespace is not None and not namespace.enabled
def is_namespace_enabled(self, namespace_name):
""" Returns whether the given namespace exists and is enabled. """
namespace = model.user.get_namespace_user(namespace_name)
return namespace is not None and namespace.enabled
def get_derived_image_signature(self, derived_image, signer_name):
"""
Returns the signature associated with the derived image and a specific signer or None if none.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage = derived_storage.derivative
signature_entry = model.storage.lookup_storage_signature(storage, signer_name)
if signature_entry is None:
return None
return signature_entry.signature
def set_derived_image_signature(self, derived_image, signer_name, signature):
"""
Sets the calculated signature for the given derived image and signer to that specified.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage = derived_storage.derivative
signature_entry = model.storage.find_or_create_storage_signature(storage, signer_name)
signature_entry.signature = signature
signature_entry.uploading = False
signature_entry.save()
def delete_derived_image(self, derived_image):
"""
Deletes a derived image and all of its storage.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
model.image.delete_derived_storage(derived_storage)
def set_derived_image_size(self, derived_image, compressed_size):
"""
Sets the compressed size on the given derived image.
"""
try:
derived_storage = database.DerivedStorageForImage.get(id=derived_image._db_id)
except database.DerivedStorageForImage.DoesNotExist:
return None
storage_entry = derived_storage.derivative
storage_entry.image_size = compressed_size
storage_entry.uploading = False
storage_entry.save()
def get_torrent_info(self, blob):
"""
Returns the torrent information associated with the given blob or None if none.
"""
try:
image_storage = database.ImageStorage.get(id=blob._db_id)
except database.ImageStorage.DoesNotExist:
return None
try:
torrent_info = model.storage.get_torrent_info(image_storage)
except model.TorrentInfoDoesNotExist:
return None
return TorrentInfo.for_torrent_info(torrent_info)
def set_torrent_info(self, blob, piece_length, pieces):
"""
Sets the torrent infomation associated with the given blob to that specified.
"""
try:
image_storage = database.ImageStorage.get(id=blob._db_id)
except database.ImageStorage.DoesNotExist:
return None
torrent_info = model.storage.save_torrent_info(image_storage, piece_length, pieces)
return TorrentInfo.for_torrent_info(torrent_info)
def get_cached_repo_blob(self, model_cache, namespace_name, repo_name, blob_digest):
"""
Returns the blob in the repository with the given digest if any or None if none.
Caches the result in the caching system.
"""
def load_blob():
repository_ref = self.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
return None
blob_found = self.get_repo_blob_by_digest(repository_ref, blob_digest,
include_placements=True)
if blob_found is None:
return None
return blob_found.asdict()
blob_cache_key = cache_key.for_repository_blob(namespace_name, repo_name, blob_digest, 2)
blob_dict = model_cache.retrieve(blob_cache_key, load_blob)
try:
return Blob.from_dict(blob_dict) if blob_dict is not None else None
except FromDictionaryException:
# The data was stale in some way. Simply reload.
repository_ref = self.lookup_repository(namespace_name, repo_name)
if repository_ref is None:
return None
return self.get_repo_blob_by_digest(repository_ref, blob_digest, include_placements=True)
def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
"""
Returns the blob in the repository with the given digest, if any or None if none. Note that
there may be multiple records in the same repository for the same blob digest, so the return
value of this function may change.
"""
try:
image_storage = model.blob.get_repository_blob_by_digest(repository_ref._db_id, blob_digest)
except model.BlobDoesNotExist:
return None
assert image_storage.cas_path is not None
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(image_storage.uuid))
return Blob.for_image_storage(image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
def create_blob_upload(self, repository_ref, new_upload_id, location_name, storage_metadata):
""" Creates a new blob upload and returns a reference. If the blob upload could not be
created, returns None. """
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
try:
upload_record = model.blob.initiate_upload(repo.namespace_user.username, repo.name,
new_upload_id, location_name, storage_metadata)
return BlobUpload.for_upload(upload_record)
except database.Repository.DoesNotExist:
return None
def lookup_blob_upload(self, repository_ref, blob_upload_id):
""" Looks up the blob upload withn the given ID under the specified repository and returns it
or None if none.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload_id)
if upload_record is None:
return None
return BlobUpload.for_upload(upload_record)
def update_blob_upload(self, blob_upload, uncompressed_byte_count, piece_hashes, piece_sha_state,
storage_metadata, byte_count, chunk_count, sha_state):
""" Updates the fields of the blob upload to match those given. Returns the updated blob upload
or None if the record does not exists.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is None:
return None
upload_record.uncompressed_byte_count = uncompressed_byte_count
upload_record.piece_hashes = piece_hashes
upload_record.piece_sha_state = piece_sha_state
upload_record.storage_metadata = storage_metadata
upload_record.byte_count = byte_count
upload_record.chunk_count = chunk_count
upload_record.sha_state = sha_state
upload_record.save()
return BlobUpload.for_upload(upload_record)
def delete_blob_upload(self, blob_upload):
""" Deletes a blob upload record. """
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is not None:
upload_record.delete_instance()
def commit_blob_upload(self, blob_upload, blob_digest_str, blob_expiration_seconds):
""" Commits the blob upload into a blob and sets an expiration before that blob will be GCed.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is None:
return None
repository = upload_record.repository
namespace_name = repository.namespace_user.username
repo_name = repository.name
# Create the blob and temporarily tag it.
location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
blob_record = model.blob.store_blob_record_and_temp_link(
namespace_name, repo_name, blob_digest_str, location_obj.id, blob_upload.byte_count,
blob_expiration_seconds, blob_upload.uncompressed_byte_count)
# Delete the blob upload.
upload_record.delete_instance()
return Blob.for_image_storage(blob_record,
storage_path=model.storage.get_layer_path(blob_record))
def mount_blob_into_repository(self, blob, target_repository_ref, expiration_sec):
"""
Mounts the blob from another repository into the specified target repository, and adds an
expiration before that blob is automatically GCed. This function is useful during push
operations if an existing blob from another repository is being pushed. Returns False if
the mounting fails.
"""
repo = model.repository.lookup_repository(target_repository_ref._db_id)
if repo is None:
return False
namespace_name = repo.namespace_user.username
repo_name = repo.name
storage = model.blob.temp_link_blob(namespace_name, repo_name, blob.digest,
expiration_sec)
return bool(storage)

367
data/registry_model/test/test_pre_oci_model.py → data/registry_model/test/test_interface.py
View file

@ -13,7 +13,7 @@ from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob, from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image, ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
TagManifestLabel, TagManifest, TagManifestLabel, DerivedStorageForImage, TagManifestLabel, TagManifest, TagManifestLabel, DerivedStorageForImage,
TorrentInfo, close_db_filter) TorrentInfo, Tag, TagToRepositoryTag, close_db_filter)
from data.cache.impl import InMemoryDataModelCache from data.cache.impl import InMemoryDataModelCache
from data.registry_model.registry_pre_oci_model import PreOCIModel from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.datatypes import RepositoryReference from data.registry_model.datatypes import RepositoryReference
@ -21,9 +21,10 @@ from image.docker.schema1 import DockerSchema1ManifestBuilder
from test.fixtures import * from test.fixtures import *
@pytest.fixture()
def pre_oci_model(initialized_db): @pytest.fixture(params=[PreOCIModel])
return PreOCIModel() def registry_model(request, initialized_db):
return request.param()
@pytest.mark.parametrize('names, expected', [ @pytest.mark.parametrize('names, expected', [
@ -33,10 +34,10 @@ def pre_oci_model(initialized_db):
(['latest', 'prod', 'another'], {'latest', 'prod'}), (['latest', 'prod', 'another'], {'latest', 'prod'}),
(['foo', 'prod'], {'prod'}), (['foo', 'prod'], {'prod'}),
]) ])
def test_find_matching_tag(names, expected, pre_oci_model): def test_find_matching_tag(names, expected, registry_model):
repo = model.repository.get_repository('devtable', 'simple') repo = model.repository.get_repository('devtable', 'simple')
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found = pre_oci_model.find_matching_tag(repository_ref, names) found = registry_model.find_matching_tag(repository_ref, names)
if expected is None: if expected is None:
assert found is None assert found is None
else: else:
@ -49,10 +50,10 @@ def test_find_matching_tag(names, expected, pre_oci_model):
('devtable', 'simple', {'latest', 'prod'}), ('devtable', 'simple', {'latest', 'prod'}),
('buynlarge', 'orgrepo', {'latest', 'prod'}), ('buynlarge', 'orgrepo', {'latest', 'prod'}),
]) ])
def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model): def test_get_most_recent_tag(repo_namespace, repo_name, expected, registry_model):
repo = model.repository.get_repository(repo_namespace, repo_name) repo = model.repository.get_repository(repo_namespace, repo_name)
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found = pre_oci_model.get_most_recent_tag(repository_ref) found = registry_model.get_most_recent_tag(repository_ref)
if expected is None: if expected is None:
assert found is None assert found is None
else: else:
@ -64,8 +65,8 @@ def test_get_most_recent_tag(repo_namespace, repo_name, expected, pre_oci_model)
('buynlarge', 'orgrepo', True), ('buynlarge', 'orgrepo', True),
('buynlarge', 'unknownrepo', False), ('buynlarge', 'unknownrepo', False),
]) ])
def test_lookup_repository(repo_namespace, repo_name, expected, pre_oci_model): def test_lookup_repository(repo_namespace, repo_name, expected, registry_model):
repo_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repo_ref = registry_model.lookup_repository(repo_namespace, repo_name)
if expected: if expected:
assert repo_ref assert repo_ref
else: else:
@ -76,22 +77,22 @@ def test_lookup_repository(repo_namespace, repo_name, expected, pre_oci_model):
('devtable', 'simple'), ('devtable', 'simple'),
('buynlarge', 'orgrepo'), ('buynlarge', 'orgrepo'),
]) ])
def test_lookup_manifests(repo_namespace, repo_name, pre_oci_model): def test_lookup_manifests(repo_namespace, repo_name, registry_model):
repo = model.repository.get_repository(repo_namespace, repo_name) repo = model.repository.get_repository(repo_namespace, repo_name)
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) found_tag = registry_model.find_matching_tag(repository_ref, ['latest'])
found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) found_manifest = registry_model.get_manifest_for_tag(found_tag)
found = pre_oci_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest, found = registry_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest,
include_legacy_image=True) include_legacy_image=True)
assert found._db_id == found_manifest._db_id assert found._db_id == found_manifest._db_id
assert found.digest == found_manifest.digest assert found.digest == found_manifest.digest
assert found.legacy_image assert found.legacy_image
def test_lookup_unknown_manifest(pre_oci_model): def test_lookup_unknown_manifest(registry_model):
repo = model.repository.get_repository('devtable', 'simple') repo = model.repository.get_repository('devtable', 'simple')
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found = pre_oci_model.lookup_manifest_by_digest(repository_ref, 'sha256:deadbeef') found = registry_model.lookup_manifest_by_digest(repository_ref, 'sha256:deadbeef')
assert found is None assert found is None
@ -101,18 +102,18 @@ def test_lookup_unknown_manifest(pre_oci_model):
('devtable', 'history'), ('devtable', 'history'),
('buynlarge', 'orgrepo'), ('buynlarge', 'orgrepo'),
]) ])
def test_legacy_images(repo_namespace, repo_name, pre_oci_model): def test_legacy_images(repo_namespace, repo_name, registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
legacy_images = pre_oci_model.get_legacy_images(repository_ref) legacy_images = registry_model.get_legacy_images(repository_ref)
assert len(legacy_images) assert len(legacy_images)
found_tags = set() found_tags = set()
for image in legacy_images: for image in legacy_images:
found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True) include_parents=True)
with assert_query_count(5 if found_image.parents else 4): with assert_query_count(5 if found_image.parents else 4):
found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True, include_blob=True) include_parents=True, include_blob=True)
assert found_image.docker_image_id == image.docker_image_id assert found_image.docker_image_id == image.docker_image_id
assert found_image.parents == image.parents assert found_image.parents == image.parents
@ -130,76 +131,76 @@ def test_legacy_images(repo_namespace, repo_name, pre_oci_model):
[p._db_id for p in found_image.parents]) [p._db_id for p in found_image.parents])
# Try without parents and ensure it raises an exception. # Try without parents and ensure it raises an exception.
found_image = pre_oci_model.get_legacy_image(repository_ref, image.docker_image_id, found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=False) include_parents=False)
with pytest.raises(Exception): with pytest.raises(Exception):
assert not found_image.parents assert not found_image.parents
assert found_tags assert found_tags
unknown = pre_oci_model.get_legacy_image(repository_ref, 'unknown', include_parents=True) unknown = registry_model.get_legacy_image(repository_ref, 'unknown', include_parents=True)
assert unknown is None assert unknown is None
def test_manifest_labels(pre_oci_model): def test_manifest_labels(registry_model):
repo = model.repository.get_repository('devtable', 'simple') repo = model.repository.get_repository('devtable', 'simple')
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) found_tag = registry_model.find_matching_tag(repository_ref, ['latest'])
found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) found_manifest = registry_model.get_manifest_for_tag(found_tag)
# Create a new label. # Create a new label.
created = pre_oci_model.create_manifest_label(found_manifest, 'foo', 'bar', 'api') created = registry_model.create_manifest_label(found_manifest, 'foo', 'bar', 'api')
assert created.key == 'foo' assert created.key == 'foo'
assert created.value == 'bar' assert created.value == 'bar'
assert created.source_type_name == 'api' assert created.source_type_name == 'api'
assert created.media_type_name == 'text/plain' assert created.media_type_name == 'text/plain'
# Ensure we can look it up. # Ensure we can look it up.
assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) == created assert registry_model.get_manifest_label(found_manifest, created.uuid) == created
# Ensure it is in our list of labels. # Ensure it is in our list of labels.
assert created in pre_oci_model.list_manifest_labels(found_manifest) assert created in registry_model.list_manifest_labels(found_manifest)
assert created in pre_oci_model.list_manifest_labels(found_manifest, key_prefix='fo') assert created in registry_model.list_manifest_labels(found_manifest, key_prefix='fo')
# Ensure it is *not* in our filtered list. # Ensure it is *not* in our filtered list.
assert created not in pre_oci_model.list_manifest_labels(found_manifest, key_prefix='ba') assert created not in registry_model.list_manifest_labels(found_manifest, key_prefix='ba')
# Delete the label and ensure it is gone. # Delete the label and ensure it is gone.
assert pre_oci_model.delete_manifest_label(found_manifest, created.uuid) assert registry_model.delete_manifest_label(found_manifest, created.uuid)
assert pre_oci_model.get_manifest_label(found_manifest, created.uuid) is None assert registry_model.get_manifest_label(found_manifest, created.uuid) is None
assert created not in pre_oci_model.list_manifest_labels(found_manifest) assert created not in registry_model.list_manifest_labels(found_manifest)
def test_manifest_label_handlers(pre_oci_model): def test_manifest_label_handlers(registry_model):
repo = model.repository.get_repository('devtable', 'simple') repo = model.repository.get_repository('devtable', 'simple')
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found_tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') found_tag = registry_model.get_repo_tag(repository_ref, 'latest')
found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) found_manifest = registry_model.get_manifest_for_tag(found_tag)
# Ensure the tag has no expiration. # Ensure the tag has no expiration.
assert found_tag.lifetime_end_ts is None assert found_tag.lifetime_end_ts is None
# Create a new label with an expires-after. # Create a new label with an expires-after.
pre_oci_model.create_manifest_label(found_manifest, 'quay.expires-after', '2h', 'api') registry_model.create_manifest_label(found_manifest, 'quay.expires-after', '2h', 'api')
# Ensure the tag now has an expiration. # Ensure the tag now has an expiration.
updated_tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') updated_tag = registry_model.get_repo_tag(repository_ref, 'latest')
assert updated_tag.lifetime_end_ts == (updated_tag.lifetime_start_ts + (60 * 60 * 2)) assert updated_tag.lifetime_end_ts == (updated_tag.lifetime_start_ts + (60 * 60 * 2))
def test_batch_labels(pre_oci_model): def test_batch_labels(registry_model):
repo = model.repository.get_repository('devtable', 'history') repo = model.repository.get_repository('devtable', 'history')
repository_ref = RepositoryReference.for_repo_obj(repo) repository_ref = RepositoryReference.for_repo_obj(repo)
found_tag = pre_oci_model.find_matching_tag(repository_ref, ['latest']) found_tag = registry_model.find_matching_tag(repository_ref, ['latest'])
found_manifest = pre_oci_model.get_manifest_for_tag(found_tag) found_manifest = registry_model.get_manifest_for_tag(found_tag)
with pre_oci_model.batch_create_manifest_labels(found_manifest) as add_label: with registry_model.batch_create_manifest_labels(found_manifest) as add_label:
add_label('foo', '1', 'api') add_label('foo', '1', 'api')
add_label('bar', '2', 'api') add_label('bar', '2', 'api')
add_label('baz', '3', 'api') add_label('baz', '3', 'api')
# Ensure we can look them up. # Ensure we can look them up.
assert len(pre_oci_model.list_manifest_labels(found_manifest)) == 3 assert len(registry_model.list_manifest_labels(found_manifest)) == 3
@pytest.mark.parametrize('repo_namespace, repo_name', [ @pytest.mark.parametrize('repo_namespace, repo_name', [
@ -208,38 +209,38 @@ def test_batch_labels(pre_oci_model):
('devtable', 'history'), ('devtable', 'history'),
('buynlarge', 'orgrepo'), ('buynlarge', 'orgrepo'),
]) ])
def test_repository_tags(repo_namespace, repo_name, pre_oci_model): def test_repository_tags(repo_namespace, repo_name, registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
with assert_query_count(1): with assert_query_count(1):
tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True) tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags) assert len(tags)
for tag in tags: for tag in tags:
with assert_query_count(2): with assert_query_count(2):
found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag == tag assert found_tag == tag
if found_tag.legacy_image is None: if found_tag.legacy_image is None:
continue continue
with assert_query_count(2): with assert_query_count(2):
found_image = pre_oci_model.get_legacy_image(repository_ref, found_image = registry_model.get_legacy_image(repository_ref,
found_tag.legacy_image.docker_image_id) found_tag.legacy_image.docker_image_id)
assert found_image == found_tag.legacy_image assert found_image == found_tag.legacy_image
def test_repository_tag_history(pre_oci_model): def test_repository_tag_history(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'history') repository_ref = registry_model.lookup_repository('devtable', 'history')
with assert_query_count(2): with assert_query_count(2):
history, has_more = pre_oci_model.list_repository_tag_history(repository_ref) history, has_more = registry_model.list_repository_tag_history(repository_ref)
assert not has_more assert not has_more
assert len(history) == 2 assert len(history) == 2
# Ensure the latest tag is marked expired, since there is an expired one. # Ensure the latest tag is marked expired, since there is an expired one.
with assert_query_count(1): with assert_query_count(1):
assert pre_oci_model.has_expired_tag(repository_ref, 'latest') assert registry_model.has_expired_tag(repository_ref, 'latest')
@pytest.mark.parametrize('repo_namespace, repo_name', [ @pytest.mark.parametrize('repo_namespace, repo_name', [
@ -252,35 +253,35 @@ def test_repository_tag_history(pre_oci_model):
False, False,
True, True,
]) ])
def test_delete_tags(repo_namespace, repo_name, via_manifest, pre_oci_model): def test_delete_tags(repo_namespace, repo_name, via_manifest, registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref) tags = registry_model.list_repository_tags(repository_ref)
assert len(tags) assert len(tags)
# Save history before the deletions. # Save history before the deletions.
previous_history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000) previous_history, _ = registry_model.list_repository_tag_history(repository_ref, size=1000)
assert len(previous_history) >= len(tags) assert len(previous_history) >= len(tags)
# Delete every tag in the repository. # Delete every tag in the repository.
for tag in tags: for tag in tags:
if via_manifest: if via_manifest:
assert pre_oci_model.delete_tag(repository_ref, tag.name) assert registry_model.delete_tag(repository_ref, tag.name)
else: else:
manifest = pre_oci_model.get_manifest_for_tag(tag) manifest = registry_model.get_manifest_for_tag(tag)
if manifest is not None: if manifest is not None:
assert pre_oci_model.delete_tags_for_manifest(manifest) assert registry_model.delete_tags_for_manifest(manifest)
# Make sure the tag is no longer found. # Make sure the tag is no longer found.
with assert_query_count(1): with assert_query_count(1):
found_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True) found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag is None assert found_tag is None
# Ensure all tags have been deleted. # Ensure all tags have been deleted.
tags = pre_oci_model.list_repository_tags(repository_ref) tags = registry_model.list_repository_tags(repository_ref)
assert not len(tags) assert not len(tags)
# Ensure that the tags all live in history. # Ensure that the tags all live in history.
history, _ = pre_oci_model.list_repository_tag_history(repository_ref, size=1000) history, _ = registry_model.list_repository_tag_history(repository_ref, size=1000)
assert len(history) == len(previous_history) assert len(history) == len(previous_history)
@ -288,12 +289,12 @@ def test_delete_tags(repo_namespace, repo_name, via_manifest, pre_oci_model):
True, True,
False, False,
]) ])
def test_retarget_tag_history(use_manifest, pre_oci_model): def test_retarget_tag_history(use_manifest, registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'history') repository_ref = registry_model.lookup_repository('devtable', 'history')
history, _ = pre_oci_model.list_repository_tag_history(repository_ref) history, _ = registry_model.list_repository_tag_history(repository_ref)
if use_manifest: if use_manifest:
manifest_or_legacy_image = pre_oci_model.lookup_manifest_by_digest(repository_ref, manifest_or_legacy_image = registry_model.lookup_manifest_by_digest(repository_ref,
history[1].manifest_digest, history[1].manifest_digest,
allow_dead=True) allow_dead=True)
else: else:
@ -301,7 +302,7 @@ def test_retarget_tag_history(use_manifest, pre_oci_model):
# Retarget the tag. # Retarget the tag.
assert manifest_or_legacy_image assert manifest_or_legacy_image
updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image, updated_tag = registry_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image,
is_reversion=True) is_reversion=True)
# Ensure the tag has changed targets. # Ensure the tag has changed targets.
@ -311,39 +312,39 @@ def test_retarget_tag_history(use_manifest, pre_oci_model):
assert updated_tag.legacy_image == manifest_or_legacy_image assert updated_tag.legacy_image == manifest_or_legacy_image
# Ensure history has been updated. # Ensure history has been updated.
new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref) new_history, _ = registry_model.list_repository_tag_history(repository_ref)
assert len(new_history) == len(history) + 1 assert len(new_history) == len(history) + 1
def test_retarget_tag(pre_oci_model): def test_retarget_tag(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex') repository_ref = registry_model.lookup_repository('devtable', 'complex')
history, _ = pre_oci_model.list_repository_tag_history(repository_ref) history, _ = registry_model.list_repository_tag_history(repository_ref)
prod_tag = pre_oci_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True) prod_tag = registry_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True)
# Retarget the tag. # Retarget the tag.
updated_tag = pre_oci_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image) updated_tag = registry_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image)
# Ensure the tag has changed targets. # Ensure the tag has changed targets.
assert updated_tag.legacy_image == prod_tag.legacy_image assert updated_tag.legacy_image == prod_tag.legacy_image
# Ensure history has been updated. # Ensure history has been updated.
new_history, _ = pre_oci_model.list_repository_tag_history(repository_ref) new_history, _ = registry_model.list_repository_tag_history(repository_ref)
assert len(new_history) == len(history) + 1 assert len(new_history) == len(history) + 1
def test_change_repository_tag_expiration(pre_oci_model): def test_change_repository_tag_expiration(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') tag = registry_model.get_repo_tag(repository_ref, 'latest')
assert tag.lifetime_end_ts is None assert tag.lifetime_end_ts is None
new_datetime = datetime.utcnow() + timedelta(days=2) new_datetime = datetime.utcnow() + timedelta(days=2)
previous, okay = pre_oci_model.change_repository_tag_expiration(tag, new_datetime) previous, okay = registry_model.change_repository_tag_expiration(tag, new_datetime)
assert okay assert okay
assert previous is None assert previous is None
tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') tag = registry_model.get_repo_tag(repository_ref, 'latest')
assert tag.lifetime_end_ts is not None assert tag.lifetime_end_ts is not None
@ -355,31 +356,33 @@ def test_change_repository_tag_expiration(pre_oci_model):
('devtable', 'gargantuan', ['v2.0', 'v3.0', 'v4.0', 'v5.0', 'v6.0']), ('devtable', 'gargantuan', ['v2.0', 'v3.0', 'v4.0', 'v5.0', 'v6.0']),
]) ])
def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_empty, def test_get_legacy_images_owned_by_tag(repo_namespace, repo_name, expected_non_empty,
pre_oci_model): registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref) tags = registry_model.list_repository_tags(repository_ref)
assert len(tags) assert len(tags)
non_empty = set() non_empty = set()
for tag in tags: for tag in tags:
if pre_oci_model.get_legacy_images_owned_by_tag(tag): if registry_model.get_legacy_images_owned_by_tag(tag):
non_empty.add(tag.name) non_empty.add(tag.name)
assert non_empty == set(expected_non_empty) assert non_empty == set(expected_non_empty)
def test_get_security_status(pre_oci_model): def test_get_security_status(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True) tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags) assert len(tags)
for tag in tags: for tag in tags:
assert pre_oci_model.get_security_status(tag.legacy_image) assert registry_model.get_security_status(tag.legacy_image)
@pytest.fixture() @pytest.fixture()
def clear_rows(initialized_db): def clear_rows(initialized_db):
# Remove all new-style rows so we can backfill. # Remove all new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
TagManifestLabelMap.delete().execute() TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute() ManifestLabel.delete().execute()
ManifestBlob.delete().execute() ManifestBlob.delete().execute()
@ -396,24 +399,24 @@ def clear_rows(initialized_db):
('devtable', 'history'), ('devtable', 'history'),
('buynlarge', 'orgrepo'), ('buynlarge', 'orgrepo'),
]) ])
def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, pre_oci_model): def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref) tags = registry_model.list_repository_tags(repository_ref)
assert tags assert tags
for tag in tags: for tag in tags:
assert not tag.manifest_digest assert not tag.manifest_digest
assert pre_oci_model.backfill_manifest_for_tag(tag) assert registry_model.backfill_manifest_for_tag(tag)
tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True) tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert tags assert tags
for tag in tags: for tag in tags:
assert tag.manifest_digest assert tag.manifest_digest
manifest = pre_oci_model.get_manifest_for_tag(tag) manifest = registry_model.get_manifest_for_tag(tag)
assert manifest assert manifest
legacy_image = pre_oci_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id, legacy_image = registry_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id,
include_parents=True) include_parents=True)
parsed_manifest = manifest.get_parsed_manifest() parsed_manifest = manifest.get_parsed_manifest()
@ -427,19 +430,19 @@ def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, pre_oc
('devtable', 'history'), ('devtable', 'history'),
('buynlarge', 'orgrepo'), ('buynlarge', 'orgrepo'),
]) ])
def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, pre_oci_model): def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref) tags = registry_model.list_repository_tags(repository_ref)
assert tags assert tags
for tag in tags: for tag in tags:
assert not tag.manifest_digest assert not tag.manifest_digest
assert not pre_oci_model.get_manifest_for_tag(tag) assert not registry_model.get_manifest_for_tag(tag)
manifest = pre_oci_model.get_manifest_for_tag(tag, backfill_if_necessary=True) manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
assert manifest assert manifest
updated_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name) updated_tag = registry_model.get_repo_tag(repository_ref, tag.name)
assert updated_tag.manifest_digest == manifest.digest assert updated_tag.manifest_digest == manifest.digest
@ -449,8 +452,8 @@ def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, pre_
('disabled', False), ('disabled', False),
]) ])
def test_is_namespace_enabled(namespace, expect_enabled, pre_oci_model): def test_is_namespace_enabled(namespace, expect_enabled, registry_model):
assert pre_oci_model.is_namespace_enabled(namespace) == expect_enabled assert registry_model.is_namespace_enabled(namespace) == expect_enabled
@pytest.mark.parametrize('repo_namespace, repo_name', [ @pytest.mark.parametrize('repo_namespace, repo_name', [
@ -459,20 +462,20 @@ def test_is_namespace_enabled(namespace, expect_enabled, pre_oci_model):
('devtable', 'history'), ('devtable', 'history'),
('buynlarge', 'orgrepo'), ('buynlarge', 'orgrepo'),
]) ])
def test_list_manifest_layers(repo_namespace, repo_name, pre_oci_model): def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name) repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref) tags = registry_model.list_repository_tags(repository_ref)
assert tags assert tags
for tag in tags: for tag in tags:
manifest = pre_oci_model.get_manifest_for_tag(tag) manifest = registry_model.get_manifest_for_tag(tag)
assert manifest assert manifest
with assert_query_count(4): with assert_query_count(4):
layers = pre_oci_model.list_manifest_layers(manifest) layers = registry_model.list_manifest_layers(manifest)
assert layers assert layers
layers = pre_oci_model.list_manifest_layers(manifest, include_placements=True) layers = registry_model.list_manifest_layers(manifest, include_placements=True)
assert layers assert layers
parsed_layers = list(manifest.get_parsed_manifest().layers) parsed_layers = list(manifest.get_parsed_manifest().layers)
@ -484,123 +487,123 @@ def test_list_manifest_layers(repo_namespace, repo_name, pre_oci_model):
assert manifest_layer.blob.storage_path assert manifest_layer.blob.storage_path
assert manifest_layer.blob.placements assert manifest_layer.blob.placements
repo_blob = pre_oci_model.get_repo_blob_by_digest(repository_ref, manifest_layer.blob.digest) repo_blob = registry_model.get_repo_blob_by_digest(repository_ref, manifest_layer.blob.digest)
assert repo_blob.digest == manifest_layer.blob.digest assert repo_blob.digest == manifest_layer.blob.digest
assert manifest_layer.estimated_size(1) is not None assert manifest_layer.estimated_size(1) is not None
def test_derived_image(pre_oci_model): def test_derived_image(registry_model):
# Clear all existing derived storage. # Clear all existing derived storage.
DerivedStorageForImage.delete().execute() DerivedStorageForImage.delete().execute()
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') tag = registry_model.get_repo_tag(repository_ref, 'latest')
manifest = pre_oci_model.get_manifest_for_tag(tag) manifest = registry_model.get_manifest_for_tag(tag)
# Ensure the squashed image doesn't exist. # Ensure the squashed image doesn't exist.
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {}) is None assert registry_model.lookup_derived_image(manifest, 'squash', {}) is None
# Create a new one. # Create a new one.
squashed = pre_oci_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {}) squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {})
assert pre_oci_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {}) == squashed assert registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {}) == squashed
assert squashed.unique_id assert squashed.unique_id
# Check and set the size. # Check and set the size.
assert squashed.blob.compressed_size is None assert squashed.blob.compressed_size is None
pre_oci_model.set_derived_image_size(squashed, 1234) registry_model.set_derived_image_size(squashed, 1234)
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {}).blob.compressed_size == 1234 assert registry_model.lookup_derived_image(manifest, 'squash', {}).blob.compressed_size == 1234
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {}).unique_id == squashed.unique_id assert registry_model.lookup_derived_image(manifest, 'squash', {}).unique_id == squashed.unique_id
# Ensure its returned now. # Ensure its returned now.
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {}) == squashed assert registry_model.lookup_derived_image(manifest, 'squash', {}) == squashed
# Ensure different metadata results in a different derived image. # Ensure different metadata results in a different derived image.
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) is None assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) is None
squashed_foo = pre_oci_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', squashed_foo = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us',
{'foo': 'bar'}) {'foo': 'bar'})
assert squashed_foo != squashed assert squashed_foo != squashed
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) == squashed_foo assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) == squashed_foo
assert squashed.unique_id != squashed_foo.unique_id assert squashed.unique_id != squashed_foo.unique_id
# Lookup with placements. # Lookup with placements.
squashed = pre_oci_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {}, squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {},
include_placements=True) include_placements=True)
assert squashed.blob.placements assert squashed.blob.placements
# Delete the derived image. # Delete the derived image.
pre_oci_model.delete_derived_image(squashed) registry_model.delete_derived_image(squashed)
assert pre_oci_model.lookup_derived_image(manifest, 'squash', {}) is None assert registry_model.lookup_derived_image(manifest, 'squash', {}) is None
def test_derived_image_signatures(pre_oci_model): def test_derived_image_signatures(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') tag = registry_model.get_repo_tag(repository_ref, 'latest')
manifest = pre_oci_model.get_manifest_for_tag(tag) manifest = registry_model.get_manifest_for_tag(tag)
derived = pre_oci_model.lookup_derived_image(manifest, 'squash', {}) derived = registry_model.lookup_derived_image(manifest, 'squash', {})
assert derived assert derived
signature = pre_oci_model.get_derived_image_signature(derived, 'gpg2') signature = registry_model.get_derived_image_signature(derived, 'gpg2')
assert signature is None assert signature is None
pre_oci_model.set_derived_image_signature(derived, 'gpg2', 'foo') registry_model.set_derived_image_signature(derived, 'gpg2', 'foo')
assert pre_oci_model.get_derived_image_signature(derived, 'gpg2') == 'foo' assert registry_model.get_derived_image_signature(derived, 'gpg2') == 'foo'
def test_torrent_info(pre_oci_model): def test_torrent_info(registry_model):
# Remove all existing info. # Remove all existing info.
TorrentInfo.delete().execute() TorrentInfo.delete().execute()
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') tag = registry_model.get_repo_tag(repository_ref, 'latest')
manifest = pre_oci_model.get_manifest_for_tag(tag) manifest = registry_model.get_manifest_for_tag(tag)
layers = pre_oci_model.list_manifest_layers(manifest) layers = registry_model.list_manifest_layers(manifest)
assert layers assert layers
assert pre_oci_model.get_torrent_info(layers[0].blob) is None assert registry_model.get_torrent_info(layers[0].blob) is None
pre_oci_model.set_torrent_info(layers[0].blob, 2, 'foo') registry_model.set_torrent_info(layers[0].blob, 2, 'foo')
# Set it again exactly, which should be a no-op. # Set it again exactly, which should be a no-op.
pre_oci_model.set_torrent_info(layers[0].blob, 2, 'foo') registry_model.set_torrent_info(layers[0].blob, 2, 'foo')
# Check the information we've set. # Check the information we've set.
torrent_info = pre_oci_model.get_torrent_info(layers[0].blob) torrent_info = registry_model.get_torrent_info(layers[0].blob)
assert torrent_info is not None assert torrent_info is not None
assert torrent_info.piece_length == 2 assert torrent_info.piece_length == 2
assert torrent_info.pieces == 'foo' assert torrent_info.pieces == 'foo'
# Try setting it again. Nothing should happen. # Try setting it again. Nothing should happen.
pre_oci_model.set_torrent_info(layers[0].blob, 3, 'bar') registry_model.set_torrent_info(layers[0].blob, 3, 'bar')
torrent_info = pre_oci_model.get_torrent_info(layers[0].blob) torrent_info = registry_model.get_torrent_info(layers[0].blob)
assert torrent_info is not None assert torrent_info is not None
assert torrent_info.piece_length == 2 assert torrent_info.piece_length == 2
assert torrent_info.pieces == 'foo' assert torrent_info.pieces == 'foo'
def test_blob_uploads(pre_oci_model): def test_blob_uploads(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
blob_upload = pre_oci_model.create_blob_upload(repository_ref, str(uuid.uuid4()), blob_upload = registry_model.create_blob_upload(repository_ref, str(uuid.uuid4()),
'local_us', {'some': 'metadata'}) 'local_us', {'some': 'metadata'})
assert blob_upload assert blob_upload
assert blob_upload.storage_metadata == {'some': 'metadata'} assert blob_upload.storage_metadata == {'some': 'metadata'}
assert blob_upload.location_name == 'local_us' assert blob_upload.location_name == 'local_us'
# Ensure we can find the blob upload. # Ensure we can find the blob upload.
assert pre_oci_model.lookup_blob_upload(repository_ref, blob_upload.upload_id) == blob_upload assert registry_model.lookup_blob_upload(repository_ref, blob_upload.upload_id) == blob_upload
# Update and ensure the changes are saved. # Update and ensure the changes are saved.
assert pre_oci_model.update_blob_upload(blob_upload, 1, 'the-pieces_hash', assert registry_model.update_blob_upload(blob_upload, 1, 'the-pieces_hash',
blob_upload.piece_sha_state, blob_upload.piece_sha_state,
{'new': 'metadata'}, 2, 3, {'new': 'metadata'}, 2, 3,
blob_upload.sha_state) blob_upload.sha_state)
updated = pre_oci_model.lookup_blob_upload(repository_ref, blob_upload.upload_id) updated = registry_model.lookup_blob_upload(repository_ref, blob_upload.upload_id)
assert updated assert updated
assert updated.uncompressed_byte_count == 1 assert updated.uncompressed_byte_count == 1
assert updated.piece_hashes == 'the-pieces_hash' assert updated.piece_hashes == 'the-pieces_hash'
@ -609,45 +612,45 @@ def test_blob_uploads(pre_oci_model):
assert updated.chunk_count == 3 assert updated.chunk_count == 3
# Delete the upload. # Delete the upload.
pre_oci_model.delete_blob_upload(blob_upload) registry_model.delete_blob_upload(blob_upload)
# Ensure it can no longer be found. # Ensure it can no longer be found.
assert not pre_oci_model.lookup_blob_upload(repository_ref, blob_upload.upload_id) assert not registry_model.lookup_blob_upload(repository_ref, blob_upload.upload_id)
def test_commit_blob_upload(pre_oci_model): def test_commit_blob_upload(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
blob_upload = pre_oci_model.create_blob_upload(repository_ref, str(uuid.uuid4()), blob_upload = registry_model.create_blob_upload(repository_ref, str(uuid.uuid4()),
'local_us', {'some': 'metadata'}) 'local_us', {'some': 'metadata'})
# Commit the blob upload and make sure it is written as a blob. # Commit the blob upload and make sure it is written as a blob.
digest = 'sha256:' + hashlib.sha256('hello').hexdigest() digest = 'sha256:' + hashlib.sha256('hello').hexdigest()
blob = pre_oci_model.commit_blob_upload(blob_upload, digest, 60) blob = registry_model.commit_blob_upload(blob_upload, digest, 60)
assert blob.digest == digest assert blob.digest == digest
# Ensure the upload can no longer be found. # Ensure the upload can no longer be found.
assert not pre_oci_model.lookup_blob_upload(repository_ref, blob_upload.upload_id) assert not registry_model.lookup_blob_upload(repository_ref, blob_upload.upload_id)
def test_mount_blob_into_repository(pre_oci_model): def test_mount_blob_into_repository(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
latest_tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') latest_tag = registry_model.get_repo_tag(repository_ref, 'latest')
manifest = pre_oci_model.get_manifest_for_tag(latest_tag) manifest = registry_model.get_manifest_for_tag(latest_tag)
target_repository_ref = pre_oci_model.lookup_repository('devtable', 'complex') target_repository_ref = registry_model.lookup_repository('devtable', 'complex')
layers = pre_oci_model.list_manifest_layers(manifest, include_placements=True) layers = registry_model.list_manifest_layers(manifest, include_placements=True)
assert layers assert layers
for layer in layers: for layer in layers:
# Ensure the blob doesn't exist under the repository. # Ensure the blob doesn't exist under the repository.
assert not pre_oci_model.get_repo_blob_by_digest(target_repository_ref, layer.blob.digest) assert not registry_model.get_repo_blob_by_digest(target_repository_ref, layer.blob.digest)
# Mount the blob into the repository. # Mount the blob into the repository.
assert pre_oci_model.mount_blob_into_repository(layer.blob, target_repository_ref, 60) assert registry_model.mount_blob_into_repository(layer.blob, target_repository_ref, 60)
# Ensure it now exists. # Ensure it now exists.
found = pre_oci_model.get_repo_blob_by_digest(target_repository_ref, layer.blob.digest) found = registry_model.get_repo_blob_by_digest(target_repository_ref, layer.blob.digest)
assert found == layer.blob assert found == layer.blob
@ -655,20 +658,20 @@ class SomeException(Exception):
pass pass
def test_get_cached_repo_blob(pre_oci_model): def test_get_cached_repo_blob(registry_model):
model_cache = InMemoryDataModelCache() model_cache = InMemoryDataModelCache()
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
latest_tag = pre_oci_model.get_repo_tag(repository_ref, 'latest') latest_tag = registry_model.get_repo_tag(repository_ref, 'latest')
manifest = pre_oci_model.get_manifest_for_tag(latest_tag) manifest = registry_model.get_manifest_for_tag(latest_tag)
layers = pre_oci_model.list_manifest_layers(manifest, include_placements=True) layers = registry_model.list_manifest_layers(manifest, include_placements=True)
assert layers assert layers
blob = layers[0].blob blob = layers[0].blob
# Load a blob to add it to the cache. # Load a blob to add it to the cache.
found = pre_oci_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', blob.digest) found = registry_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', blob.digest)
assert found.digest == blob.digest assert found.digest == blob.digest
assert found.uuid == blob.uuid assert found.uuid == blob.uuid
assert found.compressed_size == blob.compressed_size assert found.compressed_size == blob.compressed_size
@ -683,7 +686,7 @@ def test_get_cached_repo_blob(pre_oci_model):
with patch('data.registry_model.registry_pre_oci_model.model.blob.get_repository_blob_by_digest', with patch('data.registry_model.registry_pre_oci_model.model.blob.get_repository_blob_by_digest',
fail): fail):
# Make sure we can load again, which should hit the cache. # Make sure we can load again, which should hit the cache.
cached = pre_oci_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', blob.digest) cached = registry_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', blob.digest)
assert cached.digest == blob.digest assert cached.digest == blob.digest
assert cached.uuid == blob.uuid assert cached.uuid == blob.uuid
assert cached.compressed_size == blob.compressed_size assert cached.compressed_size == blob.compressed_size
@ -694,13 +697,13 @@ def test_get_cached_repo_blob(pre_oci_model):
# Try another blob, which should fail since the DB is not connected and the cache # Try another blob, which should fail since the DB is not connected and the cache
# does not contain the blob. # does not contain the blob.
with pytest.raises(SomeException): with pytest.raises(SomeException):
pre_oci_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', 'some other digest') registry_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', 'some other digest')
def test_create_manifest_and_retarget_tag(pre_oci_model): def test_create_manifest_and_retarget_tag(registry_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple') repository_ref = registry_model.lookup_repository('devtable', 'simple')
latest_tag = pre_oci_model.get_repo_tag(repository_ref, 'latest', include_legacy_image=True) latest_tag = registry_model.get_repo_tag(repository_ref, 'latest', include_legacy_image=True)
manifest = pre_oci_model.get_manifest_for_tag(latest_tag).get_parsed_manifest() manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag') builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
builder.add_layer(manifest.blob_digests[0], builder.add_layer(manifest.blob_digests[0],
@ -708,7 +711,7 @@ def test_create_manifest_and_retarget_tag(pre_oci_model):
sample_manifest = builder.build(docker_v2_signing_key) sample_manifest = builder.build(docker_v2_signing_key)
assert sample_manifest is not None assert sample_manifest is not None
another_manifest, tag = pre_oci_model.create_manifest_and_retarget_tag(repository_ref, another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest, sample_manifest,
'anothertag') 'anothertag')
assert another_manifest is not None assert another_manifest is not None
@ -717,5 +720,5 @@ def test_create_manifest_and_retarget_tag(pre_oci_model):
assert tag.name == 'anothertag' assert tag.name == 'anothertag'
assert another_manifest.get_parsed_manifest().manifest_dict == sample_manifest.manifest_dict assert another_manifest.get_parsed_manifest().manifest_dict == sample_manifest.manifest_dict
layers = pre_oci_model.list_manifest_layers(another_manifest) layers = registry_model.list_manifest_layers(another_manifest)
assert len(layers) == 1 assert len(layers) == 1

4
endpoints/v2/test/test_manifest.py
View file

@ -46,10 +46,10 @@ def test_e2e_query_count_manifest_norewrite(client, app):
return timecode + 10 return timecode + 10
with patch('time.time', get_time): with patch('time.time', get_time):
# Necessary in order to have the tag updates not occurr in the same second, which is the # Necessary in order to have the tag updates not occur in the same second, which is the
# granularity supported currently. # granularity supported currently.
with count_queries() as counter: with count_queries() as counter:
conduct_call(client, 'v2.write_manifest_by_digest', url_for, 'PUT', params, expected_code=202, conduct_call(client, 'v2.write_manifest_by_digest', url_for, 'PUT', params, expected_code=202,
headers=headers, raw_body=tag_manifest.json_data) headers=headers, raw_body=tag_manifest.json_data)
assert counter.count <= 16 assert counter.count <= 25

7
initdb.py
View file

@ -20,7 +20,8 @@ from data.database import (db, all_models, Role, TeamRole, Visibility, LoginServ
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind, ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes, RepositoryKind, User, DisableReason, DeletedNamespace, appr_classes,
ApprTagKind, ApprBlobPlacementLocation, Repository) ApprTagKind, ApprBlobPlacementLocation, Repository, TagKind,
ManifestChild)
from data import model from data import model
from data.queue import WorkQueue from data.queue import WorkQueue
from data.registry_model import registry_model from data.registry_model import registry_model
@ -450,6 +451,8 @@ def initialize_database():
DisableReason.create(name='successive_build_failures') DisableReason.create(name='successive_build_failures')
DisableReason.create(name='successive_build_internal_errors') DisableReason.create(name='successive_build_internal_errors')
TagKind.create(name='tag')
def wipe_database(): def wipe_database():
logger.debug('Wiping all data from the DB.') logger.debug('Wiping all data from the DB.')
@ -910,7 +913,7 @@ def populate_database(minimal=False, with_storage=False):
model.repositoryactioncount.update_repository_score(to_count) model.repositoryactioncount.update_repository_score(to_count)
WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2'] WHITELISTED_EMPTY_MODELS = ['DeletedNamespace', 'LogEntry2', 'ManifestChild']
def find_models_missing_data(): def find_models_missing_data():
# As a sanity check we are going to make sure that all db tables have some data, unless explicitly # As a sanity check we are going to make sure that all db tables have some data, unless explicitly

4
workers/test/test_manifestbackfillworker.py
View file

@ -2,7 +2,7 @@ from app import docker_v2_signing_key
from data import model from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob, from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image, ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
TagManifestLabel) TagManifestLabel, Tag, TagToRepositoryTag)
from image.docker.schema1 import DockerSchema1ManifestBuilder from image.docker.schema1 import DockerSchema1ManifestBuilder
from workers.manifestbackfillworker import backfill_manifest from workers.manifestbackfillworker import backfill_manifest
from workers.labelbackfillworker import backfill_label from workers.labelbackfillworker import backfill_label
@ -12,6 +12,8 @@ from test.fixtures import *
@pytest.fixture() @pytest.fixture()
def clear_rows(initialized_db): def clear_rows(initialized_db):
# Remove all new-style rows so we can backfill. # Remove all new-style rows so we can backfill.
TagToRepositoryTag.delete().execute()
Tag.delete().execute()
TagManifestLabelMap.delete().execute() TagManifestLabelMap.delete().execute()
ManifestLabel.delete().execute() ManifestLabel.delete().execute()
ManifestBlob.delete().execute() ManifestBlob.delete().execute()