vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implement the new OCI-based registry data model

Browse source 
Note that this change does *not* enable the new data model by default, but does allow it to be used when a special environment variable is specified.
This commit is contained in:
Joseph Schorr 2018-11-05 13:03:08 -05:00
parent 924b386437
commit fdcb8bad23
23 changed files with 1847 additions and 209 deletions
Download patch file Download diff file Expand all files Collapse all files

134
data/registry_model/test/test_interface.py
View file

@ -1,4 +1,5 @@
import hashlib
import json
import uuid
from datetime import datetime, timedelta
@ -16,16 +17,21 @@ from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest,
TorrentInfo, Tag, TagToRepositoryTag, close_db_filter)
from data.cache.impl import InMemoryDataModelCache
from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.registry_oci_model import OCIModel
from data.registry_model.datatypes import RepositoryReference
from image.docker.schema1 import DockerSchema1ManifestBuilder
from test.fixtures import *
@pytest.fixture(params=[PreOCIModel])
@pytest.fixture(params=[PreOCIModel, OCIModel])
def registry_model(request, initialized_db):
return request.param()
@pytest.fixture()
def pre_oci_model(initialized_db):
return PreOCIModel()
@pytest.mark.parametrize('names, expected', [
(['unknown'], None),
@ -83,10 +89,11 @@ def test_lookup_manifests(repo_namespace, repo_name, registry_model):
found_tag = registry_model.find_matching_tag(repository_ref, ['latest'])
found_manifest = registry_model.get_manifest_for_tag(found_tag)
found = registry_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest,
include_legacy_image=True)
include_legacy_image=True)
assert found._db_id == found_manifest._db_id
assert found.digest == found_manifest.digest
assert found.legacy_image
assert found.legacy_image.parents
def test_lookup_unknown_manifest(registry_model):
@ -110,11 +117,11 @@ def test_legacy_images(repo_namespace, repo_name, registry_model):
found_tags = set()
for image in legacy_images:
found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True)
include_parents=True)
with assert_query_count(5 if found_image.parents else 4):
found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True, include_blob=True)
include_parents=True, include_blob=True)
assert found_image.docker_image_id == image.docker_image_id
assert found_image.parents == image.parents
assert found_image.blob
@ -132,7 +139,7 @@ def test_legacy_images(repo_namespace, repo_name, registry_model):
# Try without parents and ensure it raises an exception.
found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=False)
include_parents=False)
with pytest.raises(Exception):
assert not found_image.parents
@ -211,23 +218,19 @@ def test_batch_labels(registry_model):
])
def test_repository_tags(repo_namespace, repo_name, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
with assert_query_count(1):
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags)
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags)
for tag in tags:
with assert_query_count(2):
found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag == tag
found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag == tag
if found_tag.legacy_image is None:
continue
with assert_query_count(2):
found_image = registry_model.get_legacy_image(repository_ref,
found_tag.legacy_image.docker_image_id)
assert found_image == found_tag.legacy_image
found_image = registry_model.get_legacy_image(repository_ref,
found_tag.legacy_image.docker_image_id)
assert found_image == found_tag.legacy_image
def test_repository_tag_history(registry_model):
@ -295,15 +298,15 @@ def test_retarget_tag_history(use_manifest, registry_model):
if use_manifest:
manifest_or_legacy_image = registry_model.lookup_manifest_by_digest(repository_ref,
history[1].manifest_digest,
allow_dead=True)
history[0].manifest_digest,
allow_dead=True)
else:
manifest_or_legacy_image = history[1].legacy_image
manifest_or_legacy_image = history[0].legacy_image
# Retarget the tag.
assert manifest_or_legacy_image
updated_tag = registry_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image,
is_reversion=True)
is_reversion=True)
# Ensure the tag has changed targets.
if use_manifest:
@ -316,23 +319,6 @@ def test_retarget_tag_history(use_manifest, registry_model):
assert len(new_history) == len(history) + 1
def test_retarget_tag(registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'complex')
history, _ = registry_model.list_repository_tag_history(repository_ref)
prod_tag = registry_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True)
# Retarget the tag.
updated_tag = registry_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image)
# Ensure the tag has changed targets.
assert updated_tag.legacy_image == prod_tag.legacy_image
# Ensure history has been updated.
new_history, _ = registry_model.list_repository_tag_history(repository_ref)
assert len(new_history) == len(history) + 1
def test_change_repository_tag_expiration(registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = registry_model.get_repo_tag(repository_ref, 'latest')
@ -399,24 +385,24 @@ def clear_rows(initialized_db):
('devtable', 'history'),
('buynlarge', 'orgrepo'),
])
def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = registry_model.list_repository_tags(repository_ref)
def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref)
assert tags
for tag in tags:
assert not tag.manifest_digest
assert registry_model.backfill_manifest_for_tag(tag)
assert pre_oci_model.backfill_manifest_for_tag(tag)
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert tags
for tag in tags:
assert tag.manifest_digest
manifest = registry_model.get_manifest_for_tag(tag)
manifest = pre_oci_model.get_manifest_for_tag(tag)
assert manifest
legacy_image = registry_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id,
legacy_image = pre_oci_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id,
include_parents=True)
parsed_manifest = manifest.get_parsed_manifest()
@ -430,19 +416,19 @@ def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, regist
('devtable', 'history'),
('buynlarge', 'orgrepo'),
])
def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = registry_model.list_repository_tags(repository_ref)
def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref)
assert tags
for tag in tags:
assert not tag.manifest_digest
assert not registry_model.get_manifest_for_tag(tag)
assert not pre_oci_model.get_manifest_for_tag(tag)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
manifest = pre_oci_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
assert manifest
updated_tag = registry_model.get_repo_tag(repository_ref, tag.name)
updated_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name)
assert updated_tag.manifest_digest == manifest.digest
@ -471,9 +457,8 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
with assert_query_count(4):
layers = registry_model.list_manifest_layers(manifest)
assert layers
layers = registry_model.list_manifest_layers(manifest)
assert layers
layers = registry_model.list_manifest_layers(manifest, include_placements=True)
assert layers
@ -522,7 +507,7 @@ def test_derived_image(registry_model):
assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) is None
squashed_foo = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us',
{'foo': 'bar'})
{'foo': 'bar'})
assert squashed_foo != squashed
assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) == squashed_foo
@ -530,7 +515,7 @@ def test_derived_image(registry_model):
# Lookup with placements.
squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {},
include_placements=True)
include_placements=True)
assert squashed.blob.placements
# Delete the derived image.
@ -712,8 +697,8 @@ def test_create_manifest_and_retarget_tag(registry_model):
assert sample_manifest is not None
another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
sample_manifest,
'anothertag')
assert another_manifest is not None
assert tag is not None
@ -722,3 +707,38 @@ def test_create_manifest_and_retarget_tag(registry_model):
layers = registry_model.list_manifest_layers(another_manifest)
assert len(layers) == 1
def test_create_manifest_and_retarget_tag_with_labels(registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'simple')
latest_tag = registry_model.get_repo_tag(repository_ref, 'latest', include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
json_metadata = {
'id': latest_tag.legacy_image.docker_image_id,
'config': {
'Labels': {
'quay.expires-after': '2w',
},
},
}
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
builder.add_layer(manifest.blob_digests[0], json.dumps(json_metadata))
sample_manifest = builder.build(docker_v2_signing_key)
assert sample_manifest is not None
another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
assert another_manifest is not None
assert tag is not None
assert tag.name == 'anothertag'
assert another_manifest.get_parsed_manifest().manifest_dict == sample_manifest.manifest_dict
layers = registry_model.list_manifest_layers(another_manifest)
assert len(layers) == 1
# Ensure the labels were applied.
assert tag.lifetime_end_ms is not None

19
data/registry_model/test/test_manifestbuilder.py
View file

@ -10,15 +10,16 @@ from mock import patch
from data.registry_model.blobuploader import BlobUploadSettings, upload_blob
from data.registry_model.manifestbuilder import create_manifest_builder, lookup_manifest_builder
from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.registry_oci_model import OCIModel
from storage.distributedstorage import DistributedStorage
from storage.fakestorage import FakeStorage
from test.fixtures import *
@pytest.fixture()
def pre_oci_model(initialized_db):
return PreOCIModel()
@pytest.fixture(params=[PreOCIModel, OCIModel])
def registry_model(request, initialized_db):
return request.param()
@pytest.fixture()
@ -33,8 +34,8 @@ def fake_session():
('someid', 'parentid', 'some data')],
id='Multi layer'),
])
def test_build_manifest(layers, fake_session, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
def test_build_manifest(layers, fake_session, registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'complex')
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
settings = BlobUploadSettings('2M', 512 * 1024, 3600)
app_config = {'TESTING': True}
@ -67,13 +68,13 @@ def test_build_manifest(layers, fake_session, pre_oci_model):
assert tag in builder.committed_tags
# Verify the legacy image for the tag.
found = pre_oci_model.get_repo_tag(repository_ref, 'somenewtag', include_legacy_image=True)
found = registry_model.get_repo_tag(repository_ref, 'somenewtag', include_legacy_image=True)
assert found
assert found.name == 'somenewtag'
assert found.legacy_image.docker_image_id == layers[-1][0]
# Verify the blob and manifest.
manifest = pre_oci_model.get_manifest_for_tag(found)
manifest = registry_model.get_manifest_for_tag(found)
assert manifest
parsed = manifest.get_parsed_manifest()
@ -87,8 +88,8 @@ def test_build_manifest(layers, fake_session, pre_oci_model):
assert parsed.leaf_layer_v1_image_id == layers[-1][0]
def test_build_manifest_missing_parent(fake_session, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
def test_build_manifest_missing_parent(fake_session, registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'complex')
builder = create_manifest_builder(repository_ref)
assert builder.start_layer('somelayer', json.dumps({'id': 'somelayer', 'parent': 'someparent'}),