vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Convert V2's manifest endpoints to use the new data model interface

Browse source
This commit is contained in:
Joseph Schorr 2018-10-05 17:30:47 -04:00
parent a172de4fdc
commit 6b5064aba4
10 changed files with 197 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

5
data/registry_model/datatypes.py
View file

@ -7,7 +7,7 @@ from cachetools import lru_cache
from data import model
from data.registry_model.datatype import datatype, requiresinput, optionalinput
from image.docker.schema1 import DockerSchema1Manifest
from image.docker.schema1 import DockerSchema1Manifest, DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE
class RepositoryReference(datatype('Repository', [])):
@ -118,7 +118,7 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta
return legacy_image
class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])):
class Manifest(datatype('Manifest', ['digest', 'media_type', 'manifest_bytes'])):
""" Manifest represents a manifest in a repository. """
@classmethod
def for_tag_manifest(cls, tag_manifest, legacy_image=None):
@ -127,6 +127,7 @@ class Manifest(datatype('Manifest', ['digest', 'manifest_bytes'])):
return Manifest(db_id=tag_manifest.id, digest=tag_manifest.digest,
manifest_bytes=tag_manifest.json_data,
media_type=DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE, # Always in legacy.
inputs=dict(legacy_image=legacy_image))
@property

6
data/registry_model/interface.py
View file

@ -117,6 +117,12 @@ class RegistryDataInterface(object):
or None if none.
"""
@abstractmethod
def has_expired_tag(self, repository_ref, tag_name):
"""
Returns true if and only if the repository contains a tag with the given name that is expired.
"""
@abstractmethod
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
is_reversion=False):

46
data/registry_model/registry_pre_oci_model.py
View file

@ -106,7 +106,7 @@ class PreOCIModel(RegistryDataInterface):
# Ensure all the blobs in the manifest exist.
digests = manifest_interface_instance.checksums
query = model.storage.lookup_repo_storages_by_content_checksum(repository_ref._db_id, digests)
blob_map = {s.content_checksum: s.id for s in query}
blob_map = {s.content_checksum: s for s in query}
for layer in manifest_interface_instance.layers:
digest_str = str(layer.digest)
if digest_str not in blob_map:
@ -116,23 +116,38 @@ class PreOCIModel(RegistryDataInterface):
# This will let us know which v1 images we need to synthesize and which ones are invalid.
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
images_query = model.image.lookup_repository_images(repository_ref._db_id, docker_image_ids)
images_map = {i.docker_image_id: i.storage for i in images_query}
image_storage_map = {i.docker_image_id: i.storage for i in images_query}
# Rewrite any v1 image IDs that do not match the checksum in the database.
try:
rewritten_images = list(manifest_interface_instance.rewrite_invalid_image_ids(images_map))
rewritten_images = manifest_interface_instance.rewrite_invalid_image_ids(image_storage_map)
rewritten_images = list(rewritten_images)
parent_image_map = {}
for rewritten_image in rewritten_images:
if not rewritten_image.image_id in images_map:
model.image.synthesize_v1_image(
if not rewritten_image.image_id in image_storage_map:
parent_image = None
if rewritten_image.parent_image_id:
parent_image = parent_image_map.get(rewritten_image.parent_image_id)
if parent_image is None:
parent_image = model.image.get_image(repository_ref._db_id,
rewritten_image.parent_image_id)
if parent_image is None:
return None, None
synthesized = model.image.synthesize_v1_image(
repository_ref._db_id,
blob_map[rewritten_image.content_checksum],
blob_map[rewritten_image.content_checksum].id,
blob_map[rewritten_image.content_checksum].image_size,
rewritten_image.image_id,
rewritten_image.created,
rewritten_image.comment,
rewritten_image.command,
rewritten_image.compat_json,
rewritten_image.parent_image_id,
parent_image,
)
parent_image_map[rewritten_image.image_id] = synthesized
except ManifestException:
logger.exception("exception when rewriting v1 metadata")
return None, None
@ -150,7 +165,7 @@ class PreOCIModel(RegistryDataInterface):
# Save the labels on the manifest.
if newly_created:
with self.batch_create_manifest_labels(manifest) as add_label:
for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
for key, value in manifest_interface_instance.layers[-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
add_label(key, value, 'manifest', media_type)
@ -289,7 +304,8 @@ class PreOCIModel(RegistryDataInterface):
else None))
for tag in tags]
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None, active_tags_only=False):
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None,
active_tags_only=False):
"""
Returns the history of all tags in the repository (unless filtered). This includes tags that
have been made in-active due to newer versions of those tags coming into service.
@ -302,6 +318,16 @@ class PreOCIModel(RegistryDataInterface):
legacy_image=LegacyImage.for_image(tag.image))
for tag in tags], has_more
def has_expired_tag(self, repository_ref, tag_name):
"""
Returns true if and only if the repository contains a tag with the given name that is expired.
"""
try:
model.tag.get_expired_tag_in_repo(repository_ref._db_id, tag_name)
return True
except database.RepositoryTag.DoesNotExist:
return False
def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
"""
Returns the latest, *active* tag found in the repository, with the matching name
@ -706,7 +732,7 @@ class PreOCIModel(RegistryDataInterface):
blob_cache_key = cache_key.for_repository_blob(namespace_name, repo_name, blob_digest)
blob_dict = model_cache.retrieve(blob_cache_key, load_blob)
try:
return Blob.from_dict(blob_dict) if blob_dict is not None else None
except FromDictionaryException:

30
data/registry_model/test/test_pre_oci_model.py
View file

@ -8,6 +8,7 @@ import pytest
from mock import patch
from playhouse.test_utils import assert_query_count
from app import docker_v2_signing_key
from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
@ -16,6 +17,7 @@ from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest,
from data.cache.impl import InMemoryDataModelCache
from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.datatypes import RepositoryReference
from image.docker.schema1 import DockerSchema1ManifestBuilder
from test.fixtures import *
@ -235,6 +237,10 @@ def test_repository_tag_history(pre_oci_model):
assert not has_more
assert len(history) == 2
# Ensure the latest tag is marked expired, since there is an expired one.
with assert_query_count(1):
assert pre_oci_model.has_expired_tag(repository_ref, 'latest')
@pytest.mark.parametrize('repo_namespace, repo_name', [
('devtable', 'simple'),
@ -685,3 +691,27 @@ def test_get_cached_repo_blob(pre_oci_model):
# does not contain the blob.
with pytest.raises(SomeException):
pre_oci_model.get_cached_repo_blob(model_cache, 'devtable', 'simple', 'some other digest')
def test_create_manifest_and_retarget_tag(pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'simple')
latest_tag = pre_oci_model.get_repo_tag(repository_ref, 'latest', include_legacy_image=True)
manifest = pre_oci_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
builder.add_layer(manifest.blob_digests[0],
'{"id": "%s"}' % latest_tag.legacy_image.docker_image_id)
sample_manifest = builder.build(docker_v2_signing_key)
assert sample_manifest is not None
another_manifest, tag = pre_oci_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
assert another_manifest is not None
assert tag is not None
assert tag.name == 'anothertag'
assert another_manifest.get_parsed_manifest().manifest_dict == sample_manifest.manifest_dict
layers = pre_oci_model.list_manifest_layers(another_manifest)
assert len(layers) == 1