vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add support for creating schema 2 manifests and manifest lists via the OCI model

Browse source
This commit is contained in:
Joseph Schorr 2018-11-12 23:27:49 +02:00
parent e344d4a5cf
commit 30f072aeff
16 changed files with 398 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

5
data/registry_model/datatypes.py
View file

@ -7,7 +7,8 @@ from cachetools import lru_cache
from data import model
from data.registry_model.datatype import datatype, requiresinput, optionalinput
from image.docker.schema1 import DockerSchema1Manifest, DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE
from image.docker.schemas import parse_manifest_from_bytes
from image.docker.schema1 import DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE
class RepositoryReference(datatype('Repository', [])):
@ -191,7 +192,7 @@ class Manifest(datatype('Manifest', ['digest', 'media_type', 'manifest_bytes']))
def get_parsed_manifest(self, validate=True):
""" Returns the parsed manifest for this manifest. """
return DockerSchema1Manifest(self.manifest_bytes, validate=validate)
return parse_manifest_from_bytes(self.manifest_bytes, self.media_type, validate=validate)
class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'comment', 'command',

5
data/registry_model/interface.py
View file

@ -36,7 +36,8 @@ class RegistryDataInterface(object):
or None if none. """
@abstractmethod
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name):
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name,
storage):
""" Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
@ -127,7 +128,7 @@ class RegistryDataInterface(object):
@abstractmethod
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
is_reversion=False):
storage, is_reversion=False):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
legacy image specified. If is_reversion is set to True, this operation is considered a

13
data/registry_model/manifestbuilder.py
View file

@ -18,17 +18,17 @@ _BuilderState = namedtuple('_BuilderState', ['builder_id', 'images', 'tags', 'ch
_SESSION_KEY = '__manifestbuilder'
def create_manifest_builder(repository_ref):
def create_manifest_builder(repository_ref, storage):
""" Creates a new manifest builder for populating manifests under the specified repository
and returns it. Returns None if the builder could not be constructed.
"""
builder_id = str(uuid.uuid4())
builder = _ManifestBuilder(repository_ref, _BuilderState(builder_id, {}, {}, {}))
builder = _ManifestBuilder(repository_ref, _BuilderState(builder_id, {}, {}, {}), storage)
builder._save_to_session()
return builder
def lookup_manifest_builder(repository_ref, builder_id):
def lookup_manifest_builder(repository_ref, builder_id, storage):
""" Looks up the manifest builder with the given ID under the specified repository and returns
it or None if none.
"""
@ -40,16 +40,17 @@ def lookup_manifest_builder(repository_ref, builder_id):
if builder_state.builder_id != builder_id:
return None
return _ManifestBuilder(repository_ref, builder_state)
return _ManifestBuilder(repository_ref, builder_state, storage)
class _ManifestBuilder(object):
""" Helper class which provides an interface for bookkeeping the layers and configuration of
manifests being constructed.
"""
def __init__(self, repository_ref, builder_state):
def __init__(self, repository_ref, builder_state, storage):
self._repository_ref = repository_ref
self._builder_state = builder_state
self._storage = storage
@property
def builder_id(self):
@ -183,7 +184,7 @@ class _ManifestBuilder(object):
if legacy_image is None:
return None
tag = registry_model.retarget_tag(self._repository_ref, tag_name, legacy_image)
tag = registry_model.retarget_tag(self._repository_ref, tag_name, legacy_image, self._storage)
if tag is None:
return None

57
data/registry_model/registry_oci_model.py
View file

@ -11,7 +11,6 @@ from data.registry_model.interface import RegistryDataInterface
from data.registry_model.datatypes import Tag, Manifest, LegacyImage, Label, SecurityScanStatus
from data.registry_model.shared import SharedModel
from data.registry_model.label_handlers import apply_label_to_manifest
from util.validation import is_json
logger = logging.getLogger(__name__)
@ -176,7 +175,8 @@ class OCIModel(SharedModel, RegistryDataInterface):
return Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_image))
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name):
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name,
storage):
""" Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
@ -187,41 +187,47 @@ class OCIModel(SharedModel, RegistryDataInterface):
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
def _retrieve_repo_blob(digest):
blob_found = self.get_repo_blob_by_digest(repository_ref, digest, include_placements=True)
if blob_found is None:
return None
try:
return storage.get_content(blob_found.placements, blob_found.storage_path)
except IOError:
logger.exception('Could not retrieve configuration blob `%s`', digest)
return None
# Get or create the manifest itself.
manifest, newly_created = oci.manifest.get_or_create_manifest(repository_ref._db_id,
manifest_interface_instance)
if manifest is None:
created_manifest = oci.manifest.get_or_create_manifest(repository_ref._db_id,
manifest_interface_instance,
storage)
if created_manifest is None:
return (None, None)
# Re-target the tag to it.
tag = oci.tag.retarget_tag(tag_name, manifest)
tag = oci.tag.retarget_tag(tag_name, created_manifest.manifest)
if tag is None:
return (None, None)
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest)
legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
if legacy_image is None:
return (None, None)
# Save the labels on the manifest. Note that order is important here: This must come after the
# tag has been changed.
# TODO(jschorr): Support schema2 here when we're ready.
if newly_created:
has_labels = False
li = LegacyImage.for_image(legacy_image)
wrapped_manifest = Manifest.for_manifest(created_manifest.manifest, li)
with self.batch_create_manifest_labels(Manifest.for_manifest(manifest, None)) as add_label:
for key, value in manifest_interface_instance.layers[-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
add_label(key, value, 'manifest', media_type)
has_labels = True
# Apply any labels that should modify the created tag.
if created_manifest.labels_to_apply:
for key, value in created_manifest.labels_to_apply.iteritems():
apply_label_to_manifest(dict(key=key, value=value), wrapped_manifest, self)
# Reload the tag in case any updates were applied.
if has_labels:
tag = database.Tag.get(id=tag.id)
tag = database.Tag.get(id=tag.id)
li = LegacyImage.for_image(legacy_image)
return (Manifest.for_manifest(manifest, li), Tag.for_tag(tag, li))
return (wrapped_manifest, Tag.for_tag(tag, li))
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image, storage,
is_reversion=False):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
@ -240,11 +246,12 @@ class OCIModel(SharedModel, RegistryDataInterface):
if manifest_instance is None:
return None
manifest, _ = oci.manifest.get_or_create_manifest(repository_ref._db_id, manifest_instance)
if manifest is None:
created = oci.manifest.get_or_create_manifest(repository_ref._db_id, manifest_instance,
storage)
if created is None:
return None
manifest_id = manifest.id
manifest_id = created.manifest.id
tag = oci.tag.retarget_tag(tag_name, manifest_id, is_reversion=is_reversion)
legacy_image = LegacyImage.for_image(oci.shared.get_legacy_image_for_manifest(manifest_id))

5
data/registry_model/registry_pre_oci_model.py
View file

@ -79,7 +79,8 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
return Manifest.for_tag_manifest(tag_manifest, legacy_image)
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name):
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name,
storage):
""" Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
@ -298,7 +299,7 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
manifest_digest = tag_manifest.digest if tag_manifest else None
return Tag.for_repository_tag(tag, legacy_image=legacy_image, manifest_digest=manifest_digest)
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image, storage,
is_reversion=False):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or

10
data/registry_model/test/test_interface.py
View file

@ -9,7 +9,7 @@ import pytest
from mock import patch
from playhouse.test_utils import assert_query_count
from app import docker_v2_signing_key
from app import docker_v2_signing_key, storage
from data import model
from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest, ManifestBlob,
ManifestLegacyImage, ManifestLabel, TagManifest, RepositoryTag, Image,
@ -306,7 +306,7 @@ def test_retarget_tag_history(use_manifest, registry_model):
# Retarget the tag.
assert manifest_or_legacy_image
updated_tag = registry_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image,
is_reversion=True)
storage, is_reversion=True)
# Ensure the tag has changed targets.
if use_manifest:
@ -698,7 +698,8 @@ def test_create_manifest_and_retarget_tag(registry_model):
another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
'anothertag',
storage)
assert another_manifest is not None
assert tag is not None
@ -730,7 +731,8 @@ def test_create_manifest_and_retarget_tag_with_labels(registry_model):
another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
'anothertag',
storage)
assert another_manifest is not None
assert tag is not None

9
data/registry_model/test/test_manifestbuilder.py
View file

@ -40,9 +40,9 @@ def test_build_manifest(layers, fake_session, registry_model):
settings = BlobUploadSettings('2M', 512 * 1024, 3600)
app_config = {'TESTING': True}
builder = create_manifest_builder(repository_ref)
assert lookup_manifest_builder(repository_ref, 'anotherid') is None
assert lookup_manifest_builder(repository_ref, builder.builder_id) is not None
builder = create_manifest_builder(repository_ref, storage)
assert lookup_manifest_builder(repository_ref, 'anotherid', storage) is None
assert lookup_manifest_builder(repository_ref, builder.builder_id, storage) is not None
blobs_by_layer = {}
for layer_id, parent_id, layer_bytes in layers:
@ -89,8 +89,9 @@ def test_build_manifest(layers, fake_session, registry_model):
def test_build_manifest_missing_parent(fake_session, registry_model):
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
repository_ref = registry_model.lookup_repository('devtable', 'complex')
builder = create_manifest_builder(repository_ref)
builder = create_manifest_builder(repository_ref, storage)
assert builder.start_layer('somelayer', json.dumps({'id': 'somelayer', 'parent': 'someparent'}),
'local_us', None, 60) is None