vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix pulling of squashed versions of the legacy image in a manifest lists

Browse source
This commit is contained in:
Joseph Schorr 2018-11-20 16:36:49 +02:00
parent 001768c043
commit 1f03fdb27e
10 changed files with 198 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

11
data/registry_model/interface.py
View file

@ -197,6 +197,13 @@ class RegistryDataInterface(object):
Returns None if the manifest could not be parsed and validated.
"""
@abstractmethod
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, include_placements=False):
""" Returns an *ordered list* of the layers found in the parsed manifest, starting at the base
and working towards the leaf, including the associated Blob and its placements
(if specified).
"""
@abstractmethod
def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
"""
@ -205,8 +212,8 @@ class RegistryDataInterface(object):
"""
@abstractmethod
def lookup_or_create_derived_image(self, manifest, verb, storage_location, varying_metadata=None,
include_placements=False):
def lookup_or_create_derived_image(self, manifest, verb, storage_location, storage,
varying_metadata=None, include_placements=False):
"""
Looks up the derived image for the given maniest, verb and optional varying metadata
and returns it. If none exists, a new derived image is created.

37
data/registry_model/registry_oci_model.py
View file

@ -54,6 +54,30 @@ class OCIModel(SharedModel, RegistryDataInterface):
return tags_map
def _get_legacy_compatible_image_for_manifest(self, manifest, storage):
# Check for a legacy image directly on the manifest.
if manifest.media_type != DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE:
return oci.shared.get_legacy_image_for_manifest(manifest._db_id)
# Otherwise, lookup a legacy image associated with the v1-compatible manifest
# in the list.
try:
manifest_obj = database.Manifest.get(id=manifest._db_id)
except database.Manifest.DoesNotExist:
logger.exception('Could not find manifest for manifest `%s`', manifest._db_id)
return None
# See if we can lookup a schema1 legacy image.
v1_compatible = self.get_schema1_parsed_manifest(manifest, '', '', '', storage)
if v1_compatible is None:
return None
v1_id = v1_compatible.leaf_layer_v1_image_id
if v1_id is None:
return None
return model.image.get_image(manifest_obj.repository_id, v1_id)
def find_matching_tag(self, repository_ref, tag_names):
""" Finds an alive tag in the repository matching one of the given tag names and returns it
or None if none.
@ -400,7 +424,13 @@ class OCIModel(SharedModel, RegistryDataInterface):
logger.exception('Could not find manifest for manifest `%s`', manifest._db_id)
return None
return self._list_manifest_layers(manifest, manifest_obj.repository_id, include_placements)
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
return None
return self._list_manifest_layers(manifest_obj.repository_id, parsed, include_placements)
def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
"""
@ -414,13 +444,14 @@ class OCIModel(SharedModel, RegistryDataInterface):
derived = model.image.find_derived_storage_for_image(legacy_image, verb, varying_metadata)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def lookup_or_create_derived_image(self, manifest, verb, storage_location, varying_metadata=None,
def lookup_or_create_derived_image(self, manifest, verb, storage_location, storage,
varying_metadata=None,
include_placements=False):
"""
Looks up the derived image for the given maniest, verb and optional varying metadata
and returns it. If none exists, a new derived image is created.
"""
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest._db_id)
legacy_image = self._get_legacy_compatible_image_for_manifest(manifest, storage)
if legacy_image is None:
return None

16
data/registry_model/registry_pre_oci_model.py
View file

@ -11,7 +11,8 @@ from data import model
from data.database import db_transaction
from data.registry_model.interface import RegistryDataInterface
from data.registry_model.datatypes import (Tag, Manifest, LegacyImage, Label,
SecurityScanStatus, ManifestLayer, Blob, DerivedImage)
SecurityScanStatus, ManifestLayer, Blob, DerivedImage,
RepositoryReference)
from data.registry_model.shared import SharedModel
from data.registry_model.label_handlers import apply_label_to_manifest
from image.docker.schema1 import (DockerSchema1ManifestBuilder, ManifestException,
@ -489,7 +490,14 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
logger.exception('Could not find tag manifest for manifest `%s`', manifest._db_id)
return None
return self._list_manifest_layers(manifest, tag_manifest.tag.repository_id, include_placements)
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
return None
repo_ref = RepositoryReference.for_id(tag_manifest.tag.repository_id)
return self.list_parsed_manifest_layers(repo_ref, parsed, include_placements)
def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
"""
@ -506,8 +514,8 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
derived = model.image.find_derived_storage_for_image(repo_image, verb, varying_metadata)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def lookup_or_create_derived_image(self, manifest, verb, storage_location, varying_metadata=None,
include_placements=False):
def lookup_or_create_derived_image(self, manifest, verb, storage_location, storage,
varying_metadata=None, include_placements=False):
"""
Looks up the derived image for the given maniest, verb and optional varying metadata
and returns it. If none exists, a new derived image is created.

18
data/registry_model/shared.py
View file

@ -306,17 +306,18 @@ class SharedModel:
return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
def _list_manifest_layers(self, manifest, repo_id, include_placements=False):
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, include_placements=False):
""" Returns an *ordered list* of the layers found in the parsed manifest, starting at the base
and working towards the leaf, including the associated Blob and its placements
(if specified).
"""
return self._list_manifest_layers(repository_ref._db_id, parsed_manifest, include_placements)
def _list_manifest_layers(self, repo_id, parsed, include_placements=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
"""
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
return None
storage_map = {}
if parsed.local_blob_digests:
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
@ -331,11 +332,12 @@ class SharedModel:
digest_str = str(layer.digest)
if digest_str not in storage_map:
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, parsed.digest)
return None
image_storage = storage_map[digest_str]
assert image_storage.cas_path is not None
assert image_storage.image_size is not None
placements = None
if include_placements:

34
data/registry_model/test/test_blobuploader.py
View file

@ -1,5 +1,6 @@
import hashlib
import os
import tarfile
from io import BytesIO
@ -113,3 +114,36 @@ def test_extra_blob_stream_handlers(pre_oci_model):
assert ''.join(handler1_result) == 'hello world'
assert ''.join(handler2_result) == 'hello world'
def valid_tar_gz(contents):
layer_data = BytesIO()
tar_file = tarfile.open(fileobj=layer_data, mode='w|gz')
tar_file_info = tarfile.TarInfo(name='somefile')
tar_file_info.type = tarfile.REGTYPE
tar_file_info.size = len(contents)
tar_file_info.mtime = 1
tar_file.addfile(tar_file_info, BytesIO(contents))
tar_file.close()
layer_bytes = layer_data.getvalue()
layer_data.close()
return layer_bytes
def test_uncompressed_size(pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
settings = BlobUploadSettings('1K', 512 * 1024, 3600)
app_config = {'TESTING': True}
with upload_blob(repository_ref, storage, settings) as manager:
manager.upload_chunk(app_config, BytesIO(valid_tar_gz('hello world')))
blob = manager.commit_to_blob(app_config)
assert blob.compressed_size is not None
assert blob.uncompressed_size is not None

12
data/registry_model/test/test_interface.py
View file

@ -543,8 +543,10 @@ def test_derived_image(registry_model):
assert registry_model.lookup_derived_image(manifest, 'squash', {}) is None
# Create a new one.
squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {})
assert registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {}) == squashed
squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash',
'local_us', storage, {})
assert registry_model.lookup_or_create_derived_image(manifest, 'squash',
'local_us', storage, {}) == squashed
assert squashed.unique_id
# Check and set the size.
@ -560,15 +562,15 @@ def test_derived_image(registry_model):
assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) is None
squashed_foo = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us',
{'foo': 'bar'})
storage, {'foo': 'bar'})
assert squashed_foo != squashed
assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) == squashed_foo
assert squashed.unique_id != squashed_foo.unique_id
# Lookup with placements.
squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {},
include_placements=True)
squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us',
storage, {}, include_placements=True)
assert squashed.blob.placements
# Delete the derived image.