vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Unify the get_layers calls across all implements of manifest schemas to ensure we have a common type returned

Browse source 
Also renames some methods to make it more clear what kind of information they return
This commit is contained in:
Joseph Schorr 2018-11-26 17:58:48 +02:00
parent 180d8847db
commit 4e1ff90cb2
17 changed files with 210 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

6
data/registry_model/interface.py
View file

@ -195,10 +195,12 @@ class RegistryDataInterface(object):
""" Returns the set of local blobs for the given manifest or None if none. """
@abstractmethod
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, include_placements=False):
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, storage,
include_placements=False):
""" Returns an *ordered list* of the layers found in the parsed manifest, starting at the base
and working towards the leaf, including the associated Blob and its placements
(if specified).
(if specified). The layer information in `layer_info` will be of type
`image.docker.types.ManifestImageLayer`.
"""
@abstractmethod

6
data/registry_model/registry_oci_model.py
View file

@ -523,12 +523,14 @@ class OCIModel(SharedModel, RegistryDataInterface):
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, include_placements=False):
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, storage,
include_placements=False):
""" Returns an *ordered list* of the layers found in the parsed manifest, starting at the base
and working towards the leaf, including the associated Blob and its placements
(if specified).
"""
return self._list_manifest_layers(repository_ref._db_id, parsed_manifest, include_placements,
return self._list_manifest_layers(repository_ref._db_id, parsed_manifest, storage,
include_placements=include_placements,
by_manifest=True)
def get_manifest_local_blobs(self, manifest, include_placements=False):

6
data/registry_model/registry_pre_oci_model.py
View file

@ -577,12 +577,14 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, include_placements=False):
def list_parsed_manifest_layers(self, repository_ref, parsed_manifest, storage,
include_placements=False):
""" Returns an *ordered list* of the layers found in the parsed manifest, starting at the base
and working towards the leaf, including the associated Blob and its placements
(if specified).
"""
return self._list_manifest_layers(repository_ref._db_id, parsed_manifest, include_placements)
return self._list_manifest_layers(repository_ref._db_id, parsed_manifest, storage,
include_placements=include_placements)
def get_manifest_local_blobs(self, manifest, include_placements=False):
""" Returns the set of local blobs for the given manifest or None if none. """

16
data/registry_model/shared.py
View file

@ -7,6 +7,7 @@ from collections import defaultdict
from data import database
from data import model
from data.cache import cache_key
from data.model.oci.retriever import RepositoryContentRetriever
from data.registry_model.datatype import FromDictionaryException
from data.registry_model.datatypes import (RepositoryReference, Blob, TorrentInfo, BlobUpload,
LegacyImage, ManifestLayer, DerivedImage)
@ -316,7 +317,8 @@ class SharedModel:
return blobs
def _list_manifest_layers(self, repo_id, parsed, include_placements=False, by_manifest=False):
def _list_manifest_layers(self, repo_id, parsed, storage, include_placements=False,
by_manifest=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
@ -328,15 +330,21 @@ class SharedModel:
by_manifest=by_manifest)
storage_map = {blob.content_checksum: blob for blob in blob_query}
retriever = RepositoryContentRetriever(repo_id, storage)
layers = parsed.get_layers(retriever)
if layers is None:
logger.error('Could not load layers for manifest `%s`', parsed.digest)
return None
manifest_layers = []
for layer in parsed.layers:
for layer in layers:
if layer.is_remote:
manifest_layers.append(ManifestLayer(layer, None))
continue
digest_str = str(layer.digest)
digest_str = str(layer.blob_digest)
if digest_str not in storage_map:
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, parsed.digest)
logger.error('Missing digest `%s` for manifest `%s`', layer.blob_digest, parsed.digest)
return None
image_storage = storage_map[digest_str]

14
data/registry_model/test/test_interface.py
View file

@ -21,6 +21,7 @@ from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.registry_oci_model import OCIModel
from data.registry_model.datatypes import RepositoryReference
from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
from image.docker.types import ManifestImageLayer
from image.docker.schema1 import DockerSchema1ManifestBuilder
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
@ -474,19 +475,14 @@ def test_layers_and_blobs(repo_namespace, repo_name, registry_model):
parsed = manifest.get_parsed_manifest()
assert parsed
layers = registry_model.list_parsed_manifest_layers(repository_ref, parsed)
layers = registry_model.list_parsed_manifest_layers(repository_ref, parsed, storage)
assert layers
layers = registry_model.list_parsed_manifest_layers(repository_ref, parsed,
layers = registry_model.list_parsed_manifest_layers(repository_ref, parsed, storage,
include_placements=True)
assert layers
parsed_layers = list(manifest.get_parsed_manifest().layers)
assert len(layers) == len(parsed_layers)
for index, manifest_layer in enumerate(layers):
assert manifest_layer.layer_info == parsed_layers[index]
assert manifest_layer.blob.digest == str(parsed_layers[index].digest)
assert manifest_layer.blob.storage_path
assert manifest_layer.blob.placements
@ -494,6 +490,7 @@ def test_layers_and_blobs(repo_namespace, repo_name, registry_model):
assert repo_blob.digest == manifest_layer.blob.digest
assert manifest_layer.estimated_size(1) is not None
assert isinstance(manifest_layer.layer_info, ManifestImageLayer)
blobs = registry_model.get_manifest_local_blobs(manifest, include_placements=True)
assert {b.digest for b in blobs} == set(parsed.local_blob_digests)
@ -532,7 +529,8 @@ def test_manifest_remote_layers(oci_model):
assert created_manifest
layers = oci_model.list_parsed_manifest_layers(repository_ref,
created_manifest.get_parsed_manifest())
created_manifest.get_parsed_manifest(),
storage)
assert len(layers) == 1
assert layers[0].layer_info.is_remote
assert layers[0].layer_info.urls == ['http://hello/world']