Add support for pushing and pulling schema 2 manifests with remote layers
This is required for windows image support
This commit is contained in:
parent
d97055e2ba
commit
37b20010aa
19 changed files with 339 additions and 29 deletions
|
@ -57,7 +57,7 @@ def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
|
|||
|
||||
|
||||
def _create_manifest(repository_id, manifest_interface_instance, storage):
|
||||
digests = set(manifest_interface_instance.blob_digests)
|
||||
digests = set(manifest_interface_instance.local_blob_digests)
|
||||
|
||||
def _lookup_digest(digest):
|
||||
return _retrieve_bytes_in_storage(repository_id, digest, storage)
|
||||
|
@ -111,13 +111,15 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
|
|||
child_manifest_label_dicts.append(labels)
|
||||
|
||||
# Ensure all the blobs in the manifest exist.
|
||||
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
|
||||
blob_map = {s.content_checksum: s for s in query}
|
||||
for digest_str in digests:
|
||||
if digest_str not in blob_map:
|
||||
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
|
||||
manifest_interface_instance.digest, repository_id)
|
||||
return None
|
||||
blob_map = {}
|
||||
if digests:
|
||||
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
|
||||
blob_map = {s.content_checksum: s for s in query}
|
||||
for digest_str in digests:
|
||||
if digest_str not in blob_map:
|
||||
logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str,
|
||||
manifest_interface_instance.digest, repository_id)
|
||||
return None
|
||||
|
||||
# Determine and populate the legacy image if necessary. Manifest lists will not have a legacy
|
||||
# image.
|
||||
|
|
|
@ -239,3 +239,68 @@ def test_get_or_create_manifest_list(initialized_db):
|
|||
|
||||
assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type
|
||||
assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type
|
||||
|
||||
|
||||
def test_get_or_create_manifest_with_remote_layers(initialized_db):
|
||||
repository = create_repository('devtable', 'newrepo', None)
|
||||
|
||||
layer_json = json.dumps({
|
||||
'config': {},
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"created": "2018-04-03T18:37:09.284840891Z",
|
||||
"created_by": "do something",
|
||||
},
|
||||
{
|
||||
"created": "2018-04-03T18:37:09.284840891Z",
|
||||
"created_by": "do something",
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
# Add a blob containing the config.
|
||||
_, config_digest = _populate_blob(layer_json)
|
||||
|
||||
# Add a blob of random data.
|
||||
random_data = 'hello world'
|
||||
_, random_digest = _populate_blob(random_data)
|
||||
|
||||
remote_digest = sha256_digest('something')
|
||||
|
||||
builder = DockerSchema2ManifestBuilder()
|
||||
builder.set_config_digest(config_digest, len(layer_json))
|
||||
builder.add_layer(remote_digest, 1234, urls=['http://hello/world'])
|
||||
builder.add_layer(random_digest, len(random_data))
|
||||
manifest = builder.build()
|
||||
|
||||
assert remote_digest in manifest.blob_digests
|
||||
assert remote_digest not in manifest.local_blob_digests
|
||||
|
||||
assert manifest.has_remote_layer
|
||||
assert manifest.leaf_layer_v1_image_id is None
|
||||
assert manifest.get_v1_compatible_manifest('foo', 'bar', 'baz', None) is None
|
||||
|
||||
# Write the manifest.
|
||||
created_tuple = get_or_create_manifest(repository, manifest, storage)
|
||||
assert created_tuple is not None
|
||||
|
||||
created_manifest = created_tuple.manifest
|
||||
assert created_manifest
|
||||
assert created_manifest.media_type.name == manifest.media_type
|
||||
assert created_manifest.digest == manifest.digest
|
||||
|
||||
# Verify the legacy image.
|
||||
legacy_image = get_legacy_image_for_manifest(created_manifest)
|
||||
assert legacy_image is None
|
||||
|
||||
# Verify the linked blobs.
|
||||
blob_digests = {mb.blob.content_checksum for mb
|
||||
in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)}
|
||||
|
||||
assert random_digest in blob_digests
|
||||
assert config_digest in blob_digests
|
||||
assert remote_digest not in blob_digests
|
||||
|
|
|
@ -766,7 +766,7 @@ def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_laye
|
|||
raise DataModelException('Invalid image with id: %s' % leaf_layer_id)
|
||||
|
||||
storage_ids = set()
|
||||
for blob_digest in manifest.blob_digests:
|
||||
for blob_digest in manifest.local_blob_digests:
|
||||
image_storage_id = storage_id_map.get(blob_digest)
|
||||
if image_storage_id is None:
|
||||
logger.error('Missing blob for manifest `%s` in: %s', blob_digest, storage_id_map)
|
||||
|
|
|
@ -153,6 +153,14 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta
|
|||
"""
|
||||
return legacy_image
|
||||
|
||||
@property
|
||||
@optionalinput('legacy_image')
|
||||
def legacy_image_if_present(self, legacy_image):
|
||||
""" Returns the legacy Docker V1-style image for this tag. Note that this
|
||||
will be None for tags whose manifests point to other manifests instead of images.
|
||||
"""
|
||||
return legacy_image
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
""" The ID of this tag for pagination purposes only. """
|
||||
|
@ -266,7 +274,8 @@ class SecurityScanStatus(Enum):
|
|||
class ManifestLayer(namedtuple('ManifestLayer', ['layer_info', 'blob'])):
|
||||
""" Represents a single layer in a manifest. The `layer_info` data will be manifest-type specific,
|
||||
but will have a few expected fields (such as `digest`). The `blob` represents the associated
|
||||
blob for this layer, optionally with placements.
|
||||
blob for this layer, optionally with placements. If the layer is a remote layer, the blob will
|
||||
be None.
|
||||
"""
|
||||
|
||||
def estimated_size(self, estimate_multiplier):
|
||||
|
|
|
@ -317,12 +317,18 @@ class SharedModel:
|
|||
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
|
||||
return None
|
||||
|
||||
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
|
||||
parsed.blob_digests)
|
||||
storage_map = {blob.content_checksum: blob for blob in blob_query}
|
||||
storage_map = {}
|
||||
if parsed.local_blob_digests:
|
||||
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id,
|
||||
parsed.local_blob_digests)
|
||||
storage_map = {blob.content_checksum: blob for blob in blob_query}
|
||||
|
||||
manifest_layers = []
|
||||
for layer in parsed.layers:
|
||||
if layer.is_remote:
|
||||
manifest_layers.append(ManifestLayer(layer, None))
|
||||
continue
|
||||
|
||||
digest_str = str(layer.digest)
|
||||
if digest_str not in storage_map:
|
||||
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
|
||||
|
|
|
@ -3,6 +3,7 @@ import json
|
|||
import uuid
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
|
||||
|
@ -19,7 +20,9 @@ from data.cache.impl import InMemoryDataModelCache
|
|||
from data.registry_model.registry_pre_oci_model import PreOCIModel
|
||||
from data.registry_model.registry_oci_model import OCIModel
|
||||
from data.registry_model.datatypes import RepositoryReference
|
||||
from data.registry_model.blobuploader import upload_blob, BlobUploadSettings
|
||||
from image.docker.schema1 import DockerSchema1ManifestBuilder
|
||||
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
|
@ -32,6 +35,10 @@ def registry_model(request, initialized_db):
|
|||
def pre_oci_model(initialized_db):
|
||||
return PreOCIModel()
|
||||
|
||||
@pytest.fixture()
|
||||
def oci_model(initialized_db):
|
||||
return OCIModel()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('names, expected', [
|
||||
(['unknown'], None),
|
||||
|
@ -481,6 +488,45 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
|
|||
assert manifest_layer.estimated_size(1) is not None
|
||||
|
||||
|
||||
def test_manifest_remote_layers(oci_model):
|
||||
# Create a config blob for testing.
|
||||
config_json = json.dumps({
|
||||
'config': {},
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"created": "2018-04-03T18:37:09.284840891Z",
|
||||
"created_by": "do something",
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
app_config = {'TESTING': True}
|
||||
repository_ref = oci_model.lookup_repository('devtable', 'simple')
|
||||
with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500, 500)) as upload:
|
||||
upload.upload_chunk(app_config, BytesIO(config_json))
|
||||
blob = upload.commit_to_blob(app_config)
|
||||
|
||||
# Create the manifest in the repo.
|
||||
builder = DockerSchema2ManifestBuilder()
|
||||
builder.set_config_digest(blob.digest, blob.compressed_size)
|
||||
builder.add_layer('sha256:abcd', 1234, urls=['http://hello/world'])
|
||||
manifest = builder.build()
|
||||
|
||||
created_manifest, _ = oci_model.create_manifest_and_retarget_tag(repository_ref, manifest,
|
||||
'sometag', storage)
|
||||
assert created_manifest
|
||||
|
||||
layers = oci_model.list_manifest_layers(created_manifest)
|
||||
assert len(layers) == 1
|
||||
assert layers[0].layer_info.is_remote
|
||||
assert layers[0].layer_info.urls == ['http://hello/world']
|
||||
assert layers[0].blob is None
|
||||
|
||||
|
||||
def test_derived_image(registry_model):
|
||||
# Clear all existing derived storage.
|
||||
DerivedStorageForImage.delete().execute()
|
||||
|
|
|
@ -30,7 +30,8 @@ def get_tags(namespace_name, repo_name):
|
|||
# TODO(jschorr): Change this to normalize manifest lists back to their legacy image
|
||||
# (if applicable).
|
||||
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
|
||||
tag_map = {tag.name: tag.legacy_image.docker_image_id for tag in tags if tag.legacy_image}
|
||||
tag_map = {tag.name: tag.legacy_image.docker_image_id
|
||||
for tag in tags if tag.legacy_image_if_present}
|
||||
return jsonify(tag_map)
|
||||
|
||||
abort(403)
|
||||
|
|
|
@ -53,6 +53,13 @@ class ManifestInterface(object):
|
|||
config as a blob, the blob will be included here.
|
||||
"""
|
||||
|
||||
@abstractproperty
|
||||
def local_blob_digests(self):
|
||||
""" Returns an iterator over all the *non-remote* blob digests referenced by this manifest,
|
||||
from base to leaf. The blob digests are strings with prefixes. For manifests that reference
|
||||
config as a blob, the blob will be included here.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def child_manifests(self, lookup_manifest_fn):
|
||||
""" Returns an iterator of all manifests that live under this manifest, if any or None if not
|
||||
|
|
|
@ -73,7 +73,7 @@ class InvalidSchema1Signature(ManifestException):
|
|||
|
||||
|
||||
class Schema1Layer(namedtuple('Schema1Layer', ['digest', 'v1_metadata', 'raw_v1_metadata',
|
||||
'compressed_size'])):
|
||||
'compressed_size', 'is_remote'])):
|
||||
"""
|
||||
Represents all of the data about an individual layer in a given Manifest.
|
||||
This is the union of the fsLayers (digest) and the history entries (v1_compatibility).
|
||||
|
@ -302,6 +302,10 @@ class DockerSchema1Manifest(ManifestInterface):
|
|||
def blob_digests(self):
|
||||
return [str(layer.digest) for layer in self.layers]
|
||||
|
||||
@property
|
||||
def local_blob_digests(self):
|
||||
return self.blob_digests
|
||||
|
||||
def child_manifests(self, lookup_manifest_fn):
|
||||
return None
|
||||
|
||||
|
@ -349,7 +353,7 @@ class DockerSchema1Manifest(ManifestInterface):
|
|||
command, labels)
|
||||
|
||||
compressed_size = v1_metadata.get('Size')
|
||||
yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size)
|
||||
yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size, False)
|
||||
|
||||
@property
|
||||
def _payload(self):
|
||||
|
|
|
@ -224,6 +224,10 @@ class DockerSchema2ManifestList(ManifestInterface):
|
|||
manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY]
|
||||
return [m[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY] for m in manifests]
|
||||
|
||||
@property
|
||||
def local_blob_digests(self):
|
||||
return self.blob_digests
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def manifests(self, lookup_manifest_fn):
|
||||
""" Returns the manifests in the list. The `lookup_manifest_fn` is a function
|
||||
|
|
|
@ -136,6 +136,10 @@ class DockerSchema2Manifest(ManifestInterface):
|
|||
except ValidationError as ve:
|
||||
raise MalformedSchema2Manifest('manifest data does not match schema: %s' % ve)
|
||||
|
||||
for layer in self.layers:
|
||||
if layer.is_remote and not layer.urls:
|
||||
raise MalformedSchema2Manifest('missing `urls` for remote layer')
|
||||
|
||||
@property
|
||||
def schema_version(self):
|
||||
return 2
|
||||
|
@ -169,18 +173,39 @@ class DockerSchema2Manifest(ManifestInterface):
|
|||
def leaf_layer(self):
|
||||
return self.layers[-1]
|
||||
|
||||
@property
|
||||
def has_remote_layer(self):
|
||||
for layer in self.layers:
|
||||
if layer.is_remote:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@property
|
||||
def leaf_layer_v1_image_id(self):
|
||||
# NOTE: If there exists a layer with remote content, then we consider this manifest
|
||||
# to not support legacy images.
|
||||
if self.has_remote_layer:
|
||||
return None
|
||||
|
||||
return list(self.layers_with_v1_ids)[-1].v1_id
|
||||
|
||||
@property
|
||||
def legacy_image_ids(self):
|
||||
if self.has_remote_layer:
|
||||
return None
|
||||
|
||||
return [l.v1_id for l in self.layers_with_v1_ids]
|
||||
|
||||
@property
|
||||
def blob_digests(self):
|
||||
return [str(layer.digest) for layer in self.layers] + [str(self.config.digest)]
|
||||
|
||||
@property
|
||||
def local_blob_digests(self):
|
||||
return ([str(layer.digest) for layer in self.layers if not layer.urls] +
|
||||
[str(self.config.digest)])
|
||||
|
||||
def get_manifest_labels(self, lookup_config_fn):
|
||||
return self._get_built_config(lookup_config_fn).labels
|
||||
|
||||
|
@ -218,6 +243,7 @@ class DockerSchema2Manifest(ManifestInterface):
|
|||
|
||||
@property
|
||||
def layers_with_v1_ids(self):
|
||||
assert not self.has_remote_layer
|
||||
digest_history = hashlib.sha256()
|
||||
v1_layer_parent_id = None
|
||||
v1_layer_id = None
|
||||
|
@ -240,6 +266,7 @@ class DockerSchema2Manifest(ManifestInterface):
|
|||
this schema. The `lookup_config_fn` is a function that, when given the config
|
||||
digest SHA, returns the associated configuration JSON bytes for this schema.
|
||||
"""
|
||||
assert not self.has_remote_layer
|
||||
schema2_config = self._get_built_config(lookup_config_fn)
|
||||
|
||||
# Build the V1 IDs for the layers.
|
||||
|
@ -253,6 +280,8 @@ class DockerSchema2Manifest(ManifestInterface):
|
|||
return v1_builder
|
||||
|
||||
def generate_legacy_layers(self, images_map, lookup_config_fn):
|
||||
assert not self.has_remote_layer
|
||||
|
||||
# NOTE: We use the DockerSchema1ManifestBuilder here because it already contains
|
||||
# the logic for generating the DockerV1Metadata. All of this will go away once we get
|
||||
# rid of legacy images in the database, so this is a temporary solution.
|
||||
|
@ -261,6 +290,9 @@ class DockerSchema2Manifest(ManifestInterface):
|
|||
return v1_builder.build().generate_legacy_layers(images_map, lookup_config_fn)
|
||||
|
||||
def get_v1_compatible_manifest(self, namespace_name, repo_name, tag_name, lookup_fn):
|
||||
if self.has_remote_layer:
|
||||
return None
|
||||
|
||||
v1_builder = DockerSchema1ManifestBuilder(namespace_name, repo_name, tag_name)
|
||||
self.populate_schema1_builder(v1_builder, lookup_fn)
|
||||
return v1_builder.build()
|
||||
|
|
|
@ -28,7 +28,7 @@ MANIFESTLIST_BYTES = json.dumps({
|
|||
"manifests": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"size": 983,
|
||||
"size": 946,
|
||||
"digest": "sha256:e6",
|
||||
"platform": {
|
||||
"architecture": "ppc64le",
|
||||
|
@ -56,7 +56,7 @@ NO_AMD_MANIFESTLIST_BYTES = json.dumps({
|
|||
"manifests": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"size": 983,
|
||||
"size": 946,
|
||||
"digest": "sha256:e6",
|
||||
"platform": {
|
||||
"architecture": "ppc64le",
|
||||
|
@ -76,7 +76,7 @@ def test_valid_manifestlist():
|
|||
manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES)
|
||||
assert len(manifestlist.manifests(_get_manifest)) == 2
|
||||
assert (manifestlist.digest ==
|
||||
'sha256:7e22fdbe49736329786c9b4fdc154cc9251b190ca6b4cf33aed00efc0fc3df25')
|
||||
'sha256:340d7dadea77035533a2d43e8ff711ecca1965978a6e7699b87e32b35f76038d')
|
||||
|
||||
assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
|
||||
assert manifestlist.bytes == MANIFESTLIST_BYTES
|
||||
|
@ -109,7 +109,7 @@ def test_get_v1_compatible_manifest_no_matching_list():
|
|||
manifestlist = DockerSchema2ManifestList(NO_AMD_MANIFESTLIST_BYTES)
|
||||
assert len(manifestlist.manifests(_get_manifest)) == 1
|
||||
assert (manifestlist.digest ==
|
||||
'sha256:50150251101420a020ab4a3e77e9d167a18b09bd4eeb0cc65e0eafab95cf79cf')
|
||||
'sha256:40ed1cfe692333bfa519a9bfed9676975a990fff5afd35efa628320c39c793ca')
|
||||
|
||||
assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json'
|
||||
assert manifestlist.bytes == NO_AMD_MANIFESTLIST_BYTES
|
||||
|
|
|
@ -31,6 +31,38 @@ MANIFEST_BYTES = json.dumps({
|
|||
"size": 1885,
|
||||
"digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 1234,
|
||||
"digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736",
|
||||
},
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 32654,
|
||||
"digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"
|
||||
},
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 16724,
|
||||
"digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
|
||||
},
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 73109,
|
||||
"digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
REMOTE_MANIFEST_BYTES = json.dumps({
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": 1885,
|
||||
"digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
|
||||
|
@ -61,12 +93,12 @@ def test_valid_manifest():
|
|||
assert manifest.config.size == 1885
|
||||
assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
|
||||
assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
|
||||
assert not manifest.has_remote_layer
|
||||
|
||||
assert len(manifest.layers) == 4
|
||||
assert manifest.layers[0].is_remote
|
||||
assert manifest.layers[0].compressed_size == 1234
|
||||
assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
|
||||
assert manifest.layers[0].urls
|
||||
assert not manifest.layers[0].is_remote
|
||||
|
||||
assert manifest.leaf_layer == manifest.layers[3]
|
||||
assert not manifest.leaf_layer.is_remote
|
||||
|
@ -75,6 +107,37 @@ def test_valid_manifest():
|
|||
blob_digests = list(manifest.blob_digests)
|
||||
expected = [str(layer.digest) for layer in manifest.layers] + [manifest.config.digest]
|
||||
assert blob_digests == expected
|
||||
assert list(manifest.local_blob_digests) == expected
|
||||
|
||||
|
||||
def test_valid_remote_manifest():
|
||||
manifest = DockerSchema2Manifest(REMOTE_MANIFEST_BYTES)
|
||||
assert manifest.config.size == 1885
|
||||
assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7'
|
||||
assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json"
|
||||
assert manifest.has_remote_layer
|
||||
|
||||
assert len(manifest.layers) == 4
|
||||
assert manifest.layers[0].compressed_size == 1234
|
||||
assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736'
|
||||
assert manifest.layers[0].is_remote
|
||||
assert manifest.layers[0].urls == ['http://some/url']
|
||||
|
||||
assert manifest.leaf_layer == manifest.layers[3]
|
||||
assert not manifest.leaf_layer.is_remote
|
||||
assert manifest.leaf_layer.compressed_size == 73109
|
||||
|
||||
expected = set([str(layer.digest) for layer in manifest.layers] + [manifest.config.digest])
|
||||
|
||||
blob_digests = set(manifest.blob_digests)
|
||||
local_digests = set(manifest.local_blob_digests)
|
||||
|
||||
assert blob_digests == expected
|
||||
assert local_digests == (expected - {manifest.layers[0].digest})
|
||||
|
||||
assert manifest.has_remote_layer
|
||||
assert manifest.leaf_layer_v1_image_id is None
|
||||
assert manifest.legacy_image_ids is None
|
||||
|
||||
|
||||
def test_schema2_builder():
|
||||
|
@ -110,6 +173,7 @@ def test_get_manifest_labels():
|
|||
|
||||
def test_build_schema1():
|
||||
manifest = DockerSchema2Manifest(MANIFEST_BYTES)
|
||||
assert not manifest.has_remote_layer
|
||||
|
||||
builder = DockerSchema1ManifestBuilder('somenamespace', 'somename', 'sometag')
|
||||
manifest.populate_schema1_builder(builder, lambda digest: CONFIG_BYTES)
|
||||
|
@ -209,3 +273,22 @@ def test_generate_legacy_layers():
|
|||
assert legacy_layers[0].parent_image_id is None
|
||||
|
||||
assert legacy_layers[0].image_id != legacy_layers[1]
|
||||
|
||||
|
||||
def test_remote_layer_manifest():
|
||||
builder = DockerSchema2ManifestBuilder()
|
||||
builder.set_config_digest('sha256:abcd', 1234)
|
||||
builder.add_layer('sha256:adef', 1234, urls=['http://some/url'])
|
||||
builder.add_layer('sha256:1352', 4567)
|
||||
builder.add_layer('sha256:1353', 4567)
|
||||
manifest = builder.build()
|
||||
|
||||
assert manifest.has_remote_layer
|
||||
assert manifest.leaf_layer_v1_image_id is None
|
||||
assert manifest.legacy_image_ids is None
|
||||
|
||||
schema1 = manifest.get_v1_compatible_manifest('somenamespace', 'somename', 'sometag', None)
|
||||
assert schema1 is None
|
||||
|
||||
assert set(manifest.blob_digests) == {'sha256:adef', 'sha256:abcd', 'sha256:1352', 'sha256:1353'}
|
||||
assert set(manifest.local_blob_digests) == {'sha256:abcd', 'sha256:1352', 'sha256:1353'}
|
||||
|
|
|
@ -92,6 +92,20 @@ def multi_layer_images():
|
|||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def remote_images():
|
||||
""" Returns images with at least one remote layer for push and pull testing. """
|
||||
# Note: order is from base layer down to leaf.
|
||||
remote_bytes = layer_bytes_for_contents('remote contents')
|
||||
parent_bytes = layer_bytes_for_contents('parent contents')
|
||||
image_bytes = layer_bytes_for_contents('some contents')
|
||||
return [
|
||||
Image(id='remoteid', bytes=remote_bytes, parent_id=None, urls=['http://some/url']),
|
||||
Image(id='parentid', bytes=parent_bytes, parent_id='remoteid'),
|
||||
Image(id='someid', bytes=image_bytes, parent_id='parentid'),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def jwk():
|
||||
return RSAKey(key=RSA.generate(2048))
|
||||
|
|
|
@ -93,7 +93,6 @@ class V1Protocol(RegistryProtocol):
|
|||
|
||||
# GET /v1/repositories/{namespace}/{repository}/tags
|
||||
image_ids = self.conduct(session, 'GET', prefix + 'tags', headers=headers).json()
|
||||
assert len(image_ids.values()) >= len(tag_names)
|
||||
|
||||
for tag_name in tag_names:
|
||||
if tag_name not in image_ids:
|
||||
|
@ -145,6 +144,8 @@ class V1Protocol(RegistryProtocol):
|
|||
headers['Authorization'] = 'token ' + result.headers['www-authenticate']
|
||||
|
||||
for image in images:
|
||||
assert image.urls is None
|
||||
|
||||
# PUT /v1/images/{imageID}/json
|
||||
image_json_data = {'id': image.id}
|
||||
if image.size is not None:
|
||||
|
|
|
@ -209,13 +209,15 @@ class V2Protocol(RegistryProtocol):
|
|||
builder = DockerSchema2ManifestBuilder()
|
||||
for image in images:
|
||||
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
if image.urls is None:
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
# If invalid blob references were requested, just make it up.
|
||||
if options.manifest_invalid_blob_references:
|
||||
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
|
||||
|
||||
builder.add_layer(checksum, len(image.bytes))
|
||||
builder.add_layer(checksum, len(image.bytes), urls=image.urls)
|
||||
|
||||
config = {
|
||||
"os": "linux",
|
||||
|
@ -245,6 +247,8 @@ class V2Protocol(RegistryProtocol):
|
|||
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
|
||||
|
||||
for image in reversed(images):
|
||||
assert image.urls is None
|
||||
|
||||
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
|
@ -498,12 +502,16 @@ class V2Protocol(RegistryProtocol):
|
|||
|
||||
# Verify the layers.
|
||||
for index, layer in enumerate(manifest.layers):
|
||||
# If the layer is remote, then we expect the blob to *not* exist in the system.
|
||||
expected_status = 404 if images[index].urls else 200
|
||||
result = self.conduct(session, 'GET',
|
||||
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name),
|
||||
layer.digest),
|
||||
expected_status=200,
|
||||
expected_status=expected_status,
|
||||
headers=headers)
|
||||
assert result.content == images[index].bytes
|
||||
|
||||
if expected_status == 200:
|
||||
assert result.content == images[index].bytes
|
||||
|
||||
return PullResult(manifests=manifests, image_ids=image_ids)
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@ from cStringIO import StringIO
|
|||
from enum import Enum, unique
|
||||
from six import add_metaclass
|
||||
|
||||
Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created'])
|
||||
Image.__new__.__defaults__ = (None, None, None)
|
||||
Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created', 'urls'])
|
||||
Image.__new__.__defaults__ = (None, None, None, None)
|
||||
|
||||
PushResult = namedtuple('PushResult', ['manifests', 'headers'])
|
||||
PullResult = namedtuple('PullResult', ['manifests', 'image_ids'])
|
||||
|
|
|
@ -21,6 +21,7 @@ from app import instance_keys
|
|||
from data.model.tag import list_repository_tags
|
||||
from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
|
||||
from image.docker.schema2.list import DockerSchema2ManifestListBuilder
|
||||
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
|
||||
from util.security.registry_jwt import decode_bearer_header
|
||||
from util.timedeltastring import convert_to_timedelta
|
||||
|
||||
|
@ -1450,3 +1451,26 @@ def test_push_pull_manifest_list(v22_protocol, basic_images, different_images, l
|
|||
# Pull and verify the manifest list.
|
||||
v22_protocol.pull_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist,
|
||||
credentials=credentials, options=options)
|
||||
|
||||
|
||||
def test_push_pull_manifest_remote_layers(v22_protocol, legacy_puller, liveserver_session,
|
||||
app_reloader, remote_images, data_model):
|
||||
""" Test: Push a new tag with a manifest which contains at least one remote layer, and then
|
||||
pull that manifest back.
|
||||
"""
|
||||
if data_model != 'oci_model':
|
||||
return
|
||||
|
||||
credentials = ('devtable', 'password')
|
||||
|
||||
# Push a new repository.
|
||||
v22_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
|
||||
credentials=credentials)
|
||||
|
||||
# Pull the repository to verify.
|
||||
v22_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
|
||||
credentials=credentials)
|
||||
|
||||
# Ensure that the image cannot be pulled by a legacy protocol.
|
||||
legacy_puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images,
|
||||
credentials=credentials, expected_failure=Failures.UNKNOWN_TAG)
|
||||
|
|
|
@ -63,6 +63,10 @@ class BrokenManifest(ManifestInterface):
|
|||
def blob_digests(self):
|
||||
return []
|
||||
|
||||
@property
|
||||
def local_blob_digests(self):
|
||||
return []
|
||||
|
||||
def child_manifests(self, lookup_manifest_fn):
|
||||
return None
|
||||
|
||||
|
|
Reference in a new issue