diff --git a/data/model/oci/manifest.py b/data/model/oci/manifest.py index 3d2d217a5..c63437e9f 100644 --- a/data/model/oci/manifest.py +++ b/data/model/oci/manifest.py @@ -57,7 +57,7 @@ def get_or_create_manifest(repository_id, manifest_interface_instance, storage): def _create_manifest(repository_id, manifest_interface_instance, storage): - digests = set(manifest_interface_instance.blob_digests) + digests = set(manifest_interface_instance.local_blob_digests) def _lookup_digest(digest): return _retrieve_bytes_in_storage(repository_id, digest, storage) @@ -111,13 +111,15 @@ def _create_manifest(repository_id, manifest_interface_instance, storage): child_manifest_label_dicts.append(labels) # Ensure all the blobs in the manifest exist. - query = lookup_repo_storages_by_content_checksum(repository_id, digests) - blob_map = {s.content_checksum: s for s in query} - for digest_str in digests: - if digest_str not in blob_map: - logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str, - manifest_interface_instance.digest, repository_id) - return None + blob_map = {} + if digests: + query = lookup_repo_storages_by_content_checksum(repository_id, digests) + blob_map = {s.content_checksum: s for s in query} + for digest_str in digests: + if digest_str not in blob_map: + logger.warning('Unknown blob `%s` under manifest `%s` for repository `%s`', digest_str, + manifest_interface_instance.digest, repository_id) + return None # Determine and populate the legacy image if necessary. Manifest lists will not have a legacy # image. diff --git a/data/model/oci/test/test_oci_manifest.py b/data/model/oci/test/test_oci_manifest.py index 8d3b77b2f..45df20597 100644 --- a/data/model/oci/test/test_oci_manifest.py +++ b/data/model/oci/test/test_oci_manifest.py @@ -239,3 +239,68 @@ def test_get_or_create_manifest_list(initialized_db): assert child_manifests[v1_manifest.digest].media_type.name == v1_manifest.media_type assert child_manifests[v2_manifest.digest].media_type.name == v2_manifest.media_type + + +def test_get_or_create_manifest_with_remote_layers(initialized_db): + repository = create_repository('devtable', 'newrepo', None) + + layer_json = json.dumps({ + 'config': {}, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2018-04-03T18:37:09.284840891Z", + "created_by": "do something", + }, + { + "created": "2018-04-03T18:37:09.284840891Z", + "created_by": "do something", + }, + ], + }) + + # Add a blob containing the config. + _, config_digest = _populate_blob(layer_json) + + # Add a blob of random data. + random_data = 'hello world' + _, random_digest = _populate_blob(random_data) + + remote_digest = sha256_digest('something') + + builder = DockerSchema2ManifestBuilder() + builder.set_config_digest(config_digest, len(layer_json)) + builder.add_layer(remote_digest, 1234, urls=['http://hello/world']) + builder.add_layer(random_digest, len(random_data)) + manifest = builder.build() + + assert remote_digest in manifest.blob_digests + assert remote_digest not in manifest.local_blob_digests + + assert manifest.has_remote_layer + assert manifest.leaf_layer_v1_image_id is None + assert manifest.get_v1_compatible_manifest('foo', 'bar', 'baz', None) is None + + # Write the manifest. + created_tuple = get_or_create_manifest(repository, manifest, storage) + assert created_tuple is not None + + created_manifest = created_tuple.manifest + assert created_manifest + assert created_manifest.media_type.name == manifest.media_type + assert created_manifest.digest == manifest.digest + + # Verify the legacy image. + legacy_image = get_legacy_image_for_manifest(created_manifest) + assert legacy_image is None + + # Verify the linked blobs. + blob_digests = {mb.blob.content_checksum for mb + in ManifestBlob.select().where(ManifestBlob.manifest == created_manifest)} + + assert random_digest in blob_digests + assert config_digest in blob_digests + assert remote_digest not in blob_digests diff --git a/data/model/tag.py b/data/model/tag.py index a509df979..068641e8a 100644 --- a/data/model/tag.py +++ b/data/model/tag.py @@ -766,7 +766,7 @@ def _populate_manifest_and_blobs(repository, manifest, storage_id_map, leaf_laye raise DataModelException('Invalid image with id: %s' % leaf_layer_id) storage_ids = set() - for blob_digest in manifest.blob_digests: + for blob_digest in manifest.local_blob_digests: image_storage_id = storage_id_map.get(blob_digest) if image_storage_id is None: logger.error('Missing blob for manifest `%s` in: %s', blob_digest, storage_id_map) diff --git a/data/registry_model/datatypes.py b/data/registry_model/datatypes.py index 8c9fde328..224beae26 100644 --- a/data/registry_model/datatypes.py +++ b/data/registry_model/datatypes.py @@ -153,6 +153,14 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta """ return legacy_image + @property + @optionalinput('legacy_image') + def legacy_image_if_present(self, legacy_image): + """ Returns the legacy Docker V1-style image for this tag. Note that this + will be None for tags whose manifests point to other manifests instead of images. + """ + return legacy_image + @property def id(self): """ The ID of this tag for pagination purposes only. """ @@ -266,7 +274,8 @@ class SecurityScanStatus(Enum): class ManifestLayer(namedtuple('ManifestLayer', ['layer_info', 'blob'])): """ Represents a single layer in a manifest. The `layer_info` data will be manifest-type specific, but will have a few expected fields (such as `digest`). The `blob` represents the associated - blob for this layer, optionally with placements. + blob for this layer, optionally with placements. If the layer is a remote layer, the blob will + be None. """ def estimated_size(self, estimate_multiplier): diff --git a/data/registry_model/shared.py b/data/registry_model/shared.py index 82488986f..36bc566a9 100644 --- a/data/registry_model/shared.py +++ b/data/registry_model/shared.py @@ -317,12 +317,18 @@ class SharedModel: logger.exception('Could not parse and validate manifest `%s`', manifest._db_id) return None - blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id, - parsed.blob_digests) - storage_map = {blob.content_checksum: blob for blob in blob_query} + storage_map = {} + if parsed.local_blob_digests: + blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id, + parsed.local_blob_digests) + storage_map = {blob.content_checksum: blob for blob in blob_query} manifest_layers = [] for layer in parsed.layers: + if layer.is_remote: + manifest_layers.append(ManifestLayer(layer, None)) + continue + digest_str = str(layer.digest) if digest_str not in storage_map: logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id) diff --git a/data/registry_model/test/test_interface.py b/data/registry_model/test/test_interface.py index 2f6647c65..f6034a448 100644 --- a/data/registry_model/test/test_interface.py +++ b/data/registry_model/test/test_interface.py @@ -3,6 +3,7 @@ import json import uuid from datetime import datetime, timedelta +from io import BytesIO import pytest @@ -19,7 +20,9 @@ from data.cache.impl import InMemoryDataModelCache from data.registry_model.registry_pre_oci_model import PreOCIModel from data.registry_model.registry_oci_model import OCIModel from data.registry_model.datatypes import RepositoryReference +from data.registry_model.blobuploader import upload_blob, BlobUploadSettings from image.docker.schema1 import DockerSchema1ManifestBuilder +from image.docker.schema2.manifest import DockerSchema2ManifestBuilder from test.fixtures import * @@ -32,6 +35,10 @@ def registry_model(request, initialized_db): def pre_oci_model(initialized_db): return PreOCIModel() +@pytest.fixture() +def oci_model(initialized_db): + return OCIModel() + @pytest.mark.parametrize('names, expected', [ (['unknown'], None), @@ -481,6 +488,45 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model): assert manifest_layer.estimated_size(1) is not None +def test_manifest_remote_layers(oci_model): + # Create a config blob for testing. + config_json = json.dumps({ + 'config': {}, + "rootfs": { + "type": "layers", + "diff_ids": [] + }, + "history": [ + { + "created": "2018-04-03T18:37:09.284840891Z", + "created_by": "do something", + }, + ], + }) + + app_config = {'TESTING': True} + repository_ref = oci_model.lookup_repository('devtable', 'simple') + with upload_blob(repository_ref, storage, BlobUploadSettings(500, 500, 500)) as upload: + upload.upload_chunk(app_config, BytesIO(config_json)) + blob = upload.commit_to_blob(app_config) + + # Create the manifest in the repo. + builder = DockerSchema2ManifestBuilder() + builder.set_config_digest(blob.digest, blob.compressed_size) + builder.add_layer('sha256:abcd', 1234, urls=['http://hello/world']) + manifest = builder.build() + + created_manifest, _ = oci_model.create_manifest_and_retarget_tag(repository_ref, manifest, + 'sometag', storage) + assert created_manifest + + layers = oci_model.list_manifest_layers(created_manifest) + assert len(layers) == 1 + assert layers[0].layer_info.is_remote + assert layers[0].layer_info.urls == ['http://hello/world'] + assert layers[0].blob is None + + def test_derived_image(registry_model): # Clear all existing derived storage. DerivedStorageForImage.delete().execute() diff --git a/endpoints/v1/tag.py b/endpoints/v1/tag.py index fb8cb77d6..7b23309df 100644 --- a/endpoints/v1/tag.py +++ b/endpoints/v1/tag.py @@ -30,7 +30,8 @@ def get_tags(namespace_name, repo_name): # TODO(jschorr): Change this to normalize manifest lists back to their legacy image # (if applicable). tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True) - tag_map = {tag.name: tag.legacy_image.docker_image_id for tag in tags if tag.legacy_image} + tag_map = {tag.name: tag.legacy_image.docker_image_id + for tag in tags if tag.legacy_image_if_present} return jsonify(tag_map) abort(403) diff --git a/image/docker/interfaces.py b/image/docker/interfaces.py index e8cbd8dd9..a275caba7 100644 --- a/image/docker/interfaces.py +++ b/image/docker/interfaces.py @@ -53,6 +53,13 @@ class ManifestInterface(object): config as a blob, the blob will be included here. """ + @abstractproperty + def local_blob_digests(self): + """ Returns an iterator over all the *non-remote* blob digests referenced by this manifest, + from base to leaf. The blob digests are strings with prefixes. For manifests that reference + config as a blob, the blob will be included here. + """ + @abstractmethod def child_manifests(self, lookup_manifest_fn): """ Returns an iterator of all manifests that live under this manifest, if any or None if not diff --git a/image/docker/schema1.py b/image/docker/schema1.py index 11ace16e8..02f557fdd 100644 --- a/image/docker/schema1.py +++ b/image/docker/schema1.py @@ -73,7 +73,7 @@ class InvalidSchema1Signature(ManifestException): class Schema1Layer(namedtuple('Schema1Layer', ['digest', 'v1_metadata', 'raw_v1_metadata', - 'compressed_size'])): + 'compressed_size', 'is_remote'])): """ Represents all of the data about an individual layer in a given Manifest. This is the union of the fsLayers (digest) and the history entries (v1_compatibility). @@ -302,6 +302,10 @@ class DockerSchema1Manifest(ManifestInterface): def blob_digests(self): return [str(layer.digest) for layer in self.layers] + @property + def local_blob_digests(self): + return self.blob_digests + def child_manifests(self, lookup_manifest_fn): return None @@ -349,7 +353,7 @@ class DockerSchema1Manifest(ManifestInterface): command, labels) compressed_size = v1_metadata.get('Size') - yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size) + yield Schema1Layer(image_digest, extracted, metadata_string, compressed_size, False) @property def _payload(self): diff --git a/image/docker/schema2/list.py b/image/docker/schema2/list.py index fcc228ada..a7f383d7b 100644 --- a/image/docker/schema2/list.py +++ b/image/docker/schema2/list.py @@ -224,6 +224,10 @@ class DockerSchema2ManifestList(ManifestInterface): manifests = self._parsed[DOCKER_SCHEMA2_MANIFESTLIST_MANIFESTS_KEY] return [m[DOCKER_SCHEMA2_MANIFESTLIST_DIGEST_KEY] for m in manifests] + @property + def local_blob_digests(self): + return self.blob_digests + @lru_cache(maxsize=1) def manifests(self, lookup_manifest_fn): """ Returns the manifests in the list. The `lookup_manifest_fn` is a function diff --git a/image/docker/schema2/manifest.py b/image/docker/schema2/manifest.py index e8e2c332c..8f48e0f9d 100644 --- a/image/docker/schema2/manifest.py +++ b/image/docker/schema2/manifest.py @@ -136,6 +136,10 @@ class DockerSchema2Manifest(ManifestInterface): except ValidationError as ve: raise MalformedSchema2Manifest('manifest data does not match schema: %s' % ve) + for layer in self.layers: + if layer.is_remote and not layer.urls: + raise MalformedSchema2Manifest('missing `urls` for remote layer') + @property def schema_version(self): return 2 @@ -169,18 +173,39 @@ class DockerSchema2Manifest(ManifestInterface): def leaf_layer(self): return self.layers[-1] + @property + def has_remote_layer(self): + for layer in self.layers: + if layer.is_remote: + return True + + return False + @property def leaf_layer_v1_image_id(self): + # NOTE: If there exists a layer with remote content, then we consider this manifest + # to not support legacy images. + if self.has_remote_layer: + return None + return list(self.layers_with_v1_ids)[-1].v1_id @property def legacy_image_ids(self): + if self.has_remote_layer: + return None + return [l.v1_id for l in self.layers_with_v1_ids] @property def blob_digests(self): return [str(layer.digest) for layer in self.layers] + [str(self.config.digest)] + @property + def local_blob_digests(self): + return ([str(layer.digest) for layer in self.layers if not layer.urls] + + [str(self.config.digest)]) + def get_manifest_labels(self, lookup_config_fn): return self._get_built_config(lookup_config_fn).labels @@ -218,6 +243,7 @@ class DockerSchema2Manifest(ManifestInterface): @property def layers_with_v1_ids(self): + assert not self.has_remote_layer digest_history = hashlib.sha256() v1_layer_parent_id = None v1_layer_id = None @@ -240,6 +266,7 @@ class DockerSchema2Manifest(ManifestInterface): this schema. The `lookup_config_fn` is a function that, when given the config digest SHA, returns the associated configuration JSON bytes for this schema. """ + assert not self.has_remote_layer schema2_config = self._get_built_config(lookup_config_fn) # Build the V1 IDs for the layers. @@ -253,6 +280,8 @@ class DockerSchema2Manifest(ManifestInterface): return v1_builder def generate_legacy_layers(self, images_map, lookup_config_fn): + assert not self.has_remote_layer + # NOTE: We use the DockerSchema1ManifestBuilder here because it already contains # the logic for generating the DockerV1Metadata. All of this will go away once we get # rid of legacy images in the database, so this is a temporary solution. @@ -261,6 +290,9 @@ class DockerSchema2Manifest(ManifestInterface): return v1_builder.build().generate_legacy_layers(images_map, lookup_config_fn) def get_v1_compatible_manifest(self, namespace_name, repo_name, tag_name, lookup_fn): + if self.has_remote_layer: + return None + v1_builder = DockerSchema1ManifestBuilder(namespace_name, repo_name, tag_name) self.populate_schema1_builder(v1_builder, lookup_fn) return v1_builder.build() diff --git a/image/docker/schema2/test/test_list.py b/image/docker/schema2/test/test_list.py index 2794e04f4..2dc3e2ad5 100644 --- a/image/docker/schema2/test/test_list.py +++ b/image/docker/schema2/test/test_list.py @@ -28,7 +28,7 @@ MANIFESTLIST_BYTES = json.dumps({ "manifests": [ { "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "size": 983, + "size": 946, "digest": "sha256:e6", "platform": { "architecture": "ppc64le", @@ -56,7 +56,7 @@ NO_AMD_MANIFESTLIST_BYTES = json.dumps({ "manifests": [ { "mediaType": "application/vnd.docker.distribution.manifest.v2+json", - "size": 983, + "size": 946, "digest": "sha256:e6", "platform": { "architecture": "ppc64le", @@ -76,7 +76,7 @@ def test_valid_manifestlist(): manifestlist = DockerSchema2ManifestList(MANIFESTLIST_BYTES) assert len(manifestlist.manifests(_get_manifest)) == 2 assert (manifestlist.digest == - 'sha256:7e22fdbe49736329786c9b4fdc154cc9251b190ca6b4cf33aed00efc0fc3df25') + 'sha256:340d7dadea77035533a2d43e8ff711ecca1965978a6e7699b87e32b35f76038d') assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json' assert manifestlist.bytes == MANIFESTLIST_BYTES @@ -109,7 +109,7 @@ def test_get_v1_compatible_manifest_no_matching_list(): manifestlist = DockerSchema2ManifestList(NO_AMD_MANIFESTLIST_BYTES) assert len(manifestlist.manifests(_get_manifest)) == 1 assert (manifestlist.digest == - 'sha256:50150251101420a020ab4a3e77e9d167a18b09bd4eeb0cc65e0eafab95cf79cf') + 'sha256:40ed1cfe692333bfa519a9bfed9676975a990fff5afd35efa628320c39c793ca') assert manifestlist.media_type == 'application/vnd.docker.distribution.manifest.list.v2+json' assert manifestlist.bytes == NO_AMD_MANIFESTLIST_BYTES diff --git a/image/docker/schema2/test/test_manifest.py b/image/docker/schema2/test/test_manifest.py index 3961be53f..560c4bb26 100644 --- a/image/docker/schema2/test/test_manifest.py +++ b/image/docker/schema2/test/test_manifest.py @@ -31,6 +31,38 @@ MANIFEST_BYTES = json.dumps({ "size": 1885, "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7" }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 1234, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736", + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 32654, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 16724, + "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b" + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 73109, + "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736" + }, + ], +}) + +REMOTE_MANIFEST_BYTES = json.dumps({ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1885, + "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7", + }, "layers": [ { "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", @@ -61,12 +93,12 @@ def test_valid_manifest(): assert manifest.config.size == 1885 assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7' assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json" + assert not manifest.has_remote_layer assert len(manifest.layers) == 4 - assert manifest.layers[0].is_remote assert manifest.layers[0].compressed_size == 1234 assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736' - assert manifest.layers[0].urls + assert not manifest.layers[0].is_remote assert manifest.leaf_layer == manifest.layers[3] assert not manifest.leaf_layer.is_remote @@ -75,6 +107,37 @@ def test_valid_manifest(): blob_digests = list(manifest.blob_digests) expected = [str(layer.digest) for layer in manifest.layers] + [manifest.config.digest] assert blob_digests == expected + assert list(manifest.local_blob_digests) == expected + + +def test_valid_remote_manifest(): + manifest = DockerSchema2Manifest(REMOTE_MANIFEST_BYTES) + assert manifest.config.size == 1885 + assert str(manifest.config.digest) == 'sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7' + assert manifest.media_type == "application/vnd.docker.distribution.manifest.v2+json" + assert manifest.has_remote_layer + + assert len(manifest.layers) == 4 + assert manifest.layers[0].compressed_size == 1234 + assert str(manifest.layers[0].digest) == 'sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736' + assert manifest.layers[0].is_remote + assert manifest.layers[0].urls == ['http://some/url'] + + assert manifest.leaf_layer == manifest.layers[3] + assert not manifest.leaf_layer.is_remote + assert manifest.leaf_layer.compressed_size == 73109 + + expected = set([str(layer.digest) for layer in manifest.layers] + [manifest.config.digest]) + + blob_digests = set(manifest.blob_digests) + local_digests = set(manifest.local_blob_digests) + + assert blob_digests == expected + assert local_digests == (expected - {manifest.layers[0].digest}) + + assert manifest.has_remote_layer + assert manifest.leaf_layer_v1_image_id is None + assert manifest.legacy_image_ids is None def test_schema2_builder(): @@ -110,6 +173,7 @@ def test_get_manifest_labels(): def test_build_schema1(): manifest = DockerSchema2Manifest(MANIFEST_BYTES) + assert not manifest.has_remote_layer builder = DockerSchema1ManifestBuilder('somenamespace', 'somename', 'sometag') manifest.populate_schema1_builder(builder, lambda digest: CONFIG_BYTES) @@ -209,3 +273,22 @@ def test_generate_legacy_layers(): assert legacy_layers[0].parent_image_id is None assert legacy_layers[0].image_id != legacy_layers[1] + + +def test_remote_layer_manifest(): + builder = DockerSchema2ManifestBuilder() + builder.set_config_digest('sha256:abcd', 1234) + builder.add_layer('sha256:adef', 1234, urls=['http://some/url']) + builder.add_layer('sha256:1352', 4567) + builder.add_layer('sha256:1353', 4567) + manifest = builder.build() + + assert manifest.has_remote_layer + assert manifest.leaf_layer_v1_image_id is None + assert manifest.legacy_image_ids is None + + schema1 = manifest.get_v1_compatible_manifest('somenamespace', 'somename', 'sometag', None) + assert schema1 is None + + assert set(manifest.blob_digests) == {'sha256:adef', 'sha256:abcd', 'sha256:1352', 'sha256:1353'} + assert set(manifest.local_blob_digests) == {'sha256:abcd', 'sha256:1352', 'sha256:1353'} diff --git a/test/registry/protocol_fixtures.py b/test/registry/protocol_fixtures.py index bd00921c6..a418e90de 100644 --- a/test/registry/protocol_fixtures.py +++ b/test/registry/protocol_fixtures.py @@ -92,6 +92,20 @@ def multi_layer_images(): ] +@pytest.fixture(scope="session") +def remote_images(): + """ Returns images with at least one remote layer for push and pull testing. """ + # Note: order is from base layer down to leaf. + remote_bytes = layer_bytes_for_contents('remote contents') + parent_bytes = layer_bytes_for_contents('parent contents') + image_bytes = layer_bytes_for_contents('some contents') + return [ + Image(id='remoteid', bytes=remote_bytes, parent_id=None, urls=['http://some/url']), + Image(id='parentid', bytes=parent_bytes, parent_id='remoteid'), + Image(id='someid', bytes=image_bytes, parent_id='parentid'), + ] + + @pytest.fixture(scope="session") def jwk(): return RSAKey(key=RSA.generate(2048)) diff --git a/test/registry/protocol_v1.py b/test/registry/protocol_v1.py index 849a9b07b..acd86547d 100644 --- a/test/registry/protocol_v1.py +++ b/test/registry/protocol_v1.py @@ -93,7 +93,6 @@ class V1Protocol(RegistryProtocol): # GET /v1/repositories/{namespace}/{repository}/tags image_ids = self.conduct(session, 'GET', prefix + 'tags', headers=headers).json() - assert len(image_ids.values()) >= len(tag_names) for tag_name in tag_names: if tag_name not in image_ids: @@ -145,6 +144,8 @@ class V1Protocol(RegistryProtocol): headers['Authorization'] = 'token ' + result.headers['www-authenticate'] for image in images: + assert image.urls is None + # PUT /v1/images/{imageID}/json image_json_data = {'id': image.id} if image.size is not None: diff --git a/test/registry/protocol_v2.py b/test/registry/protocol_v2.py index f8377e929..2c37aa4f3 100644 --- a/test/registry/protocol_v2.py +++ b/test/registry/protocol_v2.py @@ -209,13 +209,15 @@ class V2Protocol(RegistryProtocol): builder = DockerSchema2ManifestBuilder() for image in images: checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() - blobs[checksum] = image.bytes + + if image.urls is None: + blobs[checksum] = image.bytes # If invalid blob references were requested, just make it up. if options.manifest_invalid_blob_references: checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest() - builder.add_layer(checksum, len(image.bytes)) + builder.add_layer(checksum, len(image.bytes), urls=image.urls) config = { "os": "linux", @@ -245,6 +247,8 @@ class V2Protocol(RegistryProtocol): builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name) for image in reversed(images): + assert image.urls is None + checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() blobs[checksum] = image.bytes @@ -498,12 +502,16 @@ class V2Protocol(RegistryProtocol): # Verify the layers. for index, layer in enumerate(manifest.layers): + # If the layer is remote, then we expect the blob to *not* exist in the system. + expected_status = 404 if images[index].urls else 200 result = self.conduct(session, 'GET', '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), layer.digest), - expected_status=200, + expected_status=expected_status, headers=headers) - assert result.content == images[index].bytes + + if expected_status == 200: + assert result.content == images[index].bytes return PullResult(manifests=manifests, image_ids=image_ids) diff --git a/test/registry/protocols.py b/test/registry/protocols.py index f179ddd2b..4cb521c47 100644 --- a/test/registry/protocols.py +++ b/test/registry/protocols.py @@ -7,8 +7,8 @@ from cStringIO import StringIO from enum import Enum, unique from six import add_metaclass -Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created']) -Image.__new__.__defaults__ = (None, None, None) +Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created', 'urls']) +Image.__new__.__defaults__ = (None, None, None, None) PushResult = namedtuple('PushResult', ['manifests', 'headers']) PullResult = namedtuple('PullResult', ['manifests', 'image_ids']) diff --git a/test/registry/registry_tests.py b/test/registry/registry_tests.py index 14e0ae891..41205173e 100644 --- a/test/registry/registry_tests.py +++ b/test/registry/registry_tests.py @@ -21,6 +21,7 @@ from app import instance_keys from data.model.tag import list_repository_tags from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE from image.docker.schema2.list import DockerSchema2ManifestListBuilder +from image.docker.schema2.manifest import DockerSchema2ManifestBuilder from util.security.registry_jwt import decode_bearer_header from util.timedeltastring import convert_to_timedelta @@ -1450,3 +1451,26 @@ def test_push_pull_manifest_list(v22_protocol, basic_images, different_images, l # Pull and verify the manifest list. v22_protocol.pull_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist, credentials=credentials, options=options) + + +def test_push_pull_manifest_remote_layers(v22_protocol, legacy_puller, liveserver_session, + app_reloader, remote_images, data_model): + """ Test: Push a new tag with a manifest which contains at least one remote layer, and then + pull that manifest back. + """ + if data_model != 'oci_model': + return + + credentials = ('devtable', 'password') + + # Push a new repository. + v22_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images, + credentials=credentials) + + # Pull the repository to verify. + v22_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images, + credentials=credentials) + + # Ensure that the image cannot be pulled by a legacy protocol. + legacy_puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', remote_images, + credentials=credentials, expected_failure=Failures.UNKNOWN_TAG) diff --git a/workers/manifestbackfillworker.py b/workers/manifestbackfillworker.py index f1260305b..c5e5fe613 100644 --- a/workers/manifestbackfillworker.py +++ b/workers/manifestbackfillworker.py @@ -63,6 +63,10 @@ class BrokenManifest(ManifestInterface): def blob_digests(self): return [] + @property + def local_blob_digests(self): + return [] + def child_manifests(self, lookup_manifest_fn): return None