vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add tests for manifest lists and fix some issues encountered while testing

Browse source
This commit is contained in:
Joseph Schorr 2018-11-13 19:05:45 +02:00
parent 9994f0ae61
commit 7a794e29c0
9 changed files with 311 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
data/model/oci/tag.py
View file

@ -98,7 +98,7 @@ def get_legacy_images_for_tags(tags):
.where(ManifestLegacyImage.manifest << [tag.manifest_id for tag in tags]))
by_manifest = {mli.manifest_id: mli.image for mli in query}
return {tag.id: by_manifest[tag.manifest_id] for tag in tags}
return {tag.id: by_manifest[tag.manifest_id] for tag in tags if tag.manifest_id in by_manifest}
def find_matching_tag(repository_id, tag_names, tag_kinds=None):

3
data/registry_model/registry_oci_model.py
View file

@ -217,9 +217,6 @@ class OCIModel(SharedModel, RegistryDataInterface):
return (None, None)
legacy_image = oci.shared.get_legacy_image_for_manifest(created_manifest.manifest)
if legacy_image is None:
return (None, None)
li = LegacyImage.for_image(legacy_image)
wrapped_manifest = Manifest.for_manifest(created_manifest.manifest, li)

4
endpoints/v1/tag.py
View file

@ -27,8 +27,10 @@ def get_tags(namespace_name, repo_name):
if repository_ref is None:
abort(404)
# TODO(jschorr): Change this to normalize manifest lists back to their legacy image
# (if applicable).
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
tag_map = {tag.name: tag.legacy_image.docker_image_id for tag in tags}
tag_map = {tag.name: tag.legacy_image.docker_image_id for tag in tags if tag.legacy_image}
return jsonify(tag_map)
abort(403)

1
endpoints/v2/manifest.py
View file

@ -278,6 +278,7 @@ def _write_manifest(namespace_name, repo_name, tag_name, manifest_impl):
if repository_ref is None:
raise NameUnknown()
# Create the manifest(s) and retarget the tag to point to it.
manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref, manifest_impl,
tag_name, storage)
if manifest is None:

4
image/docker/interfaces.py
View file

@ -4,6 +4,10 @@ from six import add_metaclass
@add_metaclass(ABCMeta)
class ManifestInterface(object):
""" Defines the interface for the various manifests types supported. """
@abstractproperty
def schema_version(self):
""" The version of the schema, or None for lists. """
@abstractproperty
def digest(self):
""" The digest of the manifest, including type prefix. """

6
image/docker/schema2/list.py
View file

@ -183,6 +183,11 @@ class DockerSchema2ManifestList(ManifestInterface):
except ValidationError as ve:
raise MalformedSchema2ManifestList('manifest data does not match schema: %s' % ve)
@property
def schema_version(self):
""" The version of the schema, or None for lists. """
return None
@property
def digest(self):
""" The digest of the manifest, including type prefix. """
@ -241,7 +246,6 @@ class DockerSchema2ManifestList(ManifestInterface):
platform = manifest_ref._manifest_data[DOCKER_SCHEMA2_MANIFESTLIST_PLATFORM_KEY]
architecture = platform[DOCKER_SCHEMA2_MANIFESTLIST_ARCHITECTURE_KEY]
os = platform[DOCKER_SCHEMA2_MANIFESTLIST_OS_KEY]
if architecture != 'amd64' or os != 'linux':
continue

14
test/registry/protocol_fixtures.py
View file

@ -102,6 +102,11 @@ def v2_protocol(request, jwk):
return request.param(jwk)
@pytest.fixture()
def v22_protocol(request, jwk):
return V2Protocol(jwk, schema2=True)
@pytest.fixture(params=[V1Protocol])
def v1_protocol(request, jwk):
return request.param(jwk)
@ -123,6 +128,15 @@ def pusher(request, data_model, jwk):
return V2Protocol(jwk)
@pytest.fixture(params=['v1', 'v2_1'])
def legacy_puller(request, data_model, jwk):
if request.param == 'v1':
return V1Protocol(jwk)
return V2Protocol(jwk)
@pytest.fixture(params=['v1', 'v2_1'])
def legacy_pusher(request, data_model, jwk):
if request.param == 'v1':

271
test/registry/protocol_v2.py
View file

@ -3,10 +3,12 @@ import json
from enum import Enum, unique
from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
from image.docker.schema2.list import DockerSchema2ManifestListBuilder
from image.docker.schema1 import (DockerSchema1ManifestBuilder, DockerSchema1Manifest,
DOCKER_SCHEMA1_CONTENT_TYPES)
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
from image.docker.schema2.config import DockerSchema2Config
from image.docker.schema2.list import DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
from image.docker.schemas import parse_manifest_from_bytes
from test.registry.protocols import (RegistryProtocol, Failures, ProtocolOptions, PushResult,
PullResult)
@ -123,6 +125,148 @@ class V2Protocol(RegistryProtocol):
return None, response
def pull_list(self, session, namespace, repo_name, tag_names, manifestlist,
credentials=None, expected_failure=None, options=None):
options = options or ProtocolOptions()
scopes = options.scopes or ['repository:%s:push,pull' % self.repo_name(namespace, repo_name)]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes,
expected_failure=expected_failure)
if token is None:
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
return
headers = {
'Authorization': 'Bearer ' + token,
'Accept': DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
}
for tag_name in tag_names:
# Retrieve the manifest for the tag or digest.
response = self.conduct(session, 'GET',
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name),
tag_name),
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
headers=headers)
if expected_failure is not None:
return None
# Parse the returned manifest list and ensure it matches.
assert response.headers['Content-Type'] == DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
manifest = parse_manifest_from_bytes(response.text, response.headers['Content-Type'])
assert manifest.schema_version is None
assert manifest.digest == manifestlist.digest
def push_list(self, session, namespace, repo_name, tag_names, manifestlist, blobs,
credentials=None, expected_failure=None, options=None):
options = options or ProtocolOptions()
scopes = options.scopes or ['repository:%s:push,pull' % self.repo_name(namespace, repo_name)]
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
# Ping!
self.ping(session)
# Perform auth and retrieve a token.
token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes,
expected_failure=expected_failure)
if token is None:
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
return
headers = {
'Authorization': 'Bearer ' + token,
'Accept': ','.join(options.accept_mimetypes or []),
}
# Push all blobs.
if not self._push_blobs(blobs, session, namespace, repo_name, headers, options,
expected_failure):
return
# Push the manifest list.
for tag_name in tag_names:
manifest_headers = {'Content-Type': manifestlist.media_type}
manifest_headers.update(headers)
if options.manifest_content_type is not None:
manifest_headers['Content-Type'] = options.manifest_content_type
self.conduct(session, 'PUT',
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name),
data=manifestlist.bytes,
expected_status=(202, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
headers=manifest_headers)
return PushResult(manifests=None, headers=headers)
def build_schema2(self, images, blobs, options):
builder = DockerSchema2ManifestBuilder()
for image in images:
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
blobs[checksum] = image.bytes
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
builder.add_layer(checksum, len(image.bytes))
config = {
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [{
'created': '2018-04-03T18:37:09.284840891Z',
'created_by': (('/bin/sh -c #(nop) ENTRYPOINT %s' % image.config['Entrypoint'])
if image.config and image.config.get('Entrypoint')
else '/bin/sh -c #(nop) %s' % image.id),
} for image in images],
}
if images[-1].config:
config['config'] = images[-1].config
config_json = json.dumps(config)
schema2_config = DockerSchema2Config(config_json)
builder.set_config(schema2_config)
blobs[schema2_config.digest] = schema2_config.bytes
return builder.build()
def build_schema1(self, namespace, repo_name, tag_name, images, blobs, options):
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
for image in reversed(images):
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
blobs[checksum] = image.bytes
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
layer_dict = {'id': image.id, 'parent': image.parent_id}
if image.config is not None:
layer_dict['config'] = image.config
if image.size is not None:
layer_dict['Size'] = image.size
if image.created is not None:
layer_dict['created'] = image.created
builder.add_layer(checksum, json.dumps(layer_dict))
# Build the manifest.
return builder.build(self.jwk)
def push(self, session, namespace, repo_name, tag_names, images, credentials=None,
expected_failure=None, options=None):
options = options or ProtocolOptions()
@ -141,7 +285,7 @@ class V2Protocol(RegistryProtocol):
headers = {
'Authorization': 'Bearer ' + token,
'Accept': options.accept_mimetypes,
'Accept': ','.join(options.accept_mimetypes or []),
}
# Build fake manifests.
@ -149,67 +293,39 @@ class V2Protocol(RegistryProtocol):
blobs = {}
for tag_name in tag_names:
if self.schema2:
builder = DockerSchema2ManifestBuilder()
for image in images:
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
blobs[checksum] = image.bytes
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
builder.add_layer(checksum, len(image.bytes))
config = {
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [{
'created': '2018-04-03T18:37:09.284840891Z',
'created_by': (('/bin/sh -c #(nop) ENTRYPOINT %s' % image.config['Entrypoint'])
if image.config and image.config.get('Entrypoint')
else '/bin/sh -c #(nop) %s' % image.id),
} for image in images],
}
if images[-1].config:
config['config'] = images[-1].config
config_json = json.dumps(config)
schema2_config = DockerSchema2Config(config_json)
builder.set_config(schema2_config)
blobs[schema2_config.digest] = schema2_config.bytes
manifests[tag_name] = builder.build()
manifests[tag_name] = self.build_schema2(images, blobs, options)
else:
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
for image in reversed(images):
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
blobs[checksum] = image.bytes
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
layer_dict = {'id': image.id, 'parent': image.parent_id}
if image.config is not None:
layer_dict['config'] = image.config
if image.size is not None:
layer_dict['Size'] = image.size
if image.created is not None:
layer_dict['created'] = image.created
builder.add_layer(checksum, json.dumps(layer_dict))
# Build the manifest.
manifests[tag_name] = builder.build(self.jwk)
manifests[tag_name] = self.build_schema1(namespace, repo_name, tag_name, images, blobs,
options)
# Push the blob data.
if not self._push_blobs(blobs, session, namespace, repo_name, headers, options,
expected_failure):
return
# Write a manifest for each tag.
for tag_name in tag_names:
manifest = manifests[tag_name]
# Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we
# expect a 202 response for success.
put_code = 404 if options.manifest_invalid_blob_references else 202
manifest_headers = {'Content-Type': manifest.media_type}
manifest_headers.update(headers)
if options.manifest_content_type is not None:
manifest_headers['Content-Type'] = options.manifest_content_type
tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest
self.conduct(session, 'PUT',
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest),
data=manifest.bytes,
expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
headers=manifest_headers)
return PushResult(manifests=manifests, headers=headers)
def _push_blobs(self, blobs, session, namespace, repo_name, headers, options, expected_failure):
for blob_digest, blob_bytes in blobs.iteritems():
if not options.skip_head_checks:
# Blob data should not yet exist.
@ -270,7 +386,7 @@ class V2Protocol(RegistryProtocol):
expected_status=expected_code,
headers=patch_headers)
if expected_code != 204:
return
return False
# Retrieve the upload status at each point, and ensure it is valid.
status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name),
@ -288,7 +404,7 @@ class V2Protocol(RegistryProtocol):
status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name),
upload_uuid)
self.conduct(session, 'GET', status_url, expected_status=404, headers=headers)
return
return False
# Finish the blob upload with a PUT.
response = self.conduct(session, 'PUT', location, params=dict(digest=blob_digest),
@ -310,28 +426,7 @@ class V2Protocol(RegistryProtocol):
headers=headers, expected_status=200)
assert result.content == blob_bytes
# Write a manifest for each tag.
for tag_name in tag_names:
manifest = manifests[tag_name]
# Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we
# expect a 202 response for success.
put_code = 404 if options.manifest_invalid_blob_references else 202
manifest_headers = {'Content-Type': manifest.media_type}
manifest_headers.update(headers)
if options.manifest_content_type is not None:
manifest_headers['Content-Type'] = options.manifest_content_type
tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest
self.conduct(session, 'PUT',
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest),
data=manifest.bytes,
expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
headers=manifest_headers)
return PushResult(manifests=manifests, headers=headers)
return True
def delete(self, session, namespace, repo_name, tag_names, credentials=None,
expected_failure=None, options=None):
@ -379,8 +474,7 @@ class V2Protocol(RegistryProtocol):
}
if self.schema2:
headers['Accept'] = [('application/vnd.docker.distribution.manifest.v2+json', 1),
('application/vnd.docker.distribution.manifest.list.v2+json', 1)]
headers['Accept'] = ','.join(options.accept_mimetypes or DOCKER_SCHEMA2_CONTENT_TYPES)
manifests = {}
image_ids = {}
@ -395,6 +489,9 @@ class V2Protocol(RegistryProtocol):
return None
# Ensure the manifest returned by us is valid.
if not self.schema2:
assert response.headers['Content-Type'] in DOCKER_SCHEMA1_CONTENT_TYPES
manifest = parse_manifest_from_bytes(response.text, response.headers['Content-Type'])
manifests[tag_name] = manifest
image_ids[tag_name] = manifest.leaf_layer_v1_image_id

104
test/registry/registry_tests.py
View file

@ -20,6 +20,7 @@ from test.registry.protocols import Failures, Image, layer_bytes_for_contents, P
from app import instance_keys
from data.model.tag import list_repository_tags
from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
from image.docker.schema2.list import DockerSchema2ManifestListBuilder
from util.security.registry_jwt import decode_bearer_header
from util.timedeltastring import convert_to_timedelta
@ -613,10 +614,10 @@ def test_unsupported_manifest_content_type(content_type, manifest_protocol, basi
@pytest.mark.parametrize('accept_mimetypes', [
[('application/vnd.oci.image.manifest.v1+json', 1)],
[('application/vnd.docker.distribution.manifest.v2+json', 1),
('application/vnd.docker.distribution.manifest.list.v2+json', 1)],
[('application/vnd.foo.bar', 1)],
['application/vnd.oci.image.manifest.v1+json'],
['application/vnd.docker.distribution.manifest.v2+json',
'application/vnd.docker.distribution.manifest.list.v2+json'],
['application/vnd.foo.bar'],
])
def test_unsupported_manifest_accept_headers(accept_mimetypes, manifest_protocol, basic_images,
data_model, liveserver_session, app_reloader):
@ -629,7 +630,7 @@ def test_unsupported_manifest_accept_headers(accept_mimetypes, manifest_protocol
options = ProtocolOptions()
options.manifest_content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
options.accept_mimetypes = str(Accept(accept_mimetypes))
options.accept_mimetypes = accept_mimetypes
# Attempt to push a new repository.
manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
@ -1356,3 +1357,96 @@ def test_push_tag_existing_image(v1_protocol, puller, basic_images, liveserver_s
# Pull the repository to verify.
puller.pull(liveserver_session, 'devtable', 'newrepo', 'anothertag', basic_images,
credentials=credentials)
@pytest.mark.parametrize('schema_version', [
1,
2,
])
@pytest.mark.parametrize('is_amd', [
True,
False
])
def test_push_pull_manifest_list_back_compat(v22_protocol, v2_protocol, basic_images,
different_images, liveserver_session, app_reloader,
schema_version, data_model, is_amd):
""" Test: Push a new tag with a manifest list containing two manifests, one (possibly) legacy
and one not, and, if there is a legacy manifest, ensure it can be pulled.
"""
if data_model != 'oci_model':
return
credentials = ('devtable', 'password')
options = ProtocolOptions()
# Build the manifests that will go in the list.
blobs = {}
signed = v22_protocol.build_schema1('devtable', 'newrepo', 'latest', basic_images, blobs, options)
first_manifest = signed.unsigned()
if schema_version == 2:
first_manifest = v22_protocol.build_schema2(basic_images, blobs, options)
second_manifest = v22_protocol.build_schema2(different_images, blobs, options)
# Add the manifests themselves to the blobs map.
blobs[str(first_manifest.digest)] = first_manifest.bytes
blobs[str(second_manifest.digest)] = second_manifest.bytes
# Create and push the manifest list.
builder = DockerSchema2ManifestListBuilder()
builder.add_manifest(first_manifest, 'amd64' if is_amd else 'something', 'linux')
builder.add_manifest(second_manifest, 'arm', 'linux')
manifestlist = builder.build()
v22_protocol.push_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist, blobs,
credentials=credentials, options=options)
# Pull the tag and ensure we (don't) get back the basic images, since they are(n't) part of the
# amd64+linux manifest.
v2_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
credentials=credentials,
expected_failure=Failures.UNKNOWN_TAG if not is_amd else None)
@pytest.mark.parametrize('schema_version', [
1,
2,
])
def test_push_pull_manifest_list(v22_protocol, basic_images, different_images, liveserver_session,
app_reloader, schema_version, data_model):
""" Test: Push a new tag with a manifest list containing two manifests, one (possibly) legacy
and one not, and pull it.
"""
if data_model != 'oci_model':
return
credentials = ('devtable', 'password')
options = ProtocolOptions()
# Build the manifests that will go in the list.
blobs = {}
signed = v22_protocol.build_schema1('devtable', 'newrepo', 'latest', basic_images, blobs, options)
first_manifest = signed.unsigned()
if schema_version == 2:
first_manifest = v22_protocol.build_schema2(basic_images, blobs, options)
second_manifest = v22_protocol.build_schema2(different_images, blobs, options)
# Add the manifests themselves to the blobs map.
blobs[str(first_manifest.digest)] = first_manifest.bytes
blobs[str(second_manifest.digest)] = second_manifest.bytes
# Create and push the manifest list.
builder = DockerSchema2ManifestListBuilder()
builder.add_manifest(first_manifest, 'amd64', 'linux')
builder.add_manifest(second_manifest, 'arm', 'linux')
manifestlist = builder.build()
v22_protocol.push_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist, blobs,
credentials=credentials, options=options)
# Pull and verify the manifest list.
v22_protocol.pull_list(liveserver_session, 'devtable', 'newrepo', 'latest', manifestlist,
credentials=credentials, options=options)