Update registry tests to test schema 2 manifest pushes and pulls

Note that tests for manifest *lists* will be in a following commit
This commit is contained in:
Joseph Schorr 2018-11-13 17:15:00 +02:00
parent 7b9f56eff3
commit e752a9a73f
5 changed files with 171 additions and 98 deletions

View file

@ -6,6 +6,7 @@ import pytest
from Crypto.PublicKey import RSA
from jwkest.jwk import RSAKey
from test.registry.fixtures import data_model
from test.registry.protocols import Image, layer_bytes_for_contents
from test.registry.protocol_v1 import V1Protocol
from test.registry.protocol_v2 import V2Protocol
@ -47,7 +48,8 @@ def sized_images():
Image(id='parentid', bytes=parent_bytes, parent_id=None, size=len(parent_bytes),
config={'foo': 'bar'}),
Image(id='someid', bytes=image_bytes, parent_id='parentid', size=len(image_bytes),
config={'foo': 'childbar', 'Entrypoint': ['hello']}),
config={'foo': 'childbar', 'Entrypoint': ['hello']},
created='2018-04-03T18:37:09.284840891Z'),
]
@ -105,9 +107,39 @@ def v1_protocol(request, jwk):
return request.param(jwk)
@pytest.fixture(params=[V2Protocol])
def manifest_protocol(request, jwk):
return request.param(jwk)
@pytest.fixture(params=['schema1', 'schema2'])
def manifest_protocol(request, data_model, jwk):
return V2Protocol(jwk, schema2=(request == 'schema2' and data_model == 'oci_model'))
@pytest.fixture(params=['v1', 'v2_1', 'v2_2'])
def pusher(request, data_model, jwk):
if request.param == 'v1':
return V1Protocol(jwk)
if request.param == 'v2_2' and data_model == 'oci_model':
return V2Protocol(jwk, schema2=True)
return V2Protocol(jwk)
@pytest.fixture(params=['v1', 'v2_1'])
def legacy_pusher(request, data_model, jwk):
if request.param == 'v1':
return V1Protocol(jwk)
return V2Protocol(jwk)
@pytest.fixture(params=['v1', 'v2_1', 'v2_2'])
def puller(request, data_model, jwk):
if request == 'v1':
return V1Protocol(jwk)
if request == 'v2_2' and data_model == 'oci_model':
return V2Protocol(jwk, schema2=True)
return V2Protocol(jwk)
@pytest.fixture(params=[V1Protocol, V2Protocol])
@ -115,16 +147,6 @@ def loginer(request, jwk):
return request.param(jwk)
@pytest.fixture(params=[V1Protocol, V2Protocol])
def pusher(request, jwk):
return request.param(jwk)
@pytest.fixture(params=[V1Protocol, V2Protocol])
def puller(request, jwk):
return request.param(jwk)
@pytest.fixture(scope="session")
def random_layer_data():
size = 4096

View file

@ -101,10 +101,6 @@ class V1Protocol(RegistryProtocol):
return None
tag_image_id = image_ids[tag_name]
if not options.munge_shas:
# Ensure we have a matching image ID.
known_ids = {image.id for image in images}
assert tag_image_id in known_ids
# Retrieve the ancestry of the tagged image.
image_prefix = '/v1/images/%s/' % tag_image_id
@ -160,15 +156,19 @@ class V1Protocol(RegistryProtocol):
if image.config is not None:
image_json_data['config'] = image.config
if image.created is not None:
image_json_data['created'] = image.created
image_json = json.dumps(image_json_data)
response = self.conduct(session, 'PUT', '/v1/images/%s/json' % image.id,
json_data=image_json_data, headers=headers,
data=image_json, headers=headers,
expected_status=(200, expected_failure,
V1ProtocolSteps.PUT_IMAGE_JSON))
if response.status_code != 200:
return
# PUT /v1/images/{imageID}/checksum (old style)
old_checksum = compute_tarsum(StringIO(image.bytes), json.dumps(image_json_data))
old_checksum = compute_tarsum(StringIO(image.bytes), image_json)
checksum_headers = {'X-Docker-Checksum': old_checksum}
checksum_headers.update(headers)
@ -180,7 +180,7 @@ class V1Protocol(RegistryProtocol):
data=StringIO(image.bytes), headers=headers)
# PUT /v1/images/{imageID}/checksum (new style)
checksum = compute_simple(StringIO(image.bytes), json.dumps(image_json_data))
checksum = compute_simple(StringIO(image.bytes), image_json)
checksum_headers = {'X-Docker-Checksum-Payload': checksum}
checksum_headers.update(headers)
@ -200,7 +200,7 @@ class V1Protocol(RegistryProtocol):
'/v1/repositories/%s/images' % self.repo_name(namespace, repo_name),
expected_status=204, headers=headers)
return PushResult(checksums=None, manifests=None, headers=headers)
return PushResult(manifests=None, headers=headers)
def delete(self, session, namespace, repo_name, tag_names, credentials=None,
expected_failure=None, options=None):

View file

@ -4,6 +4,10 @@ import json
from enum import Enum, unique
from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
from image.docker.schema2.list import DockerSchema2ManifestListBuilder
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
from image.docker.schema2.config import DockerSchema2Config
from image.docker.schemas import parse_manifest_from_bytes
from test.registry.protocols import (RegistryProtocol, Failures, ProtocolOptions, PushResult,
PullResult)
@ -56,8 +60,9 @@ class V2Protocol(RegistryProtocol):
},
}
def __init__(self, jwk):
def __init__(self, jwk, schema2=False):
self.jwk = jwk
self.schema2 = schema2
def ping(self, session):
result = session.get('/v2/')
@ -141,55 +146,92 @@ class V2Protocol(RegistryProtocol):
# Build fake manifests.
manifests = {}
blobs = {}
for tag_name in tag_names:
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
if self.schema2:
builder = DockerSchema2ManifestBuilder()
for image in images:
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
blobs[checksum] = image.bytes
for image in reversed(images):
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
builder.add_layer(checksum, len(image.bytes))
layer_dict = {'id': image.id, 'parent': image.parent_id}
if image.config is not None:
layer_dict['config'] = image.config
config = {
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": []
},
"history": [{
'created': '2018-04-03T18:37:09.284840891Z',
'created_by': (('/bin/sh -c #(nop) ENTRYPOINT %s' % image.config['Entrypoint'])
if image.config and image.config.get('Entrypoint')
else '/bin/sh -c #(nop) %s' % image.id),
} for image in images],
}
if image.size is not None:
layer_dict['Size'] = image.size
if images[-1].config:
config['config'] = images[-1].config
builder.add_layer(checksum, json.dumps(layer_dict))
config_json = json.dumps(config)
schema2_config = DockerSchema2Config(config_json)
builder.set_config(schema2_config)
# Build the manifest.
manifests[tag_name] = builder.build(self.jwk)
blobs[schema2_config.digest] = schema2_config.bytes
manifests[tag_name] = builder.build()
else:
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
# Push the layer data.
checksums = {}
for image in reversed(images):
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
checksums[image.id] = checksum
for image in reversed(images):
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
blobs[checksum] = image.bytes
# If invalid blob references were requested, just make it up.
if options.manifest_invalid_blob_references:
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
layer_dict = {'id': image.id, 'parent': image.parent_id}
if image.config is not None:
layer_dict['config'] = image.config
if image.size is not None:
layer_dict['Size'] = image.size
if image.created is not None:
layer_dict['created'] = image.created
builder.add_layer(checksum, json.dumps(layer_dict))
# Build the manifest.
manifests[tag_name] = builder.build(self.jwk)
# Push the blob data.
for blob_digest, blob_bytes in blobs.iteritems():
if not options.skip_head_checks:
# Layer data should not yet exist.
# Blob data should not yet exist.
self.conduct(session, 'HEAD',
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), checksum),
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), blob_digest),
expected_status=(404, expected_failure, V2ProtocolSteps.BLOB_HEAD_CHECK),
headers=headers)
# Check for mounting of blobs.
if options.mount_blobs and image.id in options.mount_blobs:
if options.mount_blobs and blob_digest in options.mount_blobs:
self.conduct(session, 'POST',
'/v2/%s/blobs/uploads/' % self.repo_name(namespace, repo_name),
params={
'mount': checksum,
'from': options.mount_blobs[image.id],
'mount': blob_digest,
'from': options.mount_blobs[blob_digest],
},
expected_status=(201, expected_failure, V2ProtocolSteps.MOUNT_BLOB),
headers=headers)
if expected_failure is not None:
return
else:
# Start a new upload of the layer data.
# Start a new upload of the blob data.
response = self.conduct(session, 'POST',
'/v2/%s/blobs/uploads/' % self.repo_name(namespace, repo_name),
expected_status=(202, expected_failure,
@ -206,9 +248,9 @@ class V2Protocol(RegistryProtocol):
# case modifies the port.
location = response.headers['Location'][len('http://localhost:5000'):]
# PATCH the image data into the layer.
# PATCH the data into the blob.
if options.chunks_for_upload is None:
self.conduct(session, 'PATCH', location, data=image.bytes, expected_status=204,
self.conduct(session, 'PATCH', location, data=blob_bytes, expected_status=204,
headers=headers)
else:
# If chunked upload is requested, upload the data as a series of chunks, checking
@ -223,7 +265,7 @@ class V2Protocol(RegistryProtocol):
patch_headers = {'Range': 'bytes=%s-%s' % (start_byte, end_byte)}
patch_headers.update(headers)
contents_chunk = image.bytes[start_byte:end_byte]
contents_chunk = blob_bytes[start_byte:end_byte]
self.conduct(session, 'PATCH', location, data=contents_chunk,
expected_status=expected_code,
headers=patch_headers)
@ -239,7 +281,7 @@ class V2Protocol(RegistryProtocol):
assert response.headers['Range'] == "bytes=0-%s" % end_byte
if options.cancel_blob_upload:
self.conduct(session, 'DELETE', location, params=dict(digest=checksum),
self.conduct(session, 'DELETE', location, params=dict(digest=blob_digest),
expected_status=204, headers=headers)
# Ensure the upload was canceled.
@ -248,24 +290,25 @@ class V2Protocol(RegistryProtocol):
self.conduct(session, 'GET', status_url, expected_status=404, headers=headers)
return
# Finish the layer upload with a PUT.
response = self.conduct(session, 'PUT', location, params=dict(digest=checksum),
# Finish the blob upload with a PUT.
response = self.conduct(session, 'PUT', location, params=dict(digest=blob_digest),
expected_status=201, headers=headers)
assert response.headers['Docker-Content-Digest'] == checksum
assert response.headers['Docker-Content-Digest'] == blob_digest
# Ensure the layer exists now.
# Ensure the blob exists now.
response = self.conduct(session, 'HEAD',
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), checksum),
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name),
blob_digest),
expected_status=200, headers=headers)
assert response.headers['Docker-Content-Digest'] == checksum
assert response.headers['Content-Length'] == str(len(image.bytes))
assert response.headers['Docker-Content-Digest'] == blob_digest
assert response.headers['Content-Length'] == str(len(blob_bytes))
# And retrieve the layer data.
# And retrieve the blob data.
result = self.conduct(session, 'GET',
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), checksum),
'/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), blob_digest),
headers=headers, expected_status=200)
assert result.content == image.bytes
assert result.content == blob_bytes
# Write a manifest for each tag.
for tag_name in tag_names:
@ -274,7 +317,7 @@ class V2Protocol(RegistryProtocol):
# Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we
# expect a 202 response for success.
put_code = 404 if options.manifest_invalid_blob_references else 202
manifest_headers = {'Content-Type': 'application/json'}
manifest_headers = {'Content-Type': manifest.media_type}
manifest_headers.update(headers)
if options.manifest_content_type is not None:
@ -287,7 +330,7 @@ class V2Protocol(RegistryProtocol):
expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
headers=manifest_headers)
return PushResult(checksums=checksums, manifests=manifests, headers=headers)
return PushResult(manifests=manifests, headers=headers)
def delete(self, session, namespace, repo_name, tag_names, credentials=None,
@ -335,6 +378,10 @@ class V2Protocol(RegistryProtocol):
'Authorization': 'Bearer ' + token,
}
if self.schema2:
headers['Accept'] = [('application/vnd.docker.distribution.manifest.v2+json', 1),
('application/vnd.docker.distribution.manifest.list.v2+json', 1)]
manifests = {}
image_ids = {}
for tag_name in tag_names:
@ -348,9 +395,9 @@ class V2Protocol(RegistryProtocol):
return None
# Ensure the manifest returned by us is valid.
manifest = DockerSchema1Manifest(response.text)
manifest = parse_manifest_from_bytes(response.text, response.headers['Content-Type'])
manifests[tag_name] = manifest
image_ids[tag_name] = manifest.leaf_layer.v1_metadata.image_id
image_ids[tag_name] = manifest.leaf_layer_v1_image_id
# Verify the layers.
for index, layer in enumerate(manifest.layers):

View file

@ -7,10 +7,10 @@ from cStringIO import StringIO
from enum import Enum, unique
from six import add_metaclass
Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config'])
Image.__new__.__defaults__ = (None, None)
Image = namedtuple('Image', ['id', 'parent_id', 'bytes', 'size', 'config', 'created'])
Image.__new__.__defaults__ = (None, None, None)
PushResult = namedtuple('PushResult', ['checksums', 'manifests', 'headers'])
PushResult = namedtuple('PushResult', ['manifests', 'headers'])
PullResult = namedtuple('PullResult', ['manifests', 'image_ids'])
@ -62,7 +62,6 @@ class Failures(Enum):
class ProtocolOptions(object):
def __init__(self):
self.munge_shas = False
self.scopes = None
self.cancel_blob_upload = False
self.manifest_invalid_blob_references = False

View file

@ -1,5 +1,4 @@
# pylint: disable=W0401, W0621, W0613, W0614, R0913
import os
import hashlib
import tarfile
@ -176,7 +175,7 @@ def test_application_repo(pusher, puller, basic_images, liveserver_session, app_
credentials=credentials, expected_failure=Failures.APP_REPOSITORY)
def test_middle_layer_different_sha(manifest_protocol, v1_protocol, liveserver_session,
def test_middle_layer_different_sha(v2_protocol, v1_protocol, liveserver_session,
app_reloader):
""" Test: Pushing of a 3-layer image with the *same* V1 ID's, but the middle layer having
different bytes, must result in new IDs being generated for the leaf layer, as
@ -192,10 +191,10 @@ def test_middle_layer_different_sha(manifest_protocol, v1_protocol, liveserver_s
]
# First push and pull the images, to ensure we have the basics setup and working.
manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', first_images,
credentials=credentials)
first_pull_result = manifest_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest',
first_images, credentials=credentials)
v2_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', first_images,
credentials=credentials)
first_pull_result = v2_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest',
first_images, credentials=credentials)
first_manifest = first_pull_result.manifests['latest']
assert set([image.id for image in first_images]) == set(first_manifest.image_ids)
assert first_pull_result.image_ids['latest'] == 'leafimage'
@ -208,11 +207,10 @@ def test_middle_layer_different_sha(manifest_protocol, v1_protocol, liveserver_s
# Push and pull the image, ensuring that the produced ID for the middle and leaf layers
# are synthesized.
options = ProtocolOptions()
options.munge_shas = True
options.skip_head_checks = True
manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', second_images,
credentials=credentials, options=options)
v2_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', second_images,
credentials=credentials, options=options)
second_pull_result = v1_protocol.pull(liveserver_session, 'devtable', 'newrepo', 'latest',
second_images, credentials=credentials, options=options)
@ -428,8 +426,8 @@ def test_pull_library_with_support_disabled(puller, basic_images, liveserver_ses
expected_failure=Failures.DISALLOWED_LIBRARY_NAMESPACE)
def test_image_replication(pusher, basic_images, liveserver_session, app_reloader, liveserver,
registry_server_executor):
def test_image_replication(pusher, puller, basic_images, liveserver_session, app_reloader,
liveserver, registry_server_executor):
""" Test: Ensure that entries are created for replication of the images pushed. """
credentials = ('devtable', 'password')
@ -437,9 +435,12 @@ def test_image_replication(pusher, basic_images, liveserver_session, app_reloade
pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
credentials=credentials)
result = puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
credentials=credentials)
# Ensure that entries were created for each image.
for image in basic_images:
r = registry_server_executor.on(liveserver).get_storage_replication_entry(image.id)
for image_id in result.image_ids.values():
r = registry_server_executor.on(liveserver).get_storage_replication_entry(image_id)
assert r.text == 'OK'
@ -484,7 +485,7 @@ def test_tag_validaton(tag_name, expected_failure, pusher, basic_images, liveser
expected_failure=expected_failure)
def test_invalid_parent(pusher, liveserver_session, app_reloader):
def test_invalid_parent(legacy_pusher, liveserver_session, app_reloader):
""" Test: Attempt to push an image with an invalid/missing parent. """
images = [
Image(id='childimage', parent_id='parentimage', size=None,
@ -493,12 +494,12 @@ def test_invalid_parent(pusher, liveserver_session, app_reloader):
credentials = ('devtable', 'password')
pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', images,
credentials=credentials,
expected_failure=Failures.INVALID_IMAGES)
legacy_pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', images,
credentials=credentials,
expected_failure=Failures.INVALID_IMAGES)
def test_wrong_image_order(pusher, liveserver_session, app_reloader):
def test_wrong_image_order(legacy_pusher, liveserver_session, app_reloader):
""" Test: Attempt to push an image with its layers in the wrong order. """
images = [
Image(id='childimage', parent_id='parentimage', size=None,
@ -509,9 +510,9 @@ def test_wrong_image_order(pusher, liveserver_session, app_reloader):
credentials = ('devtable', 'password')
pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', images,
credentials=credentials,
expected_failure=Failures.INVALID_IMAGES)
legacy_pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', images,
credentials=credentials,
expected_failure=Failures.INVALID_IMAGES)
@pytest.mark.parametrize('labels', [
@ -651,15 +652,18 @@ def test_invalid_blob_reference(manifest_protocol, basic_images, liveserver_sess
expected_failure=Failures.INVALID_BLOB)
def test_delete_tag(pusher, puller, basic_images, liveserver_session,
def test_delete_tag(pusher, puller, basic_images, different_images, liveserver_session,
app_reloader):
""" Test: Push a repository, delete a tag, and attempt to pull. """
credentials = ('devtable', 'password')
# Push the tags.
result = pusher.push(liveserver_session, 'devtable', 'newrepo', ['one', 'two'],
result = pusher.push(liveserver_session, 'devtable', 'newrepo', 'one',
basic_images, credentials=credentials)
pusher.push(liveserver_session, 'devtable', 'newrepo', 'two',
different_images, credentials=credentials)
# Delete tag `one` by digest or tag.
pusher.delete(liveserver_session, 'devtable', 'newrepo',
result.manifests['one'].digest if result.manifests else 'one',
@ -671,7 +675,7 @@ def test_delete_tag(pusher, puller, basic_images, liveserver_session,
expected_failure=Failures.UNKNOWN_TAG)
# Pull tag `two` to verify it works.
puller.pull(liveserver_session, 'devtable', 'newrepo', 'two', basic_images,
puller.pull(liveserver_session, 'devtable', 'newrepo', 'two', different_images,
credentials=credentials)
@ -1097,7 +1101,7 @@ EXPECTED_ACI_MANIFEST = {
"eventHandlers": [],
"ports": [],
"annotations": [
{"name": "created", "value": ""},
{"name": "created", "value": "2018-04-03T18:37:09.284840891Z"},
{"name": "homepage", "value": "http://localhost:5000/devtable/newrepo:latest"},
{"name": "quay.io/derived-image",
"value": "035333848582cdb72d2bac4a0809bc7eed9d88004cfb3463562013fce53c2499"},
@ -1162,7 +1166,8 @@ def test_blob_mounting(push_user, push_namespace, push_repo, mount_repo_name, ex
options = ProtocolOptions()
options.scopes = ['repository:devtable/newrepo:push,pull',
'repository:%s:pull' % (mount_repo_name)]
options.mount_blobs = {image.id: mount_repo_name for image in basic_images}
options.mount_blobs = {'sha256:' + hashlib.sha256(image.bytes).hexdigest(): mount_repo_name
for image in basic_images}
manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
credentials=('devtable', 'password'),
@ -1341,8 +1346,8 @@ def test_push_tag_existing_image(v1_protocol, puller, basic_images, liveserver_s
credentials = ('devtable', 'password')
# Push a new repository.
result = v1_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
credentials=credentials)
v1_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images,
credentials=credentials)
# Push the same image/manifest to another tag in the repository.
v1_protocol.tag(liveserver_session, 'devtable', 'newrepo', 'anothertag', basic_images[-1],