Add tests for manifest lists and fix some issues encountered while testing
This commit is contained in:
Joseph Schorr
parent
9994f0ae61
commit
7a794e29c0
9 changed files with 311 additions and 98 deletions
|
|
@ -3,10 +3,12 @@ import json
|
|||
|
||||
from enum import Enum, unique
|
||||
|
||||
from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
|
||||
from image.docker.schema2.list import DockerSchema2ManifestListBuilder
|
||||
from image.docker.schema1 import (DockerSchema1ManifestBuilder, DockerSchema1Manifest,
|
||||
DOCKER_SCHEMA1_CONTENT_TYPES)
|
||||
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES
|
||||
from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
|
||||
from image.docker.schema2.config import DockerSchema2Config
|
||||
from image.docker.schema2.list import DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
|
||||
from image.docker.schemas import parse_manifest_from_bytes
|
||||
from test.registry.protocols import (RegistryProtocol, Failures, ProtocolOptions, PushResult,
|
||||
PullResult)
|
||||
|
|
@ -123,6 +125,148 @@ class V2Protocol(RegistryProtocol):
|
|||
|
||||
return None, response
|
||||
|
||||
def pull_list(self, session, namespace, repo_name, tag_names, manifestlist,
|
||||
credentials=None, expected_failure=None, options=None):
|
||||
options = options or ProtocolOptions()
|
||||
scopes = options.scopes or ['repository:%s:push,pull' % self.repo_name(namespace, repo_name)]
|
||||
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
|
||||
|
||||
# Ping!
|
||||
self.ping(session)
|
||||
|
||||
# Perform auth and retrieve a token.
|
||||
token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes,
|
||||
expected_failure=expected_failure)
|
||||
if token is None:
|
||||
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
|
||||
return
|
||||
|
||||
headers = {
|
||||
'Authorization': 'Bearer ' + token,
|
||||
'Accept': DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE,
|
||||
}
|
||||
|
||||
for tag_name in tag_names:
|
||||
# Retrieve the manifest for the tag or digest.
|
||||
response = self.conduct(session, 'GET',
|
||||
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name),
|
||||
tag_name),
|
||||
expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST),
|
||||
headers=headers)
|
||||
if expected_failure is not None:
|
||||
return None
|
||||
|
||||
# Parse the returned manifest list and ensure it matches.
|
||||
assert response.headers['Content-Type'] == DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
|
||||
manifest = parse_manifest_from_bytes(response.text, response.headers['Content-Type'])
|
||||
assert manifest.schema_version is None
|
||||
assert manifest.digest == manifestlist.digest
|
||||
|
||||
|
||||
def push_list(self, session, namespace, repo_name, tag_names, manifestlist, blobs,
|
||||
credentials=None, expected_failure=None, options=None):
|
||||
options = options or ProtocolOptions()
|
||||
scopes = options.scopes or ['repository:%s:push,pull' % self.repo_name(namespace, repo_name)]
|
||||
tag_names = [tag_names] if isinstance(tag_names, str) else tag_names
|
||||
|
||||
# Ping!
|
||||
self.ping(session)
|
||||
|
||||
# Perform auth and retrieve a token.
|
||||
token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes,
|
||||
expected_failure=expected_failure)
|
||||
if token is None:
|
||||
assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)
|
||||
return
|
||||
|
||||
headers = {
|
||||
'Authorization': 'Bearer ' + token,
|
||||
'Accept': ','.join(options.accept_mimetypes or []),
|
||||
}
|
||||
|
||||
# Push all blobs.
|
||||
if not self._push_blobs(blobs, session, namespace, repo_name, headers, options,
|
||||
expected_failure):
|
||||
return
|
||||
|
||||
# Push the manifest list.
|
||||
for tag_name in tag_names:
|
||||
manifest_headers = {'Content-Type': manifestlist.media_type}
|
||||
manifest_headers.update(headers)
|
||||
|
||||
if options.manifest_content_type is not None:
|
||||
manifest_headers['Content-Type'] = options.manifest_content_type
|
||||
|
||||
self.conduct(session, 'PUT',
|
||||
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name),
|
||||
data=manifestlist.bytes,
|
||||
expected_status=(202, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
|
||||
headers=manifest_headers)
|
||||
|
||||
return PushResult(manifests=None, headers=headers)
|
||||
|
||||
def build_schema2(self, images, blobs, options):
|
||||
builder = DockerSchema2ManifestBuilder()
|
||||
for image in images:
|
||||
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
# If invalid blob references were requested, just make it up.
|
||||
if options.manifest_invalid_blob_references:
|
||||
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
|
||||
|
||||
builder.add_layer(checksum, len(image.bytes))
|
||||
|
||||
config = {
|
||||
"os": "linux",
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
},
|
||||
"history": [{
|
||||
'created': '2018-04-03T18:37:09.284840891Z',
|
||||
'created_by': (('/bin/sh -c #(nop) ENTRYPOINT %s' % image.config['Entrypoint'])
|
||||
if image.config and image.config.get('Entrypoint')
|
||||
else '/bin/sh -c #(nop) %s' % image.id),
|
||||
} for image in images],
|
||||
}
|
||||
|
||||
if images[-1].config:
|
||||
config['config'] = images[-1].config
|
||||
|
||||
config_json = json.dumps(config)
|
||||
schema2_config = DockerSchema2Config(config_json)
|
||||
builder.set_config(schema2_config)
|
||||
|
||||
blobs[schema2_config.digest] = schema2_config.bytes
|
||||
return builder.build()
|
||||
|
||||
def build_schema1(self, namespace, repo_name, tag_name, images, blobs, options):
|
||||
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
|
||||
|
||||
for image in reversed(images):
|
||||
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
# If invalid blob references were requested, just make it up.
|
||||
if options.manifest_invalid_blob_references:
|
||||
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
|
||||
|
||||
layer_dict = {'id': image.id, 'parent': image.parent_id}
|
||||
if image.config is not None:
|
||||
layer_dict['config'] = image.config
|
||||
|
||||
if image.size is not None:
|
||||
layer_dict['Size'] = image.size
|
||||
|
||||
if image.created is not None:
|
||||
layer_dict['created'] = image.created
|
||||
|
||||
builder.add_layer(checksum, json.dumps(layer_dict))
|
||||
|
||||
# Build the manifest.
|
||||
return builder.build(self.jwk)
|
||||
|
||||
def push(self, session, namespace, repo_name, tag_names, images, credentials=None,
|
||||
expected_failure=None, options=None):
|
||||
options = options or ProtocolOptions()
|
||||
|
|
@ -141,7 +285,7 @@ class V2Protocol(RegistryProtocol):
|
|||
|
||||
headers = {
|
||||
'Authorization': 'Bearer ' + token,
|
||||
'Accept': options.accept_mimetypes,
|
||||
'Accept': ','.join(options.accept_mimetypes or []),
|
||||
}
|
||||
|
||||
# Build fake manifests.
|
||||
|
|
@ -149,67 +293,39 @@ class V2Protocol(RegistryProtocol):
|
|||
blobs = {}
|
||||
for tag_name in tag_names:
|
||||
if self.schema2:
|
||||
builder = DockerSchema2ManifestBuilder()
|
||||
for image in images:
|
||||
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
# If invalid blob references were requested, just make it up.
|
||||
if options.manifest_invalid_blob_references:
|
||||
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
|
||||
|
||||
builder.add_layer(checksum, len(image.bytes))
|
||||
|
||||
config = {
|
||||
"os": "linux",
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": []
|
||||
},
|
||||
"history": [{
|
||||
'created': '2018-04-03T18:37:09.284840891Z',
|
||||
'created_by': (('/bin/sh -c #(nop) ENTRYPOINT %s' % image.config['Entrypoint'])
|
||||
if image.config and image.config.get('Entrypoint')
|
||||
else '/bin/sh -c #(nop) %s' % image.id),
|
||||
} for image in images],
|
||||
}
|
||||
|
||||
if images[-1].config:
|
||||
config['config'] = images[-1].config
|
||||
|
||||
config_json = json.dumps(config)
|
||||
schema2_config = DockerSchema2Config(config_json)
|
||||
builder.set_config(schema2_config)
|
||||
|
||||
blobs[schema2_config.digest] = schema2_config.bytes
|
||||
manifests[tag_name] = builder.build()
|
||||
manifests[tag_name] = self.build_schema2(images, blobs, options)
|
||||
else:
|
||||
builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
|
||||
|
||||
for image in reversed(images):
|
||||
checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest()
|
||||
blobs[checksum] = image.bytes
|
||||
|
||||
# If invalid blob references were requested, just make it up.
|
||||
if options.manifest_invalid_blob_references:
|
||||
checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest()
|
||||
|
||||
layer_dict = {'id': image.id, 'parent': image.parent_id}
|
||||
if image.config is not None:
|
||||
layer_dict['config'] = image.config
|
||||
|
||||
if image.size is not None:
|
||||
layer_dict['Size'] = image.size
|
||||
|
||||
if image.created is not None:
|
||||
layer_dict['created'] = image.created
|
||||
|
||||
builder.add_layer(checksum, json.dumps(layer_dict))
|
||||
|
||||
# Build the manifest.
|
||||
manifests[tag_name] = builder.build(self.jwk)
|
||||
manifests[tag_name] = self.build_schema1(namespace, repo_name, tag_name, images, blobs,
|
||||
options)
|
||||
|
||||
# Push the blob data.
|
||||
if not self._push_blobs(blobs, session, namespace, repo_name, headers, options,
|
||||
expected_failure):
|
||||
return
|
||||
|
||||
# Write a manifest for each tag.
|
||||
for tag_name in tag_names:
|
||||
manifest = manifests[tag_name]
|
||||
|
||||
# Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we
|
||||
# expect a 202 response for success.
|
||||
put_code = 404 if options.manifest_invalid_blob_references else 202
|
||||
manifest_headers = {'Content-Type': manifest.media_type}
|
||||
manifest_headers.update(headers)
|
||||
|
||||
if options.manifest_content_type is not None:
|
||||
manifest_headers['Content-Type'] = options.manifest_content_type
|
||||
|
||||
tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest
|
||||
self.conduct(session, 'PUT',
|
||||
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest),
|
||||
data=manifest.bytes,
|
||||
expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
|
||||
headers=manifest_headers)
|
||||
|
||||
return PushResult(manifests=manifests, headers=headers)
|
||||
|
||||
def _push_blobs(self, blobs, session, namespace, repo_name, headers, options, expected_failure):
|
||||
for blob_digest, blob_bytes in blobs.iteritems():
|
||||
if not options.skip_head_checks:
|
||||
# Blob data should not yet exist.
|
||||
|
|
@ -270,7 +386,7 @@ class V2Protocol(RegistryProtocol):
|
|||
expected_status=expected_code,
|
||||
headers=patch_headers)
|
||||
if expected_code != 204:
|
||||
return
|
||||
return False
|
||||
|
||||
# Retrieve the upload status at each point, and ensure it is valid.
|
||||
status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name),
|
||||
|
|
@ -288,7 +404,7 @@ class V2Protocol(RegistryProtocol):
|
|||
status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name),
|
||||
upload_uuid)
|
||||
self.conduct(session, 'GET', status_url, expected_status=404, headers=headers)
|
||||
return
|
||||
return False
|
||||
|
||||
# Finish the blob upload with a PUT.
|
||||
response = self.conduct(session, 'PUT', location, params=dict(digest=blob_digest),
|
||||
|
|
@ -310,28 +426,7 @@ class V2Protocol(RegistryProtocol):
|
|||
headers=headers, expected_status=200)
|
||||
assert result.content == blob_bytes
|
||||
|
||||
# Write a manifest for each tag.
|
||||
for tag_name in tag_names:
|
||||
manifest = manifests[tag_name]
|
||||
|
||||
# Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we
|
||||
# expect a 202 response for success.
|
||||
put_code = 404 if options.manifest_invalid_blob_references else 202
|
||||
manifest_headers = {'Content-Type': manifest.media_type}
|
||||
manifest_headers.update(headers)
|
||||
|
||||
if options.manifest_content_type is not None:
|
||||
manifest_headers['Content-Type'] = options.manifest_content_type
|
||||
|
||||
tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest
|
||||
self.conduct(session, 'PUT',
|
||||
'/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest),
|
||||
data=manifest.bytes,
|
||||
expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
|
||||
headers=manifest_headers)
|
||||
|
||||
return PushResult(manifests=manifests, headers=headers)
|
||||
|
||||
return True
|
||||
|
||||
def delete(self, session, namespace, repo_name, tag_names, credentials=None,
|
||||
expected_failure=None, options=None):
|
||||
|
|
@ -379,8 +474,7 @@ class V2Protocol(RegistryProtocol):
|
|||
}
|
||||
|
||||
if self.schema2:
|
||||
headers['Accept'] = [('application/vnd.docker.distribution.manifest.v2+json', 1),
|
||||
('application/vnd.docker.distribution.manifest.list.v2+json', 1)]
|
||||
headers['Accept'] = ','.join(options.accept_mimetypes or DOCKER_SCHEMA2_CONTENT_TYPES)
|
||||
|
||||
manifests = {}
|
||||
image_ids = {}
|
||||
|
|
@ -395,6 +489,9 @@ class V2Protocol(RegistryProtocol):
|
|||
return None
|
||||
|
||||
# Ensure the manifest returned by us is valid.
|
||||
if not self.schema2:
|
||||
assert response.headers['Content-Type'] in DOCKER_SCHEMA1_CONTENT_TYPES
|
||||
|
||||
manifest = parse_manifest_from_bytes(response.text, response.headers['Content-Type'])
|
||||
manifests[tag_name] = manifest
|
||||
image_ids[tag_name] = manifest.leaf_layer_v1_image_id
|
||||
|
|
|
|||
Reference in a new issue