import hashlib import json from enum import Enum, unique from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest from image.docker.schema2.list import DockerSchema2ManifestListBuilder from image.docker.schema2.manifest import DockerSchema2ManifestBuilder from image.docker.schema2.config import DockerSchema2Config from image.docker.schemas import parse_manifest_from_bytes from test.registry.protocols import (RegistryProtocol, Failures, ProtocolOptions, PushResult, PullResult) @unique class V2ProtocolSteps(Enum): """ Defines the various steps of the protocol, for matching failures. """ AUTH = 'auth' BLOB_HEAD_CHECK = 'blob-head-check' GET_MANIFEST = 'get-manifest' PUT_MANIFEST = 'put-manifest' MOUNT_BLOB = 'mount-blob' CATALOG = 'catalog' LIST_TAGS = 'list-tags' START_UPLOAD = 'start-upload' class V2Protocol(RegistryProtocol): FAILURE_CODES = { V2ProtocolSteps.AUTH: { Failures.UNAUTHENTICATED: 401, Failures.INVALID_REGISTRY: 400, Failures.APP_REPOSITORY: 405, Failures.ANONYMOUS_NOT_ALLOWED: 401, Failures.INVALID_REPOSITORY: 400, Failures.SLASH_REPOSITORY: 400, Failures.NAMESPACE_DISABLED: 400, }, V2ProtocolSteps.MOUNT_BLOB: { Failures.UNAUTHORIZED_FOR_MOUNT: 202, }, V2ProtocolSteps.GET_MANIFEST: { Failures.UNKNOWN_TAG: 404, Failures.UNAUTHORIZED: 401, Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, }, V2ProtocolSteps.BLOB_HEAD_CHECK: { Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, }, V2ProtocolSteps.START_UPLOAD: { Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, }, V2ProtocolSteps.PUT_MANIFEST: { Failures.DISALLOWED_LIBRARY_NAMESPACE: 400, Failures.MISSING_TAG: 404, Failures.INVALID_TAG: 404, Failures.INVALID_IMAGES: 400, Failures.INVALID_BLOB: 400, Failures.UNSUPPORTED_CONTENT_TYPE: 415, }, } def __init__(self, jwk, schema2=False): self.jwk = jwk self.schema2 = schema2 def ping(self, session): result = session.get('/v2/') assert result.status_code == 401 assert result.headers['Docker-Distribution-API-Version'] == 'registry/2.0' def login(self, session, username, password, scopes, expect_success): scopes = scopes if isinstance(scopes, list) else [scopes] params = { 'account': username, 'service': 'localhost:5000', 'scope': scopes, } auth = (username, password) if not username or not password: auth = None response = session.get('/v2/auth', params=params, auth=auth) if expect_success: assert response.status_code / 100 == 2 else: assert response.status_code / 100 == 4 return response def auth(self, session, credentials, namespace, repo_name, scopes=None, expected_failure=None): """ Performs the V2 Auth flow, returning the token (if any) and the response. Spec: https://docs.docker.com/registry/spec/auth/token/ """ scopes = scopes or [] auth = None username = None if credentials is not None: username, _ = credentials auth = credentials params = { 'account': username, 'service': 'localhost:5000', } if scopes: params['scope'] = scopes response = self.conduct(session, 'GET', '/v2/auth', params=params, auth=auth, expected_status=(200, expected_failure, V2ProtocolSteps.AUTH)) expect_token = (expected_failure is None or not V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure)) if expect_token: assert response.json().get('token') is not None return response.json().get('token'), response return None, response def push(self, session, namespace, repo_name, tag_names, images, credentials=None, expected_failure=None, options=None): options = options or ProtocolOptions() scopes = options.scopes or ['repository:%s:push,pull' % self.repo_name(namespace, repo_name)] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure) if token is None: assert V2Protocol.FAILURE_CODES[V2ProtocolSteps.AUTH].get(expected_failure) return headers = { 'Authorization': 'Bearer ' + token, 'Accept': options.accept_mimetypes, } # Build fake manifests. manifests = {} blobs = {} for tag_name in tag_names: if self.schema2: builder = DockerSchema2ManifestBuilder() for image in images: checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() blobs[checksum] = image.bytes # If invalid blob references were requested, just make it up. if options.manifest_invalid_blob_references: checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest() builder.add_layer(checksum, len(image.bytes)) config = { "os": "linux", "rootfs": { "type": "layers", "diff_ids": [] }, "history": [{ 'created': '2018-04-03T18:37:09.284840891Z', 'created_by': (('/bin/sh -c #(nop) ENTRYPOINT %s' % image.config['Entrypoint']) if image.config and image.config.get('Entrypoint') else '/bin/sh -c #(nop) %s' % image.id), } for image in images], } if images[-1].config: config['config'] = images[-1].config config_json = json.dumps(config) schema2_config = DockerSchema2Config(config_json) builder.set_config(schema2_config) blobs[schema2_config.digest] = schema2_config.bytes manifests[tag_name] = builder.build() else: builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name) for image in reversed(images): checksum = 'sha256:' + hashlib.sha256(image.bytes).hexdigest() blobs[checksum] = image.bytes # If invalid blob references were requested, just make it up. if options.manifest_invalid_blob_references: checksum = 'sha256:' + hashlib.sha256('notarealthing').hexdigest() layer_dict = {'id': image.id, 'parent': image.parent_id} if image.config is not None: layer_dict['config'] = image.config if image.size is not None: layer_dict['Size'] = image.size if image.created is not None: layer_dict['created'] = image.created builder.add_layer(checksum, json.dumps(layer_dict)) # Build the manifest. manifests[tag_name] = builder.build(self.jwk) # Push the blob data. for blob_digest, blob_bytes in blobs.iteritems(): if not options.skip_head_checks: # Blob data should not yet exist. self.conduct(session, 'HEAD', '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), blob_digest), expected_status=(404, expected_failure, V2ProtocolSteps.BLOB_HEAD_CHECK), headers=headers) # Check for mounting of blobs. if options.mount_blobs and blob_digest in options.mount_blobs: self.conduct(session, 'POST', '/v2/%s/blobs/uploads/' % self.repo_name(namespace, repo_name), params={ 'mount': blob_digest, 'from': options.mount_blobs[blob_digest], }, expected_status=(201, expected_failure, V2ProtocolSteps.MOUNT_BLOB), headers=headers) if expected_failure is not None: return else: # Start a new upload of the blob data. response = self.conduct(session, 'POST', '/v2/%s/blobs/uploads/' % self.repo_name(namespace, repo_name), expected_status=(202, expected_failure, V2ProtocolSteps.START_UPLOAD), headers=headers) if response.status_code != 202: continue upload_uuid = response.headers['Docker-Upload-UUID'] new_upload_location = response.headers['Location'] assert new_upload_location.startswith('http://localhost:5000') # We need to make this relative just for the tests because the live server test # case modifies the port. location = response.headers['Location'][len('http://localhost:5000'):] # PATCH the data into the blob. if options.chunks_for_upload is None: self.conduct(session, 'PATCH', location, data=blob_bytes, expected_status=204, headers=headers) else: # If chunked upload is requested, upload the data as a series of chunks, checking # status at every point. for chunk_data in options.chunks_for_upload: if len(chunk_data) == 3: (start_byte, end_byte, expected_code) = chunk_data else: (start_byte, end_byte) = chunk_data expected_code = 204 patch_headers = {'Range': 'bytes=%s-%s' % (start_byte, end_byte)} patch_headers.update(headers) contents_chunk = blob_bytes[start_byte:end_byte] self.conduct(session, 'PATCH', location, data=contents_chunk, expected_status=expected_code, headers=patch_headers) if expected_code != 204: return # Retrieve the upload status at each point, and ensure it is valid. status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name), upload_uuid) response = self.conduct(session, 'GET', status_url, expected_status=204, headers=headers) assert response.headers['Docker-Upload-UUID'] == upload_uuid assert response.headers['Range'] == "bytes=0-%s" % end_byte if options.cancel_blob_upload: self.conduct(session, 'DELETE', location, params=dict(digest=blob_digest), expected_status=204, headers=headers) # Ensure the upload was canceled. status_url = '/v2/%s/blobs/uploads/%s' % (self.repo_name(namespace, repo_name), upload_uuid) self.conduct(session, 'GET', status_url, expected_status=404, headers=headers) return # Finish the blob upload with a PUT. response = self.conduct(session, 'PUT', location, params=dict(digest=blob_digest), expected_status=201, headers=headers) assert response.headers['Docker-Content-Digest'] == blob_digest # Ensure the blob exists now. response = self.conduct(session, 'HEAD', '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), blob_digest), expected_status=200, headers=headers) assert response.headers['Docker-Content-Digest'] == blob_digest assert response.headers['Content-Length'] == str(len(blob_bytes)) # And retrieve the blob data. result = self.conduct(session, 'GET', '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), blob_digest), headers=headers, expected_status=200) assert result.content == blob_bytes # Write a manifest for each tag. for tag_name in tag_names: manifest = manifests[tag_name] # Write the manifest. If we expect it to be invalid, we expect a 404 code. Otherwise, we # expect a 202 response for success. put_code = 404 if options.manifest_invalid_blob_references else 202 manifest_headers = {'Content-Type': manifest.media_type} manifest_headers.update(headers) if options.manifest_content_type is not None: manifest_headers['Content-Type'] = options.manifest_content_type tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest self.conduct(session, 'PUT', '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest), data=manifest.bytes, expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST), headers=manifest_headers) return PushResult(manifests=manifests, headers=headers) def delete(self, session, namespace, repo_name, tag_names, credentials=None, expected_failure=None, options=None): options = options or ProtocolOptions() scopes = options.scopes or ['repository:%s:*' % self.repo_name(namespace, repo_name)] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure) if token is None: return None headers = { 'Authorization': 'Bearer ' + token, } for tag_name in tag_names: self.conduct(session, 'DELETE', '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name), headers=headers, expected_status=202) def pull(self, session, namespace, repo_name, tag_names, images, credentials=None, expected_failure=None, options=None): options = options or ProtocolOptions() scopes = options.scopes or ['repository:%s:pull' % self.repo_name(namespace, repo_name)] tag_names = [tag_names] if isinstance(tag_names, str) else tag_names # Ping! self.ping(session) # Perform auth and retrieve a token. token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure) if token is None: return None headers = { 'Authorization': 'Bearer ' + token, } if self.schema2: headers['Accept'] = [('application/vnd.docker.distribution.manifest.v2+json', 1), ('application/vnd.docker.distribution.manifest.list.v2+json', 1)] manifests = {} image_ids = {} for tag_name in tag_names: # Retrieve the manifest for the tag or digest. response = self.conduct(session, 'GET', '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name), expected_status=(200, expected_failure, V2ProtocolSteps.GET_MANIFEST), headers=headers) if expected_failure is not None: return None # Ensure the manifest returned by us is valid. manifest = parse_manifest_from_bytes(response.text, response.headers['Content-Type']) manifests[tag_name] = manifest image_ids[tag_name] = manifest.leaf_layer_v1_image_id # Verify the layers. for index, layer in enumerate(manifest.layers): result = self.conduct(session, 'GET', '/v2/%s/blobs/%s' % (self.repo_name(namespace, repo_name), layer.digest), expected_status=200, headers=headers) assert result.content == images[index].bytes return PullResult(manifests=manifests, image_ids=image_ids) def tags(self, session, namespace, repo_name, page_size=2, credentials=None, options=None, expected_failure=None): options = options or ProtocolOptions() scopes = options.scopes or ['repository:%s:pull' % self.repo_name(namespace, repo_name)] # Ping! self.ping(session) # Perform auth and retrieve a token. headers = {} if credentials is not None: token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure) if token is None: return None headers = { 'Authorization': 'Bearer ' + token, } results = [] url = '/v2/%s/tags/list' % (self.repo_name(namespace, repo_name)) params = {} if page_size is not None: params['n'] = page_size while True: response = self.conduct(session, 'GET', url, headers=headers, params=params, expected_status=(200, expected_failure, V2ProtocolSteps.LIST_TAGS)) data = response.json() assert len(data['tags']) <= page_size results.extend(data['tags']) if not response.headers.get('Link'): return results link_url = response.headers['Link'] v2_index = link_url.find('/v2/') url = link_url[v2_index:] return results def catalog(self, session, page_size=2, credentials=None, options=None, expected_failure=None, namespace=None, repo_name=None, bearer_token=None): options = options or ProtocolOptions() scopes = options.scopes or [] # Ping! self.ping(session) # Perform auth and retrieve a token. headers = {} if credentials is not None: token, _ = self.auth(session, credentials, namespace, repo_name, scopes=scopes, expected_failure=expected_failure) if token is None: return None headers = { 'Authorization': 'Bearer ' + token, } if bearer_token is not None: headers = { 'Authorization': 'Bearer ' + bearer_token, } results = [] url = '/v2/_catalog' params = {} if page_size is not None: params['n'] = page_size while True: response = self.conduct(session, 'GET', url, headers=headers, params=params, expected_status=(200, expected_failure, V2ProtocolSteps.CATALOG)) data = response.json() assert len(data['repositories']) <= page_size results.extend(data['repositories']) if not response.headers.get('Link'): return results link_url = response.headers['Link'] v2_index = link_url.find('/v2/') url = link_url[v2_index:] return results