import logging from functools import wraps from flask import request, url_for, Response import features from app import docker_v2_signing_key, app, metric_queue from auth.registry_jwt_auth import process_registry_jwt_auth from data import model from digest import digest_tools from endpoints.common import parse_repository_name from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, require_repo_read, require_repo_write from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid) from endpoints.trackhelper import track_and_log from endpoints.notificationhelper import spawn_notification from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES from util.registry.replication import queue_storage_replication from util.names import VALID_TAG_PATTERN logger = logging.getLogger(__name__) BASE_MANIFEST_ROUTE = '//manifests/' MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN) MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN) @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET']) @parse_repository_name() @process_registry_jwt_auth(scopes=['pull']) @require_repo_read @anon_protect def fetch_manifest_by_tagname(namespace_name, repo_name, tag_name): manifest = v2.get_manifest_by_tag(namespace_name, repo_name, tag_name) if manifest is None: tag = v2.get_active_tag(namespace_name, repo_name, tag_name) if tag is None: raise ManifestUnknown() manifest = _generate_and_store_manifest(namespace_name, repo_name, tag_name) if manifest is None: raise ManifestUnknown() repo = v2.get_repository(namespace_name, repo_name) if repo is not None: track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2']) return Response( manifest.bytes, status=200, headers={'Content-Type': manifest.content_type, 'Docker-Content-Digest': manifest.digest}, ) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET']) @parse_repository_name() @process_registry_jwt_auth(scopes=['pull']) @require_repo_read @anon_protect def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref): manifest = model.tag.load_manifest_by_digest(namespace_name, repo_name, manifest_ref) if manifest is None: # Without a tag name to reference, we can't make an attempt to generate the manifest raise ManifestUnknown() repo = v2.get_repository(namespace_name, repo_name) if repo is not None: track_and_log('pull_repo', repo) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2']) return Response(manifest.json, status=200, headers={'Content-Type': manifest.content_type, 'Docker-Content-Digest': manifest.digest}) def _reject_manifest2_schema2(func): @wraps(func) def wrapped(*args, **kwargs): if request.content_type in DOCKER_SCHEMA2_CONTENT_TYPES: raise ManifestInvalid(detail={'message': 'manifest schema version not supported'}, http_status_code=415) return func(*args, **kwargs) return wrapped @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT']) @_reject_manifest2_schema2 @parse_repository_name() @process_registry_jwt_auth(scopes=['pull', 'push']) @require_repo_write @anon_protect def write_manifest_by_tagname(namespace_name, repo_name, tag_name): try: manifest = DockerSchema1Manifest(request.data) except ManifestException as me: raise ManifestInvalid(detail={'message': me.message}) if manifest.tag != tag_name: raise TagInvalid() return _write_manifest(namespace_name, repo_name, manifest) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT']) @_reject_manifest2_schema2 @parse_repository_name() @process_registry_jwt_auth(scopes=['pull', 'push']) @require_repo_write @anon_protect def write_manifest_by_digest(namespace_name, repo_name, digest): try: manifest = DockerSchema1Manifest(request.data) except ManifestException as me: raise ManifestInvalid(detail={'message': me.message}) if manifest.digest != digest: raise ManifestInvalid(detail={'message': 'manifest digest mismatch'}) return _write_manifest(namespace_name, repo_name, manifest) def _write_manifest(namespace_name, repo_name, manifest): if (manifest.namespace == '' and features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']): pass elif manifest.namespace != namespace_name: raise NameInvalid() if manifest.repo_name != repo_name: raise NameInvalid() # Ensure that the repository exists. repo = v2.get_repository(namespace_name, repo_name) if repo is None: raise NameInvalid() if not manifest.layers: raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'}) # Ensure all the blobs in the manifest exist. storage_query = model.storage.lookup_repo_storages_by_content_checksum(repo, manifest.checksums) storage_map = {storage.content_checksum: storage for storage in storage_query} for layer in manifest.layers: digest_str = str(layer.digest) if digest_str not in storage_map: raise BlobUnknown(detail={'digest': digest_str}) # Lookup all the images and their parent images (if any) inside the manifest. # This will let us know which v1 images we need to synthesize and which ones are invalid. all_image_ids = list(manifest.docker_image_ids | manifest.parent_image_ids) images_map = v2.get_docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids) # Rewrite any v1 image IDs that do not match the checksum in the database. try: rewritten_images = manifest.rewrite_invalid_image_ids(images_map) for rewritten_image in rewritten_images: image = v2.synthesize_v1_image( repo, storage_map[rewritten_image.content_checksum], rewritten_image.image_id, rewritten_image.created, rewritten_image.comment, rewritten_image.command, rewritten_image.compat_json, rewritten_image.parent_image_id, ) images_map[image.image_id] = image except ManifestException as me: raise ManifestInvalid(detail={'message': me.message}) # Store the manifest pointing to the tag. leaf_layer_id = images_map[manifest.leaf_layer.v1_metadata.image_id].image_id v2.save_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest.digest, manifest.bytes) # Queue all blob manifests for replication. # TODO(jschorr): Find a way to optimize this insertion. if features.STORAGE_REPLICATION: for layer in manifest.layers: digest_str = str(layer.digest) queue_storage_replication(namespace_name, storage_map[digest_str]) track_and_log('push_repo', repo, tag=manifest.tag) spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]}) metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2']) return Response( 'OK', status=202, headers={ 'Docker-Content-Digest': manifest.digest, 'Location': url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name), manifest_ref=manifest.digest), }, ) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE']) @parse_repository_name() @process_registry_jwt_auth(scopes=['pull', 'push']) @require_repo_write @anon_protect def delete_manifest_by_digest(namespace_name, repo_name, digest): """ Delete the manifest specified by the digest. Note: there is no equivalent method for deleting by tag name because it is forbidden by the spec. """ tag = v2.get_tag_by_manifest_digest(namespace_name, repo_name, digest) if tag is None: # TODO(jzelinskie): disambiguate between no manifest and no tag raise ManifestUnknown() # Mark the tag as no longer alive. deleted = v2.delete_tag(namespace_name, repo_name, tag.name) if not deleted: # Tag was not alive. raise ManifestUnknown() track_and_log('delete_tag', tag.repository, tag=tag.name, digest=digest) return Response(status=202) def _generate_and_store_manifest(namespace_name, repo_name, tag_name): # Find the v1 metadata for this image and its parents. v1_metadata = v2.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name) parents_v1_metadata = v2.get_parents_docker_v1_metadata(namespace_name, repo_name, v1_metadata.image_id) # If the manifest is being generated under the library namespace, then we make its namespace # empty. manifest_namespace = namespace_name if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']: manifest_namespace = '' # Create and populate the manifest builder builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name) # Add the leaf layer builder.add_layer(v1_metadata.content_checksum, v1_metadata.compat_json) for parent_v1_metadata in parents_v1_metadata: builder.add_layer(parent_v1_metadata.content_checksum, parent_v1_metadata.compat_json) # Sign the manifest with our signing key. manifest = builder.build(docker_v2_signing_key) # Write the manifest to the DB. v2.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, manifest.bytes) return manifest