import logging from functools import wraps from flask import request, url_for, Response import features from app import docker_v2_signing_key, app, metric_queue from auth.registry_jwt_auth import process_registry_jwt_auth from digest import digest_tools from endpoints.decorators import anon_protect, parse_repository_name from endpoints.v2 import v2_bp, require_repo_read, require_repo_write from endpoints.v2.models_interface import Label from endpoints.v2.models_pre_oci import data_model as model from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid, NameInvalid, TagExpired) from endpoints.v2.labelhandlers import handle_label from image.docker import ManifestException from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES, OCI_CONTENT_TYPES from notifications import spawn_notification from util.audit import track_and_log from util.names import VALID_TAG_PATTERN from util.registry.replication import queue_replication_batch from util.validation import is_json logger = logging.getLogger(__name__) BASE_MANIFEST_ROUTE = '//manifests/' MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN) MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN) @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET']) @parse_repository_name() @process_registry_jwt_auth(scopes=['pull']) @require_repo_read @anon_protect def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref): manifest = model.get_manifest_by_tag(namespace_name, repo_name, manifest_ref) if manifest is None: has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref) if not has_tag: has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref) if has_expired_tag: logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name, repo_name) msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref raise TagExpired(msg) else: raise ManifestUnknown() manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref) if manifest is None: raise ManifestUnknown() repo = model.get_repository(namespace_name, repo_name) if repo is not None: track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01, tag=manifest_ref) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True]) return Response( manifest.json, status=200, headers={'Content-Type': manifest.media_type, 'Docker-Content-Digest': manifest.digest},) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET']) @parse_repository_name() @process_registry_jwt_auth(scopes=['pull']) @require_repo_read @anon_protect def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref): manifest = model.get_manifest_by_digest(namespace_name, repo_name, manifest_ref) if manifest is None: # Without a tag name to reference, we can't make an attempt to generate the manifest raise ManifestUnknown() repo = model.get_repository(namespace_name, repo_name) if repo is not None: track_and_log('pull_repo', repo, manifest_digest=manifest_ref) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True]) return Response(manifest.json, status=200, headers={ 'Content-Type': manifest.media_type, 'Docker-Content-Digest': manifest.digest}) def _reject_manifest2_schema2(func): @wraps(func) def wrapped(*args, **kwargs): if request.content_type in (DOCKER_SCHEMA2_CONTENT_TYPES | OCI_CONTENT_TYPES): raise ManifestInvalid(detail={'message': 'manifest schema version not supported'}, http_status_code=415) return func(*args, **kwargs) return wrapped @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT']) @_reject_manifest2_schema2 @parse_repository_name() @process_registry_jwt_auth(scopes=['pull', 'push']) @require_repo_write @anon_protect def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref): try: manifest = DockerSchema1Manifest(request.data) except ManifestException as me: logger.exception("failed to parse manifest when writing by tagname") raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message}) if manifest.tag != manifest_ref: raise TagInvalid() return _write_manifest_and_log(namespace_name, repo_name, manifest) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT']) @_reject_manifest2_schema2 @parse_repository_name() @process_registry_jwt_auth(scopes=['pull', 'push']) @require_repo_write @anon_protect def write_manifest_by_digest(namespace_name, repo_name, manifest_ref): try: manifest = DockerSchema1Manifest(request.data) except ManifestException as me: logger.exception("failed to parse manifest when writing by digest") raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message}) if manifest.digest != manifest_ref: raise ManifestInvalid(detail={'message': 'manifest digest mismatch'}) return _write_manifest_and_log(namespace_name, repo_name, manifest) def _write_manifest(namespace_name, repo_name, manifest): if (manifest.namespace == '' and features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']): pass elif manifest.namespace != namespace_name: raise NameInvalid() if manifest.repo_name != repo_name: raise NameInvalid() # Ensure that the repository exists. repo = model.get_repository(namespace_name, repo_name) if repo is None: raise NameInvalid() if not manifest.layers: raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'}) # Ensure all the blobs in the manifest exist. storage_map = model.lookup_blobs_by_digest(repo, manifest.checksums) for layer in manifest.layers: digest_str = str(layer.digest) if digest_str not in storage_map: raise BlobUnknown(detail={'digest': digest_str}) # Lookup all the images and their parent images (if any) inside the manifest. # This will let us know which v1 images we need to synthesize and which ones are invalid. all_image_ids = list(manifest.parent_image_ids | manifest.image_ids) images_map = model.get_docker_v1_metadata_by_image_id(repo, all_image_ids) # Rewrite any v1 image IDs that do not match the checksum in the database. try: # TODO: make this batch and read the parent image from the previous iteration, rather than # reloading it. rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map)) for rewritten_image in rewritten_images: if not rewritten_image.image_id in images_map: model.synthesize_v1_image( repo, storage_map[rewritten_image.content_checksum], rewritten_image.image_id, rewritten_image.created, rewritten_image.comment, rewritten_image.command, rewritten_image.compat_json, rewritten_image.parent_image_id, ) except ManifestException as me: logger.exception("exception when rewriting v1 metadata") raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message}) # Store the manifest pointing to the tag. leaf_layer_id = rewritten_images[-1].image_id newly_created = model.save_manifest(repo, manifest.tag, leaf_layer_id, manifest.digest, manifest.bytes) if newly_created: # TODO: make this batch labels = [] for key, value in manifest.layers[-1].v1_metadata.labels.iteritems(): media_type = 'application/json' if is_json(value) else 'text/plain' labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type)) handle_label(key, value, namespace_name, repo_name, manifest.digest) model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels) return repo, storage_map def _write_manifest_and_log(namespace_name, repo_name, manifest): repo, storage_map = _write_manifest(namespace_name, repo_name, manifest) # Queue all blob manifests for replication. if features.STORAGE_REPLICATION: with queue_replication_batch(namespace_name) as queue_storage_replication: for layer in manifest.layers: digest_str = str(layer.digest) queue_storage_replication(storage_map[digest_str]) track_and_log('push_repo', repo, tag=manifest.tag) spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]}) metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2', True]) return Response( 'OK', status=202, headers={ 'Docker-Content-Digest': manifest.digest, 'Location': url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name), manifest_ref=manifest.digest), }, ) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE']) @parse_repository_name() @process_registry_jwt_auth(scopes=['pull', 'push']) @require_repo_write @anon_protect def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref): """ Delete the manifest specified by the digest. Note: there is no equivalent method for deleting by tag name because it is forbidden by the spec. """ tags = model.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref) if not tags: raise ManifestUnknown() for tag in tags: track_and_log('delete_tag', tag.repository, tag=tag.name, digest=manifest_ref) return Response(status=202) def _generate_and_store_manifest(namespace_name, repo_name, tag_name): # Find the v1 metadata for this image and its parents. v1_metadata = model.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name) parents_v1_metadata = model.get_parents_docker_v1_metadata(namespace_name, repo_name, v1_metadata.image_id) # If the manifest is being generated under the library namespace, then we make its namespace # empty. manifest_namespace = namespace_name if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']: manifest_namespace = '' # Create and populate the manifest builder builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name) # Add the leaf layer builder.add_layer(v1_metadata.content_checksum, v1_metadata.compat_json) for parent_v1_metadata in parents_v1_metadata: builder.add_layer(parent_v1_metadata.content_checksum, parent_v1_metadata.compat_json) # Sign the manifest with our signing key. manifest = builder.build(docker_v2_signing_key) # Write the manifest to the DB. model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest, manifest.bytes) return manifest