9e16a989f5
Instead of 41 queries now for the simple manifest, we are down to 14. The biggest changes: - Only synthesize the V1 image rows if we haven't already found them in the database - Thread the repository object through to the other model method calls, and use it instead of loading again and again
284 lines
11 KiB
Python
284 lines
11 KiB
Python
import logging
|
|
|
|
from functools import wraps
|
|
|
|
from flask import request, url_for, Response
|
|
|
|
import features
|
|
|
|
from app import docker_v2_signing_key, app, metric_queue
|
|
from auth.registry_jwt_auth import process_registry_jwt_auth
|
|
from digest import digest_tools
|
|
from endpoints.decorators import anon_protect, parse_repository_name
|
|
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
|
|
from endpoints.v2.models_interface import Label
|
|
from endpoints.v2.models_pre_oci import data_model as model
|
|
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
|
|
NameInvalid, TagExpired)
|
|
from endpoints.v2.labelhandlers import handle_label
|
|
from image.docker import ManifestException
|
|
from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
|
|
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES, OCI_CONTENT_TYPES
|
|
from notifications import spawn_notification
|
|
from util.audit import track_and_log
|
|
from util.names import VALID_TAG_PATTERN
|
|
from util.registry.replication import queue_replication_batch
|
|
from util.validation import is_json
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>'
|
|
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
|
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|
manifest = model.get_manifest_by_tag(namespace_name, repo_name, manifest_ref)
|
|
if manifest is None:
|
|
has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref)
|
|
if not has_tag:
|
|
has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref)
|
|
if has_expired_tag:
|
|
logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name,
|
|
repo_name)
|
|
msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref
|
|
raise TagExpired(msg)
|
|
else:
|
|
raise ManifestUnknown()
|
|
|
|
manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref)
|
|
if manifest is None:
|
|
raise ManifestUnknown()
|
|
|
|
repo = model.get_repository(namespace_name, repo_name)
|
|
if repo is not None:
|
|
track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01,
|
|
tag=manifest_ref)
|
|
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
|
|
|
|
return Response(
|
|
manifest.json,
|
|
status=200,
|
|
headers={'Content-Type': manifest.media_type,
|
|
'Docker-Content-Digest': manifest.digest},)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
manifest = model.get_manifest_by_digest(namespace_name, repo_name, manifest_ref)
|
|
if manifest is None:
|
|
# Without a tag name to reference, we can't make an attempt to generate the manifest
|
|
raise ManifestUnknown()
|
|
|
|
repo = model.get_repository(namespace_name, repo_name)
|
|
if repo is not None:
|
|
track_and_log('pull_repo', repo, manifest_digest=manifest_ref)
|
|
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
|
|
|
|
return Response(manifest.json, status=200, headers={
|
|
'Content-Type': manifest.media_type,
|
|
'Docker-Content-Digest': manifest.digest})
|
|
|
|
|
|
def _reject_manifest2_schema2(func):
|
|
@wraps(func)
|
|
def wrapped(*args, **kwargs):
|
|
if request.content_type in (DOCKER_SCHEMA2_CONTENT_TYPES | OCI_CONTENT_TYPES):
|
|
raise ManifestInvalid(detail={'message': 'manifest schema version not supported'},
|
|
http_status_code=415)
|
|
return func(*args, **kwargs)
|
|
|
|
return wrapped
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
|
|
@_reject_manifest2_schema2
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|
try:
|
|
manifest = DockerSchema1Manifest(request.data)
|
|
except ManifestException as me:
|
|
logger.exception("failed to parse manifest when writing by tagname")
|
|
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
|
|
|
|
if manifest.tag != manifest_ref:
|
|
raise TagInvalid()
|
|
|
|
return _write_manifest_and_log(namespace_name, repo_name, manifest)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
|
|
@_reject_manifest2_schema2
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
try:
|
|
manifest = DockerSchema1Manifest(request.data)
|
|
except ManifestException as me:
|
|
logger.exception("failed to parse manifest when writing by digest")
|
|
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
|
|
|
|
if manifest.digest != manifest_ref:
|
|
raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})
|
|
|
|
return _write_manifest_and_log(namespace_name, repo_name, manifest)
|
|
|
|
|
|
def _write_manifest(namespace_name, repo_name, manifest):
|
|
if (manifest.namespace == '' and features.LIBRARY_SUPPORT and
|
|
namespace_name == app.config['LIBRARY_NAMESPACE']):
|
|
pass
|
|
elif manifest.namespace != namespace_name:
|
|
raise NameInvalid()
|
|
|
|
if manifest.repo_name != repo_name:
|
|
raise NameInvalid()
|
|
|
|
# Ensure that the repository exists.
|
|
repo = model.get_repository(namespace_name, repo_name)
|
|
if repo is None:
|
|
raise NameInvalid()
|
|
|
|
if not manifest.layers:
|
|
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
|
|
|
|
# Ensure all the blobs in the manifest exist.
|
|
storage_map = model.lookup_blobs_by_digest(repo, manifest.checksums)
|
|
for layer in manifest.layers:
|
|
digest_str = str(layer.digest)
|
|
if digest_str not in storage_map:
|
|
raise BlobUnknown(detail={'digest': digest_str})
|
|
|
|
# Lookup all the images and their parent images (if any) inside the manifest.
|
|
# This will let us know which v1 images we need to synthesize and which ones are invalid.
|
|
all_image_ids = list(manifest.parent_image_ids | manifest.image_ids)
|
|
images_map = model.get_docker_v1_metadata_by_image_id(repo, all_image_ids)
|
|
|
|
# Rewrite any v1 image IDs that do not match the checksum in the database.
|
|
try:
|
|
# TODO: make this batch and read the parent image from the previous iteration, rather than
|
|
# reloading it.
|
|
rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map))
|
|
for rewritten_image in rewritten_images:
|
|
if not rewritten_image.image_id in images_map:
|
|
model.synthesize_v1_image(
|
|
repo,
|
|
storage_map[rewritten_image.content_checksum],
|
|
rewritten_image.image_id,
|
|
rewritten_image.created,
|
|
rewritten_image.comment,
|
|
rewritten_image.command,
|
|
rewritten_image.compat_json,
|
|
rewritten_image.parent_image_id,
|
|
)
|
|
except ManifestException as me:
|
|
logger.exception("exception when rewriting v1 metadata")
|
|
raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message})
|
|
|
|
# Store the manifest pointing to the tag.
|
|
leaf_layer_id = rewritten_images[-1].image_id
|
|
newly_created = model.save_manifest(repo, manifest.tag, leaf_layer_id, manifest.digest,
|
|
manifest.bytes)
|
|
if newly_created:
|
|
# TODO: make this batch
|
|
labels = []
|
|
for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
|
|
media_type = 'application/json' if is_json(value) else 'text/plain'
|
|
labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type))
|
|
handle_label(key, value, namespace_name, repo_name, manifest.digest)
|
|
|
|
model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels)
|
|
|
|
return repo, storage_map
|
|
|
|
|
|
def _write_manifest_and_log(namespace_name, repo_name, manifest):
|
|
repo, storage_map = _write_manifest(namespace_name, repo_name, manifest)
|
|
|
|
# Queue all blob manifests for replication.
|
|
if features.STORAGE_REPLICATION:
|
|
with queue_replication_batch(namespace_name) as queue_storage_replication:
|
|
for layer in manifest.layers:
|
|
digest_str = str(layer.digest)
|
|
queue_storage_replication(storage_map[digest_str])
|
|
|
|
track_and_log('push_repo', repo, tag=manifest.tag)
|
|
spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]})
|
|
metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
|
|
|
|
return Response(
|
|
'OK',
|
|
status=202,
|
|
headers={
|
|
'Docker-Content-Digest': manifest.digest,
|
|
'Location':
|
|
url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name),
|
|
manifest_ref=manifest.digest),
|
|
},
|
|
)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
"""
|
|
Delete the manifest specified by the digest.
|
|
|
|
Note: there is no equivalent method for deleting by tag name because it is
|
|
forbidden by the spec.
|
|
"""
|
|
tags = model.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref)
|
|
if not tags:
|
|
raise ManifestUnknown()
|
|
|
|
for tag in tags:
|
|
track_and_log('delete_tag', tag.repository, tag=tag.name, digest=manifest_ref)
|
|
|
|
return Response(status=202)
|
|
|
|
|
|
def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
|
|
# Find the v1 metadata for this image and its parents.
|
|
v1_metadata = model.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name)
|
|
parents_v1_metadata = model.get_parents_docker_v1_metadata(namespace_name, repo_name,
|
|
v1_metadata.image_id)
|
|
|
|
# If the manifest is being generated under the library namespace, then we make its namespace
|
|
# empty.
|
|
manifest_namespace = namespace_name
|
|
if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
|
|
manifest_namespace = ''
|
|
|
|
# Create and populate the manifest builder
|
|
builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)
|
|
|
|
# Add the leaf layer
|
|
builder.add_layer(v1_metadata.content_checksum, v1_metadata.compat_json)
|
|
|
|
for parent_v1_metadata in parents_v1_metadata:
|
|
builder.add_layer(parent_v1_metadata.content_checksum, parent_v1_metadata.compat_json)
|
|
|
|
# Sign the manifest with our signing key.
|
|
manifest = builder.build(docker_v2_signing_key)
|
|
|
|
# Write the manifest to the DB.
|
|
model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest.digest,
|
|
manifest.bytes)
|
|
return manifest
|