a46660a06f
This change also starts passing in the manifest interface, rather than the raw data, to the model for writing. Note that this change does *not* backfill the existing rows in to the new tables; that will occur in a followup PR. The new columns in `tagmanifest` and `tagmanifestlabel` will be used to track the backfill, as it will occur in a worker.
282 lines
11 KiB
Python
282 lines
11 KiB
Python
import logging
|
|
|
|
from functools import wraps
|
|
|
|
from flask import request, url_for, Response
|
|
|
|
import features
|
|
|
|
from app import docker_v2_signing_key, app, metric_queue
|
|
from auth.registry_jwt_auth import process_registry_jwt_auth
|
|
from digest import digest_tools
|
|
from endpoints.decorators import anon_protect, parse_repository_name
|
|
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
|
|
from endpoints.v2.models_interface import Label
|
|
from endpoints.v2.models_pre_oci import data_model as model
|
|
from endpoints.v2.errors import (BlobUnknown, ManifestInvalid, ManifestUnknown, TagInvalid,
|
|
NameInvalid, TagExpired)
|
|
from endpoints.v2.labelhandlers import handle_label
|
|
from image.docker import ManifestException
|
|
from image.docker.schema1 import DockerSchema1Manifest, DockerSchema1ManifestBuilder
|
|
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES, OCI_CONTENT_TYPES
|
|
from notifications import spawn_notification
|
|
from util.audit import track_and_log
|
|
from util.names import VALID_TAG_PATTERN
|
|
from util.registry.replication import queue_replication_batch
|
|
from util.validation import is_json
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>'
|
|
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
|
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|
manifest = model.get_manifest_by_tag(namespace_name, repo_name, manifest_ref)
|
|
if manifest is None:
|
|
has_tag = model.has_active_tag(namespace_name, repo_name, manifest_ref)
|
|
if not has_tag:
|
|
has_expired_tag = model.has_tag(namespace_name, repo_name, manifest_ref)
|
|
if has_expired_tag:
|
|
logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name,
|
|
repo_name)
|
|
msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref
|
|
raise TagExpired(msg)
|
|
else:
|
|
raise ManifestUnknown()
|
|
|
|
manifest = _generate_and_store_manifest(namespace_name, repo_name, manifest_ref)
|
|
if manifest is None:
|
|
raise ManifestUnknown()
|
|
|
|
repo = model.get_repository(namespace_name, repo_name)
|
|
if repo is not None:
|
|
track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01,
|
|
tag=manifest_ref)
|
|
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
|
|
|
|
return Response(
|
|
manifest.json,
|
|
status=200,
|
|
headers={'Content-Type': manifest.media_type,
|
|
'Docker-Content-Digest': manifest.digest},)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
manifest = model.get_manifest_by_digest(namespace_name, repo_name, manifest_ref)
|
|
if manifest is None:
|
|
# Without a tag name to reference, we can't make an attempt to generate the manifest
|
|
raise ManifestUnknown()
|
|
|
|
repo = model.get_repository(namespace_name, repo_name)
|
|
if repo is not None:
|
|
track_and_log('pull_repo', repo, manifest_digest=manifest_ref)
|
|
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
|
|
|
|
return Response(manifest.json, status=200, headers={
|
|
'Content-Type': manifest.media_type,
|
|
'Docker-Content-Digest': manifest.digest})
|
|
|
|
|
|
def _reject_manifest2_schema2(func):
|
|
@wraps(func)
|
|
def wrapped(*args, **kwargs):
|
|
if request.content_type in (DOCKER_SCHEMA2_CONTENT_TYPES | OCI_CONTENT_TYPES):
|
|
raise ManifestInvalid(detail={'message': 'manifest schema version not supported'},
|
|
http_status_code=415)
|
|
return func(*args, **kwargs)
|
|
|
|
return wrapped
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
|
|
@_reject_manifest2_schema2
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|
try:
|
|
manifest = DockerSchema1Manifest(request.data)
|
|
except ManifestException as me:
|
|
logger.exception("failed to parse manifest when writing by tagname")
|
|
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
|
|
|
|
if manifest.tag != manifest_ref:
|
|
raise TagInvalid()
|
|
|
|
return _write_manifest_and_log(namespace_name, repo_name, manifest)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
|
|
@_reject_manifest2_schema2
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
try:
|
|
manifest = DockerSchema1Manifest(request.data)
|
|
except ManifestException as me:
|
|
logger.exception("failed to parse manifest when writing by digest")
|
|
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me.message})
|
|
|
|
if manifest.digest != manifest_ref:
|
|
raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})
|
|
|
|
return _write_manifest_and_log(namespace_name, repo_name, manifest)
|
|
|
|
|
|
def _write_manifest(namespace_name, repo_name, manifest):
|
|
if (manifest.namespace == '' and features.LIBRARY_SUPPORT and
|
|
namespace_name == app.config['LIBRARY_NAMESPACE']):
|
|
pass
|
|
elif manifest.namespace != namespace_name:
|
|
raise NameInvalid()
|
|
|
|
if manifest.repo_name != repo_name:
|
|
raise NameInvalid()
|
|
|
|
# Ensure that the repository exists.
|
|
repo = model.get_repository(namespace_name, repo_name)
|
|
if repo is None:
|
|
raise NameInvalid()
|
|
|
|
if not manifest.layers:
|
|
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
|
|
|
|
# Ensure all the blobs in the manifest exist.
|
|
storage_map = model.lookup_blobs_by_digest(repo, manifest.checksums)
|
|
for layer in manifest.layers:
|
|
digest_str = str(layer.digest)
|
|
if digest_str not in storage_map:
|
|
raise BlobUnknown(detail={'digest': digest_str})
|
|
|
|
# Lookup all the images and their parent images (if any) inside the manifest.
|
|
# This will let us know which v1 images we need to synthesize and which ones are invalid.
|
|
all_image_ids = list(manifest.parent_image_ids | manifest.image_ids)
|
|
images_map = model.get_docker_v1_metadata_by_image_id(repo, all_image_ids)
|
|
|
|
# Rewrite any v1 image IDs that do not match the checksum in the database.
|
|
try:
|
|
# TODO: make this batch and read the parent image from the previous iteration, rather than
|
|
# reloading it.
|
|
rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map))
|
|
for rewritten_image in rewritten_images:
|
|
if not rewritten_image.image_id in images_map:
|
|
model.synthesize_v1_image(
|
|
repo,
|
|
storage_map[rewritten_image.content_checksum],
|
|
rewritten_image.image_id,
|
|
rewritten_image.created,
|
|
rewritten_image.comment,
|
|
rewritten_image.command,
|
|
rewritten_image.compat_json,
|
|
rewritten_image.parent_image_id,
|
|
)
|
|
except ManifestException as me:
|
|
logger.exception("exception when rewriting v1 metadata")
|
|
raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message})
|
|
|
|
# Store the manifest pointing to the tag.
|
|
leaf_layer_id = rewritten_images[-1].image_id
|
|
newly_created = model.save_manifest(repo, manifest.tag, manifest, leaf_layer_id)
|
|
if newly_created:
|
|
# TODO: make this batch
|
|
labels = []
|
|
for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
|
|
media_type = 'application/json' if is_json(value) else 'text/plain'
|
|
labels.append(Label(key=key, value=value, source_type='manifest', media_type=media_type))
|
|
handle_label(key, value, namespace_name, repo_name, manifest.digest)
|
|
|
|
model.create_manifest_labels(namespace_name, repo_name, manifest.digest, labels)
|
|
|
|
return repo, storage_map
|
|
|
|
|
|
def _write_manifest_and_log(namespace_name, repo_name, manifest):
|
|
repo, storage_map = _write_manifest(namespace_name, repo_name, manifest)
|
|
|
|
# Queue all blob manifests for replication.
|
|
if features.STORAGE_REPLICATION:
|
|
with queue_replication_batch(namespace_name) as queue_storage_replication:
|
|
for layer in manifest.layers:
|
|
digest_str = str(layer.digest)
|
|
queue_storage_replication(storage_map[digest_str])
|
|
|
|
track_and_log('push_repo', repo, tag=manifest.tag)
|
|
spawn_notification(repo, 'repo_push', {'updated_tags': [manifest.tag]})
|
|
metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
|
|
|
|
return Response(
|
|
'OK',
|
|
status=202,
|
|
headers={
|
|
'Docker-Content-Digest': manifest.digest,
|
|
'Location':
|
|
url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name),
|
|
manifest_ref=manifest.digest),
|
|
},
|
|
)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
"""
|
|
Delete the manifest specified by the digest.
|
|
|
|
Note: there is no equivalent method for deleting by tag name because it is
|
|
forbidden by the spec.
|
|
"""
|
|
tags = model.delete_manifest_by_digest(namespace_name, repo_name, manifest_ref)
|
|
if not tags:
|
|
raise ManifestUnknown()
|
|
|
|
for tag in tags:
|
|
track_and_log('delete_tag', tag.repository, tag=tag.name, digest=manifest_ref)
|
|
|
|
return Response(status=202)
|
|
|
|
|
|
def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
|
|
# Find the v1 metadata for this image and its parents.
|
|
v1_metadata = model.get_docker_v1_metadata_by_tag(namespace_name, repo_name, tag_name)
|
|
parents_v1_metadata = model.get_parents_docker_v1_metadata(namespace_name, repo_name,
|
|
v1_metadata.image_id)
|
|
|
|
# If the manifest is being generated under the library namespace, then we make its namespace
|
|
# empty.
|
|
manifest_namespace = namespace_name
|
|
if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
|
|
manifest_namespace = ''
|
|
|
|
# Create and populate the manifest builder
|
|
builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)
|
|
|
|
# Add the leaf layer
|
|
builder.add_layer(v1_metadata.content_checksum, v1_metadata.compat_json)
|
|
|
|
for parent_v1_metadata in parents_v1_metadata:
|
|
builder.add_layer(parent_v1_metadata.content_checksum, parent_v1_metadata.compat_json)
|
|
|
|
# Sign the manifest with our signing key.
|
|
manifest = builder.build(docker_v2_signing_key)
|
|
|
|
# Write the manifest to the DB.
|
|
model.create_manifest_and_update_tag(namespace_name, repo_name, tag_name, manifest)
|
|
return manifest
|