4bf4ce33c9
This change replaces the metricqueue library with a native Prometheus client implementation with the intention to aggregated results with the Prometheus PushGateway. This change also adds instrumentation for greenlet context switches.
337 lines
13 KiB
Python
337 lines
13 KiB
Python
import logging
|
|
|
|
from functools import wraps
|
|
|
|
from flask import request, url_for, Response
|
|
|
|
import features
|
|
|
|
from app import app, storage
|
|
from auth.registry_jwt_auth import process_registry_jwt_auth
|
|
from digest import digest_tools
|
|
from data.registry_model import registry_model
|
|
from data.model.oci.manifest import CreateManifestException
|
|
from endpoints.decorators import anon_protect, parse_repository_name, check_readonly
|
|
from endpoints.metrics import image_pulls, image_pushes
|
|
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write
|
|
from endpoints.v2.errors import (ManifestInvalid, ManifestUnknown, NameInvalid, TagExpired,
|
|
NameUnknown)
|
|
from image.docker import ManifestException
|
|
from image.docker.schema1 import DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE, DOCKER_SCHEMA1_CONTENT_TYPES
|
|
from image.docker.schema2 import DOCKER_SCHEMA2_CONTENT_TYPES, OCI_CONTENT_TYPES
|
|
from image.docker.schemas import parse_manifest_from_bytes
|
|
from notifications import spawn_notification
|
|
from util.audit import track_and_log
|
|
from util.bytes import Bytes
|
|
from util.names import VALID_TAG_PATTERN
|
|
from util.registry.replication import queue_replication_batch
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>'
|
|
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
|
|
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
|
|
if repository_ref is None:
|
|
image_pulls.labels('v2_1', 'tag', 404).inc()
|
|
raise NameUnknown()
|
|
|
|
tag = registry_model.get_repo_tag(repository_ref, manifest_ref)
|
|
if tag is None:
|
|
if registry_model.has_expired_tag(repository_ref, manifest_ref):
|
|
logger.debug('Found expired tag %s for repository %s/%s', manifest_ref, namespace_name,
|
|
repo_name)
|
|
msg = 'Tag %s was deleted or has expired. To pull, revive via time machine' % manifest_ref
|
|
image_pulls.labels('v2_1', 'tag', 404).inc()
|
|
raise TagExpired(msg)
|
|
|
|
image_pulls.labels('v2_1', 'tag', 404).inc()
|
|
raise ManifestUnknown()
|
|
|
|
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
|
|
if manifest is None:
|
|
# Something went wrong.
|
|
image_pulls.labels('v2_1', 'tag', 400).inc()
|
|
raise ManifestInvalid()
|
|
|
|
manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary(
|
|
namespace_name, repo_name, manifest_ref, manifest)
|
|
if manifest_bytes is None:
|
|
image_pulls.labels('v2_1', 'tag', 404).inc()
|
|
raise ManifestUnknown()
|
|
|
|
track_and_log('pull_repo', repository_ref, analytics_name='pull_repo_100x', analytics_sample=0.01,
|
|
tag=manifest_ref)
|
|
image_pulls.labels('v2_1', 'tag', 200).inc()
|
|
|
|
return Response(
|
|
manifest_bytes.as_unicode(),
|
|
status=200,
|
|
headers={
|
|
'Content-Type': manifest_media_type,
|
|
'Docker-Content-Digest': manifest_digest,
|
|
},
|
|
)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull'])
|
|
@require_repo_read
|
|
@anon_protect
|
|
def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
|
|
if repository_ref is None:
|
|
image_pulls.labels('v2_1', 'manifest', 404).inc()
|
|
raise NameUnknown()
|
|
|
|
manifest = registry_model.lookup_manifest_by_digest(repository_ref, manifest_ref)
|
|
if manifest is None:
|
|
image_pulls.labels('v2_1', 'manifest', 404).inc()
|
|
raise ManifestUnknown()
|
|
|
|
manifest_bytes, manifest_digest, manifest_media_type = _rewrite_schema_if_necessary(
|
|
namespace_name, repo_name, '$digest', manifest)
|
|
if manifest_digest is None:
|
|
image_pulls.labels('v2_1', 'manifest', 404).inc()
|
|
raise ManifestUnknown()
|
|
|
|
track_and_log('pull_repo', repository_ref, manifest_digest=manifest_ref)
|
|
image_pulls.labels('v2_1', 'manifest', 200).inc()
|
|
|
|
return Response(manifest_bytes.as_unicode(), status=200, headers={
|
|
'Content-Type': manifest_media_type,
|
|
'Docker-Content-Digest': manifest_digest,
|
|
})
|
|
|
|
|
|
def _rewrite_schema_if_necessary(namespace_name, repo_name, tag_name, manifest):
|
|
# As per the Docker protocol, if the manifest is not schema version 1 and the manifest's
|
|
# media type is not in the Accept header, we return a schema 1 version of the manifest for
|
|
# the amd64+linux platform, if any, or None if none.
|
|
# See: https://docs.docker.com/registry/spec/manifest-v2-2
|
|
mimetypes = [mimetype for mimetype, _ in request.accept_mimetypes]
|
|
if manifest.media_type in mimetypes:
|
|
return manifest.internal_manifest_bytes, manifest.digest, manifest.media_type
|
|
|
|
# Short-circuit check: If the mimetypes is empty or just `application/json`, verify we have
|
|
# a schema 1 manifest and return it.
|
|
if not mimetypes or mimetypes == ['application/json']:
|
|
if manifest.media_type in DOCKER_SCHEMA1_CONTENT_TYPES:
|
|
return manifest.internal_manifest_bytes, manifest.digest, manifest.media_type
|
|
|
|
logger.debug('Manifest `%s` not compatible against %s; checking for conversion', manifest.digest,
|
|
request.accept_mimetypes)
|
|
converted = registry_model.convert_manifest(manifest, namespace_name, repo_name, tag_name,
|
|
mimetypes, storage)
|
|
if converted is not None:
|
|
return converted.bytes, converted.digest, converted.media_type
|
|
|
|
# For back-compat, we always default to schema 1 if the manifest could not be converted.
|
|
schema1 = registry_model.get_schema1_parsed_manifest(manifest, namespace_name, repo_name,
|
|
tag_name, storage)
|
|
if schema1 is None:
|
|
return None, None, None
|
|
|
|
return schema1.bytes, schema1.digest, schema1.media_type
|
|
|
|
|
|
def _reject_manifest2_schema2(func):
|
|
@wraps(func)
|
|
def wrapped(*args, **kwargs):
|
|
namespace_name = kwargs['namespace_name']
|
|
if registry_model.supports_schema2(namespace_name):
|
|
return func(*args, **kwargs)
|
|
|
|
if _doesnt_accept_schema_v1() or \
|
|
request.content_type in DOCKER_SCHEMA2_CONTENT_TYPES | OCI_CONTENT_TYPES:
|
|
raise ManifestInvalid(detail={'message': 'manifest schema version not supported'},
|
|
http_status_code=415)
|
|
return func(*args, **kwargs)
|
|
|
|
return wrapped
|
|
|
|
|
|
def _doesnt_accept_schema_v1():
|
|
# If the client doesn't specify anything, still give them Schema v1.
|
|
return len(request.accept_mimetypes) != 0 and \
|
|
DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE not in request.accept_mimetypes
|
|
|
|
|
|
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['PUT'])
|
|
@parse_repository_name()
|
|
@_reject_manifest2_schema2
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
@check_readonly
|
|
def write_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
|
|
parsed = _parse_manifest()
|
|
return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, parsed)
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['PUT'])
|
|
@parse_repository_name()
|
|
@_reject_manifest2_schema2
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
@check_readonly
|
|
def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
parsed = _parse_manifest()
|
|
if parsed.digest != manifest_ref:
|
|
image_pushes.labels('v2_invalid', 400).inc()
|
|
raise ManifestInvalid(detail={'message': 'manifest digest mismatch'})
|
|
|
|
if parsed.schema_version != 2:
|
|
return _write_manifest_and_log(namespace_name, repo_name, parsed.tag, parsed)
|
|
|
|
# If the manifest is schema version 2, then this cannot be a normal tag-based push, as the
|
|
# manifest does not contain the tag and this call was not given a tag name. Instead, we write the
|
|
# manifest with a temporary tag, as it is being pushed as part of a call for a manifest list.
|
|
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
|
|
if repository_ref is None:
|
|
image_pushes.labels('v2_2', 404).inc()
|
|
raise NameUnknown()
|
|
|
|
expiration_sec = app.config['PUSH_TEMP_TAG_EXPIRATION_SEC']
|
|
manifest = registry_model.create_manifest_with_temp_tag(repository_ref, parsed, expiration_sec,
|
|
storage)
|
|
if manifest is None:
|
|
image_pushes.labels('v2_2', 400).inc()
|
|
raise ManifestInvalid()
|
|
|
|
image_pushes.labels('v2_2', 202).inc()
|
|
return Response(
|
|
'OK',
|
|
status=202,
|
|
headers={
|
|
'Docker-Content-Digest': manifest.digest,
|
|
'Location':
|
|
url_for('v2.fetch_manifest_by_digest',
|
|
repository='%s/%s' % (namespace_name, repo_name),
|
|
manifest_ref=manifest.digest),
|
|
},
|
|
)
|
|
|
|
|
|
def _parse_manifest():
|
|
content_type = request.content_type or DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
|
|
if content_type == 'application/json':
|
|
# For back-compat.
|
|
content_type = DOCKER_SCHEMA1_MANIFEST_CONTENT_TYPE
|
|
|
|
try:
|
|
return parse_manifest_from_bytes(Bytes.for_string_or_unicode(request.data), content_type)
|
|
except ManifestException as me:
|
|
logger.exception("failed to parse manifest when writing by tagname")
|
|
raise ManifestInvalid(detail={'message': 'failed to parse manifest: %s' % me})
|
|
|
|
|
|
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
|
|
@parse_repository_name()
|
|
@process_registry_jwt_auth(scopes=['pull', 'push'])
|
|
@require_repo_write
|
|
@anon_protect
|
|
@check_readonly
|
|
def delete_manifest_by_digest(namespace_name, repo_name, manifest_ref):
|
|
"""
|
|
Delete the manifest specified by the digest.
|
|
|
|
Note: there is no equivalent method for deleting by tag name because it is
|
|
forbidden by the spec.
|
|
"""
|
|
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
|
|
if repository_ref is None:
|
|
raise NameUnknown()
|
|
|
|
manifest = registry_model.lookup_manifest_by_digest(repository_ref, manifest_ref)
|
|
if manifest is None:
|
|
raise ManifestUnknown()
|
|
|
|
tags = registry_model.delete_tags_for_manifest(manifest)
|
|
if not tags:
|
|
raise ManifestUnknown()
|
|
|
|
for tag in tags:
|
|
track_and_log('delete_tag', repository_ref, tag=tag.name, digest=manifest_ref)
|
|
|
|
return Response(status=202)
|
|
|
|
|
|
def _write_manifest_and_log(namespace_name, repo_name, tag_name, manifest_impl):
|
|
repository_ref, manifest, tag = _write_manifest(namespace_name, repo_name, tag_name,
|
|
manifest_impl)
|
|
|
|
# Queue all blob manifests for replication.
|
|
if features.STORAGE_REPLICATION:
|
|
blobs = registry_model.get_manifest_local_blobs(manifest)
|
|
if blobs is None:
|
|
logger.error('Could not lookup blobs for manifest `%s`', manifest.digest)
|
|
else:
|
|
with queue_replication_batch(namespace_name) as queue_storage_replication:
|
|
for blob_digest in blobs:
|
|
queue_storage_replication(blob_digest)
|
|
|
|
track_and_log('push_repo', repository_ref, tag=tag_name)
|
|
spawn_notification(repository_ref, 'repo_push', {'updated_tags': [tag_name]})
|
|
image_pushes.labels('v2_1', 202).inc()
|
|
|
|
return Response(
|
|
'OK',
|
|
status=202,
|
|
headers={
|
|
'Docker-Content-Digest': manifest.digest,
|
|
'Location':
|
|
url_for('v2.fetch_manifest_by_digest',
|
|
repository='%s/%s' % (namespace_name, repo_name),
|
|
manifest_ref=manifest.digest),
|
|
},
|
|
)
|
|
|
|
|
|
def _write_manifest(namespace_name, repo_name, tag_name, manifest_impl):
|
|
# NOTE: These extra checks are needed for schema version 1 because the manifests
|
|
# contain the repo namespace, name and tag name.
|
|
if manifest_impl.schema_version == 1:
|
|
if (manifest_impl.namespace == '' and features.LIBRARY_SUPPORT and
|
|
namespace_name == app.config['LIBRARY_NAMESPACE']):
|
|
pass
|
|
elif manifest_impl.namespace != namespace_name:
|
|
raise NameInvalid()
|
|
|
|
if manifest_impl.repo_name != repo_name:
|
|
raise NameInvalid()
|
|
|
|
try:
|
|
if not manifest_impl.layers:
|
|
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
|
|
except ManifestException as me:
|
|
raise ManifestInvalid(detail={'message': str(me)})
|
|
|
|
# Ensure that the repository exists.
|
|
repository_ref = registry_model.lookup_repository(namespace_name, repo_name)
|
|
if repository_ref is None:
|
|
raise NameUnknown()
|
|
|
|
# Create the manifest(s) and retarget the tag to point to it.
|
|
try:
|
|
manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref, manifest_impl,
|
|
tag_name, storage,
|
|
raise_on_error=True)
|
|
except CreateManifestException as cme:
|
|
raise ManifestInvalid(detail={'message': str(cme)})
|
|
|
|
if manifest is None:
|
|
raise ManifestInvalid()
|
|
|
|
return repository_ref, manifest, tag
|