vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

endpoints.v2: yapf format

Browse source
This commit is contained in:
Jimmy Zelinskie 2017-06-26 18:16:15 -04:00
parent 0e26a03f7e
commit b1434b0380
9 changed files with 152 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
endpoints/v2/__init__.py
View file

@ -12,8 +12,8 @@ import features
from app import app, metric_queue, get_app_url, license_validator from app import app, metric_queue, get_app_url, license_validator
from auth.auth_context import get_grant_context from auth.auth_context import get_grant_context
from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, from auth.permissions import (
AdministerRepositoryPermission) ReadRepositoryPermission, ModifyRepositoryPermission, AdministerRepositoryPermission)
from auth.registry_jwt_auth import process_registry_jwt_auth, get_auth_headers from auth.registry_jwt_auth import process_registry_jwt_auth, get_auth_headers
from endpoints.decorators import anon_protect, anon_allowed from endpoints.decorators import anon_protect, anon_allowed
from endpoints.v2.errors import V2RegistryException, Unauthorized, Unsupported, NameUnknown from endpoints.v2.errors import V2RegistryException, Unauthorized, Unsupported, NameUnknown
@ -23,10 +23,8 @@ from util.metrics.metricqueue import time_blueprint
from util.registry.dockerver import docker_version from util.registry.dockerver import docker_version
from util.pagination import encrypt_page_token, decrypt_page_token from util.pagination import encrypt_page_token, decrypt_page_token
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
v2_bp = Blueprint('v2', __name__) v2_bp = Blueprint('v2', __name__)
license_validator.enforce_license_before_request(v2_bp) license_validator.enforce_license_before_request(v2_bp)
time_blueprint(v2_bp, metric_queue) time_blueprint(v2_bp, metric_queue)
@ -34,9 +32,7 @@ time_blueprint(v2_bp, metric_queue)
@v2_bp.app_errorhandler(V2RegistryException) @v2_bp.app_errorhandler(V2RegistryException)
def handle_registry_v2_exception(error): def handle_registry_v2_exception(error):
response = jsonify({ response = jsonify({'errors': [error.as_dict()]})
'errors': [error.as_dict()]
})
response.status_code = error.http_status_code response.status_code = error.http_status_code
if response.status_code == 401: if response.status_code == 401:
@ -53,6 +49,7 @@ def paginate(limit_kwarg_name='limit', offset_kwarg_name='offset',
""" """
Decorates a handler adding a parsed pagination token and a callback to encode a response token. Decorates a handler adding a parsed pagination token and a callback to encode a response token.
""" """
def wrapper(func): def wrapper(func):
@wraps(func) @wraps(func)
def wrapped(*args, **kwargs): def wrapped(*args, **kwargs):
@ -86,7 +83,9 @@ def paginate(limit_kwarg_name='limit', offset_kwarg_name='offset',
kwargs[offset_kwarg_name] = offset kwargs[offset_kwarg_name] = offset
kwargs[callback_kwarg_name] = callback kwargs[callback_kwarg_name] = callback
return func(*args, **kwargs) return func(*args, **kwargs)
return wrapped return wrapped
return wrapper return wrapper
@ -94,17 +93,15 @@ def _require_repo_permission(permission_class, scopes=None, allow_public=False):
def wrapper(func): def wrapper(func):
@wraps(func) @wraps(func)
def wrapped(namespace_name, repo_name, *args, **kwargs): def wrapped(namespace_name, repo_name, *args, **kwargs):
logger.debug('Checking permission %s for repo: %s/%s', permission_class, logger.debug('Checking permission %s for repo: %s/%s', permission_class, namespace_name,
namespace_name, repo_name) repo_name)
repository = namespace_name + '/' + repo_name repository = namespace_name + '/' + repo_name
repo = model.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if repo is None: if repo is None:
raise Unauthorized(repository=repository, scopes=scopes) raise Unauthorized(repository=repository, scopes=scopes)
permission = permission_class(namespace_name, repo_name) permission = permission_class(namespace_name, repo_name)
if (permission.can() or if (permission.can() or (allow_public and repo.is_public)):
(allow_public and
repo.is_public)):
if repo.kind != 'image': if repo.kind != 'image':
msg = 'This repository is for managing %s resources and not container images.' % repo.kind msg = 'This repository is for managing %s resources and not container images.' % repo.kind
raise Unsupported(detail=msg) raise Unsupported(detail=msg)
@ -112,16 +109,15 @@ def _require_repo_permission(permission_class, scopes=None, allow_public=False):
raise Unauthorized(repository=repository, scopes=scopes) raise Unauthorized(repository=repository, scopes=scopes)
return wrapped return wrapped
return wrapper return wrapper
require_repo_read = _require_repo_permission(ReadRepositoryPermission, require_repo_read = _require_repo_permission(ReadRepositoryPermission, scopes=['pull'],
scopes=['pull'],
allow_public=True) allow_public=True)
require_repo_write = _require_repo_permission(ModifyRepositoryPermission, require_repo_write = _require_repo_permission(ModifyRepositoryPermission, scopes=['pull', 'push'])
scopes=['pull', 'push']) require_repo_admin = _require_repo_permission(AdministerRepositoryPermission, scopes=[
require_repo_admin = _require_repo_permission(AdministerRepositoryPermission, 'pull', 'push'])
scopes=['pull', 'push'])
def get_input_stream(flask_request): def get_input_stream(flask_request):
@ -138,7 +134,9 @@ def route_show_if(value):
abort(404) abort(404)
return f(*args, **kwargs) return f(*args, **kwargs)
return decorated_function return decorated_function
return decorator return decorator
@ -169,5 +167,4 @@ from endpoints.v2 import (
catalog, catalog,
manifest, manifest,
tag, tag,
v2auth, v2auth,)
)

84
endpoints/v2/blob.py
View file

@ -14,18 +14,16 @@ from digest import digest_tools
from endpoints.common import parse_repository_name from endpoints.common import parse_repository_name
from endpoints.decorators import anon_protect from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream from endpoints.v2 import v2_bp, require_repo_read, require_repo_write, get_input_stream
from endpoints.v2.errors import (BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported, from endpoints.v2.errors import (
NameUnknown, LayerTooLarge) BlobUnknown, BlobUploadInvalid, BlobUploadUnknown, Unsupported, NameUnknown, LayerTooLarge)
from endpoints.v2.models_pre_oci import data_model as model from endpoints.v2.models_pre_oci import data_model as model
from util.cache import cache_control from util.cache import cache_control
from util.registry.filelike import wrap_with_handler, StreamSlice from util.registry.filelike import wrap_with_handler, StreamSlice
from util.registry.gzipstream import calculate_size_handler from util.registry.gzipstream import calculate_size_handler
from util.registry.torrent import PieceHasher from util.registry.torrent import PieceHasher
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
BASE_BLOB_ROUTE = '/<repopath:repository>/blobs/<regex("{0}"):digest>' BASE_BLOB_ROUTE = '/<repopath:repository>/blobs/<regex("{0}"):digest>'
BLOB_DIGEST_ROUTE = BASE_BLOB_ROUTE.format(digest_tools.DIGEST_PATTERN) BLOB_DIGEST_ROUTE = BASE_BLOB_ROUTE.format(digest_tools.DIGEST_PATTERN)
RANGE_HEADER_REGEX = re.compile(r'^bytes=([0-9]+)-([0-9]+)$') RANGE_HEADER_REGEX = re.compile(r'^bytes=([0-9]+)-([0-9]+)$')
@ -52,8 +50,7 @@ def check_blob_exists(namespace_name, repo_name, digest):
headers = { headers = {
'Docker-Content-Digest': digest, 'Docker-Content-Digest': digest,
'Content-Length': blob.size, 'Content-Length': blob.size,
'Content-Type': BLOB_CONTENT_TYPE, 'Content-Type': BLOB_CONTENT_TYPE,}
}
# If our storage supports range requests, let the client know. # If our storage supports range requests, let the client know.
if storage.get_supports_resumable_downloads(blob.locations): if storage.get_supports_resumable_downloads(blob.locations):
@ -102,10 +99,7 @@ def download_blob(namespace_name, repo_name, digest):
storage.stream_read(blob.locations, path), storage.stream_read(blob.locations, path),
headers=headers.update({ headers=headers.update({
'Content-Length': blob.size, 'Content-Length': blob.size,
'Content-Type': BLOB_CONTENT_TYPE, 'Content-Type': BLOB_CONTENT_TYPE,}),)
}),
)
@v2_bp.route('/<repopath:repository>/blobs/uploads/', methods=['POST']) @v2_bp.route('/<repopath:repository>/blobs/uploads/', methods=['POST'])
@ -128,13 +122,13 @@ def start_blob_upload(namespace_name, repo_name):
return Response( return Response(
status=202, status=202,
headers={ headers={
'Docker-Upload-UUID': new_upload_uuid, 'Docker-Upload-UUID':
'Range': _render_range(0), new_upload_uuid,
'Location': get_app_url() + url_for('v2.upload_chunk', 'Range':
repository='%s/%s' % (namespace_name, repo_name), _render_range(0),
upload_uuid=new_upload_uuid) 'Location':
}, get_app_url() + url_for('v2.upload_chunk', repository='%s/%s' %
) (namespace_name, repo_name), upload_uuid=new_upload_uuid)},)
# The user plans to send us the entire body right now. # The user plans to send us the entire body right now.
# Find the upload. # Find the upload.
@ -158,12 +152,11 @@ def start_blob_upload(namespace_name, repo_name):
return Response( return Response(
status=201, status=201,
headers={ headers={
'Docker-Content-Digest': digest, 'Docker-Content-Digest':
'Location': get_app_url() + url_for('v2.download_blob', digest,
repository='%s/%s' % (namespace_name, repo_name), 'Location':
digest=digest), get_app_url() + url_for('v2.download_blob', repository='%s/%s' %
}, (namespace_name, repo_name), digest=digest),},)
)
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['GET']) @v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['GET'])
@ -180,9 +173,8 @@ def fetch_existing_upload(namespace_name, repo_name, upload_uuid):
status=204, status=204,
headers={ headers={
'Docker-Upload-UUID': upload_uuid, 'Docker-Upload-UUID': upload_uuid,
'Range': _render_range(blob_upload.byte_count+1), # byte ranges are exclusive 'Range': _render_range(blob_upload.byte_count + 1), # byte ranges are exclusive
}, },)
)
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['PATCH']) @v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['PATCH'])
@ -211,9 +203,7 @@ def upload_chunk(namespace_name, repo_name, upload_uuid):
headers={ headers={
'Location': _current_request_url(), 'Location': _current_request_url(),
'Range': _render_range(updated_blob_upload.byte_count, with_bytes_prefix=False), 'Range': _render_range(updated_blob_upload.byte_count, with_bytes_prefix=False),
'Docker-Upload-UUID': upload_uuid, 'Docker-Upload-UUID': upload_uuid,},)
},
)
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['PUT']) @v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['PUT'])
@ -242,15 +232,12 @@ def monolithic_upload_or_last_chunk(namespace_name, repo_name, upload_uuid):
_finish_upload(namespace_name, repo_name, updated_blob_upload, digest) _finish_upload(namespace_name, repo_name, updated_blob_upload, digest)
# Write the response to the client. # Write the response to the client.
return Response( return Response(status=201, headers={
status=201, 'Docker-Content-Digest':
headers={ digest,
'Docker-Content-Digest': digest, 'Location':
'Location': get_app_url() + url_for('v2.download_blob', get_app_url() + url_for('v2.download_blob', repository='%s/%s' %
repository='%s/%s' % (namespace_name, repo_name), (namespace_name, repo_name), digest=digest),})
digest=digest),
}
)
@v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['DELETE']) @v2_bp.route('/<repopath:repository>/blobs/uploads/<upload_uuid>', methods=['DELETE'])
@ -300,9 +287,11 @@ def _abort_range_not_satisfiable(valid_end, upload_uuid):
TODO(jzelinskie): Unify this with the V2RegistryException class. TODO(jzelinskie): Unify this with the V2RegistryException class.
""" """
flask_abort(Response(status=416, headers={'Location': _current_request_url(), flask_abort(
'Range': '0-{0}'.format(valid_end), Response(status=416, headers={
'Docker-Upload-UUID': upload_uuid})) 'Location': _current_request_url(),
'Range': '0-{0}'.format(valid_end),
'Docker-Upload-UUID': upload_uuid}))
def _parse_range_header(range_header_text): def _parse_range_header(range_header_text):
@ -415,16 +404,15 @@ def _upload_chunk(blob_upload, range_header):
length, length,
input_fp, input_fp,
blob_upload.storage_metadata, blob_upload.storage_metadata,
content_type=BLOB_CONTENT_TYPE, content_type=BLOB_CONTENT_TYPE,)
)
if upload_error is not None: if upload_error is not None:
logger.error('storage.stream_upload_chunk returned error %s', upload_error) logger.error('storage.stream_upload_chunk returned error %s', upload_error)
return None return None
# Update the chunk upload time metric. # Update the chunk upload time metric.
metric_queue.chunk_upload_time.Observe(time.time() - start_time, metric_queue.chunk_upload_time.Observe(time.time() - start_time, labelvalues=[
labelvalues=[length_written, list(location_set)[0]]) length_written, list(location_set)[0]])
# If we determined an uncompressed size and this is the first chunk, add it to the blob. # If we determined an uncompressed size and this is the first chunk, add it to the blob.
# Otherwise, we clear the size from the blob as it was uploaded in multiple chunks. # Otherwise, we clear the size from the blob as it was uploaded in multiple chunks.
@ -499,8 +487,7 @@ def _finalize_blob_database(namespace_name, repo_name, blob_upload, digest, alre
repo_name, repo_name,
digest, digest,
blob_upload, blob_upload,
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'], app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'],)
)
# If it doesn't already exist, create the BitTorrent pieces for the blob. # If it doesn't already exist, create the BitTorrent pieces for the blob.
if blob_upload.piece_sha_state is not None and not already_existed: if blob_upload.piece_sha_state is not None and not already_existed:
@ -521,5 +508,4 @@ def _finish_upload(namespace_name, repo_name, blob_upload, digest):
repo_name, repo_name,
blob_upload, blob_upload,
digest, digest,
_finalize_blob_storage(blob_upload, digest), _finalize_blob_storage(blob_upload, digest),)
)

6
endpoints/v2/catalog.py
View file

@ -7,6 +7,7 @@ from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, paginate from endpoints.v2 import v2_bp, paginate
from endpoints.v2.models_pre_oci import data_model as model from endpoints.v2.models_pre_oci import data_model as model
@v2_bp.route('/_catalog', methods=['GET']) @v2_bp.route('/_catalog', methods=['GET'])
@process_registry_jwt_auth() @process_registry_jwt_auth()
@anon_protect @anon_protect
@ -18,12 +19,11 @@ def catalog_search(limit, offset, pagination_callback):
username = entity.user.username username = entity.user.username
include_public = bool(features.PUBLIC_CATALOG) include_public = bool(features.PUBLIC_CATALOG)
visible_repositories = model.get_visible_repositories(username, limit+1, offset, visible_repositories = model.get_visible_repositories(username, limit + 1, offset,
include_public=include_public) include_public=include_public)
response = jsonify({ response = jsonify({
'repositories': ['%s/%s' % (repo.namespace_name, repo.name) 'repositories': ['%s/%s' % (repo.namespace_name, repo.name)
for repo in visible_repositories][0:limit], for repo in visible_repositories][0:limit],})
})
pagination_callback(len(visible_repositories), response) pagination_callback(len(visible_repositories), response)
return response return response

96
endpoints/v2/errors.py
View file

@ -1,8 +1,9 @@
import bitmath import bitmath
class V2RegistryException(Exception): class V2RegistryException(Exception):
def __init__(self, error_code_str, message, detail, http_status_code=400, def __init__(self, error_code_str, message, detail, http_status_code=400, repository=None,
repository=None, scopes=None): scopes=None):
super(V2RegistryException, self).__init__(message) super(V2RegistryException, self).__init__(message)
self.http_status_code = http_status_code self.http_status_code = http_status_code
self.repository = repository self.repository = repository
@ -15,104 +16,81 @@ class V2RegistryException(Exception):
return { return {
'code': self._error_code_str, 'code': self._error_code_str,
'message': self.message, 'message': self.message,
'detail': self._detail if self._detail is not None else {}, 'detail': self._detail if self._detail is not None else {},}
}
class BlobUnknown(V2RegistryException): class BlobUnknown(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(BlobUnknown, self).__init__('BLOB_UNKNOWN', super(BlobUnknown, self).__init__('BLOB_UNKNOWN', 'blob unknown to registry', detail, 404)
'blob unknown to registry',
detail,
404)
class BlobUploadInvalid(V2RegistryException): class BlobUploadInvalid(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(BlobUploadInvalid, self).__init__('BLOB_UPLOAD_INVALID', super(BlobUploadInvalid, self).__init__('BLOB_UPLOAD_INVALID', 'blob upload invalid', detail)
'blob upload invalid',
detail)
class BlobUploadUnknown(V2RegistryException): class BlobUploadUnknown(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(BlobUploadUnknown, self).__init__('BLOB_UPLOAD_UNKNOWN', super(BlobUploadUnknown, self).__init__('BLOB_UPLOAD_UNKNOWN',
'blob upload unknown to registry', 'blob upload unknown to registry', detail, 404)
detail,
404)
class DigestInvalid(V2RegistryException): class DigestInvalid(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(DigestInvalid, self).__init__('DIGEST_INVALID', super(DigestInvalid, self).__init__('DIGEST_INVALID',
'provided digest did not match uploaded content', 'provided digest did not match uploaded content', detail)
detail)
class ManifestBlobUnknown(V2RegistryException): class ManifestBlobUnknown(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(ManifestBlobUnknown, self).__init__('MANIFEST_BLOB_UNKNOWN', super(ManifestBlobUnknown, self).__init__('MANIFEST_BLOB_UNKNOWN',
'manifest blob unknown to registry', 'manifest blob unknown to registry', detail)
detail)
class ManifestInvalid(V2RegistryException): class ManifestInvalid(V2RegistryException):
def __init__(self, detail=None, http_status_code=400): def __init__(self, detail=None, http_status_code=400):
super(ManifestInvalid, self).__init__('MANIFEST_INVALID', super(ManifestInvalid, self).__init__('MANIFEST_INVALID', 'manifest invalid', detail,
'manifest invalid',
detail,
http_status_code) http_status_code)
class ManifestUnknown(V2RegistryException): class ManifestUnknown(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(ManifestUnknown, self).__init__('MANIFEST_UNKNOWN', super(ManifestUnknown, self).__init__('MANIFEST_UNKNOWN', 'manifest unknown', detail, 404)
'manifest unknown',
detail,
404)
class ManifestUnverified(V2RegistryException): class ManifestUnverified(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(ManifestUnverified, self).__init__('MANIFEST_UNVERIFIED', super(ManifestUnverified, self).__init__('MANIFEST_UNVERIFIED',
'manifest failed signature verification', 'manifest failed signature verification', detail)
detail)
class NameInvalid(V2RegistryException): class NameInvalid(V2RegistryException):
def __init__(self, detail=None, message=None): def __init__(self, detail=None, message=None):
super(NameInvalid, self).__init__('NAME_INVALID', super(NameInvalid, self).__init__('NAME_INVALID', message or 'invalid repository name', detail)
message or 'invalid repository name',
detail)
class NameUnknown(V2RegistryException): class NameUnknown(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(NameUnknown, self).__init__('NAME_UNKNOWN', super(NameUnknown, self).__init__('NAME_UNKNOWN', 'repository name not known to registry',
'repository name not known to registry', detail, 404)
detail,
404)
class SizeInvalid(V2RegistryException): class SizeInvalid(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(SizeInvalid, self).__init__('SIZE_INVALID', super(SizeInvalid, self).__init__('SIZE_INVALID',
'provided length did not match content length', 'provided length did not match content length', detail)
detail)
class TagAlreadyExists(V2RegistryException): class TagAlreadyExists(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(TagAlreadyExists, self).__init__('TAG_ALREADY_EXISTS', super(TagAlreadyExists, self).__init__('TAG_ALREADY_EXISTS', 'tag was already pushed', detail,
'tag was already pushed',
detail,
409) 409)
class TagInvalid(V2RegistryException): class TagInvalid(V2RegistryException):
def __init__(self, detail=None): def __init__(self, detail=None):
super(TagInvalid, self).__init__('TAG_INVALID', super(TagInvalid, self).__init__('TAG_INVALID', 'manifest tag did not match URI', detail)
'manifest tag did not match URI',
detail)
class LayerTooLarge(V2RegistryException): class LayerTooLarge(V2RegistryException):
def __init__(self, uploaded=None, max_allowed=None): def __init__(self, uploaded=None, max_allowed=None):
@ -123,43 +101,33 @@ class LayerTooLarge(V2RegistryException):
detail = { detail = {
'reason': '%s is greater than maximum allowed size %s' % (uploaded, max_allowed), 'reason': '%s is greater than maximum allowed size %s' % (uploaded, max_allowed),
'max_allowed': max_allowed, 'max_allowed': max_allowed,
'uploaded': uploaded, 'uploaded': uploaded,}
}
up_str = bitmath.Byte(uploaded).best_prefix().format("{value:.2f} {unit}") up_str = bitmath.Byte(uploaded).best_prefix().format("{value:.2f} {unit}")
max_str = bitmath.Byte(max_allowed).best_prefix().format("{value:.2f} {unit}") max_str = bitmath.Byte(max_allowed).best_prefix().format("{value:.2f} {unit}")
message = 'Uploaded blob of %s is larger than %s allowed by this registry' % (up_str, max_str) message = 'Uploaded blob of %s is larger than %s allowed by this registry' % (up_str,
max_str)
class Unauthorized(V2RegistryException): class Unauthorized(V2RegistryException):
def __init__(self, detail=None, repository=None, scopes=None): def __init__(self, detail=None, repository=None, scopes=None):
super(Unauthorized, self).__init__('UNAUTHORIZED', super(Unauthorized,
'access to the requested resource is not authorized', self).__init__('UNAUTHORIZED', 'access to the requested resource is not authorized',
detail, detail, 401, repository=repository, scopes=scopes)
401,
repository=repository,
scopes=scopes)
class Unsupported(V2RegistryException): class Unsupported(V2RegistryException):
def __init__(self, detail=None, message=None): def __init__(self, detail=None, message=None):
super(Unsupported, self).__init__('UNSUPPORTED', super(Unsupported, self).__init__('UNSUPPORTED', message or 'The operation is unsupported.',
message or 'The operation is unsupported.', detail, 405)
detail,
405)
class InvalidLogin(V2RegistryException): class InvalidLogin(V2RegistryException):
def __init__(self, message=None): def __init__(self, message=None):
super(InvalidLogin, self).__init__('UNAUTHORIZED', super(InvalidLogin, self).__init__('UNAUTHORIZED', message or
message or 'Specified credentials are invalid', 'Specified credentials are invalid', {}, 401)
{},
401)
class InvalidRequest(V2RegistryException): class InvalidRequest(V2RegistryException):
def __init__(self, message=None): def __init__(self, message=None):
super(InvalidRequest, self).__init__('INVALID_REQUEST', super(InvalidRequest, self).__init__('INVALID_REQUEST', message or 'Invalid request', {}, 400)
message or 'Invalid request',
{},
400)

33
endpoints/v2/manifest.py
View file

@ -25,14 +25,13 @@ from util.names import VALID_TAG_PATTERN
from util.registry.replication import queue_replication_batch from util.registry.replication import queue_replication_batch
from util.validation import is_json from util.validation import is_json
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>' BASE_MANIFEST_ROUTE = '/<repopath:repository>/manifests/<regex("{0}"):manifest_ref>'
MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN) MANIFEST_DIGEST_ROUTE = BASE_MANIFEST_ROUTE.format(digest_tools.DIGEST_PATTERN)
MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN) MANIFEST_TAGNAME_ROUTE = BASE_MANIFEST_ROUTE.format(VALID_TAG_PATTERN)
@v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET']) @v2_bp.route(MANIFEST_TAGNAME_ROUTE, methods=['GET'])
@parse_repository_name() @parse_repository_name()
@process_registry_jwt_auth(scopes=['pull']) @process_registry_jwt_auth(scopes=['pull'])
@ -52,14 +51,14 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
repo = model.get_repository(namespace_name, repo_name) repo = model.get_repository(namespace_name, repo_name)
if repo is not None: if repo is not None:
track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01, track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01,
tag=manifest_ref) tag=manifest_ref)
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True]) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
return Response( return Response(
manifest.json, manifest.json,
status=200, status=200,
headers={'Content-Type': manifest.media_type, 'Docker-Content-Digest': manifest.digest}, headers={'Content-Type': manifest.media_type,
) 'Docker-Content-Digest': manifest.digest},)
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET']) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['GET'])
@ -78,8 +77,9 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
track_and_log('pull_repo', repo, manifest_digest=manifest_ref) track_and_log('pull_repo', repo, manifest_digest=manifest_ref)
metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True]) metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2', True])
return Response(manifest.json, status=200, headers={'Content-Type': manifest.media_type, return Response(manifest.json, status=200, headers={
'Docker-Content-Digest': manifest.digest}) 'Content-Type': manifest.media_type,
'Docker-Content-Digest': manifest.digest})
def _reject_manifest2_schema2(func): def _reject_manifest2_schema2(func):
@ -89,6 +89,7 @@ def _reject_manifest2_schema2(func):
raise ManifestInvalid(detail={'message': 'manifest schema version not supported'}, raise ManifestInvalid(detail={'message': 'manifest schema version not supported'},
http_status_code=415) http_status_code=415)
return func(*args, **kwargs) return func(*args, **kwargs)
return wrapped return wrapped
@ -131,8 +132,7 @@ def write_manifest_by_digest(namespace_name, repo_name, manifest_ref):
def _write_manifest(namespace_name, repo_name, manifest): def _write_manifest(namespace_name, repo_name, manifest):
if (manifest.namespace == '' and if (manifest.namespace == '' and features.LIBRARY_SUPPORT and
features.LIBRARY_SUPPORT and
namespace_name == app.config['LIBRARY_NAMESPACE']): namespace_name == app.config['LIBRARY_NAMESPACE']):
pass pass
elif manifest.namespace != namespace_name: elif manifest.namespace != namespace_name:
@ -174,8 +174,7 @@ def _write_manifest(namespace_name, repo_name, manifest):
rewritten_image.comment, rewritten_image.comment,
rewritten_image.command, rewritten_image.command,
rewritten_image.compat_json, rewritten_image.compat_json,
rewritten_image.parent_image_id, rewritten_image.parent_image_id,)
)
except ManifestException as me: except ManifestException as me:
logger.exception("exception when rewriting v1 metadata") logger.exception("exception when rewriting v1 metadata")
raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message}) raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message})
@ -212,12 +211,11 @@ def _write_manifest_and_log(namespace_name, repo_name, manifest):
'OK', 'OK',
status=202, status=202,
headers={ headers={
'Docker-Content-Digest': manifest.digest, 'Docker-Content-Digest':
'Location': url_for('v2.fetch_manifest_by_digest', manifest.digest,
repository='%s/%s' % (namespace_name, repo_name), 'Location':
manifest_ref=manifest.digest), url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name),
}, manifest_ref=manifest.digest),},)
)
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE']) @v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])
@ -271,5 +269,6 @@ def _generate_and_store_manifest(namespace_name, repo_name, tag_name):
manifest.bytes) manifest.bytes)
return manifest return manifest
def _determine_media_type(value): def _determine_media_type(value):
media_type_name = 'application/json' if is_json(value) else 'text/plain' media_type_name = 'application/json' if is_json(value) else 'text/plain'

19
endpoints/v2/models_interface.py
View file

@ -5,8 +5,9 @@ from namedlist import namedlist
from six import add_metaclass from six import add_metaclass
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', class Repository(
'is_public', 'kind', 'trust_enabled'])): namedtuple('Repository', [
'id', 'name', 'namespace_name', 'description', 'is_public', 'kind', 'trust_enabled'])):
""" """
Repository represents a namespaced collection of tags. Repository represents a namespaced collection of tags.
:type id: int :type id: int
@ -18,6 +19,7 @@ class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'desc
:type trust_enabled: bool :type trust_enabled: bool
""" """
class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])): class ManifestJSON(namedtuple('ManifestJSON', ['digest', 'json', 'media_type'])):
""" """
ManifestJSON represents a Manifest of any format. ManifestJSON represents a Manifest of any format.
@ -30,10 +32,10 @@ class Tag(namedtuple('Tag', ['name', 'repository'])):
""" """
class BlobUpload(namedlist('BlobUpload', ['uuid', 'byte_count', 'uncompressed_byte_count', class BlobUpload(
'chunk_count', 'sha_state', 'location_name', namedlist('BlobUpload', [
'storage_metadata', 'piece_sha_state', 'piece_hashes', 'uuid', 'byte_count', 'uncompressed_byte_count', 'chunk_count', 'sha_state', 'location_name',
'repo_namespace_name', 'repo_name'])): 'storage_metadata', 'piece_sha_state', 'piece_hashes', 'repo_namespace_name', 'repo_name'])):
""" """
BlobUpload represents the current state of an Blob being uploaded. BlobUpload represents the current state of an Blob being uploaded.
""" """
@ -50,6 +52,7 @@ class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'name
RepositoryReference represents a reference to a Repository, without its full metadata. RepositoryReference represents a reference to a Repository, without its full metadata.
""" """
class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])): class Label(namedtuple('Label', ['key', 'value', 'source_type', 'media_type'])):
""" """
Label represents a key-value pair that describes a particular Manifest. Label represents a key-value pair that describes a particular Manifest.
@ -178,7 +181,8 @@ class DockerRegistryV2DataInterface(object):
pass pass
@abstractmethod @abstractmethod
def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata): def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name,
storage_metadata):
""" """
Creates a blob upload under the matching repository with the given UUID and metadata. Creates a blob upload under the matching repository with the given UUID and metadata.
Returns whether the matching repository exists. Returns whether the matching repository exists.
@ -246,7 +250,6 @@ class DockerRegistryV2DataInterface(object):
""" """
pass pass
@abstractmethod @abstractmethod
def get_blob_path(self, blob): def get_blob_path(self, blob):
""" """

67
endpoints/v2/models_pre_oci.py
View file

@ -9,11 +9,9 @@ from endpoints.v2.models_interface import (
ManifestJSON, ManifestJSON,
Repository, Repository,
RepositoryReference, RepositoryReference,
Tag, Tag,)
)
from image.docker.v1 import DockerV1Metadata from image.docker.v1 import DockerV1Metadata
_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws" _MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"
@ -22,6 +20,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema PreOCIModel implements the data model for the v2 Docker Registry protocol using a database schema
before it was changed to support the OCI specification. before it was changed to support the OCI specification.
""" """
def create_repository(self, namespace_name, repo_name, creating_user=None): def create_repository(self, namespace_name, repo_name, creating_user=None):
return model.repository.create_repository(namespace_name, repo_name, creating_user) return model.repository.create_repository(namespace_name, repo_name, creating_user)
@ -54,14 +53,10 @@ class PreOCIModel(DockerRegistryV2DataInterface):
def delete_manifest_by_digest(self, namespace_name, repo_name, digest): def delete_manifest_by_digest(self, namespace_name, repo_name, digest):
def _tag_view(tag): def _tag_view(tag):
return Tag( return Tag(name=tag.name, repository=RepositoryReference(
name=tag.name, id=tag.repository_id,
repository=RepositoryReference( name=repo_name,
id=tag.repository_id, namespace_name=namespace_name,))
name=repo_name,
namespace_name=namespace_name,
)
)
tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest) tags = model.tag.delete_manifest_by_digest(namespace_name, repo_name, digest)
return [_tag_view(tag) for tag in tags] return [_tag_view(tag) for tag in tags]
@ -79,8 +74,9 @@ class PreOCIModel(DockerRegistryV2DataInterface):
return {} return {}
images_query = model.image.lookup_repository_images(repo, docker_image_ids) images_query = model.image.lookup_repository_images(repo, docker_image_ids)
return {image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image) return {
for image in images_query} image.docker_image_id: _docker_v1_metadata(namespace_name, repo_name, image)
for image in images_query}
def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id): def get_parents_docker_v1_metadata(self, namespace_name, repo_name, docker_image_id):
repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id) repo_image = model.image.get_repo_image(namespace_name, repo_name, docker_image_id)
@ -122,21 +118,16 @@ class PreOCIModel(DockerRegistryV2DataInterface):
def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id, def save_manifest(self, namespace_name, repo_name, tag_name, leaf_layer_docker_id,
manifest_digest, manifest_bytes): manifest_digest, manifest_bytes):
(_, newly_created) = model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, (_, newly_created) = model.tag.store_tag_manifest(
leaf_layer_docker_id, manifest_digest, namespace_name, repo_name, tag_name, leaf_layer_docker_id, manifest_digest, manifest_bytes)
manifest_bytes)
return newly_created return newly_created
def repository_tags(self, namespace_name, repo_name, limit, offset): def repository_tags(self, namespace_name, repo_name, limit, offset):
def _tag_view(tag): def _tag_view(tag):
return Tag( return Tag(name=tag.name, repository=RepositoryReference(
name=tag.name, id=tag.repository_id,
repository=RepositoryReference( name=repo_name,
id=tag.repository_id, namespace_name=namespace_name,))
name=repo_name,
namespace_name=namespace_name,
)
)
tags_query = model.tag.list_repository_tags(namespace_name, repo_name) tags_query = model.tag.list_repository_tags(namespace_name, repo_name)
tags_query = tags_query.limit(limit).offset(offset) tags_query = tags_query.limit(limit).offset(offset)
@ -151,7 +142,8 @@ class PreOCIModel(DockerRegistryV2DataInterface):
query = query.limit(limit).offset(offset) query = query.limit(limit).offset(offset)
return [_repository_for_repo(repo) for repo in query] return [_repository_for_repo(repo) for repo in query]
def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name, storage_metadata): def create_blob_upload(self, namespace_name, repo_name, upload_uuid, location_name,
storage_metadata):
try: try:
model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name, model.blob.initiate_upload(namespace_name, repo_name, upload_uuid, location_name,
storage_metadata) storage_metadata)
@ -176,8 +168,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
piece_sha_state=found.piece_sha_state, piece_sha_state=found.piece_sha_state,
piece_hashes=found.piece_hashes, piece_hashes=found.piece_hashes,
location_name=found.location.name, location_name=found.location.name,
storage_metadata=found.storage_metadata, storage_metadata=found.storage_metadata,)
)
def update_blob_upload(self, blob_upload): def update_blob_upload(self, blob_upload):
# Lookup the blob upload object. # Lookup the blob upload object.
@ -206,17 +197,14 @@ class PreOCIModel(DockerRegistryV2DataInterface):
def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload, def create_blob_and_temp_tag(self, namespace_name, repo_name, blob_digest, blob_upload,
expiration_sec): expiration_sec):
location_obj = model.storage.get_image_location_for_name(blob_upload.location_name) location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
blob_record = model.blob.store_blob_record_and_temp_link(namespace_name, repo_name, blob_record = model.blob.store_blob_record_and_temp_link(
blob_digest, location_obj.id, namespace_name, repo_name, blob_digest, location_obj.id, blob_upload.byte_count,
blob_upload.byte_count, expiration_sec, blob_upload.uncompressed_byte_count)
expiration_sec,
blob_upload.uncompressed_byte_count)
return Blob( return Blob(
uuid=blob_record.uuid, uuid=blob_record.uuid,
digest=blob_digest, digest=blob_digest,
size=blob_upload.byte_count, size=blob_upload.byte_count,
locations=[blob_upload.location_name], locations=[blob_upload.location_name],)
)
def lookup_blobs_by_digest(self, namespace_name, repo_name, digests): def lookup_blobs_by_digest(self, namespace_name, repo_name, digests):
def _blob_view(blob_record): def _blob_view(blob_record):
@ -224,7 +212,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
uuid=blob_record.uuid, uuid=blob_record.uuid,
digest=blob_record.content_checksum, digest=blob_record.content_checksum,
size=blob_record.image_size, size=blob_record.image_size,
locations=None, # Note: Locations is None in this case. locations=None, # Note: Locations is None in this case.
) )
repo = model.repository.get_repository(namespace_name, repo_name) repo = model.repository.get_repository(namespace_name, repo_name)
@ -240,8 +228,7 @@ class PreOCIModel(DockerRegistryV2DataInterface):
uuid=blob_record.uuid, uuid=blob_record.uuid,
digest=digest, digest=digest,
size=blob_record.image_size, size=blob_record.image_size,
locations=blob_record.locations, locations=blob_record.locations,)
)
except model.BlobDoesNotExist: except model.BlobDoesNotExist:
return None return None
@ -282,8 +269,7 @@ def _docker_v1_metadata(namespace_name, repo_name, repo_image):
comment=repo_image.comment, comment=repo_image.comment,
command=repo_image.command, command=repo_image.command,
# TODO: make sure this isn't needed anywhere, as it is expensive to lookup # TODO: make sure this isn't needed anywhere, as it is expensive to lookup
parent_image_id=None, parent_image_id=None,)
)
def _repository_for_repo(repo): def _repository_for_repo(repo):
@ -295,8 +281,7 @@ def _repository_for_repo(repo):
description=repo.description, description=repo.description,
is_public=model.repository.is_repository_public(repo), is_public=model.repository.is_repository_public(repo),
kind=model.repository.get_repo_kind_name(repo), kind=model.repository.get_repo_kind_name(repo),
trust_enabled=repo.trust_enabled, trust_enabled=repo.trust_enabled,)
)
data_model = PreOCIModel() data_model = PreOCIModel()

4
endpoints/v2/tag.py
View file

@ -6,6 +6,7 @@ from endpoints.decorators import anon_protect
from endpoints.v2 import v2_bp, require_repo_read, paginate from endpoints.v2 import v2_bp, require_repo_read, paginate
from endpoints.v2.models_pre_oci import data_model as model from endpoints.v2.models_pre_oci import data_model as model
@v2_bp.route('/<repopath:repository>/tags/list', methods=['GET']) @v2_bp.route('/<repopath:repository>/tags/list', methods=['GET'])
@parse_repository_name() @parse_repository_name()
@process_registry_jwt_auth(scopes=['pull']) @process_registry_jwt_auth(scopes=['pull'])
@ -16,8 +17,7 @@ def list_all_tags(namespace_name, repo_name, limit, offset, pagination_callback)
tags = model.repository_tags(namespace_name, repo_name, limit, offset) tags = model.repository_tags(namespace_name, repo_name, limit, offset)
response = jsonify({ response = jsonify({
'name': '{0}/{1}'.format(namespace_name, repo_name), 'name': '{0}/{1}'.format(namespace_name, repo_name),
'tags': [tag.name for tag in tags], 'tags': [tag.name for tag in tags],})
})
pagination_callback(len(tags), response) pagination_callback(len(tags), response)
return response return response

22
endpoints/v2/v2auth.py
View file

@ -16,15 +16,15 @@ from endpoints.v2.errors import InvalidLogin, NameInvalid, InvalidRequest, Unsup
from endpoints.v2.models_pre_oci import data_model as model from endpoints.v2.models_pre_oci import data_model as model
from util.cache import no_cache from util.cache import no_cache
from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX from util.names import parse_namespace_repository, REPOSITORY_NAME_REGEX
from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject, QUAY_TUF_ROOT, from util.security.registry_jwt import (generate_bearer_token, build_context_and_subject,
SIGNER_TUF_ROOT, DISABLED_TUF_ROOT) QUAY_TUF_ROOT, SIGNER_TUF_ROOT, DISABLED_TUF_ROOT)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour TOKEN_VALIDITY_LIFETIME_S = 60 * 60 # 1 hour
SCOPE_REGEX_TEMPLATE = r'^repository:((?:{}\/)?((?:[\.a-zA-Z0-9_\-]+\/)*[\.a-zA-Z0-9_\-]+)):((?:push|pull|\*)(?:,(?:push|pull|\*))*)$' SCOPE_REGEX_TEMPLATE = r'^repository:((?:{}\/)?((?:[\.a-zA-Z0-9_\-]+\/)*[\.a-zA-Z0-9_\-]+)):((?:push|pull|\*)(?:,(?:push|pull|\*))*)$'
@lru_cache(maxsize=1) @lru_cache(maxsize=1)
def get_scope_regex(): def get_scope_regex():
hostname = re.escape(app.config['SERVER_HOSTNAME']) hostname = re.escape(app.config['SERVER_HOSTNAME'])
@ -64,8 +64,7 @@ def generate_registry_jwt(auth_result):
access = [] access = []
user_event_data = { user_event_data = {
'action': 'login', 'action': 'login',}
}
tuf_root = DISABLED_TUF_ROOT tuf_root = DISABLED_TUF_ROOT
if len(scope_param) > 0: if len(scope_param) > 0:
@ -101,8 +100,8 @@ def generate_registry_jwt(auth_result):
repo_is_public = repo is not None and repo.is_public repo_is_public = repo is not None and repo.is_public
invalid_repo_message = '' invalid_repo_message = ''
if repo is not None and repo.kind != 'image': if repo is not None and repo.kind != 'image':
invalid_repo_message = (('This repository is for managing %s resources ' + invalid_repo_message = ((
'and not container images.') % repo.kind) 'This repository is for managing %s resources ' + 'and not container images.') % repo.kind)
if 'push' in actions: if 'push' in actions:
# If there is no valid user or token, then the repository cannot be # If there is no valid user or token, then the repository cannot be
@ -150,8 +149,7 @@ def generate_registry_jwt(auth_result):
access.append({ access.append({
'type': 'repository', 'type': 'repository',
'name': registry_and_repo, 'name': registry_and_repo,
'actions': final_actions, 'actions': final_actions,})
})
# Set the user event data for the auth. # Set the user event data for the auth.
if 'push' in final_actions: if 'push' in final_actions:
@ -164,8 +162,7 @@ def generate_registry_jwt(auth_result):
user_event_data = { user_event_data = {
'action': user_action, 'action': user_action,
'repository': reponame, 'repository': reponame,
'namespace': namespace, 'namespace': namespace,}
}
tuf_root = get_tuf_root(repo, namespace, reponame) tuf_root = get_tuf_root(repo, namespace, reponame)
elif user is None and token is None: elif user is None and token is None:
@ -179,7 +176,8 @@ def generate_registry_jwt(auth_result):
event.publish_event_data('docker-cli', user_event_data) event.publish_event_data('docker-cli', user_event_data)
# Build the signed JWT. # Build the signed JWT.
context, subject = build_context_and_subject(user=user, token=token, oauthtoken=oauthtoken, tuf_root=tuf_root) context, subject = build_context_and_subject(user=user, token=token, oauthtoken=oauthtoken,
tuf_root=tuf_root)
token = generate_bearer_token(audience_param, subject, context, access, token = generate_bearer_token(audience_param, subject, context, access,
TOKEN_VALIDITY_LIFETIME_S, instance_keys) TOKEN_VALIDITY_LIFETIME_S, instance_keys)
return jsonify({'token': token}) return jsonify({'token': token})