yapf
This commit is contained in:
Joseph Schorr
parent
8b4958dbcc
commit
d07cc91dc6
4 changed files with 84 additions and 75 deletions
|
|
@ -22,16 +22,14 @@ from util.http import exact_abort
|
|||
from util.registry.filelike import wrap_with_handler
|
||||
from util.registry.queuefile import QueueFile
|
||||
from util.registry.queueprocess import QueueProcess
|
||||
from util.registry.torrent import (make_torrent, per_user_torrent_filename, public_torrent_filename,
|
||||
PieceHasher)
|
||||
|
||||
from util.registry.torrent import (
|
||||
make_torrent, per_user_torrent_filename, public_torrent_filename, PieceHasher)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
verbs = Blueprint('verbs', __name__)
|
||||
license_validator.enforce_license_before_request(verbs)
|
||||
|
||||
|
||||
LAYER_MIMETYPE = 'binary/octet-stream'
|
||||
|
||||
|
||||
|
|
@ -60,7 +58,8 @@ def _open_stream(formatter, repo_image, tag, derived_image_id, handlers):
|
|||
logger.debug('Returning image layer %s: %s', current_image.image_id, current_image_path)
|
||||
yield current_image_stream
|
||||
|
||||
stream = formatter.build_stream(repo_image, tag, derived_image_id, get_next_image, get_next_layer)
|
||||
stream = formatter.build_stream(repo_image, tag, derived_image_id, get_next_image,
|
||||
get_next_layer)
|
||||
|
||||
for handler_fn in handlers:
|
||||
stream = wrap_with_handler(stream, handler_fn)
|
||||
|
|
@ -89,6 +88,7 @@ def _write_derived_image_to_storage(verb, derived_image, queue_file):
|
|||
""" Read from the generated stream and write it back to the storage engine. This method runs in a
|
||||
separate process.
|
||||
"""
|
||||
|
||||
def handle_exception(ex):
|
||||
logger.debug('Exception when building %s derived image %s: %s', verb, derived_image.ref, ex)
|
||||
|
||||
|
|
@ -139,7 +139,8 @@ def _torrent_for_blob(blob, is_public):
|
|||
torrent_file = make_torrent(name, webseed, blob.size, torrent_info.piece_length,
|
||||
torrent_info.pieces)
|
||||
|
||||
headers = {'Content-Type': 'application/x-bittorrent',
|
||||
headers = {
|
||||
'Content-Type': 'application/x-bittorrent',
|
||||
'Content-Disposition': 'attachment; filename={0}.torrent'.format(name)}
|
||||
|
||||
return make_response(torrent_file, 200, headers)
|
||||
|
|
@ -158,8 +159,7 @@ def _torrent_repo_verb(repo_image, tag, verb, **kwargs):
|
|||
abort(406)
|
||||
|
||||
# Return the torrent.
|
||||
repo = model.get_repository(repo_image.repository.namespace_name,
|
||||
repo_image.repository.name)
|
||||
repo = model.get_repository(repo_image.repository.namespace_name, repo_image.repository.name)
|
||||
repo_is_public = repo is not None and repo.is_public
|
||||
torrent = _torrent_for_blob(derived_image.blob, repo_is_public)
|
||||
|
||||
|
|
@ -229,15 +229,14 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb, True])
|
||||
|
||||
# Lookup/create the derived image for the verb and repo image.
|
||||
derived_image = model.lookup_or_create_derived_image(repo_image, verb,
|
||||
storage.preferred_locations[0],
|
||||
varying_metadata={'tag': tag})
|
||||
derived_image = model.lookup_or_create_derived_image(
|
||||
repo_image, verb, storage.preferred_locations[0], varying_metadata={'tag': tag})
|
||||
if not derived_image.blob.uploading:
|
||||
logger.debug('Derived %s image %s exists in storage', verb, derived_image.ref)
|
||||
derived_layer_path = model.get_blob_path(derived_image.blob)
|
||||
is_head_request = request.method == 'HEAD'
|
||||
download_url = storage.get_direct_download_url(derived_image.blob.locations, derived_layer_path,
|
||||
head=is_head_request)
|
||||
download_url = storage.get_direct_download_url(derived_image.blob.locations,
|
||||
derived_layer_path, head=is_head_request)
|
||||
if download_url:
|
||||
logger.debug('Redirecting to download URL for derived %s image %s', verb, derived_image.ref)
|
||||
return redirect(download_url)
|
||||
|
|
@ -246,7 +245,8 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
database.close_db_filter(None)
|
||||
|
||||
logger.debug('Sending cached derived %s image %s', verb, derived_image.ref)
|
||||
return send_file(storage.stream_read_file(derived_image.blob.locations, derived_layer_path),
|
||||
return send_file(
|
||||
storage.stream_read_file(derived_image.blob.locations, derived_layer_path),
|
||||
mimetype=LAYER_MIMETYPE)
|
||||
|
||||
logger.debug('Building and returning derived %s image %s', verb, derived_image.ref)
|
||||
|
|
@ -270,9 +270,12 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
# and send the results to the client and storage.
|
||||
handlers = [hasher.update]
|
||||
args = (formatter, repo_image, tag, derived_image_id, handlers)
|
||||
queue_process = QueueProcess(_open_stream,
|
||||
8 * 1024, 10 * 1024 * 1024, # 8K/10M chunk/max
|
||||
args, finished=_store_metadata_and_cleanup)
|
||||
queue_process = QueueProcess(
|
||||
_open_stream,
|
||||
8 * 1024,
|
||||
10 * 1024 * 1024, # 8K/10M chunk/max
|
||||
args,
|
||||
finished=_store_metadata_and_cleanup)
|
||||
|
||||
client_queue_file = QueueFile(queue_process.create_queue(), 'client')
|
||||
storage_queue_file = QueueFile(queue_process.create_queue(), 'storage')
|
||||
|
|
@ -336,11 +339,13 @@ def get_aci_signature(server, namespace, repository, tag, os, arch):
|
|||
|
||||
@route_show_if(features.ACI_CONVERSION)
|
||||
@anon_protect
|
||||
@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/', methods=['GET', 'HEAD'])
|
||||
@verbs.route('/aci/<server>/<namespace>/<repository>/<tag>/aci/<os>/<arch>/', methods=[
|
||||
'GET', 'HEAD'])
|
||||
@process_auth
|
||||
def get_aci_image(server, namespace, repository, tag, os, arch):
|
||||
return _repo_verb(namespace, repository, tag, 'aci', AppCImageFormatter(),
|
||||
sign=True, checker=os_arch_checker(os, arch), os=os, arch=arch)
|
||||
return _repo_verb(namespace, repository, tag, 'aci',
|
||||
AppCImageFormatter(), sign=True, checker=os_arch_checker(os, arch), os=os,
|
||||
arch=arch)
|
||||
|
||||
|
||||
@anon_protect
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ from collections import namedtuple
|
|||
|
||||
from six import add_metaclass
|
||||
|
||||
class Repository(namedtuple('Repository', ['id', 'name', 'namespace_name', 'description',
|
||||
'is_public', 'kind'])):
|
||||
|
||||
class Repository(
|
||||
namedtuple('Repository', ['id', 'name', 'namespace_name', 'description', 'is_public',
|
||||
'kind'])):
|
||||
"""
|
||||
Repository represents a namespaced collection of tags.
|
||||
:type id: int
|
||||
|
|
@ -21,22 +23,27 @@ class DerivedImage(namedtuple('DerivedImage', ['ref', 'blob', 'internal_source_i
|
|||
DerivedImage represents a user-facing alias for an image which was derived from another image.
|
||||
"""
|
||||
|
||||
|
||||
class RepositoryReference(namedtuple('RepositoryReference', ['id', 'name', 'namespace_name'])):
|
||||
"""
|
||||
RepositoryReference represents a reference to a Repository, without its full metadata.
|
||||
"""
|
||||
|
||||
class ImageWithBlob(namedtuple('Image', ['image_id', 'blob', 'compat_metadata', 'repository',
|
||||
'internal_db_id', 'v1_metadata'])):
|
||||
|
||||
class ImageWithBlob(
|
||||
namedtuple('Image', [
|
||||
'image_id', 'blob', 'compat_metadata', 'repository', 'internal_db_id', 'v1_metadata'])):
|
||||
"""
|
||||
ImageWithBlob represents a user-facing alias for referencing an image, along with its blob.
|
||||
"""
|
||||
|
||||
|
||||
class Blob(namedtuple('Blob', ['uuid', 'size', 'uncompressed_size', 'uploading', 'locations'])):
|
||||
"""
|
||||
Blob represents an opaque binary blob saved to the storage system.
|
||||
"""
|
||||
|
||||
|
||||
class TorrentInfo(namedtuple('TorrentInfo', ['piece_length', 'pieces'])):
|
||||
"""
|
||||
TorrentInfo represents the torrent piece information associated with a blob.
|
||||
|
|
@ -49,6 +56,7 @@ class VerbsDataInterface(object):
|
|||
Interface that represents all data store interactions required by the registry's custom HTTP
|
||||
verbs.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_repository(self, namespace_name, repo_name):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ from endpoints.verbs.models_interface import (
|
|||
Repository,
|
||||
RepositoryReference,
|
||||
TorrentInfo,
|
||||
VerbsDataInterface,
|
||||
)
|
||||
VerbsDataInterface,)
|
||||
|
||||
|
||||
class PreOCIModel(VerbsDataInterface):
|
||||
"""
|
||||
|
|
@ -27,13 +27,11 @@ class PreOCIModel(VerbsDataInterface):
|
|||
return _repository_for_repo(repo)
|
||||
|
||||
def get_manifest_layers_with_blobs(self, repo_image):
|
||||
repo_image_record = model.image.get_image_by_id(repo_image.repository.namespace_name,
|
||||
repo_image.repository.name,
|
||||
repo_image.image_id)
|
||||
repo_image_record = model.image.get_image_by_id(
|
||||
repo_image.repository.namespace_name, repo_image.repository.name, repo_image.image_id)
|
||||
|
||||
parents = model.image.get_parent_images_with_placements(repo_image.repository.namespace_name,
|
||||
repo_image.repository.name,
|
||||
repo_image_record)
|
||||
parents = model.image.get_parent_images_with_placements(
|
||||
repo_image.repository.namespace_name, repo_image.repository.name, repo_image_record)
|
||||
|
||||
yield repo_image
|
||||
|
||||
|
|
@ -51,8 +49,7 @@ class PreOCIModel(VerbsDataInterface):
|
|||
compat_metadata=metadata,
|
||||
v1_metadata=_docker_v1_metadata(repo_image.repository.namespace_name,
|
||||
repo_image.repository.name, parent),
|
||||
internal_db_id=parent.id,
|
||||
)
|
||||
internal_db_id=parent.id,)
|
||||
|
||||
def get_derived_image_signature(self, derived_image, signer_name):
|
||||
storage = model.storage.get_storage_by_uuid(derived_image.blob.uuid)
|
||||
|
|
@ -100,8 +97,7 @@ class PreOCIModel(VerbsDataInterface):
|
|||
|
||||
return TorrentInfo(
|
||||
pieces=torrent_info.pieces,
|
||||
piece_length=torrent_info.piece_length,
|
||||
)
|
||||
piece_length=torrent_info.piece_length,)
|
||||
|
||||
def set_torrent_info(self, blob, piece_length, pieces):
|
||||
blob_record = model.storage.get_storage_by_uuid(blob.uuid)
|
||||
|
|
@ -138,12 +134,10 @@ class PreOCIModel(VerbsDataInterface):
|
|||
repository=RepositoryReference(
|
||||
namespace_name=namespace_name,
|
||||
name=repo_name,
|
||||
id=found.repository_id,
|
||||
),
|
||||
id=found.repository_id,),
|
||||
compat_metadata=metadata,
|
||||
v1_metadata=_docker_v1_metadata(namespace_name, repo_name, found),
|
||||
internal_db_id=found.id,
|
||||
)
|
||||
internal_db_id=found.id,)
|
||||
|
||||
|
||||
pre_oci_model = PreOCIModel()
|
||||
|
|
@ -168,8 +162,7 @@ def _docker_v1_metadata(namespace_name, repo_name, repo_image):
|
|||
|
||||
# Note: These are not needed in verbs and are expensive to load, so we just skip them.
|
||||
content_checksum=None,
|
||||
parent_image_id=None,
|
||||
)
|
||||
parent_image_id=None,)
|
||||
|
||||
|
||||
def _derived_image(blob_record, repo_image):
|
||||
|
|
@ -179,8 +172,7 @@ def _derived_image(blob_record, repo_image):
|
|||
return DerivedImage(
|
||||
ref=repo_image.internal_db_id,
|
||||
blob=_blob(blob_record),
|
||||
internal_source_image_db_id=repo_image.internal_db_id,
|
||||
)
|
||||
internal_source_image_db_id=repo_image.internal_db_id,)
|
||||
|
||||
|
||||
def _blob(blob_record):
|
||||
|
|
@ -197,8 +189,8 @@ def _blob(blob_record):
|
|||
size=blob_record.image_size,
|
||||
uncompressed_size=blob_record.uncompressed_size,
|
||||
uploading=blob_record.uploading,
|
||||
locations=locations,
|
||||
)
|
||||
locations=locations,)
|
||||
|
||||
|
||||
def _repository_for_repo(repo):
|
||||
""" Returns a Repository object representing the Pre-OCI data model repo instance given. """
|
||||
|
|
@ -208,5 +200,4 @@ def _repository_for_repo(repo):
|
|||
namespace_name=repo.namespace_user.username,
|
||||
description=repo.description,
|
||||
is_public=model.repository.is_repository_public(repo),
|
||||
kind=model.repository.get_repo_kind_name(repo),
|
||||
)
|
||||
kind=model.repository.get_repo_kind_name(repo),)
|
||||
|
|
|
|||
|
|
@ -18,17 +18,18 @@ ACI_ARGS = {
|
|||
'server': 'someserver',
|
||||
'tag': 'fake',
|
||||
'os': 'linux',
|
||||
'arch': 'x64',
|
||||
}
|
||||
'arch': 'x64',}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('user', [
|
||||
(0, None),
|
||||
(1, NO_ACCESS_USER),
|
||||
(2, READ_ACCESS_USER),
|
||||
(3, CREATOR_ACCESS_USER),
|
||||
(4, ADMIN_ACCESS_USER),
|
||||
])
|
||||
@pytest.mark.parametrize('endpoint,method,repository,single_repo_path,params,expected_statuses', [
|
||||
(4, ADMIN_ACCESS_USER),])
|
||||
@pytest.mark.parametrize(
|
||||
'endpoint,method,repository,single_repo_path,params,expected_statuses',
|
||||
[
|
||||
('get_aci_signature', 'GET', PUBLIC_REPO, False, ACI_ARGS, (404, 404, 404, 404, 404)),
|
||||
('get_aci_signature', 'GET', PRIVATE_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
|
||||
('get_aci_signature', 'GET', ORG_REPO, False, ACI_ARGS, (403, 403, 404, 403, 404)),
|
||||
|
|
@ -44,14 +45,18 @@ ACI_ARGS = {
|
|||
('get_squashed_tag', 'GET', PUBLIC_REPO, False, dict(tag='fake'), (404, 404, 404, 404, 404)),
|
||||
('get_squashed_tag', 'GET', PRIVATE_REPO, False, dict(tag='fake'), (403, 403, 404, 403, 404)),
|
||||
('get_squashed_tag', 'GET', ORG_REPO, False, dict(tag='fake'), (403, 403, 404, 403, 404)),
|
||||
('get_squashed_tag', 'GET', ANOTHER_ORG_REPO, False, dict(tag='fake'), (403, 403, 403, 403, 404)),
|
||||
('get_squashed_tag', 'GET', ANOTHER_ORG_REPO, False, dict(tag='fake'), (403, 403, 403, 403,
|
||||
404)),
|
||||
|
||||
# get_tag_torrent
|
||||
('get_tag_torrent', 'GET', PUBLIC_REPO, True, dict(digest='sha256:1234'), (404, 404, 404, 404, 404)),
|
||||
('get_tag_torrent', 'GET', PRIVATE_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403, 404)),
|
||||
('get_tag_torrent', 'GET', ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403, 404)),
|
||||
('get_tag_torrent', 'GET', ANOTHER_ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 403, 403, 404)),
|
||||
])
|
||||
('get_tag_torrent', 'GET', PUBLIC_REPO, True, dict(digest='sha256:1234'), (404, 404, 404, 404,
|
||||
404)),
|
||||
('get_tag_torrent', 'GET', PRIVATE_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403,
|
||||
404)),
|
||||
('get_tag_torrent', 'GET', ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 404, 403,
|
||||
404)),
|
||||
('get_tag_torrent', 'GET', ANOTHER_ORG_REPO, True, dict(digest='sha256:1234'), (403, 403, 403,
|
||||
403, 404)),])
|
||||
def test_verbs_security(user, endpoint, method, repository, single_repo_path, params,
|
||||
expected_statuses, app, client):
|
||||
headers = {}
|
||||
|
|
|
|||
Reference in a new issue