Move verbs endpoint to use new registry data model
This commit is contained in:
Joseph Schorr
parent
bafab2e734
commit
f252b0b16f
14 changed files with 145 additions and 469 deletions
|
|
@ -10,13 +10,14 @@ from auth.auth_context import get_authenticated_user
|
|||
from auth.decorators import process_auth
|
||||
from auth.permissions import ReadRepositoryPermission
|
||||
from data import database
|
||||
from data import model
|
||||
from data.registry_model import registry_model
|
||||
from endpoints.decorators import anon_protect, anon_allowed, route_show_if, parse_repository_name
|
||||
from endpoints.verbs.models_pre_oci import pre_oci_model as model
|
||||
from endpoints.v2.blob import BLOB_DIGEST_ROUTE
|
||||
from image.appc import AppCImageFormatter
|
||||
from image.docker.squashed import SquashedDockerImageFormatter
|
||||
from storage import Storage
|
||||
from util.audit import track_and_log
|
||||
from util.audit import track_and_log, wrap_repository
|
||||
from util.http import exact_abort
|
||||
from util.registry.filelike import wrap_with_handler
|
||||
from util.registry.queuefile import QueueFile
|
||||
|
|
@ -40,7 +41,7 @@ class VerbReporter(TarLayerFormatterReporter):
|
|||
metric_queue.verb_action_passes.Inc(labelvalues=[self.kind, pass_count])
|
||||
|
||||
|
||||
def _open_stream(formatter, repo_image, tag, derived_image_id, handlers, reporter):
|
||||
def _open_stream(formatter, tag, manifest, derived_image_id, handlers, reporter):
|
||||
"""
|
||||
This method generates a stream of data which will be replicated and read from the queue files.
|
||||
This method runs in a separate process.
|
||||
|
|
@ -48,29 +49,25 @@ def _open_stream(formatter, repo_image, tag, derived_image_id, handlers, reporte
|
|||
# For performance reasons, we load the full image list here, cache it, then disconnect from
|
||||
# the database.
|
||||
with database.UseThenDisconnect(app.config):
|
||||
image_list = list(model.get_manifest_layers_with_blobs(repo_image))
|
||||
layers = registry_model.list_manifest_layers(manifest, include_placements=True)
|
||||
|
||||
def get_next_image():
|
||||
for current_image in image_list:
|
||||
yield current_image
|
||||
|
||||
def image_stream_getter(store, current_image):
|
||||
def image_stream_getter(store, blob):
|
||||
def get_stream_for_storage():
|
||||
current_image_path = model.get_blob_path(current_image.blob)
|
||||
current_image_stream = store.stream_read_file(current_image.blob.locations,
|
||||
current_image_path)
|
||||
|
||||
logger.debug('Returning image layer %s: %s', current_image.image_id, current_image_path)
|
||||
current_image_stream = store.stream_read_file(blob.placements, blob.storage_path)
|
||||
logger.debug('Returning blob %s: %s', blob.digest, blob.storage_path)
|
||||
return current_image_stream
|
||||
return get_stream_for_storage
|
||||
|
||||
def tar_stream_getter_iterator():
|
||||
# Re-Initialize the storage engine because some may not respond well to forking (e.g. S3)
|
||||
store = Storage(app, metric_queue, config_provider=config_provider, ip_resolver=ip_resolver)
|
||||
for current_image in image_list:
|
||||
yield image_stream_getter(store, current_image)
|
||||
|
||||
stream = formatter.build_stream(repo_image, tag, derived_image_id, get_next_image,
|
||||
# Note: We reverse because we have to start at the leaf layer and move upward,
|
||||
# as per the spec for the formatters.
|
||||
for layer in reversed(layers):
|
||||
yield image_stream_getter(store, layer.blob)
|
||||
|
||||
stream = formatter.build_stream(tag, manifest, derived_image_id, layers,
|
||||
tar_stream_getter_iterator, reporter=reporter)
|
||||
|
||||
for handler_fn in handlers:
|
||||
|
|
@ -86,14 +83,14 @@ def _sign_derived_image(verb, derived_image, queue_file):
|
|||
try:
|
||||
signature = signer.detached_sign(queue_file)
|
||||
except:
|
||||
logger.exception('Exception when signing %s deriving image %s', verb, derived_image.ref)
|
||||
logger.exception('Exception when signing %s deriving image %s', verb, derived_image)
|
||||
return
|
||||
|
||||
# Setup the database (since this is a new process) and then disconnect immediately
|
||||
# once the operation completes.
|
||||
if not queue_file.raised_exception:
|
||||
with database.UseThenDisconnect(app.config):
|
||||
model.set_derived_image_signature(derived_image, signer.name, signature)
|
||||
registry_model.set_derived_image_signature(derived_image, signer.name, signature)
|
||||
|
||||
|
||||
def _write_derived_image_to_storage(verb, derived_image, queue_file):
|
||||
|
|
@ -102,17 +99,16 @@ def _write_derived_image_to_storage(verb, derived_image, queue_file):
|
|||
"""
|
||||
|
||||
def handle_exception(ex):
|
||||
logger.debug('Exception when building %s derived image %s: %s', verb, derived_image.ref, ex)
|
||||
logger.debug('Exception when building %s derived image %s: %s', verb, derived_image, ex)
|
||||
|
||||
with database.UseThenDisconnect(app.config):
|
||||
model.delete_derived_image(derived_image)
|
||||
registry_model.delete_derived_image(derived_image)
|
||||
|
||||
queue_file.add_exception_handler(handle_exception)
|
||||
|
||||
# Re-Initialize the storage engine because some may not respond well to forking (e.g. S3)
|
||||
store = Storage(app, metric_queue, config_provider=config_provider, ip_resolver=ip_resolver)
|
||||
image_path = model.get_blob_path(derived_image.blob)
|
||||
store.stream_write(derived_image.blob.locations, image_path, queue_file)
|
||||
store.stream_write(derived_image.blob.placements, derived_image.blob.storage_path, queue_file)
|
||||
queue_file.close()
|
||||
|
||||
|
||||
|
|
@ -121,17 +117,16 @@ def _torrent_for_blob(blob, is_public):
|
|||
with an error if the state is not valid (e.g. non-public, non-user request).
|
||||
"""
|
||||
# Make sure the storage has a size.
|
||||
if not blob.size:
|
||||
if not blob.compressed_size:
|
||||
abort(404)
|
||||
|
||||
# Lookup the torrent information for the storage.
|
||||
torrent_info = model.get_torrent_info(blob)
|
||||
torrent_info = registry_model.get_torrent_info(blob)
|
||||
if torrent_info is None:
|
||||
abort(404)
|
||||
|
||||
# Lookup the webseed path for the storage.
|
||||
path = model.get_blob_path(blob)
|
||||
webseed = storage.get_direct_download_url(blob.locations, path,
|
||||
webseed = storage.get_direct_download_url(blob.placements, blob.storage_path,
|
||||
expires_in=app.config['BITTORRENT_WEBSEED_LIFETIME'])
|
||||
if webseed is None:
|
||||
# We cannot support webseeds for storages that cannot provide direct downloads.
|
||||
|
|
@ -151,8 +146,8 @@ def _torrent_for_blob(blob, is_public):
|
|||
name = per_user_torrent_filename(torrent_config, user.uuid, blob.uuid)
|
||||
|
||||
# Return the torrent file.
|
||||
torrent_file = make_torrent(torrent_config, name, webseed, blob.size, torrent_info.piece_length,
|
||||
torrent_info.pieces)
|
||||
torrent_file = make_torrent(torrent_config, name, webseed, blob.compressed_size,
|
||||
torrent_info.piece_length, torrent_info.pieces)
|
||||
|
||||
headers = {
|
||||
'Content-Type': 'application/x-bittorrent',
|
||||
|
|
@ -161,7 +156,7 @@ def _torrent_for_blob(blob, is_public):
|
|||
return make_response(torrent_file, 200, headers)
|
||||
|
||||
|
||||
def _torrent_repo_verb(repo_image, tag, verb, **kwargs):
|
||||
def _torrent_repo_verb(repository, tag, manifest, verb, **kwargs):
|
||||
""" Handles returning a torrent for the given verb on the given image and tag. """
|
||||
if not features.BITTORRENT:
|
||||
# Torrent feature is not enabled.
|
||||
|
|
@ -169,60 +164,73 @@ def _torrent_repo_verb(repo_image, tag, verb, **kwargs):
|
|||
|
||||
# Lookup an *existing* derived storage for the verb. If the verb's image storage doesn't exist,
|
||||
# we cannot create it here, so we 406.
|
||||
derived_image = model.lookup_derived_image(repo_image, verb, varying_metadata={'tag': tag})
|
||||
derived_image = registry_model.lookup_derived_image(manifest, verb,
|
||||
varying_metadata={'tag': tag.name},
|
||||
include_placements=True)
|
||||
if derived_image is None:
|
||||
abort(406)
|
||||
|
||||
# Return the torrent.
|
||||
repo = model.get_repository(repo_image.repository.namespace_name, repo_image.repository.name)
|
||||
repo_is_public = repo is not None and repo.is_public
|
||||
torrent = _torrent_for_blob(derived_image.blob, repo_is_public)
|
||||
torrent = _torrent_for_blob(derived_image.blob, model.repository.is_repository_public(repository))
|
||||
|
||||
# Log the action.
|
||||
track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, torrent=True, **kwargs)
|
||||
track_and_log('repo_verb', wrap_repository(repository), tag=tag.name, verb=verb, torrent=True,
|
||||
**kwargs)
|
||||
return torrent
|
||||
|
||||
|
||||
def _verify_repo_verb(_, namespace, repo_name, tag, verb, checker=None):
|
||||
def _verify_repo_verb(_, namespace, repo_name, tag_name, verb, checker=None):
|
||||
permission = ReadRepositoryPermission(namespace, repo_name)
|
||||
repo = model.get_repository(namespace, repo_name)
|
||||
repo_is_public = repo is not None and repo.is_public
|
||||
repo = model.repository.get_repository(namespace, repo_name)
|
||||
repo_is_public = repo is not None and model.repository.is_repository_public(repo)
|
||||
if not permission.can() and not repo_is_public:
|
||||
logger.debug('No permission to read repository %s/%s for user %s with verb %s', namespace,
|
||||
repo_name, get_authenticated_user(), verb)
|
||||
abort(403)
|
||||
|
||||
# Lookup the requested tag.
|
||||
tag_image = model.get_tag_image(namespace, repo_name, tag)
|
||||
if tag_image is None:
|
||||
logger.debug('Tag %s does not exist in repository %s/%s for user %s', tag, namespace, repo_name,
|
||||
get_authenticated_user())
|
||||
abort(404)
|
||||
|
||||
if repo is not None and repo.kind != 'image':
|
||||
if repo is not None and repo.kind.name != 'image':
|
||||
logger.debug('Repository %s/%s for user %s is not an image repo', namespace, repo_name,
|
||||
get_authenticated_user())
|
||||
abort(405)
|
||||
|
||||
# Make sure the repo's namespace isn't disabled.
|
||||
if not model.is_namespace_enabled(namespace):
|
||||
if not registry_model.is_namespace_enabled(namespace):
|
||||
abort(400)
|
||||
|
||||
# Lookup the requested tag.
|
||||
repo_ref = registry_model.lookup_repository(namespace, repo_name)
|
||||
tag = registry_model.get_repo_tag(repo_ref, tag_name)
|
||||
if tag is None:
|
||||
logger.debug('Tag %s does not exist in repository %s/%s for user %s', tag, namespace, repo_name,
|
||||
get_authenticated_user())
|
||||
abort(404)
|
||||
|
||||
# Get its associated manifest.
|
||||
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
|
||||
if manifest is None:
|
||||
logger.debug('Could not get manifest on %s/%s:%s::%s', namespace, repo_name, tag.name, verb)
|
||||
abort(404)
|
||||
|
||||
# If there is a data checker, call it first.
|
||||
if checker is not None:
|
||||
if not checker(tag_image):
|
||||
logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repo_name, tag, verb)
|
||||
if not checker(tag, manifest):
|
||||
logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repo_name, tag.name, verb)
|
||||
abort(404)
|
||||
|
||||
return tag_image
|
||||
# Preload the tag's repository information, so it gets cached.
|
||||
assert tag.repository.namespace_name
|
||||
assert tag.repository.name
|
||||
|
||||
return tag, manifest
|
||||
|
||||
|
||||
def _repo_verb_signature(namespace, repository, tag, verb, checker=None, **kwargs):
|
||||
# Verify that the image exists and that we have access to it.
|
||||
repo_image = _verify_repo_verb(storage, namespace, repository, tag, verb, checker)
|
||||
def _repo_verb_signature(namespace, repository, tag_name, verb, checker=None, **kwargs):
|
||||
# Verify that the tag exists and that we have access to it.
|
||||
tag, manifest = _verify_repo_verb(storage, namespace, repository, tag_name, verb, checker)
|
||||
|
||||
# derived_image the derived image storage for the verb.
|
||||
derived_image = model.lookup_derived_image(repo_image, verb, varying_metadata={'tag': tag})
|
||||
# Find the derived image storage for the verb.
|
||||
derived_image = registry_model.lookup_derived_image(manifest, verb,
|
||||
varying_metadata={'tag': tag.name})
|
||||
if derived_image is None or derived_image.blob.uploading:
|
||||
return make_response('', 202)
|
||||
|
||||
|
|
@ -231,7 +239,7 @@ def _repo_verb_signature(namespace, repository, tag, verb, checker=None, **kwarg
|
|||
abort(404)
|
||||
|
||||
# Lookup the signature for the verb.
|
||||
signature_value = model.get_derived_image_signature(derived_image, signer.name)
|
||||
signature_value = registry_model.get_derived_image_signature(derived_image, signer.name)
|
||||
if signature_value is None:
|
||||
abort(404)
|
||||
|
||||
|
|
@ -239,53 +247,57 @@ def _repo_verb_signature(namespace, repository, tag, verb, checker=None, **kwarg
|
|||
return make_response(signature_value)
|
||||
|
||||
|
||||
def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=None, **kwargs):
|
||||
def _repo_verb(namespace, repository, tag_name, verb, formatter, sign=False, checker=None,
|
||||
**kwargs):
|
||||
# Verify that the image exists and that we have access to it.
|
||||
logger.debug('Verifying repo verb %s for repository %s/%s with user %s with mimetype %s',
|
||||
verb, namespace, repository, get_authenticated_user(), request.accept_mimetypes.best)
|
||||
repo_image = _verify_repo_verb(storage, namespace, repository, tag, verb, checker)
|
||||
tag, manifest = _verify_repo_verb(storage, namespace, repository, tag_name, verb, checker)
|
||||
|
||||
# Load the repository for later.
|
||||
repo = model.repository.get_repository(namespace, repository)
|
||||
if repo is None:
|
||||
abort(404)
|
||||
|
||||
# Check for torrent. If found, we return a torrent for the repo verb image (if the derived
|
||||
# image already exists).
|
||||
if request.accept_mimetypes.best == 'application/x-bittorrent':
|
||||
metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb + '+torrent', True])
|
||||
return _torrent_repo_verb(repo_image, tag, verb, **kwargs)
|
||||
return _torrent_repo_verb(repo, tag, manifest, verb, **kwargs)
|
||||
|
||||
# Log the action.
|
||||
track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs)
|
||||
track_and_log('repo_verb', wrap_repository(repo), tag=tag.name, verb=verb, **kwargs)
|
||||
metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb, True])
|
||||
|
||||
# Lookup/create the derived image for the verb and repo image.
|
||||
derived_image = model.lookup_or_create_derived_image(
|
||||
repo_image, verb, storage.preferred_locations[0], varying_metadata={'tag': tag})
|
||||
derived_image = registry_model.lookup_or_create_derived_image(
|
||||
manifest, verb, storage.preferred_locations[0], varying_metadata={'tag': tag.name},
|
||||
include_placements=True)
|
||||
if not derived_image.blob.uploading:
|
||||
logger.debug('Derived %s image %s exists in storage', verb, derived_image.ref)
|
||||
derived_layer_path = model.get_blob_path(derived_image.blob)
|
||||
logger.debug('Derived %s image %s exists in storage', verb, derived_image)
|
||||
is_head_request = request.method == 'HEAD'
|
||||
download_url = storage.get_direct_download_url(derived_image.blob.locations,
|
||||
derived_layer_path, head=is_head_request)
|
||||
download_url = storage.get_direct_download_url(derived_image.blob.placements,
|
||||
derived_image.blob.storage_path,
|
||||
head=is_head_request)
|
||||
if download_url:
|
||||
logger.debug('Redirecting to download URL for derived %s image %s', verb, derived_image.ref)
|
||||
logger.debug('Redirecting to download URL for derived %s image %s', verb, derived_image)
|
||||
return redirect(download_url)
|
||||
|
||||
# Close the database handle here for this process before we send the long download.
|
||||
database.close_db_filter(None)
|
||||
|
||||
logger.debug('Sending cached derived %s image %s', verb, derived_image.ref)
|
||||
logger.debug('Sending cached derived %s image %s', verb, derived_image)
|
||||
return send_file(
|
||||
storage.stream_read_file(derived_image.blob.locations, derived_layer_path),
|
||||
storage.stream_read_file(derived_image.blob.placements, derived_image.blob.storage_path),
|
||||
mimetype=LAYER_MIMETYPE)
|
||||
|
||||
logger.debug('Building and returning derived %s image %s', verb, derived_image.ref)
|
||||
logger.debug('Building and returning derived %s image %s', verb, derived_image)
|
||||
|
||||
# Close the database connection before any process forking occurs. This is important because
|
||||
# the Postgres driver does not react kindly to forking, so we need to make sure it is closed
|
||||
# so that each process will get its own unique connection.
|
||||
database.close_db_filter(None)
|
||||
|
||||
# Calculate a derived image ID.
|
||||
derived_image_id = hashlib.sha256(repo_image.image_id + ':' + verb).hexdigest()
|
||||
|
||||
def _cleanup():
|
||||
# Close any existing DB connection once the process has exited.
|
||||
database.close_db_filter(None)
|
||||
|
|
@ -294,15 +306,15 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
|
||||
def _store_metadata_and_cleanup():
|
||||
with database.UseThenDisconnect(app.config):
|
||||
model.set_torrent_info(derived_image.blob, app.config['BITTORRENT_PIECE_SIZE'],
|
||||
hasher.final_piece_hashes())
|
||||
model.set_blob_size(derived_image.blob, hasher.hashed_bytes)
|
||||
registry_model.set_torrent_info(derived_image.blob, app.config['BITTORRENT_PIECE_SIZE'],
|
||||
hasher.final_piece_hashes())
|
||||
registry_model.set_derived_image_size(derived_image, hasher.hashed_bytes)
|
||||
|
||||
# Create a queue process to generate the data. The queue files will read from the process
|
||||
# and send the results to the client and storage.
|
||||
handlers = [hasher.update]
|
||||
reporter = VerbReporter(verb)
|
||||
args = (formatter, repo_image, tag, derived_image_id, handlers, reporter)
|
||||
args = (formatter, tag, manifest, derived_image.unique_id, handlers, reporter)
|
||||
queue_process = QueueProcess(
|
||||
_open_stream,
|
||||
8 * 1024,
|
||||
|
|
@ -337,8 +349,8 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
|
|||
|
||||
|
||||
def os_arch_checker(os, arch):
|
||||
def checker(repo_image):
|
||||
image_json = repo_image.compat_metadata
|
||||
def checker(tag, manifest):
|
||||
image_json = manifest.leaf_layer.v1_metadata
|
||||
|
||||
# Verify the architecture and os.
|
||||
operating_system = image_json.get('os', 'linux')
|
||||
|
|
@ -394,8 +406,8 @@ def get_squashed_tag(namespace, repository, tag):
|
|||
@process_auth
|
||||
@parse_repository_name()
|
||||
def get_tag_torrent(namespace_name, repo_name, digest):
|
||||
repo = model.get_repository(namespace_name, repo_name)
|
||||
repo_is_public = repo is not None and repo.is_public
|
||||
repo = model.repository.get_repository(namespace_name, repo_name)
|
||||
repo_is_public = repo is not None and model.repository.is_repository_public(repo)
|
||||
|
||||
permission = ReadRepositoryPermission(namespace_name, repo_name)
|
||||
if not permission.can() and not repo_is_public:
|
||||
|
|
@ -406,10 +418,14 @@ def get_tag_torrent(namespace_name, repo_name, digest):
|
|||
# We can not generate a private torrent cluster without a user uuid (e.g. token auth)
|
||||
abort(403)
|
||||
|
||||
if repo is not None and repo.kind != 'image':
|
||||
if repo is not None and repo.kind.name != 'image':
|
||||
abort(405)
|
||||
|
||||
blob = model.get_repo_blob_by_digest(namespace_name, repo_name, digest)
|
||||
repo_ref = registry_model.lookup_repository(namespace_name, repo_name)
|
||||
if repo_ref is None:
|
||||
abort(404)
|
||||
|
||||
blob = registry_model.get_repo_blob_by_digest(repo_ref, digest, include_placements=True)
|
||||
if blob is None:
|
||||
abort(404)
|
||||
|
||||
|
|
|
|||
Reference in a new issue