vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Implement the new OCI-based registry data model

Browse source 
Note that this change does *not* enable the new data model by default, but does allow it to be used when a special environment variable is specified.
This commit is contained in:
Joseph Schorr 2018-11-05 13:03:08 -05:00
parent 924b386437
commit fdcb8bad23
23 changed files with 1847 additions and 209 deletions
Download patch file Download diff file Expand all files Collapse all files

11
data/registry_model/__init__.py
View file

@ -1,3 +1,10 @@
from data.registry_model.registry_pre_oci_model import pre_oci_model
import os
import logging
registry_model = pre_oci_model
from data.registry_model.registry_pre_oci_model import pre_oci_model
from data.registry_model.registry_oci_model import oci_model
logger = logging.getLogger(__name__)
registry_model = oci_model if os.getenv('OCI_DATA_MODEL') == 'true' else pre_oci_model
logger.debug('Using registry model `%s`', registry_model)

44
data/registry_model/datatypes.py
View file

@ -95,8 +95,25 @@ class Label(datatype('Label', ['key', 'value', 'uuid', 'source_type_name', 'medi
class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_start_ts',
'lifetime_end_ts'])):
'lifetime_end_ts', 'lifetime_start_ms', 'lifetime_end_ms'])):
""" Tag represents a tag in a repository, which points to a manifest or image. """
@classmethod
def for_tag(cls, tag, legacy_image=None):
if tag is None:
return None
return Tag(db_id=tag.id,
name=tag.name,
reversion=tag.reversion,
lifetime_start_ms=tag.lifetime_start_ms,
lifetime_end_ms=tag.lifetime_end_ms,
lifetime_start_ts=tag.lifetime_start_ms / 1000,
lifetime_end_ts=tag.lifetime_end_ms / 1000 if tag.lifetime_end_ms else None,
manifest_digest=tag.manifest.digest,
inputs=dict(legacy_image=legacy_image,
manifest=tag.manifest,
repository=RepositoryReference.for_id(tag.repository_id)))
@classmethod
def for_repository_tag(cls, repository_tag, manifest_digest=None, legacy_image=None):
if repository_tag is None:
@ -107,10 +124,19 @@ class Tag(datatype('Tag', ['name', 'reversion', 'manifest_digest', 'lifetime_sta
reversion=repository_tag.reversion,
lifetime_start_ts=repository_tag.lifetime_start_ts,
lifetime_end_ts=repository_tag.lifetime_end_ts,
lifetime_start_ms=repository_tag.lifetime_start_ts * 1000,
lifetime_end_ms=(repository_tag.lifetime_end_ts * 1000
if repository_tag.lifetime_end_ts else None),
manifest_digest=manifest_digest,
inputs=dict(legacy_image=legacy_image,
repository=RepositoryReference.for_id(repository_tag.repository_id)))
@property
@requiresinput('manifest')
def _manifest(self, manifest):
""" Returns the manifest for this tag. Will only apply to new-style OCI tags. """
return manifest
@property
@requiresinput('repository')
def repository(self, repository):
@ -144,6 +170,17 @@ class Manifest(datatype('Manifest', ['digest', 'media_type', 'manifest_bytes']))
media_type=DOCKER_SCHEMA1_SIGNED_MANIFEST_CONTENT_TYPE, # Always in legacy.
inputs=dict(legacy_image=legacy_image))
@classmethod
def for_manifest(cls, manifest, legacy_image):
if manifest is None:
return None
return Manifest(db_id=manifest.id,
digest=manifest.digest,
manifest_bytes=manifest.manifest_bytes,
media_type=manifest.media_type.name,
inputs=dict(legacy_image=legacy_image))
@property
@requiresinput('legacy_image')
def legacy_image(self, legacy_image):
@ -179,6 +216,11 @@ class LegacyImage(datatype('LegacyImage', ['docker_image_id', 'created', 'commen
aggregate_size=image.aggregate_size,
uploading=image.storage.uploading)
@property
def id(self):
""" Returns the database ID of the legacy image. """
return self._db_id
@property
@requiresinput('images_map')
@requiresinput('ancestor_id_list')

413
data/registry_model/registry_oci_model.py Normal file
View file

@ -0,0 +1,413 @@
# pylint: disable=protected-access
import logging
from contextlib import contextmanager
from data import database
from data import model
from data.model import oci, DataModelException
from data.database import db_transaction, Image
from data.registry_model.interface import RegistryDataInterface
from data.registry_model.datatypes import Tag, Manifest, LegacyImage, Label, SecurityScanStatus
from data.registry_model.shared import SharedModel
from data.registry_model.label_handlers import apply_label_to_manifest
from util.validation import is_json
logger = logging.getLogger(__name__)
class OCIModel(SharedModel, RegistryDataInterface):
"""
OCIModel implements the data model for the registry API using a database schema
after it was changed to support the OCI specification.
"""
def find_matching_tag(self, repository_ref, tag_names):
""" Finds an alive tag in the repository matching one of the given tag names and returns it
or None if none.
"""
found_tag = oci.tag.find_matching_tag(repository_ref._db_id, tag_names)
assert found_tag is None or not found_tag.hidden
return Tag.for_tag(found_tag)
def get_most_recent_tag(self, repository_ref):
""" Returns the most recently pushed alive tag in the repository, if any. If none, returns
None.
"""
found_tag = oci.tag.get_most_recent_tag(repository_ref._db_id)
assert found_tag is None or not found_tag.hidden
return Tag.for_tag(found_tag)
def get_manifest_for_tag(self, tag, backfill_if_necessary=False):
""" Returns the manifest associated with the given tag. """
legacy_image = oci.shared.get_legacy_image_for_manifest(tag._manifest)
return Manifest.for_manifest(tag._manifest, LegacyImage.for_image(legacy_image))
def lookup_manifest_by_digest(self, repository_ref, manifest_digest, allow_dead=False,
include_legacy_image=False):
""" Looks up the manifest with the given digest under the given repository and returns it
or None if none. """
manifest = oci.manifest.lookup_manifest(repository_ref._db_id, manifest_digest,
allow_dead=allow_dead)
if manifest is None:
return None
legacy_image = None
if include_legacy_image:
try:
legacy_image_id = database.ManifestLegacyImage.get(manifest=manifest).image.docker_image_id
legacy_image = self.get_legacy_image(repository_ref, legacy_image_id, include_parents=True)
except database.ManifestLegacyImage.DoesNotExist:
return None
return Manifest.for_manifest(manifest, legacy_image)
def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
""" Creates a label on the manifest with the given key and value. """
label_data = dict(key=key, value=value, source_type_name=source_type_name,
media_type_name=media_type_name)
with db_transaction():
# Create the label itself.
label = oci.label.create_manifest_label(manifest._db_id, key, value, source_type_name,
media_type_name)
if label is None:
return None
# Apply any changes to the manifest that the label prescribes.
apply_label_to_manifest(label_data, manifest, self)
return Label.for_label(label)
@contextmanager
def batch_create_manifest_labels(self, manifest):
""" Returns a context manager for batch creation of labels on a manifest.
Can raise InvalidLabelKeyException or InvalidMediaTypeException depending
on the validation errors.
"""
labels_to_add = []
def add_label(key, value, source_type_name, media_type_name=None):
labels_to_add.append(dict(key=key, value=value, source_type_name=source_type_name,
media_type_name=media_type_name))
yield add_label
# TODO: make this truly batch once we've fully transitioned to V2_2 and no longer need
# the mapping tables.
for label_data in labels_to_add:
with db_transaction():
# Create the label itself.
oci.label.create_manifest_label(manifest._db_id, **label_data)
# Apply any changes to the manifest that the label prescribes.
apply_label_to_manifest(label_data, manifest, self)
def list_manifest_labels(self, manifest, key_prefix=None):
""" Returns all labels found on the manifest. If specified, the key_prefix will filter the
labels returned to those keys that start with the given prefix.
"""
labels = oci.label.list_manifest_labels(manifest._db_id, prefix_filter=key_prefix)
return [Label.for_label(l) for l in labels]
def get_manifest_label(self, manifest, label_uuid):
""" Returns the label with the specified UUID on the manifest or None if none. """
return Label.for_label(oci.label.get_manifest_label(label_uuid, manifest._db_id))
def delete_manifest_label(self, manifest, label_uuid):
""" Delete the label with the specified UUID on the manifest. Returns the label deleted
or None if none.
"""
return Label.for_label(oci.label.delete_manifest_label(label_uuid, manifest._db_id))
def list_repository_tags(self, repository_ref, include_legacy_images=False,
start_pagination_id=None,
limit=None):
"""
Returns a list of all the active tags in the repository. Note that this can be a *heavy*
operation on repositories with a lot of tags, and should be avoided for more targetted
operations wherever possible.
"""
tags = list(oci.tag.list_alive_tags(repository_ref._db_id, start_pagination_id, limit))
legacy_images_map = {}
if include_legacy_images:
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
return [Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_images_map.get(tag.id)))
for tag in tags]
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None,
active_tags_only=False):
"""
Returns the history of all tags in the repository (unless filtered). This includes tags that
have been made in-active due to newer versions of those tags coming into service.
"""
tags, has_more = oci.tag.list_repository_tag_history(repository_ref._db_id,
page, size,
specific_tag_name,
active_tags_only)
# TODO: do we need legacy images here?
legacy_images_map = oci.tag.get_legacy_images_for_tags(tags)
return [Tag.for_tag(tag, LegacyImage.for_image(legacy_images_map.get(tag.id))) for tag in tags], has_more
def has_expired_tag(self, repository_ref, tag_name):
"""
Returns true if and only if the repository contains a tag with the given name that is expired.
"""
return bool(oci.tag.get_expired_tag(repository_ref._db_id, tag_name))
def get_repo_tag(self, repository_ref, tag_name, include_legacy_image=False):
"""
Returns the latest, *active* tag found in the repository, with the matching name
or None if none.
"""
assert isinstance(tag_name, basestring)
tag = oci.tag.get_tag(repository_ref._db_id, tag_name)
if tag is None:
return None
legacy_image = None
if include_legacy_image:
legacy_images = oci.tag.get_legacy_images_for_tags([tag])
legacy_image = legacy_images.get(tag.id)
return Tag.for_tag(tag, legacy_image=LegacyImage.for_image(legacy_image))
def create_manifest_and_retarget_tag(self, repository_ref, manifest_interface_instance, tag_name):
""" Creates a manifest in a repository, adding all of the necessary data in the model.
The `manifest_interface_instance` parameter must be an instance of the manifest
interface as returned by the image/docker package.
Note that all blobs referenced by the manifest must exist under the repository or this
method will fail and return None.
Returns a reference to the (created manifest, tag) or (None, None) on error.
"""
# Get or create the manifest itself.
manifest, newly_created = oci.manifest.get_or_create_manifest(repository_ref._db_id,
manifest_interface_instance)
if manifest is None:
return (None, None)
# Re-target the tag to it.
tag = oci.tag.retarget_tag(tag_name, manifest)
if tag is None:
return (None, None)
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest)
if legacy_image is None:
return (None, None)
# Save the labels on the manifest. Note that order is important here: This must come after the
# tag has been changed.
# TODO(jschorr): Support schema2 here when we're ready.
if newly_created:
has_labels = False
with self.batch_create_manifest_labels(Manifest.for_manifest(manifest, None)) as add_label:
for key, value in manifest_interface_instance.layers[-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
add_label(key, value, 'manifest', media_type)
has_labels = True
# Reload the tag in case any updates were applied.
if has_labels:
tag = database.Tag.get(id=tag.id)
li = LegacyImage.for_image(legacy_image)
return (Manifest.for_manifest(manifest, li), Tag.for_tag(tag, li))
def retarget_tag(self, repository_ref, tag_name, manifest_or_legacy_image,
is_reversion=False):
"""
Creates, updates or moves a tag to a new entry in history, pointing to the manifest or
legacy image specified. If is_reversion is set to True, this operation is considered a
reversion over a previous tag move operation. Returns the updated Tag or None on error.
"""
manifest_id = manifest_or_legacy_image._db_id
if isinstance(manifest_or_legacy_image, LegacyImage):
# If a legacy image was required, build a new manifest for it and move the tag to that.
try:
image_row = database.Image.get(id=manifest_or_legacy_image._db_id)
except database.Image.DoesNotExist:
return None
manifest_instance = self._build_manifest_for_legacy_image(tag_name, image_row)
if manifest_instance is None:
return None
manifest, _ = oci.manifest.get_or_create_manifest(repository_ref._db_id, manifest_instance)
if manifest is None:
return None
manifest_id = manifest.id
tag = oci.tag.retarget_tag(tag_name, manifest_id, is_reversion=is_reversion)
legacy_image = LegacyImage.for_image(oci.shared.get_legacy_image_for_manifest(manifest_id))
return Tag.for_tag(tag, legacy_image)
def delete_tag(self, repository_ref, tag_name):
"""
Deletes the latest, *active* tag with the given name in the repository.
"""
deleted_tag = oci.tag.delete_tag(repository_ref._db_id, tag_name)
if deleted_tag is None:
# TODO(jschorr): This is only needed because preoci raises an exception. Remove and fix
# expected status codes once PreOCIModel is gone.
msg = ('Invalid repository tag \'%s\' on repository' % tag_name)
raise DataModelException(msg)
return Tag.for_tag(deleted_tag)
def delete_tags_for_manifest(self, manifest):
"""
Deletes all tags pointing to the given manifest, making the manifest inaccessible for pulling.
Returns the tags deleted, if any. Returns None on error.
"""
deleted_tags = oci.tag.delete_tags_for_manifest(manifest._db_id)
return [Tag.for_tag(tag) for tag in deleted_tags]
def change_repository_tag_expiration(self, tag, expiration_date):
""" Sets the expiration date of the tag under the matching repository to that given. If the
expiration date is None, then the tag will not expire. Returns a tuple of the previous
expiration timestamp in seconds (if any), and whether the operation succeeded.
"""
return oci.tag.change_tag_expiration(tag._db_id, expiration_date)
def get_legacy_images_owned_by_tag(self, tag):
""" Returns all legacy images *solely owned and used* by the given tag. """
tag_obj = oci.tag.get_tag_by_id(tag._db_id)
if tag_obj is None:
return None
tags = oci.tag.list_alive_tags(tag_obj.repository_id)
legacy_images = oci.tag.get_legacy_images_for_tags(tags)
tag_legacy_image = legacy_images.get(tag._db_id)
if tag_legacy_image is None:
return None
assert isinstance(tag_legacy_image, Image)
# Collect the IDs of all images that the tag uses.
tag_image_ids = set()
tag_image_ids.add(tag_legacy_image.id)
tag_image_ids.update(tag_legacy_image.ancestor_id_list())
# Remove any images shared by other tags.
for current in tags:
if current == tag_obj:
continue
current_image = legacy_images.get(current.id)
if current_image is None:
continue
tag_image_ids.discard(current_image.id)
tag_image_ids = tag_image_ids.difference(current_image.ancestor_id_list())
if not tag_image_ids:
return []
if not tag_image_ids:
return []
# Load the images we need to return.
images = database.Image.select().where(database.Image.id << list(tag_image_ids))
all_image_ids = set()
for image in images:
all_image_ids.add(image.id)
all_image_ids.update(image.ancestor_id_list())
# Build a map of all the images and their parents.
images_map = {}
all_images = database.Image.select().where(database.Image.id << list(all_image_ids))
for image in all_images:
images_map[image.id] = image
return [LegacyImage.for_image(image, images_map=images_map) for image in images]
def get_security_status(self, manifest_or_legacy_image):
""" Returns the security status for the given manifest or legacy image or None if none. """
image = None
if isinstance(manifest_or_legacy_image, Manifest):
image = oci.shared.get_legacy_image_for_manifest(manifest_or_legacy_image._db_id)
if image is None:
return None
else:
try:
image = database.Image.get(id=manifest_or_legacy_image._db_id)
except database.Image.DoesNotExist:
return None
if image.security_indexed_engine is not None and image.security_indexed_engine >= 0:
return SecurityScanStatus.SCANNED if image.security_indexed else SecurityScanStatus.FAILED
return SecurityScanStatus.QUEUED
def backfill_manifest_for_tag(self, tag):
""" Backfills a manifest for the V1 tag specified.
If a manifest already exists for the tag, returns that manifest.
NOTE: This method will only be necessary until we've completed the backfill, at which point
it should be removed.
"""
# Nothing to do for OCI tags.
manifest = tag.manifest
if manifest is None:
return None
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest)
return Manifest.for_manifest(manifest, LegacyImage.for_image(legacy_image))
def list_manifest_layers(self, manifest, include_placements=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
"""
try:
manifest_obj = database.Manifest.get(id=manifest._db_id)
except database.Manifest.DoesNotExist:
logger.exception('Could not find manifest for manifest `%s`', manifest._db_id)
return None
return self._list_manifest_layers(manifest, manifest_obj.repository_id, include_placements)
def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
"""
Looks up the derived image for the given manifest, verb and optional varying metadata and
returns it or None if none.
"""
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest._db_id)
if legacy_image is None:
return None
derived = model.image.find_derived_storage_for_image(legacy_image, verb, varying_metadata)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def lookup_or_create_derived_image(self, manifest, verb, storage_location, varying_metadata=None,
include_placements=False):
"""
Looks up the derived image for the given maniest, verb and optional varying metadata
and returns it. If none exists, a new derived image is created.
"""
legacy_image = oci.shared.get_legacy_image_for_manifest(manifest._db_id)
if legacy_image is None:
return None
derived = model.image.find_or_create_derived_storage(legacy_image, verb, storage_location,
varying_metadata)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
"""
Sets the expiration on all tags that point to the given manifest to that specified.
"""
oci.tag.set_tag_expiration_sec_for_manifest(manifest._db_id, expiration_sec)
oci_model = OCIModel()

130
data/registry_model/registry_pre_oci_model.py
View file

@ -155,60 +155,23 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
manifest = Manifest.for_tag_manifest(tag_manifest)
# Save the labels on the manifest.
repo_tag = tag_manifest.tag
if newly_created:
has_labels = False
with self.batch_create_manifest_labels(manifest) as add_label:
if add_label is None:
return None, None
for key, value in manifest_interface_instance.layers[-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
add_label(key, value, 'manifest', media_type)
has_labels = True
return manifest, Tag.for_repository_tag(tag_manifest.tag)
# Reload the tag in case any updates were applied.
if has_labels:
repo_tag = database.RepositoryTag.get(id=repo_tag.id)
def get_legacy_images(self, repository_ref):
"""
Returns an iterator of all the LegacyImage's defined in the matching repository.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
all_images = model.image.get_repository_images_without_placements(repo)
all_images_map = {image.id: image for image in all_images}
all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name)
tags_by_image_id = defaultdict(list)
for tag in all_tags:
tags_by_image_id[tag.image_id].append(tag)
return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
for image in all_images]
def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False,
include_blob=False):
"""
Returns the matching LegacyImages under the matching repository, if any. If none,
returns None.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
image = model.image.get_image(repository_ref._db_id, docker_image_id)
if image is None:
return None
parent_images_map = None
if include_parents:
parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image)
parent_images_map = {image.id: image for image in parent_images}
blob = None
if include_blob:
placements = list(model.storage.get_storage_locations(image.storage.uuid))
blob = Blob.for_image_storage(image.storage,
storage_path=model.storage.get_layer_path(image.storage),
placements=placements)
return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
return manifest, Tag.for_repository_tag(repo_tag)
def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
""" Creates a label on the manifest with the given key and value. """
@ -471,10 +434,6 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
NOTE: This method will only be necessary until we've completed the backfill, at which point
it should be removed.
"""
import features
from app import app, docker_v2_signing_key
# Ensure that there isn't already a manifest for the tag.
tag_manifest = model.tag.get_tag_manifest(tag._db_id)
if tag_manifest is not None:
@ -492,29 +451,8 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
namespace_name = repo.namespace_user.username
repo_name = repo.name
# Find the v1 metadata for this image and its parents.
repo_image = tag_obj.image
parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
# If the manifest is being generated under the library namespace, then we make its namespace
# empty.
manifest_namespace = namespace_name
if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
manifest_namespace = ''
# Create and populate the manifest builder
builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag.name)
# Add the leaf layer
builder.add_layer(repo_image.storage.content_checksum, repo_image.v1_json_metadata)
for parent_image in parents:
builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
# Sign the manifest with our signing key.
manifest = builder.build(docker_v2_signing_key)
# Write the manifest to the DB.
manifest = self._build_manifest_for_legacy_image(tag_obj.name, tag_obj.image)
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo,
manifest.checksums)
@ -533,42 +471,13 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
"""
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
return None
try:
tag_manifest = database.TagManifest.get(id=manifest._db_id)
except database.TagManifest.DoesNotExist:
logger.exception('Could not find tag manifest for manifest `%s`', manifest._db_id)
return None
repo = tag_manifest.tag.repository
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo, parsed.checksums)
storage_map = {blob.content_checksum: blob for blob in blob_query}
manifest_layers = []
for layer in parsed.layers:
digest_str = str(layer.digest)
if digest_str not in storage_map:
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
return None
image_storage = storage_map[digest_str]
assert image_storage.cas_path is not None
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(image_storage.uuid))
blob = Blob.for_image_storage(image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
manifest_layers.append(ManifestLayer(layer, blob))
return manifest_layers
return self._list_manifest_layers(manifest, tag_manifest.tag.repository_id, include_placements)
def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
"""
@ -602,21 +511,6 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
varying_metadata)
return self._build_derived(derived, verb, varying_metadata, include_placements)
def _build_derived(self, derived, verb, varying_metadata, include_placements):
if derived is None:
return None
derived_storage = derived.derivative
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(derived_storage.uuid))
blob = Blob.for_image_storage(derived_storage,
storage_path=model.storage.get_layer_path(derived_storage),
placements=placements)
return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
"""
Sets the expiration on all tags that point to the given manifest to that specified.

133
data/registry_model/shared.py
View file

@ -1,11 +1,15 @@
# pylint: disable=protected-access
import logging
from collections import defaultdict
from data import database
from data import model
from data.cache import cache_key
from data.registry_model.datatype import FromDictionaryException
from data.registry_model.datatypes import RepositoryReference, Blob, TorrentInfo, BlobUpload
from data.registry_model.datatypes import (RepositoryReference, Blob, TorrentInfo, BlobUpload,
LegacyImage, ManifestLayer, DerivedImage)
from image.docker.schema1 import ManifestException, DockerSchema1ManifestBuilder
logger = logging.getLogger(__name__)
@ -254,3 +258,130 @@ class SharedModel:
storage = model.blob.temp_link_blob(namespace_name, repo_name, blob.digest,
expiration_sec)
return bool(storage)
def get_legacy_images(self, repository_ref):
"""
Returns an iterator of all the LegacyImage's defined in the matching repository.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
all_images = model.image.get_repository_images_without_placements(repo)
all_images_map = {image.id: image for image in all_images}
all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name)
tags_by_image_id = defaultdict(list)
for tag in all_tags:
tags_by_image_id[tag.image_id].append(tag)
return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
for image in all_images]
def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False,
include_blob=False):
"""
Returns the matching LegacyImages under the matching repository, if any. If none,
returns None.
"""
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
image = model.image.get_image(repository_ref._db_id, docker_image_id)
if image is None:
return None
parent_images_map = None
if include_parents:
parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image)
parent_images_map = {image.id: image for image in parent_images}
blob = None
if include_blob:
placements = list(model.storage.get_storage_locations(image.storage.uuid))
blob = Blob.for_image_storage(image.storage,
storage_path=model.storage.get_layer_path(image.storage),
placements=placements)
return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
def _list_manifest_layers(self, manifest, repo_id, include_placements=False):
""" Returns an *ordered list* of the layers found in the manifest, starting at the base and
working towards the leaf, including the associated Blob and its placements (if specified).
Returns None if the manifest could not be parsed and validated.
"""
try:
parsed = manifest.get_parsed_manifest()
except ManifestException:
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
return None
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo_id, parsed.checksums)
storage_map = {blob.content_checksum: blob for blob in blob_query}
manifest_layers = []
for layer in parsed.layers:
digest_str = str(layer.digest)
if digest_str not in storage_map:
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
return None
image_storage = storage_map[digest_str]
assert image_storage.cas_path is not None
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(image_storage.uuid))
blob = Blob.for_image_storage(image_storage,
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
manifest_layers.append(ManifestLayer(layer, blob))
return manifest_layers
def _build_derived(self, derived, verb, varying_metadata, include_placements):
if derived is None:
return None
derived_storage = derived.derivative
placements = None
if include_placements:
placements = list(model.storage.get_storage_locations(derived_storage.uuid))
blob = Blob.for_image_storage(derived_storage,
storage_path=model.storage.get_layer_path(derived_storage),
placements=placements)
return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
def _build_manifest_for_legacy_image(self, tag_name, legacy_image_row):
import features
from app import app, docker_v2_signing_key
repo = legacy_image_row.repository
namespace_name = repo.namespace_user.username
repo_name = repo.name
# Find the v1 metadata for this image and its parents.
parents = model.image.get_parent_images(namespace_name, repo_name, legacy_image_row)
# If the manifest is being generated under the library namespace, then we make its namespace
# empty.
manifest_namespace = namespace_name
if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
manifest_namespace = ''
# Create and populate the manifest builder
builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag_name)
# Add the leaf layer
builder.add_layer(legacy_image_row.storage.content_checksum, legacy_image_row.v1_json_metadata)
for parent_image in parents:
builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
# Sign the manifest with our signing key.
return builder.build(docker_v2_signing_key)

134
data/registry_model/test/test_interface.py
View file

@ -1,4 +1,5 @@
import hashlib
import json
import uuid
from datetime import datetime, timedelta
@ -16,16 +17,21 @@ from data.database import (TagManifestLabelMap, TagManifestToManifest, Manifest,
TorrentInfo, Tag, TagToRepositoryTag, close_db_filter)
from data.cache.impl import InMemoryDataModelCache
from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.registry_oci_model import OCIModel
from data.registry_model.datatypes import RepositoryReference
from image.docker.schema1 import DockerSchema1ManifestBuilder
from test.fixtures import *
@pytest.fixture(params=[PreOCIModel])
@pytest.fixture(params=[PreOCIModel, OCIModel])
def registry_model(request, initialized_db):
return request.param()
@pytest.fixture()
def pre_oci_model(initialized_db):
return PreOCIModel()
@pytest.mark.parametrize('names, expected', [
(['unknown'], None),
@ -83,10 +89,11 @@ def test_lookup_manifests(repo_namespace, repo_name, registry_model):
found_tag = registry_model.find_matching_tag(repository_ref, ['latest'])
found_manifest = registry_model.get_manifest_for_tag(found_tag)
found = registry_model.lookup_manifest_by_digest(repository_ref, found_manifest.digest,
include_legacy_image=True)
include_legacy_image=True)
assert found._db_id == found_manifest._db_id
assert found.digest == found_manifest.digest
assert found.legacy_image
assert found.legacy_image.parents
def test_lookup_unknown_manifest(registry_model):
@ -110,11 +117,11 @@ def test_legacy_images(repo_namespace, repo_name, registry_model):
found_tags = set()
for image in legacy_images:
found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True)
include_parents=True)
with assert_query_count(5 if found_image.parents else 4):
found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=True, include_blob=True)
include_parents=True, include_blob=True)
assert found_image.docker_image_id == image.docker_image_id
assert found_image.parents == image.parents
assert found_image.blob
@ -132,7 +139,7 @@ def test_legacy_images(repo_namespace, repo_name, registry_model):
# Try without parents and ensure it raises an exception.
found_image = registry_model.get_legacy_image(repository_ref, image.docker_image_id,
include_parents=False)
include_parents=False)
with pytest.raises(Exception):
assert not found_image.parents
@ -211,23 +218,19 @@ def test_batch_labels(registry_model):
])
def test_repository_tags(repo_namespace, repo_name, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
with assert_query_count(1):
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags)
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert len(tags)
for tag in tags:
with assert_query_count(2):
found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag == tag
found_tag = registry_model.get_repo_tag(repository_ref, tag.name, include_legacy_image=True)
assert found_tag == tag
if found_tag.legacy_image is None:
continue
with assert_query_count(2):
found_image = registry_model.get_legacy_image(repository_ref,
found_tag.legacy_image.docker_image_id)
assert found_image == found_tag.legacy_image
found_image = registry_model.get_legacy_image(repository_ref,
found_tag.legacy_image.docker_image_id)
assert found_image == found_tag.legacy_image
def test_repository_tag_history(registry_model):
@ -295,15 +298,15 @@ def test_retarget_tag_history(use_manifest, registry_model):
if use_manifest:
manifest_or_legacy_image = registry_model.lookup_manifest_by_digest(repository_ref,
history[1].manifest_digest,
allow_dead=True)
history[0].manifest_digest,
allow_dead=True)
else:
manifest_or_legacy_image = history[1].legacy_image
manifest_or_legacy_image = history[0].legacy_image
# Retarget the tag.
assert manifest_or_legacy_image
updated_tag = registry_model.retarget_tag(repository_ref, 'latest', manifest_or_legacy_image,
is_reversion=True)
is_reversion=True)
# Ensure the tag has changed targets.
if use_manifest:
@ -316,23 +319,6 @@ def test_retarget_tag_history(use_manifest, registry_model):
assert len(new_history) == len(history) + 1
def test_retarget_tag(registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'complex')
history, _ = registry_model.list_repository_tag_history(repository_ref)
prod_tag = registry_model.get_repo_tag(repository_ref, 'prod', include_legacy_image=True)
# Retarget the tag.
updated_tag = registry_model.retarget_tag(repository_ref, 'latest', prod_tag.legacy_image)
# Ensure the tag has changed targets.
assert updated_tag.legacy_image == prod_tag.legacy_image
# Ensure history has been updated.
new_history, _ = registry_model.list_repository_tag_history(repository_ref)
assert len(new_history) == len(history) + 1
def test_change_repository_tag_expiration(registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'simple')
tag = registry_model.get_repo_tag(repository_ref, 'latest')
@ -399,24 +385,24 @@ def clear_rows(initialized_db):
('devtable', 'history'),
('buynlarge', 'orgrepo'),
])
def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = registry_model.list_repository_tags(repository_ref)
def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref)
assert tags
for tag in tags:
assert not tag.manifest_digest
assert registry_model.backfill_manifest_for_tag(tag)
assert pre_oci_model.backfill_manifest_for_tag(tag)
tags = registry_model.list_repository_tags(repository_ref, include_legacy_images=True)
tags = pre_oci_model.list_repository_tags(repository_ref, include_legacy_images=True)
assert tags
for tag in tags:
assert tag.manifest_digest
manifest = registry_model.get_manifest_for_tag(tag)
manifest = pre_oci_model.get_manifest_for_tag(tag)
assert manifest
legacy_image = registry_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id,
legacy_image = pre_oci_model.get_legacy_image(repository_ref, tag.legacy_image.docker_image_id,
include_parents=True)
parsed_manifest = manifest.get_parsed_manifest()
@ -430,19 +416,19 @@ def test_backfill_manifest_for_tag(repo_namespace, repo_name, clear_rows, regist
('devtable', 'history'),
('buynlarge', 'orgrepo'),
])
def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, registry_model):
repository_ref = registry_model.lookup_repository(repo_namespace, repo_name)
tags = registry_model.list_repository_tags(repository_ref)
def test_backfill_manifest_on_lookup(repo_namespace, repo_name, clear_rows, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository(repo_namespace, repo_name)
tags = pre_oci_model.list_repository_tags(repository_ref)
assert tags
for tag in tags:
assert not tag.manifest_digest
assert not registry_model.get_manifest_for_tag(tag)
assert not pre_oci_model.get_manifest_for_tag(tag)
manifest = registry_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
manifest = pre_oci_model.get_manifest_for_tag(tag, backfill_if_necessary=True)
assert manifest
updated_tag = registry_model.get_repo_tag(repository_ref, tag.name)
updated_tag = pre_oci_model.get_repo_tag(repository_ref, tag.name)
assert updated_tag.manifest_digest == manifest.digest
@ -471,9 +457,8 @@ def test_list_manifest_layers(repo_namespace, repo_name, registry_model):
manifest = registry_model.get_manifest_for_tag(tag)
assert manifest
with assert_query_count(4):
layers = registry_model.list_manifest_layers(manifest)
assert layers
layers = registry_model.list_manifest_layers(manifest)
assert layers
layers = registry_model.list_manifest_layers(manifest, include_placements=True)
assert layers
@ -522,7 +507,7 @@ def test_derived_image(registry_model):
assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) is None
squashed_foo = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us',
{'foo': 'bar'})
{'foo': 'bar'})
assert squashed_foo != squashed
assert registry_model.lookup_derived_image(manifest, 'squash', {'foo': 'bar'}) == squashed_foo
@ -530,7 +515,7 @@ def test_derived_image(registry_model):
# Lookup with placements.
squashed = registry_model.lookup_or_create_derived_image(manifest, 'squash', 'local_us', {},
include_placements=True)
include_placements=True)
assert squashed.blob.placements
# Delete the derived image.
@ -712,8 +697,8 @@ def test_create_manifest_and_retarget_tag(registry_model):
assert sample_manifest is not None
another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
sample_manifest,
'anothertag')
assert another_manifest is not None
assert tag is not None
@ -722,3 +707,38 @@ def test_create_manifest_and_retarget_tag(registry_model):
layers = registry_model.list_manifest_layers(another_manifest)
assert len(layers) == 1
def test_create_manifest_and_retarget_tag_with_labels(registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'simple')
latest_tag = registry_model.get_repo_tag(repository_ref, 'latest', include_legacy_image=True)
manifest = registry_model.get_manifest_for_tag(latest_tag).get_parsed_manifest()
json_metadata = {
'id': latest_tag.legacy_image.docker_image_id,
'config': {
'Labels': {
'quay.expires-after': '2w',
},
},
}
builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
builder.add_layer(manifest.blob_digests[0], json.dumps(json_metadata))
sample_manifest = builder.build(docker_v2_signing_key)
assert sample_manifest is not None
another_manifest, tag = registry_model.create_manifest_and_retarget_tag(repository_ref,
sample_manifest,
'anothertag')
assert another_manifest is not None
assert tag is not None
assert tag.name == 'anothertag'
assert another_manifest.get_parsed_manifest().manifest_dict == sample_manifest.manifest_dict
layers = registry_model.list_manifest_layers(another_manifest)
assert len(layers) == 1
# Ensure the labels were applied.
assert tag.lifetime_end_ms is not None

19
data/registry_model/test/test_manifestbuilder.py
View file

@ -10,15 +10,16 @@ from mock import patch
from data.registry_model.blobuploader import BlobUploadSettings, upload_blob
from data.registry_model.manifestbuilder import create_manifest_builder, lookup_manifest_builder
from data.registry_model.registry_pre_oci_model import PreOCIModel
from data.registry_model.registry_oci_model import OCIModel
from storage.distributedstorage import DistributedStorage
from storage.fakestorage import FakeStorage
from test.fixtures import *
@pytest.fixture()
def pre_oci_model(initialized_db):
return PreOCIModel()
@pytest.fixture(params=[PreOCIModel, OCIModel])
def registry_model(request, initialized_db):
return request.param()
@pytest.fixture()
@ -33,8 +34,8 @@ def fake_session():
('someid', 'parentid', 'some data')],
id='Multi layer'),
])
def test_build_manifest(layers, fake_session, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
def test_build_manifest(layers, fake_session, registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'complex')
storage = DistributedStorage({'local_us': FakeStorage(None)}, ['local_us'])
settings = BlobUploadSettings('2M', 512 * 1024, 3600)
app_config = {'TESTING': True}
@ -67,13 +68,13 @@ def test_build_manifest(layers, fake_session, pre_oci_model):
assert tag in builder.committed_tags
# Verify the legacy image for the tag.
found = pre_oci_model.get_repo_tag(repository_ref, 'somenewtag', include_legacy_image=True)
found = registry_model.get_repo_tag(repository_ref, 'somenewtag', include_legacy_image=True)
assert found
assert found.name == 'somenewtag'
assert found.legacy_image.docker_image_id == layers[-1][0]
# Verify the blob and manifest.
manifest = pre_oci_model.get_manifest_for_tag(found)
manifest = registry_model.get_manifest_for_tag(found)
assert manifest
parsed = manifest.get_parsed_manifest()
@ -87,8 +88,8 @@ def test_build_manifest(layers, fake_session, pre_oci_model):
assert parsed.leaf_layer_v1_image_id == layers[-1][0]
def test_build_manifest_missing_parent(fake_session, pre_oci_model):
repository_ref = pre_oci_model.lookup_repository('devtable', 'complex')
def test_build_manifest_missing_parent(fake_session, registry_model):
repository_ref = registry_model.lookup_repository('devtable', 'complex')
builder = create_manifest_builder(repository_ref)
assert builder.start_layer('somelayer', json.dumps({'id': 'somelayer', 'parent': 'someparent'}),