Implement the new OCI-based registry data model
Note that this change does *not* enable the new data model by default, but does allow it to be used when a special environment variable is specified.
This commit is contained in:
Joseph Schorr
parent
924b386437
commit
fdcb8bad23
23 changed files with 1847 additions and 209 deletions
|
|
@ -155,60 +155,23 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
|
|||
manifest = Manifest.for_tag_manifest(tag_manifest)
|
||||
|
||||
# Save the labels on the manifest.
|
||||
repo_tag = tag_manifest.tag
|
||||
if newly_created:
|
||||
has_labels = False
|
||||
with self.batch_create_manifest_labels(manifest) as add_label:
|
||||
if add_label is None:
|
||||
return None, None
|
||||
|
||||
for key, value in manifest_interface_instance.layers[-1].v1_metadata.labels.iteritems():
|
||||
media_type = 'application/json' if is_json(value) else 'text/plain'
|
||||
add_label(key, value, 'manifest', media_type)
|
||||
has_labels = True
|
||||
|
||||
return manifest, Tag.for_repository_tag(tag_manifest.tag)
|
||||
# Reload the tag in case any updates were applied.
|
||||
if has_labels:
|
||||
repo_tag = database.RepositoryTag.get(id=repo_tag.id)
|
||||
|
||||
def get_legacy_images(self, repository_ref):
|
||||
"""
|
||||
Returns an iterator of all the LegacyImage's defined in the matching repository.
|
||||
"""
|
||||
repo = model.repository.lookup_repository(repository_ref._db_id)
|
||||
if repo is None:
|
||||
return None
|
||||
|
||||
all_images = model.image.get_repository_images_without_placements(repo)
|
||||
all_images_map = {image.id: image for image in all_images}
|
||||
|
||||
all_tags = model.tag.list_repository_tags(repo.namespace_user.username, repo.name)
|
||||
tags_by_image_id = defaultdict(list)
|
||||
for tag in all_tags:
|
||||
tags_by_image_id[tag.image_id].append(tag)
|
||||
|
||||
return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
|
||||
for image in all_images]
|
||||
|
||||
def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False,
|
||||
include_blob=False):
|
||||
"""
|
||||
Returns the matching LegacyImages under the matching repository, if any. If none,
|
||||
returns None.
|
||||
"""
|
||||
repo = model.repository.lookup_repository(repository_ref._db_id)
|
||||
if repo is None:
|
||||
return None
|
||||
|
||||
image = model.image.get_image(repository_ref._db_id, docker_image_id)
|
||||
if image is None:
|
||||
return None
|
||||
|
||||
parent_images_map = None
|
||||
if include_parents:
|
||||
parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image)
|
||||
parent_images_map = {image.id: image for image in parent_images}
|
||||
|
||||
blob = None
|
||||
if include_blob:
|
||||
placements = list(model.storage.get_storage_locations(image.storage.uuid))
|
||||
blob = Blob.for_image_storage(image.storage,
|
||||
storage_path=model.storage.get_layer_path(image.storage),
|
||||
placements=placements)
|
||||
|
||||
return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
|
||||
return manifest, Tag.for_repository_tag(repo_tag)
|
||||
|
||||
def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
|
||||
""" Creates a label on the manifest with the given key and value. """
|
||||
|
|
@ -471,10 +434,6 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
|
|||
NOTE: This method will only be necessary until we've completed the backfill, at which point
|
||||
it should be removed.
|
||||
"""
|
||||
import features
|
||||
|
||||
from app import app, docker_v2_signing_key
|
||||
|
||||
# Ensure that there isn't already a manifest for the tag.
|
||||
tag_manifest = model.tag.get_tag_manifest(tag._db_id)
|
||||
if tag_manifest is not None:
|
||||
|
|
@ -492,29 +451,8 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
|
|||
namespace_name = repo.namespace_user.username
|
||||
repo_name = repo.name
|
||||
|
||||
# Find the v1 metadata for this image and its parents.
|
||||
repo_image = tag_obj.image
|
||||
parents = model.image.get_parent_images(namespace_name, repo_name, repo_image)
|
||||
|
||||
# If the manifest is being generated under the library namespace, then we make its namespace
|
||||
# empty.
|
||||
manifest_namespace = namespace_name
|
||||
if features.LIBRARY_SUPPORT and namespace_name == app.config['LIBRARY_NAMESPACE']:
|
||||
manifest_namespace = ''
|
||||
|
||||
# Create and populate the manifest builder
|
||||
builder = DockerSchema1ManifestBuilder(manifest_namespace, repo_name, tag.name)
|
||||
|
||||
# Add the leaf layer
|
||||
builder.add_layer(repo_image.storage.content_checksum, repo_image.v1_json_metadata)
|
||||
|
||||
for parent_image in parents:
|
||||
builder.add_layer(parent_image.storage.content_checksum, parent_image.v1_json_metadata)
|
||||
|
||||
# Sign the manifest with our signing key.
|
||||
manifest = builder.build(docker_v2_signing_key)
|
||||
|
||||
# Write the manifest to the DB.
|
||||
manifest = self._build_manifest_for_legacy_image(tag_obj.name, tag_obj.image)
|
||||
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo,
|
||||
manifest.checksums)
|
||||
|
||||
|
|
@ -533,42 +471,13 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
|
|||
working towards the leaf, including the associated Blob and its placements (if specified).
|
||||
Returns None if the manifest could not be parsed and validated.
|
||||
"""
|
||||
try:
|
||||
parsed = manifest.get_parsed_manifest()
|
||||
except ManifestException:
|
||||
logger.exception('Could not parse and validate manifest `%s`', manifest._db_id)
|
||||
return None
|
||||
|
||||
try:
|
||||
tag_manifest = database.TagManifest.get(id=manifest._db_id)
|
||||
except database.TagManifest.DoesNotExist:
|
||||
logger.exception('Could not find tag manifest for manifest `%s`', manifest._db_id)
|
||||
return None
|
||||
|
||||
repo = tag_manifest.tag.repository
|
||||
blob_query = model.storage.lookup_repo_storages_by_content_checksum(repo, parsed.checksums)
|
||||
storage_map = {blob.content_checksum: blob for blob in blob_query}
|
||||
|
||||
manifest_layers = []
|
||||
for layer in parsed.layers:
|
||||
digest_str = str(layer.digest)
|
||||
if digest_str not in storage_map:
|
||||
logger.error('Missing digest `%s` for manifest `%s`', layer.digest, manifest._db_id)
|
||||
return None
|
||||
|
||||
image_storage = storage_map[digest_str]
|
||||
assert image_storage.cas_path is not None
|
||||
|
||||
placements = None
|
||||
if include_placements:
|
||||
placements = list(model.storage.get_storage_locations(image_storage.uuid))
|
||||
|
||||
blob = Blob.for_image_storage(image_storage,
|
||||
storage_path=model.storage.get_layer_path(image_storage),
|
||||
placements=placements)
|
||||
manifest_layers.append(ManifestLayer(layer, blob))
|
||||
|
||||
return manifest_layers
|
||||
return self._list_manifest_layers(manifest, tag_manifest.tag.repository_id, include_placements)
|
||||
|
||||
def lookup_derived_image(self, manifest, verb, varying_metadata=None, include_placements=False):
|
||||
"""
|
||||
|
|
@ -602,21 +511,6 @@ class PreOCIModel(SharedModel, RegistryDataInterface):
|
|||
varying_metadata)
|
||||
return self._build_derived(derived, verb, varying_metadata, include_placements)
|
||||
|
||||
def _build_derived(self, derived, verb, varying_metadata, include_placements):
|
||||
if derived is None:
|
||||
return None
|
||||
|
||||
derived_storage = derived.derivative
|
||||
placements = None
|
||||
if include_placements:
|
||||
placements = list(model.storage.get_storage_locations(derived_storage.uuid))
|
||||
|
||||
blob = Blob.for_image_storage(derived_storage,
|
||||
storage_path=model.storage.get_layer_path(derived_storage),
|
||||
placements=placements)
|
||||
|
||||
return DerivedImage.for_derived_storage(derived, verb, varying_metadata, blob)
|
||||
|
||||
def set_tags_expiration_for_manifest(self, manifest, expiration_sec):
|
||||
"""
|
||||
Sets the expiration on all tags that point to the given manifest to that specified.
|
||||
|
|
|
|||
Reference in a new issue