vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Change OCI data model to use a content retriever for the various manifest operations

Browse source
This commit is contained in:
Joseph Schorr 2018-11-19 12:24:04 +02:00
parent cbfb6054e5
commit 82ee21bfbd
5 changed files with 61 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

50
data/model/oci/manifest.py
View file

@ -9,9 +9,8 @@ from data.database import (Tag, Manifest, ManifestBlob, ManifestLegacyImage, Man
from data.model import BlobDoesNotExist
from data.model.oci.tag import filter_to_alive_tags
from data.model.oci.label import create_manifest_label
from data.model.storage import (lookup_repo_storages_by_content_checksum, get_storage_locations,
get_layer_path)
from data.model.blob import get_repository_blob_by_digest
from data.model.oci.retriever import RepositoryContentRetriever
from data.model.storage import lookup_repo_storages_by_content_checksum
from data.model.image import lookup_repository_images, get_image, synthesize_v1_image
from image.docker.schema1 import ManifestException
from image.docker.schema2.list import MalformedSchema2ManifestList
@ -57,13 +56,9 @@ def get_or_create_manifest(repository_id, manifest_interface_instance, storage):
def _create_manifest(repository_id, manifest_interface_instance, storage):
digests = set(manifest_interface_instance.local_blob_digests)
def _lookup_digest(digest):
return _retrieve_bytes_in_storage(repository_id, digest, storage)
# Load, parse and get/create the child manifests, if any.
child_manifest_refs = manifest_interface_instance.child_manifests(_lookup_digest)
retriever = RepositoryContentRetriever.for_repository(repository_id, storage)
child_manifest_refs = manifest_interface_instance.child_manifests(retriever)
child_manifest_rows = []
child_manifest_label_dicts = []
@ -72,25 +67,13 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
# Load and parse the child manifest.
try:
child_manifest = child_manifest_ref.manifest_obj
except ManifestException:
logger.exception('Could not load manifest list for manifest `%s`',
manifest_interface_instance.digest)
return None
except MalformedSchema2ManifestList:
logger.exception('Could not load manifest list for manifest `%s`',
manifest_interface_instance.digest)
return None
except BlobDoesNotExist:
logger.exception('Could not load manifest list for manifest `%s`',
manifest_interface_instance.digest)
return None
except IOError:
except (ManifestException, MalformedSchema2ManifestList, BlobDoesNotExist, IOError):
logger.exception('Could not load manifest list for manifest `%s`',
manifest_interface_instance.digest)
return None
# Retrieve its labels.
labels = child_manifest.get_manifest_labels(_lookup_digest)
labels = child_manifest.get_manifest_labels(retriever)
if labels is None:
logger.exception('Could not load manifest labels for child manifest')
return None
@ -111,6 +94,7 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
child_manifest_label_dicts.append(labels)
# Ensure all the blobs in the manifest exist.
digests = set(manifest_interface_instance.local_blob_digests)
blob_map = {}
if digests:
query = lookup_repo_storages_by_content_checksum(repository_id, digests)
@ -126,7 +110,7 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
legacy_image = None
if manifest_interface_instance.leaf_layer_v1_image_id is not None:
legacy_image_id = _populate_legacy_image(repository_id, manifest_interface_instance, blob_map,
storage)
retriever)
if legacy_image_id is None:
return None
@ -167,7 +151,7 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
ManifestChild.insert_many(children_to_insert).execute()
# Define the labels for the manifest (if any).
labels = manifest_interface_instance.get_manifest_labels(_lookup_digest)
labels = manifest_interface_instance.get_manifest_labels(retriever)
if labels:
for key, value in labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
@ -190,10 +174,7 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
return CreatedManifest(manifest=manifest, newly_created=True, labels_to_apply=labels_to_apply)
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map, storage):
def _lookup_digest(digest):
return _retrieve_bytes_in_storage(repository_id, digest, storage)
def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map, retriever):
# Lookup all the images and their parent images (if any) inside the manifest.
# This will let us know which v1 images we need to synthesize and which ones are invalid.
docker_image_ids = list(manifest_interface_instance.legacy_image_ids)
@ -203,7 +184,7 @@ def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map,
# Rewrite any v1 image IDs that do not match the checksum in the database.
try:
rewritten_images = manifest_interface_instance.generate_legacy_layers(image_storage_map,
_lookup_digest)
retriever)
rewritten_images = list(rewritten_images)
parent_image_map = {}
@ -235,12 +216,3 @@ def _populate_legacy_image(repository_id, manifest_interface_instance, blob_map,
return None
return rewritten_images[-1].image_id
def _retrieve_bytes_in_storage(repository_id, digest, storage):
blob = get_repository_blob_by_digest(repository_id, digest)
if blob is None:
return None
placements = list(get_storage_locations(blob.uuid))
return storage.get_content(placements, get_layer_path(blob))