vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Change manifest creation to take in the map of blobs that form the manifest

Browse source 
We need to lookup the blobs *specific to the images in that manifest*, so we now pass them in from the locations in which we know that information
This commit is contained in:
Joseph Schorr 2018-08-07 16:28:50 -04:00
parent e3da522d26
commit 56222f95dc
8 changed files with 89 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

43
data/model/tag.py
View file

@ -4,7 +4,7 @@ from calendar import timegm
from uuid import uuid4
from peewee import IntegrityError, JOIN, fn
from data.model import (image, db_transaction, DataModelException, _basequery,
from data.model import (image, storage, db_transaction, DataModelException, _basequery,
InvalidManifestException, TagAlreadyCreatedException, StaleTagException,
config)
from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
@ -550,7 +550,7 @@ def restore_tag_to_image(repo_obj, tag_name, docker_image_id):
def store_tag_manifest_for_testing(namespace_name, repository_name, tag_name, manifest,
leaf_layer_id=None):
leaf_layer_id, storage_id_map):
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
object, as well as a boolean indicating whether the TagManifest was created.
"""
@ -559,21 +559,16 @@ def store_tag_manifest_for_testing(namespace_name, repository_name, tag_name, ma
except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
return store_tag_manifest_for_repo(repo.id, tag_name, manifest, leaf_layer_id=leaf_layer_id)
return store_tag_manifest_for_repo(repo.id, tag_name, manifest, leaf_layer_id, storage_id_map)
def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id=None,
def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id, storage_id_map,
reversion=False):
""" Stores a tag manifest for a specific tag name in the database. Returns the TagManifest
object, as well as a boolean indicating whether the TagManifest was created.
"""
# Lookup all blobs in the manifest.
blobs = ImageStorage.select().where(ImageStorage.content_checksum << list(manifest.blob_digests))
blob_map = {blob.content_checksum: blob for blob in blobs}
docker_image_id = leaf_layer_id or manifest.leaf_layer_v1_image_id
with db_transaction():
tag = create_or_update_tag_for_repo(repository_id, tag_name, docker_image_id,
tag = create_or_update_tag_for_repo(repository_id, tag_name, leaf_layer_id,
reversion=reversion)
try:
@ -582,7 +577,7 @@ def store_tag_manifest_for_repo(repository_id, tag_name, manifest, leaf_layer_id
manifest.save()
return manifest, False
except TagManifest.DoesNotExist:
return _create_manifest(tag, manifest, blob_map), True
return _create_manifest(tag, manifest, storage_id_map), True
def get_active_tag(namespace, repo_name, tag_name):
@ -603,16 +598,12 @@ def get_possibly_expired_tag(namespace, repo_name, tag_name):
Namespace.username == namespace)).get()
def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest):
def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest, storage_id_map):
tag = get_active_tag(namespace, repo_name, tag_name)
# Lookup all blobs in the manifest.
blobs = ImageStorage.select().where(ImageStorage.content_checksum << list(manifest.blob_digests))
blob_map = {blob.content_checksum: blob for blob in blobs}
return _create_manifest(tag, manifest, blob_map)
return _create_manifest(tag, manifest, storage_id_map)
def _create_manifest(tag, manifest, blob_map):
def _create_manifest(tag, manifest, storage_id_map):
media_type = Manifest.media_type.get_id(manifest.media_type)
with db_transaction():
@ -621,17 +612,17 @@ def _create_manifest(tag, manifest, blob_map):
ManifestLegacyImage.create(manifest=manifest_row, repository=tag.repository, image=tag.image)
blobs_created = set()
for index, blob_digest in enumerate(reversed(manifest.blob_digests)):
image_storage = blob_map.get(blob_digest)
if image_storage is None:
raise DataModelException('Missing blob for manifest')
for blob_digest in reversed(manifest.blob_digests):
image_storage_id = storage_id_map.get(blob_digest)
if image_storage_id is None:
logger.error('Missing blob for manifest `%s` in: %s', blob_digest, storage_id_map)
raise DataModelException('Missing blob for manifest `%s`' % blob_digest)
if image_storage.id in blobs_created:
if image_storage_id in blobs_created:
continue
blobs_created.add(image_storage.id)
ManifestBlob.create(manifest=manifest_row, repository=tag.repository, blob=image_storage,
blob_index=index)
ManifestBlob.create(manifest=manifest_row, repository=tag.repository, blob=image_storage_id)
blobs_created.add(image_storage_id)
tag_manifest = TagManifest.create(tag=tag, digest=manifest.digest, json_data=manifest.bytes)
TagManifestToManifest.create(tag_manifest=tag_manifest, manifest=manifest_row)