vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Audit the number of SQL queries we make in writing manifests, and significantly reduce in the common case

Browse source 
Instead of 41 queries now for the simple manifest, we are down to 14.

The biggest changes:
  - Only synthesize the V1 image rows if we haven't already found them in the database
  - Thread the repository object through to the other model method calls, and use it instead of loading again and again
This commit is contained in:
Joseph Schorr 2018-01-11 16:25:38 -05:00
parent 4f47808e99
commit 9e16a989f5
8 changed files with 145 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

38
endpoints/v2/manifest.py
View file

@ -157,7 +157,7 @@ def _write_manifest(namespace_name, repo_name, manifest):
raise ManifestInvalid(detail={'message': 'manifest does not reference any layers'})
# Ensure all the blobs in the manifest exist.
storage_map = model.lookup_blobs_by_digest(namespace_name, repo_name, manifest.checksums)
storage_map = model.lookup_blobs_by_digest(repo, manifest.checksums)
for layer in manifest.layers:
digest_str = str(layer.digest)
if digest_str not in storage_map:
@ -166,30 +166,35 @@ def _write_manifest(namespace_name, repo_name, manifest):
# Lookup all the images and their parent images (if any) inside the manifest.
# This will let us know which v1 images we need to synthesize and which ones are invalid.
all_image_ids = list(manifest.parent_image_ids | manifest.image_ids)
images_map = model.get_docker_v1_metadata_by_image_id(namespace_name, repo_name, all_image_ids)
images_map = model.get_docker_v1_metadata_by_image_id(repo, all_image_ids)
# Rewrite any v1 image IDs that do not match the checksum in the database.
try:
# TODO: make this batch and read the parent image from the previous iteration, rather than
# reloading it.
rewritten_images = list(manifest.rewrite_invalid_image_ids(images_map))
for rewritten_image in rewritten_images:
model.synthesize_v1_image(
repo,
storage_map[rewritten_image.content_checksum],
rewritten_image.image_id,
rewritten_image.created,
rewritten_image.comment,
rewritten_image.command,
rewritten_image.compat_json,
rewritten_image.parent_image_id,)
if not rewritten_image.image_id in images_map:
model.synthesize_v1_image(
repo,
storage_map[rewritten_image.content_checksum],
rewritten_image.image_id,
rewritten_image.created,
rewritten_image.comment,
rewritten_image.command,
rewritten_image.compat_json,
rewritten_image.parent_image_id,
)
except ManifestException as me:
logger.exception("exception when rewriting v1 metadata")
raise ManifestInvalid(detail={'message': 'failed synthesizing v1 metadata: %s' % me.message})
# Store the manifest pointing to the tag.
leaf_layer_id = rewritten_images[-1].image_id
newly_created = model.save_manifest(namespace_name, repo_name, manifest.tag, leaf_layer_id,
manifest.digest, manifest.bytes)
newly_created = model.save_manifest(repo, manifest.tag, leaf_layer_id, manifest.digest,
manifest.bytes)
if newly_created:
# TODO: make this batch
labels = []
for key, value in manifest.layers[-1].v1_metadata.labels.iteritems():
media_type = 'application/json' if is_json(value) else 'text/plain'
@ -219,11 +224,12 @@ def _write_manifest_and_log(namespace_name, repo_name, manifest):
'OK',
status=202,
headers={
'Docker-Content-Digest':
manifest.digest,
'Docker-Content-Digest': manifest.digest,
'Location':
url_for('v2.fetch_manifest_by_digest', repository='%s/%s' % (namespace_name, repo_name),
manifest_ref=manifest.digest),},)
manifest_ref=manifest.digest),
},
)
@v2_bp.route(MANIFEST_DIGEST_ROUTE, methods=['DELETE'])