vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3250 from quay/joseph.schorr/QUAY-1030/interfacing-part-9

Browse source 
Implement blob uploader and change V1 to use it
This commit is contained in:
Joseph Schorr 2018-09-24 16:03:41 -04:00 • committed by GitHub
commit 468e5a8fc2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 717 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

89
data/registry_model/registry_pre_oci_model.py
View file

@ -10,7 +10,7 @@ from data import model
from data.registry_model.interface import RegistryDataInterface
from data.registry_model.datatypes import (Tag, RepositoryReference, Manifest, LegacyImage, Label,
SecurityScanStatus, ManifestLayer, Blob, DerivedImage,
TorrentInfo)
TorrentInfo, BlobUpload)
from image.docker.schema1 import DockerSchema1ManifestBuilder, ManifestException
@ -99,7 +99,8 @@ class PreOCIModel(RegistryDataInterface):
return [LegacyImage.for_image(image, images_map=all_images_map, tags_map=tags_by_image_id)
for image in all_images]
def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False):
def get_legacy_image(self, repository_ref, docker_image_id, include_parents=False,
include_blob=False):
"""
Returns the matching LegacyImages under the matching repository, if any. If none,
returns None.
@ -117,7 +118,14 @@ class PreOCIModel(RegistryDataInterface):
parent_images = model.image.get_parent_images(repo.namespace_user.username, repo.name, image)
parent_images_map = {image.id: image for image in parent_images}
return LegacyImage.for_image(image, images_map=parent_images_map)
blob = None
if include_blob:
placements = list(model.storage.get_storage_locations(image.storage.uuid))
blob = Blob.for_image_storage(image.storage,
storage_path=model.storage.get_layer_path(image.storage),
placements=placements)
return LegacyImage.for_image(image, images_map=parent_images_map, blob=blob)
def create_manifest_label(self, manifest, key, value, source_type_name, media_type_name=None):
""" Creates a label on the manifest with the given key and value. """
@ -547,14 +555,14 @@ class PreOCIModel(RegistryDataInterface):
torrent_info = model.storage.save_torrent_info(image_storage, piece_length, pieces)
return TorrentInfo.for_torrent_info(torrent_info)
def get_repo_blob_by_digest(self, repo_ref, blob_digest, include_placements=False):
def get_repo_blob_by_digest(self, repository_ref, blob_digest, include_placements=False):
"""
Returns the blob in the repository with the given digest, if any or None if none. Note that
there may be multiple records in the same repository for the same blob digest, so the return
value of this function may change.
"""
try:
image_storage = model.blob.get_repository_blob_by_digest(repo_ref._db_id, blob_digest)
image_storage = model.blob.get_repository_blob_by_digest(repository_ref._db_id, blob_digest)
except model.BlobDoesNotExist:
return None
@ -568,5 +576,76 @@ class PreOCIModel(RegistryDataInterface):
storage_path=model.storage.get_layer_path(image_storage),
placements=placements)
def create_blob_upload(self, repository_ref, new_upload_id, location_name, storage_metadata):
""" Creates a new blob upload and returns a reference. If the blob upload could not be
created, returns None. """
repo = model.repository.lookup_repository(repository_ref._db_id)
if repo is None:
return None
try:
upload_record = model.blob.initiate_upload(repo.namespace_user.username, repo.name,
new_upload_id, location_name, storage_metadata)
return BlobUpload.for_upload(upload_record)
except database.Repository.DoesNotExist:
return None
def lookup_blob_upload(self, repository_ref, blob_upload_id):
""" Looks up the blob upload withn the given ID under the specified repository and returns it
or None if none.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload_id)
if upload_record is None:
return None
return BlobUpload.for_upload(upload_record)
def update_blob_upload(self, blob_upload, uncompressed_byte_count, piece_hashes, piece_sha_state,
storage_metadata, byte_count, chunk_count, sha_state):
""" Updates the fields of the blob upload to match those given. Returns the updated blob upload
or None if the record does not exists.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is None:
return None
upload_record.uncompressed_byte_count = uncompressed_byte_count
upload_record.piece_hashes = piece_hashes
upload_record.piece_sha_state = piece_sha_state
upload_record.storage_metadata = storage_metadata
upload_record.byte_count = byte_count
upload_record.chunk_count = chunk_count
upload_record.sha_state = sha_state
upload_record.save()
return BlobUpload.for_upload(upload_record)
def delete_blob_upload(self, blob_upload):
""" Deletes a blob upload record. """
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is not None:
upload_record.delete_instance()
def commit_blob_upload(self, blob_upload, blob_digest_str, blob_expiration_seconds):
""" Commits the blob upload into a blob and sets an expiration before that blob will be GCed.
"""
upload_record = model.blob.get_blob_upload_by_uuid(blob_upload.upload_id)
if upload_record is None:
return None
repository = upload_record.repository
namespace_name = repository.namespace_user.username
repo_name = repository.name
# Create the blob and temporarily tag it.
location_obj = model.storage.get_image_location_for_name(blob_upload.location_name)
blob_record = model.blob.store_blob_record_and_temp_link(
namespace_name, repo_name, blob_digest_str, location_obj.id, blob_upload.byte_count,
blob_expiration_seconds, blob_upload.uncompressed_byte_count)
# Delete the blob upload.
upload_record.delete_instance()
return Blob.for_image_storage(blob_record,
storage_path=model.storage.get_layer_path(blob_record))
pre_oci_model = PreOCIModel()