vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #885 from jakedt/python-registry-v2

Browse source 
Python registry v2 mega merge
This commit is contained in:
Jake Moshenko 2015-11-16 16:15:40 -05:00
commit 7205bf5e7f
54 changed files with 3256 additions and 837 deletions
Download patch file Download diff file Expand all files Collapse all files

8
data/model/__init__.py
View file

@ -9,6 +9,10 @@ class BlobDoesNotExist(DataModelException):
pass
class InvalidBlobUpload(DataModelException):
pass
class InvalidEmailAddressException(DataModelException):
pass
@ -65,6 +69,10 @@ class InvalidTeamMemberException(DataModelException):
pass
class InvalidManifestException(DataModelException):
pass
class TooManyLoginAttemptsException(Exception):
def __init__(self, message, retry_after):
super(TooManyLoginAttemptsException, self).__init__(message)

52
data/model/blob.py
View file

@ -1,8 +1,8 @@
from uuid import uuid4
from data.model import tag, _basequery, BlobDoesNotExist, db_transaction
from data.model import tag, _basequery, BlobDoesNotExist, InvalidBlobUpload, db_transaction
from data.database import (Repository, Namespace, ImageStorage, Image, ImageStorageLocation,
ImageStoragePlacement)
ImageStoragePlacement, BlobUpload)
def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
@ -15,35 +15,63 @@ def get_repo_blob_by_digest(namespace, repo_name, blob_digest):
.join(ImageStorage)
.join(Image)
.join(Repository)
.join(Namespace)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(Repository.name == repo_name, Namespace.username == namespace,
ImageStorage.content_checksum == blob_digest))
ImageStorage.content_checksum == blob_digest,
ImageStorage.uploading == False))
if not placements:
raise BlobDoesNotExist('Blob does not exist with digest: {0}'.format(blob_digest))
found = placements[0].storage
found.locations = {placement.location.name for placement in placements}
found.locations = {placement.location.name for placement in placements
if placement.storage.id == found.id}
return found
def store_blob_record_and_temp_link(namespace, repo_name, blob_digest, location_name,
def store_blob_record_and_temp_link(namespace, repo_name, blob_digest, location_obj, byte_count,
link_expiration_s):
""" Store a record of the blob and temporarily link it to the specified repository.
"""
random_image_name = str(uuid4())
with db_transaction:
with db_transaction():
repo = _basequery.get_existing_repository(namespace, repo_name)
try:
storage = ImageStorage.get(content_checksum=blob_digest)
location = ImageStorageLocation.get(name=location_name)
ImageStoragePlacement.get(storage=storage, location=location)
storage.image_size = byte_count
storage.save()
ImageStoragePlacement.get(storage=storage, location=location_obj)
except ImageStorage.DoesNotExist:
storage = ImageStorage.create(content_checksum=blob_digest)
storage = ImageStorage.create(content_checksum=blob_digest, uploading=False,
image_size=byte_count)
ImageStoragePlacement.create(storage=storage, location=location_obj)
except ImageStoragePlacement.DoesNotExist:
ImageStoragePlacement.create(storage=storage, location=location)
ImageStoragePlacement.create(storage=storage, location=location_obj)
# Create a temporary link into the repository, to be replaced by the v1 metadata later
# and create a temporary tag to reference it
image = Image.create(storage=storage, docker_image_id=random_image_name, repository=repo)
tag.create_temporary_hidden_tag(repo, image, link_expiration_s)
def get_blob_upload(namespace, repo_name, upload_uuid):
""" Load the upload which is already in progress.
"""
try:
return (BlobUpload
.select()
.join(Repository)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(Repository.name == repo_name, Namespace.username == namespace,
BlobUpload.uuid == upload_uuid)
.get())
except BlobUpload.DoesNotExist:
raise InvalidBlobUpload()
def initiate_upload(namespace, repo_name, uuid, location_name, storage_metadata):
repo = _basequery.get_existing_repository(namespace, repo_name)
location = ImageStorageLocation.get(name=location_name)
return BlobUpload.create(repository=repo, location=location, uuid=uuid,
storage_metadata=storage_metadata)

93
data/model/image.py
View file

@ -4,7 +4,8 @@ import dateutil.parser
from peewee import JOIN_LEFT_OUTER, fn, SQL
from datetime import datetime
from data.model import DataModelException, db_transaction, _basequery, storage
from data.model import (DataModelException, db_transaction, _basequery, storage,
InvalidImageException, config)
from data.database import (Image, Repository, ImageStoragePlacement, Namespace, ImageStorage,
ImageStorageLocation, RepositoryPermission, db_for_update)
@ -32,7 +33,9 @@ def get_repository_image_and_deriving(docker_image_id, storage_uuid):
def get_parent_images(namespace_name, repository_name, image_obj):
""" Returns a list of parent Image objects in chronilogical order. """
""" Returns a list of parent Image objects starting with the most recent parent
and ending with the base layer.
"""
parents = image_obj.ancestors
# Ancestors are in the format /<root>/<intermediate>/.../<parent>/, with each path section
@ -49,7 +52,7 @@ def get_parent_images(namespace_name, repository_name, image_obj):
id_to_image = {unicode(image.id): image for image in parents}
return [id_to_image[parent_id] for parent_id in parent_db_ids]
return [id_to_image[parent_id] for parent_id in reversed(parent_db_ids)]
def get_repo_image(namespace_name, repository_name, docker_image_id):
@ -97,7 +100,6 @@ def get_repository_images_base(namespace_name, repository_name, query_modifier):
.where(Repository.name == repository_name, Namespace.username == namespace_name))
query = query_modifier(query)
return invert_placement_query_results(query)
@ -127,18 +129,15 @@ def invert_placement_query_results(placement_query):
return images.values()
def lookup_repository_images(namespace_name, repository_name, docker_image_ids):
def lookup_repository_images(repo, docker_image_ids):
return (Image
.select()
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id << docker_image_ids))
.where(Image.repository == repo, Image.docker_image_id << docker_image_ids))
def get_matching_repository_images(namespace_name, repository_name, docker_image_ids):
def modify_query(query):
return query.where(Image.docker_image_id << docker_image_ids)
return query.where(Image.docker_image_id << list(docker_image_ids))
return get_repository_images_base(namespace_name, repository_name, modify_query)
@ -164,8 +163,8 @@ def get_repository_images(namespace_name, repository_name):
def get_image_by_id(namespace_name, repository_name, docker_image_id):
image = get_repo_image_extended(namespace_name, repository_name, docker_image_id)
if not image:
raise DataModelException('Unable to find image \'%s\' for repo \'%s/%s\'' %
(docker_image_id, namespace_name, repository_name))
raise InvalidImageException('Unable to find image \'%s\' for repo \'%s/%s\'' %
(docker_image_id, namespace_name, repository_name))
return image
@ -286,7 +285,7 @@ def find_create_or_link_image(docker_image_id, repo_obj, username, translations,
return repo_image
logger.debug('Creating new storage for docker id: %s', docker_image_id)
new_storage = storage.create_storage(preferred_location)
new_storage = storage.create_v1_storage(preferred_location)
return Image.create(docker_image_id=docker_image_id,
repository=repo_obj, storage=new_storage,
@ -331,7 +330,6 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
fetched.parent_id = parent.id
fetched.save()
fetched.storage.save()
return fetched
@ -379,13 +377,76 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size,
return image
def get_image(repo, dockerfile_id):
def get_image(repo, docker_image_id):
try:
return Image.get(Image.docker_image_id == dockerfile_id, Image.repository == repo)
return Image.get(Image.docker_image_id == docker_image_id, Image.repository == repo)
except Image.DoesNotExist:
return None
def get_repo_image_by_storage_checksum(namespace, repository_name, storage_checksum):
try:
return (Image
.select()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(Repository.name == repository_name, Namespace.username == namespace,
ImageStorage.content_checksum == storage_checksum,
ImageStorage.uploading == False)
.get())
except Image.DoesNotExist:
msg = 'Image with storage checksum {0} does not exist in repo {1}/{2}'.format(storage_checksum,
namespace,
repository_name)
raise InvalidImageException(msg)
def get_image_layers(image):
""" Returns a list of the full layers of an image, including itself (if specified), sorted
from base image outward. """
ancestors = image.ancestors.split('/')[1:-1]
image_ids = [ancestor_id for ancestor_id in ancestors if ancestor_id]
image_ids.append(str(image.id))
query = (ImageStoragePlacement
.select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation)
.join(ImageStorageLocation)
.switch(ImageStoragePlacement)
.join(ImageStorage, JOIN_LEFT_OUTER)
.join(Image)
.where(Image.id << image_ids))
image_list = list(_translate_placements_to_images_with_locations(query))
image_list.sort(key=lambda image: image_ids.index(str(image.id)))
return image_list
def synthesize_v1_image(repo, image_storage, docker_image_id, created_date_str,
comment, command, v1_json_metadata, parent_image=None):
""" Find an existing image with this docker image id, and if none exists, write one with the
specified metadata.
"""
ancestors = '/'
parent_id = None
if parent_image is not None:
ancestors = '{0}{1}/'.format(parent_image.ancestors, parent_image.id)
parent_id = parent_image.id
created = None
if created_date_str is not None:
try:
created = dateutil.parser.parse(created_date_str).replace(tzinfo=None)
except:
# parse raises different exceptions, so we cannot use a specific kind of handler here.
pass
return Image.create(docker_image_id=docker_image_id, ancestors=ancestors, comment=comment,
command=command, v1_json_metadata=v1_json_metadata, created=created,
storage=image_storage, repository=repo, parent_id=parent_id)
def ensure_image_locations(*names):
with db_transaction():
locations = ImageStorageLocation.select().where(ImageStorageLocation.name << names)

10
data/model/repository.py
View file

@ -7,8 +7,8 @@ from data.model import (DataModelException, tag, db_transaction, storage, image,
_basequery, config)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User,
Visibility, RepositoryPermission, TupleSelector, RepositoryActionCount,
Role, RepositoryAuthorizedEmail, db_for_update, get_epoch_timestamp,
db_random_func)
Role, RepositoryAuthorizedEmail, TagManifest, db_for_update,
get_epoch_timestamp, db_random_func)
logger = logging.getLogger(__name__)
@ -47,6 +47,12 @@ def _purge_all_repository_tags(namespace_name, repository_name):
except Repository.DoesNotExist:
raise DataModelException('Invalid repository \'%s/%s\'' %
(namespace_name, repository_name))
# Delete all manifests.
repo_tags = RepositoryTag.select().where(RepositoryTag.repository == repo.id)
TagManifest.delete().where(TagManifest.tag << repo_tags).execute()
# Delete all tags.
RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute()

70
data/model/storage.py
View file

@ -5,7 +5,7 @@ from peewee import JOIN_LEFT_OUTER, fn
from data.model import config, db_transaction, InvalidImageException
from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStoragePlacement,
ImageStorageLocation, ImageStorageTransformation, ImageStorageSignature,
ImageStorageSignatureKind)
ImageStorageSignatureKind, Repository, Namespace)
logger = logging.getLogger(__name__)
@ -24,7 +24,7 @@ def find_or_create_derived_storage(source, transformation_name, preferred_locati
logger.debug('Creating storage dervied from source: %s', source.uuid)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = create_storage(preferred_location)
new_storage = create_v1_storage(preferred_location)
DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
return new_storage
@ -34,7 +34,7 @@ def garbage_collect_storage(storage_id_whitelist):
return
def placements_query_to_paths_set(placements_query):
return {(placement.location.name, config.store.image_path(placement.storage.uuid))
return {(placement.location.name, get_layer_path(placement.storage))
for placement in placements_query}
def orphaned_storage_query(select_base_query, candidates, group_by):
@ -123,7 +123,7 @@ def garbage_collect_storage(storage_id_whitelist):
config.store.remove({location_name}, image_path)
def create_storage(location_name):
def create_v1_storage(location_name):
storage = ImageStorage.create(cas_path=False)
location = ImageStorageLocation.get(name=location_name)
ImageStoragePlacement.create(location=location, storage=storage)
@ -144,10 +144,9 @@ def lookup_storage_signature(storage, signature_kind):
kind = ImageStorageSignatureKind.get(name=signature_kind)
try:
return (ImageStorageSignature
.select()
.where(ImageStorageSignature.storage == storage,
ImageStorageSignature.kind == kind)
.get())
.select()
.where(ImageStorageSignature.storage == storage, ImageStorageSignature.kind == kind)
.get())
except ImageStorageSignature.DoesNotExist:
return None
@ -155,12 +154,12 @@ def lookup_storage_signature(storage, signature_kind):
def find_derived_storage(source, transformation_name):
try:
found = (ImageStorage
.select(ImageStorage, DerivedImageStorage)
.join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
.join(ImageStorageTransformation)
.where(DerivedImageStorage.source == source,
ImageStorageTransformation.name == transformation_name)
.get())
.select(ImageStorage, DerivedImageStorage)
.join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
.join(ImageStorageTransformation)
.where(DerivedImageStorage.source == source,
ImageStorageTransformation.name == transformation_name)
.get())
found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
return found
@ -182,16 +181,17 @@ def delete_derived_storage_by_uuid(storage_uuid):
image_storage.delete_instance(recursive=True)
def get_storage_by_uuid(storage_uuid):
placements = list(ImageStoragePlacement
.select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
.join(ImageStorageLocation)
.switch(ImageStoragePlacement)
.join(ImageStorage)
.where(ImageStorage.uuid == storage_uuid))
def _get_storage(query_modifier):
query = (ImageStoragePlacement
.select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
.join(ImageStorageLocation)
.switch(ImageStoragePlacement)
.join(ImageStorage))
placements = list(query_modifier(query))
if not placements:
raise InvalidImageException('No storage found with uuid: %s', storage_uuid)
raise InvalidImageException()
found = placements[0].storage
found.locations = {placement.location.name for placement in placements}
@ -199,3 +199,29 @@ def get_storage_by_uuid(storage_uuid):
return found
def get_storage_by_uuid(storage_uuid):
def filter_to_uuid(query):
return query.where(ImageStorage.uuid == storage_uuid)
try:
return _get_storage(filter_to_uuid)
except InvalidImageException:
raise InvalidImageException('No storage found with uuid: %s', storage_uuid)
def get_layer_path(storage_record):
""" Returns the path in the storage engine to the layer data referenced by the storage row. """
store = config.store
if not storage_record.cas_path:
logger.debug('Serving layer from legacy v1 path')
return store.v1_image_layer_path(storage_record.uuid)
return store.blob_path(storage_record.content_checksum)
def lookup_repo_storages_by_content_checksum(repo, checksums):
""" Looks up repository storages (without placements) matching the given repository
and checksum. """
return (ImageStorage
.select()
.join(Image)
.where(Image.repository == repo, ImageStorage.content_checksum << checksums))

68
data/model/tag.py
View file

@ -1,7 +1,8 @@
from uuid import uuid4
from data.model import image, db_transaction, DataModelException, _basequery
from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace,
from data.model import (image, db_transaction, DataModelException, _basequery,
InvalidManifestException)
from data.database import (RepositoryTag, Repository, Image, ImageStorage, Namespace, TagManifest,
get_epoch_timestamp, db_for_update)
@ -50,8 +51,8 @@ def list_repository_tags(namespace_name, repository_name, include_hidden=False,
return query
def create_or_update_tag(namespace_name, repository_name, tag_name,
tag_docker_image_id, reversion=False):
def create_or_update_tag(namespace_name, repository_name, tag_name, tag_docker_image_id,
reversion=False):
try:
repo = _basequery.get_existing_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
@ -174,3 +175,62 @@ def revert_tag(repo_obj, tag_name, docker_image_id):
return create_or_update_tag(repo_obj.namespace_user.username, repo_obj.name, tag_name,
docker_image_id, reversion=True)
def store_tag_manifest(namespace, repo_name, tag_name, docker_image_id, manifest_digest,
manifest_data):
with db_transaction():
tag = create_or_update_tag(namespace, repo_name, tag_name, docker_image_id)
try:
manifest = TagManifest.get(digest=manifest_digest)
manifest.tag = tag
manifest.save()
except TagManifest.DoesNotExist:
return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data)
def _get_active_tag(namespace, repo_name, tag_name):
return _tag_alive(RepositoryTag
.select()
.join(Image)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryTag.name == tag_name, Repository.name == repo_name,
Namespace.username == namespace)).get()
def associate_generated_tag_manifest(namespace, repo_name, tag_name, manifest_digest,
manifest_data):
tag = _get_active_tag(namespace, repo_name, tag_name)
return TagManifest.create(tag=tag, digest=manifest_digest, json_data=manifest_data)
def load_tag_manifest(namespace, repo_name, tag_name):
try:
return (_load_repo_manifests(namespace, repo_name)
.where(RepositoryTag.name == tag_name)
.get())
except TagManifest.DoesNotExist:
msg = 'Manifest not found for tag {0} in repo {1}/{2}'.format(tag_name, namespace, repo_name)
raise InvalidManifestException(msg)
def load_manifest_by_digest(namespace, repo_name, digest):
try:
return (_load_repo_manifests(namespace, repo_name)
.where(TagManifest.digest == digest)
.get())
except TagManifest.DoesNotExist:
msg = 'Manifest not found with digest {0} in repo {1}/{2}'.format(digest, namespace, repo_name)
raise InvalidManifestException(msg)
def _load_repo_manifests(namespace, repo_name):
return (TagManifest
.select(TagManifest, RepositoryTag)
.join(RepositoryTag)
.join(Image)
.join(Repository)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(Repository.name == repo_name, Namespace.username == namespace))

15
data/users/externaljwt.py
View file

@ -2,7 +2,6 @@ import logging
import json
import os
from datetime import datetime, timedelta
from data.users.federated import FederatedUsers, VerifiedCredentials
from util.security import strictjwt
@ -46,9 +45,11 @@ class ExternalJWTAuthN(FederatedUsers):
# Load the JWT returned.
encoded = result_data.get('token', '')
exp_limit_options = strictjwt.exp_max_s_option(self.max_fresh_s)
try:
payload = strictjwt.decode(encoded, self.public_key, algorithms=['RS256'],
audience='quay.io/jwtauthn', issuer=self.issuer)
audience='quay.io/jwtauthn', issuer=self.issuer,
options=exp_limit_options)
except strictjwt.InvalidTokenError:
logger.exception('Exception when decoding returned JWT')
return (None, 'Invalid username or password')
@ -59,16 +60,6 @@ class ExternalJWTAuthN(FederatedUsers):
if not 'email' in payload:
raise Exception('Missing email field in JWT')
if not 'exp' in payload:
raise Exception('Missing exp field in JWT')
# Verify that the expiration is no more than self.max_fresh_s seconds in the future.
expiration = datetime.utcfromtimestamp(payload['exp'])
if expiration > datetime.utcnow() + timedelta(seconds=self.max_fresh_s):
logger.debug('Payload expiration is outside of the %s second window: %s', self.max_fresh_s,
payload['exp'])
return (None, 'Invalid username or password')
# Parse out the username and email.
return (VerifiedCredentials(username=payload['sub'], email=payload['email']), None)