vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'perf'

Browse source
This commit is contained in:
Joseph Schorr 2014-11-10 21:52:27 -05:00
commit e7cbda86f7
7 changed files with 219 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

188
data/model/legacy.py
View file

@ -1035,16 +1035,26 @@ def get_repository(namespace_name, repository_name):
return None
def get_repo_image(namespace_name, repository_name, image_id):
def get_repo_image(namespace_name, repository_name, docker_image_id):
def limit_to_image_id(query):
return query.where(Image.docker_image_id == image_id)
return query.where(Image.docker_image_id == docker_image_id).limit(1)
query = _get_repository_images(namespace_name, repository_name, limit_to_image_id)
try:
return query.get()
except Image.DoesNotExist:
return None
def get_repo_image_extended(namespace_name, repository_name, docker_image_id):
def limit_to_image_id(query):
return query.where(Image.docker_image_id == docker_image_id).limit(1)
images = _get_repository_images_base(namespace_name, repository_name, limit_to_image_id)
if not images:
return None
else:
return images[0]
return images[0]
def repository_is_public(namespace_name, repository_name):
try:
@ -1137,20 +1147,21 @@ def __translate_ancestry(old_ancestry, translations, repository, username, prefe
if old_ancestry == '/':
return '/'
def translate_id(old_id):
def translate_id(old_id, docker_image_id):
logger.debug('Translating id: %s', old_id)
if old_id not in translations:
# Figure out which docker_image_id the old id refers to, then find a
# a local one
old = Image.select(Image.docker_image_id).where(Image.id == old_id).get()
image_in_repo = find_create_or_link_image(old.docker_image_id, repository, username,
image_in_repo = find_create_or_link_image(docker_image_id, repository, username,
translations, preferred_location)
translations[old_id] = image_in_repo.id
return translations[old_id]
# Select all the ancestor Docker IDs in a single query.
old_ids = [int(id_str) for id_str in old_ancestry.split('/')[1:-1]]
new_ids = [str(translate_id(old_id)) for old_id in old_ids]
query = Image.select(Image.id, Image.docker_image_id).where(Image.id << old_ids)
old_images = {i.id: i.docker_image_id for i in query}
# Translate the old images into new ones.
new_ids = [str(translate_id(old_id, old_images[old_id])) for old_id in old_ids]
return '/%s/' % '/'.join(new_ids)
@ -1162,36 +1173,22 @@ def _create_storage(location_name):
return storage
def find_create_or_link_image(docker_image_id, repository, username, translations,
preferred_location):
def _find_or_link_image(existing_image, repository, username, translations, preferred_location):
# TODO(jake): This call is currently recursively done under a single transaction. Can we make
# it instead be done under a set of transactions?
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Check for an existing image, under the transaction, to make sure it doesn't already exist.
repo_image = get_repo_image(repository.namespace_user.username, repository.name,
docker_image_id)
existing_image.docker_image_id)
if repo_image:
return repo_image
query = (Image
.select(Image, ImageStorage)
.distinct()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(ImageStorage.uploading == False))
query = (_filter_to_repos_for_user(query, username)
.where(Image.docker_image_id == docker_image_id))
new_image_ancestry = '/'
origin_image_id = None
# Make sure the existing base image still exists.
try:
to_copy = query.get()
to_copy = Image.select().join(ImageStorage).where(Image.id == existing_image.id).get()
msg = 'Linking image to existing storage with docker id: %s and uuid: %s'
logger.debug(msg, docker_image_id, to_copy.storage.uuid)
logger.debug(msg, existing_image.docker_image_id, to_copy.storage.uuid)
new_image_ancestry = __translate_ancestry(to_copy.ancestors, translations, repository,
username, preferred_location)
@ -1199,25 +1196,73 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
storage = to_copy.storage
storage.locations = {placement.location.name
for placement in storage.imagestorageplacement_set}
origin_image_id = to_copy.id
new_image = Image.create(docker_image_id=existing_image.docker_image_id,
repository=repository, storage=storage,
ancestors=new_image_ancestry)
logger.debug('Storing translation %s -> %s', existing_image.id, new_image.id)
translations[existing_image.id] = new_image.id
return new_image
except Image.DoesNotExist:
logger.debug('Creating new storage for docker id: %s', docker_image_id)
storage = _create_storage(preferred_location)
logger.debug('Storage locations: %s', storage.locations)
new_image = Image.create(docker_image_id=docker_image_id,
repository=repository, storage=storage,
ancestors=new_image_ancestry)
logger.debug('new_image storage locations: %s', new_image.storage.locations)
return None
if origin_image_id:
logger.debug('Storing translation %s -> %s', origin_image_id, new_image.id)
translations[origin_image_id] = new_image.id
def find_create_or_link_image(docker_image_id, repository, username, translations,
preferred_location):
return new_image
# First check for the image existing in the repository. If found, we simply return it.
repo_image = get_repo_image(repository.namespace_user.username, repository.name,
docker_image_id)
if repo_image:
return repo_image
# We next check to see if there is an existing storage the new image can link to.
existing_image_query = (Image
.select(Image, ImageStorage)
.distinct()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(ImageStorage.uploading == False))
existing_image_query = (_filter_to_repos_for_user(existing_image_query, username)
.where(Image.docker_image_id == docker_image_id))
# If there is an existing image, we try to translate its ancestry and copy its storage.
new_image = None
try:
logger.debug('Looking up existing image for ID: %s', docker_image_id)
existing_image = existing_image_query.get()
logger.debug('Existing image %s found for ID: %s', existing_image.id, docker_image_id)
new_image = _find_or_link_image(existing_image, repository, username, translations,
preferred_location)
if new_image:
return new_image
except Image.DoesNotExist:
logger.debug('No existing image found for ID: %s', docker_image_id)
pass
# Otherwise, create a new storage directly.
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Final check for an existing image, under the transaction.
repo_image = get_repo_image(repository.namespace_user.username, repository.name,
docker_image_id)
if repo_image:
return repo_image
logger.debug('Creating new storage for docker id: %s', docker_image_id)
storage = _create_storage(preferred_location)
return Image.create(docker_image_id=docker_image_id,
repository=repository, storage=storage,
ancestors='/')
def find_or_create_derived_storage(source, transformation_name, preferred_location):
@ -1331,6 +1376,15 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
fetched.storage.save()
return fetched
def _get_repository_images(namespace_name, repository_name, query_modifier):
query = (Image
.select()
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name))
query = query_modifier(query)
return query
def _get_repository_images_base(namespace_name, repository_name, query_modifier):
query = (ImageStoragePlacement
@ -1367,6 +1421,20 @@ def _get_repository_images_base(namespace_name, repository_name, query_modifier)
return images.values()
def lookup_repository_images(namespace_name, repository_name, docker_image_ids):
return (Image
.select()
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name,
Image.docker_image_id << docker_image_ids))
def get_matching_repository_images(namespace_name, repository_name, docker_image_ids):
def modify_query(q):
return q.where(Image.docker_image_id << docker_image_ids)
return _get_repository_images_base(namespace_name, repository_name, modify_query)
def get_repository_images(namespace_name, repository_name):
return _get_repository_images_base(namespace_name, repository_name, lambda q: q)
@ -1385,6 +1453,9 @@ def garbage_collect_repository(namespace_name, repository_name):
storage_id_whitelist = {}
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# TODO (jake): We could probably select this and all the images in a single query using
# a different kind of join.
# Get a list of all images used by tags in the repository
tag_query = (RepositoryTag
.select(RepositoryTag, Image, ImageStorage)
@ -1403,7 +1474,7 @@ def garbage_collect_repository(namespace_name, repository_name):
referenced_anscestors = referenced_anscestors.union(set(ancestor_list))
referenced_anscestors.add(tag.image.id)
all_repo_images = get_repository_images(namespace_name, repository_name)
all_repo_images = _get_repository_images(namespace_name, repository_name, lambda q: q)
all_images = {int(img.id): img for img in all_repo_images}
to_remove = set(all_images.keys()).difference(referenced_anscestors)
@ -1461,8 +1532,8 @@ def garbage_collect_storage(storage_id_whitelist):
# Note: Both of these deletes must occur in the same transaction (unfortunately) because a
# storage without any placement is invalid, and a placement cannot exist without a storage.
# TODO: We might want to allow for null storages on placements, which would allow us to delete
# the storages, then delete the placements in a non-transaction.
# TODO(jake): We might want to allow for null storages on placements, which would allow us to
# delete the storages, then delete the placements in a non-transaction.
logger.debug('Garbage collecting storages from candidates: %s', storage_id_whitelist)
with config.app_config['DB_TRANSACTION_FACTORY'](db):
# Track all of the data that should be removed from blob storage
@ -1516,7 +1587,7 @@ def get_tag_image(namespace_name, repository_name, tag_name):
def get_image_by_id(namespace_name, repository_name, docker_image_id):
image = get_repo_image(namespace_name, repository_name, docker_image_id)
image = get_repo_image_extended(namespace_name, repository_name, docker_image_id)
if not image:
raise DataModelException('Unable to find image \'%s\' for repo \'%s/%s\'' %
(docker_image_id, namespace_name, repository_name))
@ -1747,11 +1818,10 @@ def get_repository_delegate_tokens(namespace_name, repository_name):
def get_repo_delegate_token(namespace_name, repository_name, code):
repo_query = get_repository_delegate_tokens(namespace_name, repository_name)
found = list(repo_query.where(AccessToken.code == code))
if found:
return found[0]
else:
try:
return repo_query.where(AccessToken.code == code).get()
except AccessToken.DoesNotExist:
raise InvalidTokenException('Unable to find token with code: %s' % code)
@ -1920,9 +1990,9 @@ def list_logs(start_time, end_time, performer=None, repository=None, namespace=N
if namespace:
joined = joined.where(User.username == namespace)
return joined.where(
return list(joined.where(
LogEntry.datetime >= start_time,
LogEntry.datetime < end_time).order_by(LogEntry.datetime.desc())
LogEntry.datetime < end_time).order_by(LogEntry.datetime.desc()))
def log_action(kind_name, user_or_organization_name, performer=None,