vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into rustedbuilds

Browse source 
Conflicts:
	data/database.py
	endpoints/api.py
	endpoints/common.py
	test/data/test.db
This commit is contained in:
jakedt 2014-02-21 14:52:40 -05:00
commit d5304f7db0
128 changed files with 845 additions and 207343 deletions
Download patch file Download diff file Expand all files Collapse all files

404
data/model.py
View file

@ -13,7 +13,7 @@ from util.names import format_robot_username
logger = logging.getLogger(__name__)
store = app.config['STORAGE']
transaction_factory = app.config['DB_TRANSACTION_FACTORY']
class DataModelException(Exception):
pass
@ -541,26 +541,30 @@ def get_user_teams_within_org(username, organization):
def get_visible_repository_count(username=None, include_public=True,
sort=False, namespace=None):
return get_visible_repository_internal(username=username,
include_public=include_public,
sort=sort, namespace=namespace,
get_count=True)
namespace=None):
query = _visible_repository_query(username=username,
include_public=include_public,
namespace=namespace)
return query.count()
def get_visible_repositories(username=None, include_public=True, page=None,
limit=None, sort=False, namespace=None):
return get_visible_repository_internal(username=username,
include_public=include_public,
page=page, limit=limit, sort=sort,
namespace=namespace, get_count=False)
query = _visible_repository_query(username=username,
include_public=include_public, page=page,
limit=limit, namespace=namespace)
if sort:
query = query.order_by(Repository.description.desc())
if limit:
query = query.limit(limit)
return query
def get_visible_repository_internal(username=None, include_public=True,
limit=None, page=None, sort=False,
namespace=None, get_count=False):
if not username and not include_public:
return []
def _visible_repository_query(username=None, include_public=True, limit=None,
page=None, namespace=None):
query = (Repository
.select() # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains.
.distinct()
@ -568,8 +572,22 @@ def get_visible_repository_internal(username=None, include_public=True,
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
query = _filter_to_repos_for_user(query, username, namespace, include_public)
if page:
query = query.paginate(page, limit)
elif limit:
query = query.limit(limit)
return query
def _filter_to_repos_for_user(query, username=None, namespace=None,
include_public=True):
if not include_public and not username:
return Repository.select().where(Repository.id == '-1')
where_clause = None
admin_query = None
if username:
UserThroughTeam = User.alias()
Org = User.alias()
@ -578,6 +596,7 @@ def get_visible_repository_internal(username=None, include_public=True,
AdminUser = User.alias()
query = (query
.switch(RepositoryPermission)
.join(User, JOIN_LEFT_OUTER)
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER)
@ -610,19 +629,7 @@ def get_visible_repository_internal(username=None, include_public=True,
else:
where_clause = new_clause
if sort:
query = query.order_by(Repository.description.desc())
if page:
query = query.paginate(page, limit)
elif limit:
query = query.limit(limit)
where = query.where(where_clause)
if get_count:
return where.count()
else:
return where
return query.where(where_clause)
def get_matching_repositories(repo_term, username=None):
@ -786,16 +793,20 @@ def get_repository(namespace_name, repository_name):
def get_repo_image(namespace_name, repository_name, image_id):
joined = Image.select().join(Repository)
query = joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == image_id).limit(1)
result = list(query)
if not result:
query = (Image
.select()
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == image_id))
try:
return query.get()
except Image.DoesNotExist:
return None
return result[0]
def repository_is_public(namespace_name, repository_name):
joined = Repository.select().join(Visibility)
@ -877,10 +888,64 @@ def create_repository(namespace, name, creating_user, visibility='private'):
return repo
def create_image(docker_image_id, repository):
new_image = Image.create(docker_image_id=docker_image_id,
repository=repository)
return new_image
def __translate_ancestry(old_ancestry, translations, existing_images):
if old_ancestry == '/':
return '/'
def translate_id(old_id):
if old_id not in translations:
# Figure out which docker_image_id the old id refers to, then find a
# a local one
old = Image.select(Image.docker_image_id).where(Image.id == old_id).get()
translations[old_id] = existing_images[old.docker_image_id]
return translations[old_id]
old_ids = [int(id_str) for id_str in old_ancestry.split('/')[1:-1]]
new_ids = [str(translate_id(old_id)) for old_id in old_ids]
return '/%s/' % '/'.join(new_ids)
def create_or_link_image(docker_image_id, repository, username, translations,
existing_images):
with transaction_factory(db):
query = (Image
.select(Image, ImageStorage)
.distinct()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
query = (_filter_to_repos_for_user(query, username)
.where(Image.docker_image_id == docker_image_id))
new_image_ancestry = '/'
origin_image_id = None
try:
to_copy = query.get()
msg = 'Linking image to existing storage with docker id: %s and uuid: %s'
logger.debug(msg, docker_image_id, to_copy.storage.uuid)
new_image_ancestry = __translate_ancestry(to_copy.ancestors,
translations, existing_images)
storage = to_copy.storage
origin_image_id = to_copy.id
except Image.DoesNotExist:
logger.debug('Creating new storage for docker id: %s', docker_image_id)
storage = ImageStorage.create()
new_image = Image.create(docker_image_id=docker_image_id,
repository=repository, storage=storage,
ancestors=new_image_ancestry)
if origin_image_id:
translations[origin_image_id] = new_image.id
return new_image
def set_image_checksum(docker_image_id, repository, checksum):
@ -893,46 +958,67 @@ def set_image_checksum(docker_image_id, repository, checksum):
def set_image_size(docker_image_id, namespace_name, repository_name,
image_size):
joined = Image.select().join(Repository)
image_list = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
try:
image = (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id)
.get())
if not image_list:
except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository')
fetched = image_list[0]
fetched.image_size = image_size
fetched.save()
return fetched
if image.storage and image.storage.id:
image.storage.image_size = image_size
image.storage.save()
else:
image.image_size = image_size
image.save()
return image
def set_image_metadata(docker_image_id, namespace_name, repository_name,
created_date_str, comment, command, parent=None):
joined = Image.select().join(Repository)
image_list = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
with transaction_factory(db):
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
if not image_list:
raise DataModelException('No image with specified id and repository')
try:
fetched = query.get()
except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository')
fetched = image_list[0]
fetched.created = dateutil.parser.parse(created_date_str)
fetched.comment = comment
fetched.command = command
fetched.storage.created = dateutil.parser.parse(created_date_str)
fetched.storage.comment = comment
fetched.storage.command = command
if parent:
fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
if parent:
fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
fetched.save()
return fetched
fetched.save()
fetched.storage.save()
return fetched
def get_repository_images(namespace_name, repository_name):
joined = Image.select().join(Repository)
return joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name)
return (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
def list_repository_tags(namespace_name, repository_name):
@ -942,111 +1028,102 @@ def list_repository_tags(namespace_name, repository_name):
return with_image.where(Repository.name == repository_name,
Repository.namespace == namespace_name)
def delete_tag_and_images(namespace_name, repository_name, tag_name):
all_images = get_repository_images(namespace_name, repository_name)
all_tags = list_repository_tags(namespace_name, repository_name)
# Find the tag's information.
found_tag = None
for tag in all_tags:
if tag.name == tag_name:
found_tag = tag
break
if not found_tag:
return
# Build the set of database IDs corresponding to the tag's ancestor images,
# as well as the tag's image itself.
tag_image_ids = set(found_tag.image.ancestors.split('/'))
tag_image_ids.add(str(found_tag.image.id))
# Filter out any images that belong to any other tags.
for tag in all_tags:
if tag.name != tag_name:
# Remove all ancestors of the tag.
tag_image_ids = tag_image_ids - set(tag.image.ancestors.split('/'))
# Remove the current image ID.
tag_image_ids.discard(str(tag.image.id))
# Find all the images that belong to the tag.
tag_images = [image for image in all_images
if str(image.id) in tag_image_ids]
# Delete the tag found.
found_tag.delete_instance()
# Delete the images found.
for image in tag_images:
image.delete_instance()
repository_path = store.image_path(namespace_name, repository_name,
image.docker_image_id)
logger.debug('Recursively deleting image path: %s' % repository_path)
store.remove(repository_path)
def garbage_collect_repository(namespace_name, repository_name):
# Get a list of all images used by tags in the repository
tag_query = (RepositoryTag
.select(RepositoryTag, Image)
.join(Image)
.switch(RepositoryTag)
.join(Repository)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
with transaction_factory(db):
# Get a list of all images used by tags in the repository
tag_query = (RepositoryTag
.select(RepositoryTag, Image, ImageStorage)
.join(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.switch(RepositoryTag)
.join(Repository)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
referenced_anscestors = set()
for tag in tag_query:
# The anscestor list is in the format '/1/2/3/', extract just the ids
anscestor_id_strings = tag.image.ancestors.split('/')[1:-1]
ancestor_list = [int(img_id_str) for img_id_str in anscestor_id_strings]
referenced_anscestors = referenced_anscestors.union(set(ancestor_list))
referenced_anscestors.add(tag.image.id)
referenced_anscestors = set()
for tag in tag_query:
# The anscestor list is in the format '/1/2/3/', extract just the ids
anscestor_id_strings = tag.image.ancestors.split('/')[1:-1]
ancestor_list = [int(img_id_str) for img_id_str in anscestor_id_strings]
referenced_anscestors = referenced_anscestors.union(set(ancestor_list))
referenced_anscestors.add(tag.image.id)
all_repo_images = get_repository_images(namespace_name, repository_name)
all_images = {int(img.id):img for img in all_repo_images}
to_remove = set(all_images.keys()).difference(referenced_anscestors)
all_repo_images = get_repository_images(namespace_name, repository_name)
all_images = {int(img.id): img for img in all_repo_images}
to_remove = set(all_images.keys()).difference(referenced_anscestors)
logger.info('Cleaning up unreferenced images: %s', to_remove)
logger.info('Cleaning up unreferenced images: %s', to_remove)
for image_id_to_remove in to_remove:
image_to_remove = all_images[image_id_to_remove]
image_path = store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id)
image_to_remove.delete_instance()
logger.debug('Deleting image storage: %s' % image_path)
store.remove(image_path)
uuids_to_check_for_gc = set()
for image_id_to_remove in to_remove:
image_to_remove = all_images[image_id_to_remove]
return len(to_remove)
if image_to_remove.storage and image_to_remove.storage.id:
logger.debug('Adding image storage to the gc list: %s',
image_to_remove.storage.uuid)
uuids_to_check_for_gc.add(image_to_remove.storage.uuid)
else:
image_path = store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id, None)
logger.debug('Deleting image storage: %s', image_path)
store.remove(image_path)
image_to_remove.delete_instance()
if uuids_to_check_for_gc:
storage_to_remove = (ImageStorage
.select()
.join(Image, JOIN_LEFT_OUTER)
.group_by(ImageStorage)
.where(ImageStorage.uuid << list(uuids_to_check_for_gc))
.having(fn.Count(Image.id) == 0))
for storage in storage_to_remove:
logger.debug('Garbage collecting image storage: %s', storage.uuid)
storage.delete_instance()
image_path = store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id,
storage.uuid)
store.remove(image_path)
return len(to_remove)
def get_tag_image(namespace_name, repository_name, tag_name):
joined = Image.select().join(RepositoryTag).join(Repository)
fetched = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
RepositoryTag.name == tag_name))
query = (Image
.select(Image, ImageStorage)
.join(RepositoryTag)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
RepositoryTag.name == tag_name))
if not fetched:
try:
return query.get()
except Image.DoesNotExist:
raise DataModelException('Unable to find image for tag.')
return fetched[0]
def get_image_by_id(namespace_name, repository_name, docker_image_id):
joined = Image.select().join(Repository)
fetched = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
if not fetched:
try:
return query.get()
except Image.DoesNotExist:
raise DataModelException('Unable to find image \'%s\' for repo \'%s/%s\'' %
(docker_image_id, namespace_name,
repository_name))
return fetched[0]
def get_parent_images(image_obj):
""" Returns a list of parent Image objects in chronilogical order. """
@ -1056,8 +1133,11 @@ def get_parent_images(image_obj):
if parent_db_ids == ['']:
return []
or_clauses = [(Image.id == db_id) for db_id in parent_db_ids]
parent_images = Image.select().where(reduce(operator.or_, or_clauses))
parent_images = (Image
.select(Image, ImageStorage)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Image.id << parent_db_ids))
id_to_image = {unicode(image.id): image for image in parent_images}
return [id_to_image[parent_id] for parent_id in parent_db_ids]
@ -1111,7 +1191,7 @@ def delete_all_repository_tags(namespace_name, repository_name):
except Repository.DoesNotExist:
raise DataModelException('Invalid repository \'%s/%s\'' %
(namespace_name, repository_name))
RepositoryTag.delete().where(RepositoryTag.repository == repo).execute()
RepositoryTag.delete().where(RepositoryTag.repository == repo.id).execute()
def __entity_permission_repo_query(entity_id, entity_table,
@ -1215,15 +1295,17 @@ def set_team_repo_permission(team_name, namespace_name, repository_name,
def purge_repository(namespace_name, repository_name):
# Delete all tags to allow gc to reclaim storage
delete_all_repository_tags(namespace_name, repository_name)
# Gc to remove the images and storage
garbage_collect_repository(namespace_name, repository_name)
# Delete the rest of the repository metadata
fetched = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
fetched.delete_instance(recursive=True)
repository_path = store.repository_namespace_path(namespace_name,
repository_name)
logger.debug('Recursively deleting path: %s' % repository_path)
store.remove(repository_path)
def get_private_repo_count(username):
joined = Repository.select().join(Visibility)