vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Get the base image stuff working. Checkpoint before fixing the tests.

Browse source
This commit is contained in:
jakedt 2014-02-16 17:38:47 -05:00
parent 5742e6ea4e
commit b619356907
120 changed files with 305 additions and 261 deletions
Download patch file Download diff file Expand all files Collapse all files

348
data/model.py
View file

@ -537,26 +537,30 @@ def get_user_teams_within_org(username, organization):
def get_visible_repository_count(username=None, include_public=True,
sort=False, namespace=None):
return get_visible_repository_internal(username=username,
include_public=include_public,
sort=sort, namespace=namespace,
get_count=True)
namespace=None):
query = _visible_repository_query(username=username,
include_public=include_public,
namespace=namespace)
return query.count()
def get_visible_repositories(username=None, include_public=True, page=None,
limit=None, sort=False, namespace=None):
return get_visible_repository_internal(username=username,
include_public=include_public,
page=page, limit=limit, sort=sort,
namespace=namespace, get_count=False)
query = _visible_repository_query(username=username,
include_public=include_public, page=page,
limit=limit, namespace=namespace)
if sort:
query = query.order_by(Repository.description.desc())
if limit:
query = query.limit(limit)
return query
def get_visible_repository_internal(username=None, include_public=True,
limit=None, page=None, sort=False,
namespace=None, get_count=False):
if not username and not include_public:
return []
def _visible_repository_query(username=None, include_public=True, limit=None,
page=None, namespace=None):
query = (Repository
.select() # Note: We need to leave this blank for the get_count case. Otherwise, MySQL/RDS complains.
.distinct()
@ -564,8 +568,19 @@ def get_visible_repository_internal(username=None, include_public=True,
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
query = _filter_to_repos_for_user(query, username, namespace, include_public)
if page:
query = query.paginate(page, limit)
elif limit:
query = query.limit(limit)
return query
def _filter_to_repos_for_user(query, username=None, namespace=None,
include_public=True):
where_clause = None
admin_query = None
if username:
UserThroughTeam = User.alias()
Org = User.alias()
@ -574,6 +589,7 @@ def get_visible_repository_internal(username=None, include_public=True,
AdminUser = User.alias()
query = (query
.switch(RepositoryPermission)
.join(User, JOIN_LEFT_OUTER)
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER)
@ -606,19 +622,7 @@ def get_visible_repository_internal(username=None, include_public=True,
else:
where_clause = new_clause
if sort:
query = query.order_by(Repository.description.desc())
if page:
query = query.paginate(page, limit)
elif limit:
query = query.limit(limit)
where = query.where(where_clause)
if get_count:
return where.count()
else:
return where
return query.where(where_clause)
def get_matching_repositories(repo_term, username=None):
@ -779,14 +783,13 @@ def get_repo_image(namespace_name, repository_name, image_id):
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == image_id)
.limit(1))
result = list(query)
if not result:
Image.docker_image_id == image_id))
try:
return query.get()
except Image.DoesNotExist:
return None
return result[0]
def repository_is_public(namespace_name, repository_name):
joined = Repository.select().join(Visibility)
@ -868,10 +871,31 @@ def create_repository(namespace, name, creating_user, visibility='private'):
return repo
def create_image(docker_image_id, repository):
new_image = Image.create(docker_image_id=docker_image_id,
repository=repository)
return new_image
def create_or_link_image(docker_image_id, repository, username, create=True):
with db.transaction():
query = (ImageStorage
.select()
.distinct()
.join(Image)
.join(Repository)
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER))
query = (_filter_to_repos_for_user(query, username)
.where(Image.docker_image_id == docker_image_id))
try:
storage = query.get()
msg = 'Linking image to existing storage with docker id: %s and uuid: %s'
logger.debug(msg, docker_image_id, storage.uuid)
except ImageStorage.DoesNotExist:
logger.debug('Creating new storage for docker id: %s', docker_image_id)
storage = ImageStorage.create()
new_image = Image.create(docker_image_id=docker_image_id,
repository=repository, storage=storage)
return new_image
def set_image_checksum(docker_image_id, repository, checksum):
@ -884,46 +908,67 @@ def set_image_checksum(docker_image_id, repository, checksum):
def set_image_size(docker_image_id, namespace_name, repository_name,
image_size):
joined = Image.select().join(Repository)
image_list = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
try:
image = (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id)
.get())
if not image_list:
except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository')
fetched = image_list[0]
fetched.image_size = image_size
fetched.save()
return fetched
if image.storage:
image.storage.image_size = image_size
image.storage.save()
else:
image.image_size = image_size
image.save()
return image
def set_image_metadata(docker_image_id, namespace_name, repository_name,
created_date_str, comment, command, parent=None):
joined = Image.select().join(Repository)
image_list = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
with db.transaction():
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
if not image_list:
raise DataModelException('No image with specified id and repository')
try:
fetched = query.get()
except Image.DoesNotExist:
raise DataModelException('No image with specified id and repository')
fetched = image_list[0]
fetched.created = dateutil.parser.parse(created_date_str)
fetched.comment = comment
fetched.command = command
fetched.storage.created = dateutil.parser.parse(created_date_str)
fetched.storage.comment = comment
fetched.storage.command = command
if parent:
fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
if parent:
fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
fetched.save()
return fetched
fetched.save()
fetched.storage.save()
return fetched
def get_repository_images(namespace_name, repository_name):
joined = Image.select().join(Repository)
return joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name)
return (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
def list_repository_tags(namespace_name, repository_name):
@ -933,111 +978,97 @@ def list_repository_tags(namespace_name, repository_name):
return with_image.where(Repository.name == repository_name,
Repository.namespace == namespace_name)
def delete_tag_and_images(namespace_name, repository_name, tag_name):
all_images = get_repository_images(namespace_name, repository_name)
all_tags = list_repository_tags(namespace_name, repository_name)
# Find the tag's information.
found_tag = None
for tag in all_tags:
if tag.name == tag_name:
found_tag = tag
break
if not found_tag:
return
# Build the set of database IDs corresponding to the tag's ancestor images,
# as well as the tag's image itself.
tag_image_ids = set(found_tag.image.ancestors.split('/'))
tag_image_ids.add(str(found_tag.image.id))
# Filter out any images that belong to any other tags.
for tag in all_tags:
if tag.name != tag_name:
# Remove all ancestors of the tag.
tag_image_ids = tag_image_ids - set(tag.image.ancestors.split('/'))
# Remove the current image ID.
tag_image_ids.discard(str(tag.image.id))
# Find all the images that belong to the tag.
tag_images = [image for image in all_images
if str(image.id) in tag_image_ids]
# Delete the tag found.
found_tag.delete_instance()
# Delete the images found.
for image in tag_images:
image.delete_instance()
repository_path = store.image_path(namespace_name, repository_name,
image.docker_image_id)
logger.debug('Recursively deleting image path: %s' % repository_path)
store.remove(repository_path)
def garbage_collect_repository(namespace_name, repository_name):
# Get a list of all images used by tags in the repository
tag_query = (RepositoryTag
.select(RepositoryTag, Image)
.join(Image)
.switch(RepositoryTag)
.join(Repository)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
with db.transaction():
# Get a list of all images used by tags in the repository
tag_query = (RepositoryTag
.select(RepositoryTag, Image, ImageStorage)
.join(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.switch(RepositoryTag)
.join(Repository)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name))
referenced_anscestors = set()
for tag in tag_query:
# The anscestor list is in the format '/1/2/3/', extract just the ids
anscestor_id_strings = tag.image.ancestors.split('/')[1:-1]
ancestor_list = [int(img_id_str) for img_id_str in anscestor_id_strings]
referenced_anscestors = referenced_anscestors.union(set(ancestor_list))
referenced_anscestors.add(tag.image.id)
referenced_anscestors = set()
for tag in tag_query:
# The anscestor list is in the format '/1/2/3/', extract just the ids
anscestor_id_strings = tag.image.ancestors.split('/')[1:-1]
ancestor_list = [int(img_id_str) for img_id_str in anscestor_id_strings]
referenced_anscestors = referenced_anscestors.union(set(ancestor_list))
referenced_anscestors.add(tag.image.id)
all_repo_images = get_repository_images(namespace_name, repository_name)
all_images = {int(img.id):img for img in all_repo_images}
to_remove = set(all_images.keys()).difference(referenced_anscestors)
all_repo_images = get_repository_images(namespace_name, repository_name)
all_images = {int(img.id): img for img in all_repo_images}
to_remove = set(all_images.keys()).difference(referenced_anscestors)
logger.info('Cleaning up unreferenced images: %s', to_remove)
logger.info('Cleaning up unreferenced images: %s', to_remove)
for image_id_to_remove in to_remove:
image_to_remove = all_images[image_id_to_remove]
image_path = store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id)
image_to_remove.delete_instance()
logger.debug('Deleting image storage: %s' % image_path)
store.remove(image_path)
uuids_to_check_for_gc = set()
for image_id_to_remove in to_remove:
image_to_remove = all_images[image_id_to_remove]
image_to_remove.delete_instance()
if not image_to_remove.storage:
image_path = store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id, None)
logger.debug('Deleting image storage: %s', image_path)
else:
uuids_to_check_for_gc.add(image_to_remove.storage.uuid)
storage_to_remove = (ImageStorage
.select()
.join(Image, JOIN_LEFT_OUTER)
.group_by(ImageStorage)
.where(ImageStorage.uuid << list(uuids_to_check_for_gc))
.having(fn.Count(Image.id) == 0))
for storage in storage_to_remove:
logger.debug('Garbage collecting image storage: %s', storage.uuid)
storage.delete_instance()
image_path = store.image_path(namespace_name, repository_name,
image_to_remove.docker_image_id,
storage.uuid)
store.remove(image_path)
return len(to_remove)
def get_tag_image(namespace_name, repository_name, tag_name):
joined = Image.select().join(RepositoryTag).join(Repository)
fetched = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
RepositoryTag.name == tag_name))
query = (Image
.select(Image, ImageStorage)
.join(RepositoryTag)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
RepositoryTag.name == tag_name))
if not fetched:
try:
return query.get()
except Image.DoesNotExist:
raise DataModelException('Unable to find image for tag.')
return fetched[0]
def get_image_by_id(namespace_name, repository_name, docker_image_id):
joined = Image.select().join(Repository)
fetched = list(joined.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
query = (Image
.select(Image, ImageStorage)
.join(Repository)
.switch(Image)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Repository.name == repository_name,
Repository.namespace == namespace_name,
Image.docker_image_id == docker_image_id))
if not fetched:
try:
return query.get()
except Image.DoesNotExist:
raise DataModelException('Unable to find image \'%s\' for repo \'%s/%s\'' %
(docker_image_id, namespace_name,
repository_name))
return fetched[0]
def get_parent_images(image_obj):
""" Returns a list of parent Image objects in chronilogical order. """
@ -1047,8 +1078,11 @@ def get_parent_images(image_obj):
if parent_db_ids == ['']:
return []
or_clauses = [(Image.id == db_id) for db_id in parent_db_ids]
parent_images = Image.select().where(reduce(operator.or_, or_clauses))
parent_images = (Image
.select(Image, ImageStorage)
.join(ImageStorage, JOIN_LEFT_OUTER)
.where(Image.id << parent_db_ids))
id_to_image = {unicode(image.id): image for image in parent_images}
return [id_to_image[parent_id] for parent_id in parent_db_ids]
@ -1206,15 +1240,17 @@ def set_team_repo_permission(team_name, namespace_name, repository_name,
def purge_repository(namespace_name, repository_name):
# Delete all tags to allow gc to reclaim storage
delete_all_repository_tags(namespace_name, repository_name)
# Gc to remove the images and storage
garbage_collect_repository(namespace_name, repository_name)
# Delete the rest of the repository metadata
fetched = Repository.get(Repository.name == repository_name,
Repository.namespace == namespace_name)
fetched.delete_instance(recursive=True)
repository_path = store.repository_namespace_path(namespace_name,
repository_name)
logger.debug('Recursively deleting path: %s' % repository_path)
store.remove(repository_path)
def get_private_repo_count(username):
joined = Repository.select().join(Visibility)