vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

style(data, endpoints, test): ran yapf against changed files

Browse source 
### Description of Changes

Issue: https://coreosdev.atlassian.net/browse/QUAY-633

## Reviewer Checklist

- [ ] It works!
- [ ] Comments provide sufficient explanations for the next contributor
- [ ] Tests cover changes and corner cases
- [ ] Follows Quay syntax patterns and format
This commit is contained in:
Charlton Austin 2017-07-24 11:05:15 -04:00
parent 9e1106f164
commit 8f1200b00d
7 changed files with 871 additions and 1332 deletions
Download patch file Download diff file Expand all files Collapse all files

217
data/model/repository.py
View file

@ -7,17 +7,16 @@ from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
from playhouse.shortcuts import case
from cachetools import ttl_cache
from data.model import (config, DataModelException, tag, db_transaction, storage, permission,
_basequery)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User,
Visibility, RepositoryPermission, RepositoryActionCount,
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore)
from data.model import (
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import (
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore)
from data.text import prefix_search
from util.itertoolrecipes import take
logger = logging.getLogger(__name__)
SEARCH_FIELDS = Enum("SearchFields", ["name", "description"])
@ -87,8 +86,7 @@ def purge_repository(namespace_name, repository_name):
unreferenced_image_q = Image.select(Image.id).where(Image.repository == repo)
if len(previously_referenced) > 0:
unreferenced_image_q = (unreferenced_image_q
.where(~(Image.id << list(previously_referenced))))
unreferenced_image_q = (unreferenced_image_q.where(~(Image.id << list(previously_referenced))))
unreferenced_candidates = set(img[0] for img in unreferenced_image_q.tuples())
@ -116,11 +114,10 @@ def purge_repository(namespace_name, repository_name):
@ttl_cache(maxsize=1, ttl=600)
def _get_gc_expiration_policies():
policy_tuples_query = (Namespace
.select(Namespace.removed_tag_expiration_s)
.distinct()
.limit(100) # This sucks but it's the only way to limit memory
.tuples())
policy_tuples_query = (
Namespace.select(Namespace.removed_tag_expiration_s).distinct()
.limit(100) # This sucks but it's the only way to limit memory
.tuples())
return [policy[0] for policy in policy_tuples_query]
@ -134,22 +131,15 @@ def find_repository_with_garbage(limit_to_gc_policy_s):
expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s
try:
candidates = (RepositoryTag
.select(RepositoryTag.repository)
.join(Repository)
candidates = (RepositoryTag.select(RepositoryTag.repository).join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts <= expiration_timestamp),
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s))
.limit(500)
.distinct()
.alias('candidates'))
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s)).limit(500)
.distinct().alias('candidates'))
found = (RepositoryTag
.select(candidates.c.repository_id)
.from_(candidates)
.order_by(db_random_func())
.get())
found = (RepositoryTag.select(candidates.c.repository_id).from_(candidates)
.order_by(db_random_func()).get())
if found is None:
return
@ -186,10 +176,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
all_unreferenced_candidates = set()
# Remove any images directly referenced by tags, to prune the working set.
direct_referenced = (RepositoryTag
.select(RepositoryTag.image)
.where(RepositoryTag.repository == repo.id,
RepositoryTag.image << list(candidate_orphan_image_set)))
direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
RepositoryTag.repository == repo.id, RepositoryTag.image << list(candidate_orphan_image_set)))
candidate_orphan_image_set.difference_update([t.image_id for t in direct_referenced])
# Iteratively try to remove images from the database. The only images we can remove are those
@ -205,26 +193,20 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
with db_transaction():
# Any image directly referenced by a tag that still exists, cannot be GCed.
direct_referenced = (RepositoryTag
.select(RepositoryTag.image)
.where(RepositoryTag.repository == repo.id,
RepositoryTag.image << candidates_orphans))
direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
RepositoryTag.repository == repo.id, RepositoryTag.image << candidates_orphans))
# Any image which is the parent of another image, cannot be GCed.
parent_referenced = (Image
.select(Image.parent)
.where(Image.repository == repo.id,
Image.parent << candidates_orphans))
parent_referenced = (Image.select(Image.parent).where(Image.repository == repo.id,
Image.parent << candidates_orphans))
referenced_candidates = (direct_referenced | parent_referenced)
# We desire a few pieces of information from the database from the following
# query: all of the image ids which are associated with this repository,
# and the storages which are associated with those images.
unreferenced_candidates = (Image
.select(Image.id, Image.docker_image_id,
ImageStorage.id, ImageStorage.uuid)
.join(ImageStorage)
unreferenced_candidates = (Image.select(Image.id, Image.docker_image_id, ImageStorage.id,
ImageStorage.uuid).join(ImageStorage)
.where(Image.id << candidates_orphans,
~(Image.id << referenced_candidates)))
@ -238,8 +220,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
storage_id_whitelist = set([candidate.storage_id for candidate in unreferenced_candidates])
# Lookup any derived images for the images to remove.
derived = DerivedStorageForImage.select().where(
DerivedStorageForImage.source_image << image_ids_to_remove)
derived = DerivedStorageForImage.select().where(DerivedStorageForImage.source_image <<
image_ids_to_remove)
has_derived = False
for derived_image in derived:
@ -249,10 +231,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
# Delete any derived images and the images themselves.
if has_derived:
try:
(DerivedStorageForImage
.delete()
.where(DerivedStorageForImage.source_image << image_ids_to_remove)
.execute())
(DerivedStorageForImage.delete()
.where(DerivedStorageForImage.source_image << image_ids_to_remove).execute())
except IntegrityError:
logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove)
return False
@ -278,8 +258,10 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
# If any storages were removed and cleanup callbacks are registered, call them with
# the images+storages removed.
if storage_ids_removed and config.image_cleanup_callbacks:
image_storages_removed = [candidate for candidate in all_unreferenced_candidates
if candidate.storage_id in storage_ids_removed]
image_storages_removed = [
candidate for candidate in all_unreferenced_candidates
if candidate.storage_id in storage_ids_removed
]
for callback in config.image_cleanup_callbacks:
callback(image_storages_removed)
@ -295,10 +277,7 @@ def star_repository(user, repository):
def unstar_repository(user, repository):
""" Unstars a repository. """
try:
(Star
.delete()
.where(Star.repository == repository.id, Star.user == user.id)
.execute())
(Star.delete().where(Star.repository == repository.id, Star.user == user.id).execute())
except Star.DoesNotExist:
raise DataModelException('Star not found.')
@ -308,6 +287,11 @@ def set_trust(repo, trust_enabled):
repo.save()
def set_description(repo, description):
repo.description = description
repo.save()
def get_user_starred_repositories(user, kind_filter='image'):
""" Retrieves all of the repositories a user has starred. """
try:
@ -315,13 +299,8 @@ def get_user_starred_repositories(user, kind_filter='image'):
except RepositoryKind.DoesNotExist:
raise DataModelException('Unknown kind of repository')
query = (Repository
.select(Repository, User, Visibility, Repository.id.alias('rid'))
.join(Star)
.switch(Repository)
.join(User)
.switch(Repository)
.join(Visibility)
query = (Repository.select(Repository, User, Visibility, Repository.id.alias('rid')).join(Star)
.switch(Repository).join(User).switch(Repository).join(Visibility)
.where(Star.user == user, Repository.kind == repo_kind))
return query
@ -330,10 +309,7 @@ def get_user_starred_repositories(user, kind_filter='image'):
def repository_is_starred(user, repository):
""" Determines whether a user has starred a repository or not. """
try:
(Star
.select()
.where(Star.repository == repository.id, Star.user == user.id)
.get())
(Star.select().where(Star.repository == repository.id, Star.user == user.id).get())
return True
except Star.DoesNotExist:
return False
@ -346,10 +322,8 @@ def get_when_last_modified(repository_ids):
if not repository_ids:
return {}
tuples = (RepositoryTag
.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts))
.where(RepositoryTag.repository << repository_ids)
.group_by(RepositoryTag.repository)
tuples = (RepositoryTag.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts))
.where(RepositoryTag.repository << repository_ids).group_by(RepositoryTag.repository)
.tuples())
last_modified_map = {}
@ -366,11 +340,8 @@ def get_stars(repository_ids):
if not repository_ids:
return {}
tuples = (Star
.select(Star.repository, fn.Count(Star.id))
.where(Star.repository << repository_ids)
.group_by(Star.repository)
.tuples())
tuples = (Star.select(Star.repository, fn.Count(Star.id))
.where(Star.repository << repository_ids).group_by(Star.repository).tuples())
star_map = {}
for record in tuples:
@ -388,12 +359,10 @@ def get_visible_repositories(username, namespace=None, kind_filter='image', incl
# here, as it will be modified by other queries later on.
return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1)
query = (Repository
.select(Repository.name, Repository.id.alias('rid'),
Repository.description, Namespace.username, Repository.visibility,
Repository.kind)
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)))
query = (Repository.select(Repository.name,
Repository.id.alias('rid'), Repository.description,
Namespace.username, Repository.visibility, Repository.kind)
.switch(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id)))
if username:
# Note: We only need the permissions table if we will filter based on a user's permissions.
@ -422,8 +391,8 @@ def get_app_search(lookup, search_fields=None, username=None, limit=50):
search_fields = set([SEARCH_FIELDS.name.name])
return get_filtered_matching_repositories(lookup, filter_username=username,
search_fields=search_fields,
repo_kind='application', offset=0, limit=limit)
search_fields=search_fields, repo_kind='application',
offset=0, limit=limit)
def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image',
@ -460,7 +429,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
unfiltered_page = 0
iteration_count = 0
while iteration_count < 10: # Just to be safe
while iteration_count < 10: # Just to be safe
# Find the next chunk's worth of repository IDs, paginated by the chunk size.
unfiltered_page = unfiltered_page + 1
found_ids = [r.id for r in unfiltered_query.paginate(unfiltered_page, chunk_count)]
@ -476,13 +445,9 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
encountered.update(new_unfiltered_ids)
# Filter the repositories found to only those visible to the current user.
query = (Repository
.select(Repository, Namespace)
.distinct()
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.switch(Repository)
.join(RepositoryPermission)
.where(Repository.id << list(new_unfiltered_ids)))
query = (Repository.select(Repository, Namespace).distinct()
.join(Namespace, on=(Namespace.id == Repository.namespace_user)).switch(Repository)
.join(RepositoryPermission).where(Repository.id << list(new_unfiltered_ids)))
filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind)
@ -520,16 +485,12 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if SEARCH_FIELDS.description.name in search_fields:
clause = Repository.description.match(lookup_value) | clause
cases = [
(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),
]
cases = [(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),]
computed_score = case(None, cases, RepositorySearchScore.score).alias('score')
query = (Repository
.select(Repository, Namespace, computed_score)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(clause)
query = (Repository.select(Repository, Namespace, computed_score)
.join(Namespace, on=(Namespace.id == Repository.namespace_user)).where(clause)
.group_by(Repository.id, Namespace.id))
if repo_kind is not None:
@ -538,11 +499,8 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if not include_private:
query = query.where(Repository.visibility == _basequery.get_public_repo_visibility())
query = (query
.switch(Repository)
.join(RepositorySearchScore)
.group_by(Repository, Namespace, RepositorySearchScore)
.order_by(SQL('score').desc()))
query = (query.switch(Repository).join(RepositorySearchScore)
.group_by(Repository, Namespace, RepositorySearchScore).order_by(SQL('score').desc()))
return query
@ -560,15 +518,10 @@ def is_repository_public(repository):
def repository_is_public(namespace_name, repository_name):
try:
(Repository
.select()
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(Visibility)
.where(Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == 'public')
.get())
(Repository.select().join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository).join(Visibility).where(Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == 'public').get())
return True
except Repository.DoesNotExist:
return False
@ -585,14 +538,10 @@ def set_repository_visibility(repo, visibility):
def get_email_authorized_for_repo(namespace, repository, email):
try:
return (RepositoryAuthorizedEmail
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace,
Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.get())
return (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Namespace.username == namespace, Repository.name == repository,
RepositoryAuthorizedEmail.email == email).get())
except RepositoryAuthorizedEmail.DoesNotExist:
return None
@ -601,20 +550,16 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email):
try:
repo = _basequery.get_existing_repository(namespace_name, repository_name)
except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' %
(namespace_name, repository_name))
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
return RepositoryAuthorizedEmail.create(repository=repo, email=email, confirmed=False)
def confirm_email_authorization_for_repo(code):
try:
found = (RepositoryAuthorizedEmail
.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryAuthorizedEmail.code == code)
.get())
found = (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryAuthorizedEmail.code == code).get())
except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException('Invalid confirmation code.')
@ -626,17 +571,13 @@ def confirm_email_authorization_for_repo(code):
def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'):
cutoff = datetime.now() - time_span
return (Repository
.select(Namespace.username, Repository.name)
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(Repository)
.join(RepositoryActionCount)
.where(RepositoryActionCount.date >= cutoff,
Repository.visibility == get_public_repo_visibility(),
Repository.kind == Repository.kind.get_id(repo_kind))
return (Repository.select(Namespace.username, Repository.name)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)).switch(Repository)
.join(RepositoryActionCount).where(RepositoryActionCount.date >= cutoff,
Repository.visibility == get_public_repo_visibility(),
Repository.kind == Repository.kind.get_id(repo_kind))
.group_by(RepositoryActionCount.repository, Repository.name, Namespace.username)
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold)
.tuples())
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold).tuples())
def is_empty(namespace_name, repository_name):