vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

style(data, endpoints, test): ran yapf against changed files

Browse source 
### Description of Changes

Issue: https://coreosdev.atlassian.net/browse/QUAY-633

## Reviewer Checklist

- [ ] It works!
- [ ] Comments provide sufficient explanations for the next contributor
- [ ] Tests cover changes and corner cases
- [ ] Follows Quay syntax patterns and format
This commit is contained in:
Charlton Austin 2017-07-24 11:05:15 -04:00
parent 9e1106f164
commit 8f1200b00d
7 changed files with 871 additions and 1332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

217
data/model/repository.py
View file

@ -7,17 +7,16 @@ from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
from playhouse.shortcuts import case from playhouse.shortcuts import case
from cachetools import ttl_cache from cachetools import ttl_cache
from data.model import (config, DataModelException, tag, db_transaction, storage, permission, from data.model import (
_basequery) config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, from data.database import (
Visibility, RepositoryPermission, RepositoryActionCount, Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage, RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
Label, TagManifestLabel, db_for_update, get_epoch_timestamp, DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore) db_random_func, db_concat_func, RepositorySearchScore)
from data.text import prefix_search from data.text import prefix_search
from util.itertoolrecipes import take from util.itertoolrecipes import take
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
SEARCH_FIELDS = Enum("SearchFields", ["name", "description"]) SEARCH_FIELDS = Enum("SearchFields", ["name", "description"])
@ -87,8 +86,7 @@ def purge_repository(namespace_name, repository_name):
unreferenced_image_q = Image.select(Image.id).where(Image.repository == repo) unreferenced_image_q = Image.select(Image.id).where(Image.repository == repo)
if len(previously_referenced) > 0: if len(previously_referenced) > 0:
unreferenced_image_q = (unreferenced_image_q unreferenced_image_q = (unreferenced_image_q.where(~(Image.id << list(previously_referenced))))
.where(~(Image.id << list(previously_referenced))))
unreferenced_candidates = set(img[0] for img in unreferenced_image_q.tuples()) unreferenced_candidates = set(img[0] for img in unreferenced_image_q.tuples())
@ -116,11 +114,10 @@ def purge_repository(namespace_name, repository_name):
@ttl_cache(maxsize=1, ttl=600) @ttl_cache(maxsize=1, ttl=600)
def _get_gc_expiration_policies(): def _get_gc_expiration_policies():
policy_tuples_query = (Namespace policy_tuples_query = (
.select(Namespace.removed_tag_expiration_s) Namespace.select(Namespace.removed_tag_expiration_s).distinct()
.distinct() .limit(100) # This sucks but it's the only way to limit memory
.limit(100) # This sucks but it's the only way to limit memory .tuples())
.tuples())
return [policy[0] for policy in policy_tuples_query] return [policy[0] for policy in policy_tuples_query]
@ -134,22 +131,15 @@ def find_repository_with_garbage(limit_to_gc_policy_s):
expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s
try: try:
candidates = (RepositoryTag candidates = (RepositoryTag.select(RepositoryTag.repository).join(Repository)
.select(RepositoryTag.repository)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(~(RepositoryTag.lifetime_end_ts >> None), .where(~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts <= expiration_timestamp), (RepositoryTag.lifetime_end_ts <= expiration_timestamp),
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s)) (Namespace.removed_tag_expiration_s == limit_to_gc_policy_s)).limit(500)
.limit(500) .distinct().alias('candidates'))
.distinct()
.alias('candidates'))
found = (RepositoryTag found = (RepositoryTag.select(candidates.c.repository_id).from_(candidates)
.select(candidates.c.repository_id) .order_by(db_random_func()).get())
.from_(candidates)
.order_by(db_random_func())
.get())
if found is None: if found is None:
return return
@ -186,10 +176,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
all_unreferenced_candidates = set() all_unreferenced_candidates = set()
# Remove any images directly referenced by tags, to prune the working set. # Remove any images directly referenced by tags, to prune the working set.
direct_referenced = (RepositoryTag direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
.select(RepositoryTag.image) RepositoryTag.repository == repo.id, RepositoryTag.image << list(candidate_orphan_image_set)))
.where(RepositoryTag.repository == repo.id,
RepositoryTag.image << list(candidate_orphan_image_set)))
candidate_orphan_image_set.difference_update([t.image_id for t in direct_referenced]) candidate_orphan_image_set.difference_update([t.image_id for t in direct_referenced])
# Iteratively try to remove images from the database. The only images we can remove are those # Iteratively try to remove images from the database. The only images we can remove are those
@ -205,26 +193,20 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
with db_transaction(): with db_transaction():
# Any image directly referenced by a tag that still exists, cannot be GCed. # Any image directly referenced by a tag that still exists, cannot be GCed.
direct_referenced = (RepositoryTag direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
.select(RepositoryTag.image) RepositoryTag.repository == repo.id, RepositoryTag.image << candidates_orphans))
.where(RepositoryTag.repository == repo.id,
RepositoryTag.image << candidates_orphans))
# Any image which is the parent of another image, cannot be GCed. # Any image which is the parent of another image, cannot be GCed.
parent_referenced = (Image parent_referenced = (Image.select(Image.parent).where(Image.repository == repo.id,
.select(Image.parent) Image.parent << candidates_orphans))
.where(Image.repository == repo.id,
Image.parent << candidates_orphans))
referenced_candidates = (direct_referenced | parent_referenced) referenced_candidates = (direct_referenced | parent_referenced)
# We desire a few pieces of information from the database from the following # We desire a few pieces of information from the database from the following
# query: all of the image ids which are associated with this repository, # query: all of the image ids which are associated with this repository,
# and the storages which are associated with those images. # and the storages which are associated with those images.
unreferenced_candidates = (Image unreferenced_candidates = (Image.select(Image.id, Image.docker_image_id, ImageStorage.id,
.select(Image.id, Image.docker_image_id, ImageStorage.uuid).join(ImageStorage)
ImageStorage.id, ImageStorage.uuid)
.join(ImageStorage)
.where(Image.id << candidates_orphans, .where(Image.id << candidates_orphans,
~(Image.id << referenced_candidates))) ~(Image.id << referenced_candidates)))
@ -238,8 +220,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
storage_id_whitelist = set([candidate.storage_id for candidate in unreferenced_candidates]) storage_id_whitelist = set([candidate.storage_id for candidate in unreferenced_candidates])
# Lookup any derived images for the images to remove. # Lookup any derived images for the images to remove.
derived = DerivedStorageForImage.select().where( derived = DerivedStorageForImage.select().where(DerivedStorageForImage.source_image <<
DerivedStorageForImage.source_image << image_ids_to_remove) image_ids_to_remove)
has_derived = False has_derived = False
for derived_image in derived: for derived_image in derived:
@ -249,10 +231,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
# Delete any derived images and the images themselves. # Delete any derived images and the images themselves.
if has_derived: if has_derived:
try: try:
(DerivedStorageForImage (DerivedStorageForImage.delete()
.delete() .where(DerivedStorageForImage.source_image << image_ids_to_remove).execute())
.where(DerivedStorageForImage.source_image << image_ids_to_remove)
.execute())
except IntegrityError: except IntegrityError:
logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove) logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove)
return False return False
@ -278,8 +258,10 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
# If any storages were removed and cleanup callbacks are registered, call them with # If any storages were removed and cleanup callbacks are registered, call them with
# the images+storages removed. # the images+storages removed.
if storage_ids_removed and config.image_cleanup_callbacks: if storage_ids_removed and config.image_cleanup_callbacks:
image_storages_removed = [candidate for candidate in all_unreferenced_candidates image_storages_removed = [
if candidate.storage_id in storage_ids_removed] candidate for candidate in all_unreferenced_candidates
if candidate.storage_id in storage_ids_removed
]
for callback in config.image_cleanup_callbacks: for callback in config.image_cleanup_callbacks:
callback(image_storages_removed) callback(image_storages_removed)
@ -295,10 +277,7 @@ def star_repository(user, repository):
def unstar_repository(user, repository): def unstar_repository(user, repository):
""" Unstars a repository. """ """ Unstars a repository. """
try: try:
(Star (Star.delete().where(Star.repository == repository.id, Star.user == user.id).execute())
.delete()
.where(Star.repository == repository.id, Star.user == user.id)
.execute())
except Star.DoesNotExist: except Star.DoesNotExist:
raise DataModelException('Star not found.') raise DataModelException('Star not found.')
@ -308,6 +287,11 @@ def set_trust(repo, trust_enabled):
repo.save() repo.save()
def set_description(repo, description):
repo.description = description
repo.save()
def get_user_starred_repositories(user, kind_filter='image'): def get_user_starred_repositories(user, kind_filter='image'):
""" Retrieves all of the repositories a user has starred. """ """ Retrieves all of the repositories a user has starred. """
try: try:
@ -315,13 +299,8 @@ def get_user_starred_repositories(user, kind_filter='image'):
except RepositoryKind.DoesNotExist: except RepositoryKind.DoesNotExist:
raise DataModelException('Unknown kind of repository') raise DataModelException('Unknown kind of repository')
query = (Repository query = (Repository.select(Repository, User, Visibility, Repository.id.alias('rid')).join(Star)
.select(Repository, User, Visibility, Repository.id.alias('rid')) .switch(Repository).join(User).switch(Repository).join(Visibility)
.join(Star)
.switch(Repository)
.join(User)
.switch(Repository)
.join(Visibility)
.where(Star.user == user, Repository.kind == repo_kind)) .where(Star.user == user, Repository.kind == repo_kind))
return query return query
@ -330,10 +309,7 @@ def get_user_starred_repositories(user, kind_filter='image'):
def repository_is_starred(user, repository): def repository_is_starred(user, repository):
""" Determines whether a user has starred a repository or not. """ """ Determines whether a user has starred a repository or not. """
try: try:
(Star (Star.select().where(Star.repository == repository.id, Star.user == user.id).get())
.select()
.where(Star.repository == repository.id, Star.user == user.id)
.get())
return True return True
except Star.DoesNotExist: except Star.DoesNotExist:
return False return False
@ -346,10 +322,8 @@ def get_when_last_modified(repository_ids):
if not repository_ids: if not repository_ids:
return {} return {}
tuples = (RepositoryTag tuples = (RepositoryTag.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts))
.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts)) .where(RepositoryTag.repository << repository_ids).group_by(RepositoryTag.repository)
.where(RepositoryTag.repository << repository_ids)
.group_by(RepositoryTag.repository)
.tuples()) .tuples())
last_modified_map = {} last_modified_map = {}
@ -366,11 +340,8 @@ def get_stars(repository_ids):
if not repository_ids: if not repository_ids:
return {} return {}
tuples = (Star tuples = (Star.select(Star.repository, fn.Count(Star.id))
.select(Star.repository, fn.Count(Star.id)) .where(Star.repository << repository_ids).group_by(Star.repository).tuples())
.where(Star.repository << repository_ids)
.group_by(Star.repository)
.tuples())
star_map = {} star_map = {}
for record in tuples: for record in tuples:
@ -388,12 +359,10 @@ def get_visible_repositories(username, namespace=None, kind_filter='image', incl
# here, as it will be modified by other queries later on. # here, as it will be modified by other queries later on.
return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1) return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1)
query = (Repository query = (Repository.select(Repository.name,
.select(Repository.name, Repository.id.alias('rid'), Repository.id.alias('rid'), Repository.description,
Repository.description, Namespace.username, Repository.visibility, Namespace.username, Repository.visibility, Repository.kind)
Repository.kind) .switch(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id)))
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)))
if username: if username:
# Note: We only need the permissions table if we will filter based on a user's permissions. # Note: We only need the permissions table if we will filter based on a user's permissions.
@ -422,8 +391,8 @@ def get_app_search(lookup, search_fields=None, username=None, limit=50):
search_fields = set([SEARCH_FIELDS.name.name]) search_fields = set([SEARCH_FIELDS.name.name])
return get_filtered_matching_repositories(lookup, filter_username=username, return get_filtered_matching_repositories(lookup, filter_username=username,
search_fields=search_fields, search_fields=search_fields, repo_kind='application',
repo_kind='application', offset=0, limit=limit) offset=0, limit=limit)
def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image', def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image',
@ -460,7 +429,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
unfiltered_page = 0 unfiltered_page = 0
iteration_count = 0 iteration_count = 0
while iteration_count < 10: # Just to be safe while iteration_count < 10: # Just to be safe
# Find the next chunk's worth of repository IDs, paginated by the chunk size. # Find the next chunk's worth of repository IDs, paginated by the chunk size.
unfiltered_page = unfiltered_page + 1 unfiltered_page = unfiltered_page + 1
found_ids = [r.id for r in unfiltered_query.paginate(unfiltered_page, chunk_count)] found_ids = [r.id for r in unfiltered_query.paginate(unfiltered_page, chunk_count)]
@ -476,13 +445,9 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
encountered.update(new_unfiltered_ids) encountered.update(new_unfiltered_ids)
# Filter the repositories found to only those visible to the current user. # Filter the repositories found to only those visible to the current user.
query = (Repository query = (Repository.select(Repository, Namespace).distinct()
.select(Repository, Namespace) .join(Namespace, on=(Namespace.id == Repository.namespace_user)).switch(Repository)
.distinct() .join(RepositoryPermission).where(Repository.id << list(new_unfiltered_ids)))
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.switch(Repository)
.join(RepositoryPermission)
.where(Repository.id << list(new_unfiltered_ids)))
filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind) filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind)
@ -520,16 +485,12 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if SEARCH_FIELDS.description.name in search_fields: if SEARCH_FIELDS.description.name in search_fields:
clause = Repository.description.match(lookup_value) | clause clause = Repository.description.match(lookup_value) | clause
cases = [ cases = [(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),]
(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),
]
computed_score = case(None, cases, RepositorySearchScore.score).alias('score') computed_score = case(None, cases, RepositorySearchScore.score).alias('score')
query = (Repository query = (Repository.select(Repository, Namespace, computed_score)
.select(Repository, Namespace, computed_score) .join(Namespace, on=(Namespace.id == Repository.namespace_user)).where(clause)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(clause)
.group_by(Repository.id, Namespace.id)) .group_by(Repository.id, Namespace.id))
if repo_kind is not None: if repo_kind is not None:
@ -538,11 +499,8 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if not include_private: if not include_private:
query = query.where(Repository.visibility == _basequery.get_public_repo_visibility()) query = query.where(Repository.visibility == _basequery.get_public_repo_visibility())
query = (query query = (query.switch(Repository).join(RepositorySearchScore)
.switch(Repository) .group_by(Repository, Namespace, RepositorySearchScore).order_by(SQL('score').desc()))
.join(RepositorySearchScore)
.group_by(Repository, Namespace, RepositorySearchScore)
.order_by(SQL('score').desc()))
return query return query
@ -560,15 +518,10 @@ def is_repository_public(repository):
def repository_is_public(namespace_name, repository_name): def repository_is_public(namespace_name, repository_name):
try: try:
(Repository (Repository.select().join(Namespace, on=(Repository.namespace_user == Namespace.id))
.select() .switch(Repository).join(Visibility).where(Namespace.username == namespace_name,
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) Repository.name == repository_name,
.switch(Repository) Visibility.name == 'public').get())
.join(Visibility)
.where(Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == 'public')
.get())
return True return True
except Repository.DoesNotExist: except Repository.DoesNotExist:
return False return False
@ -585,14 +538,10 @@ def set_repository_visibility(repo, visibility):
def get_email_authorized_for_repo(namespace, repository, email): def get_email_authorized_for_repo(namespace, repository, email):
try: try:
return (RepositoryAuthorizedEmail return (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.select(RepositoryAuthorizedEmail, Repository, Namespace) .join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(Repository) .where(Namespace.username == namespace, Repository.name == repository,
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) RepositoryAuthorizedEmail.email == email).get())
.where(Namespace.username == namespace,
Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.get())
except RepositoryAuthorizedEmail.DoesNotExist: except RepositoryAuthorizedEmail.DoesNotExist:
return None return None
@ -601,20 +550,16 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email):
try: try:
repo = _basequery.get_existing_repository(namespace_name, repository_name) repo = _basequery.get_existing_repository(namespace_name, repository_name)
except Repository.DoesNotExist: except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
(namespace_name, repository_name))
return RepositoryAuthorizedEmail.create(repository=repo, email=email, confirmed=False) return RepositoryAuthorizedEmail.create(repository=repo, email=email, confirmed=False)
def confirm_email_authorization_for_repo(code): def confirm_email_authorization_for_repo(code):
try: try:
found = (RepositoryAuthorizedEmail found = (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.select(RepositoryAuthorizedEmail, Repository, Namespace) .join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(Repository) .where(RepositoryAuthorizedEmail.code == code).get())
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryAuthorizedEmail.code == code)
.get())
except RepositoryAuthorizedEmail.DoesNotExist: except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException('Invalid confirmation code.') raise DataModelException('Invalid confirmation code.')
@ -626,17 +571,13 @@ def confirm_email_authorization_for_repo(code):
def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'): def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'):
cutoff = datetime.now() - time_span cutoff = datetime.now() - time_span
return (Repository return (Repository.select(Namespace.username, Repository.name)
.select(Namespace.username, Repository.name) .join(Namespace, on=(Repository.namespace_user == Namespace.id)).switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) .join(RepositoryActionCount).where(RepositoryActionCount.date >= cutoff,
.switch(Repository) Repository.visibility == get_public_repo_visibility(),
.join(RepositoryActionCount) Repository.kind == Repository.kind.get_id(repo_kind))
.where(RepositoryActionCount.date >= cutoff,
Repository.visibility == get_public_repo_visibility(),
Repository.kind == Repository.kind.get_id(repo_kind))
.group_by(RepositoryActionCount.repository, Repository.name, Namespace.username) .group_by(RepositoryActionCount.repository, Repository.name, Namespace.username)
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold) .having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold).tuples())
.tuples())
def is_empty(namespace_name, repository_name): def is_empty(namespace_name, repository_name):

2
endpoints/api/logs.py
View file

@ -16,8 +16,6 @@ SERVICE_LEVEL_LOG_KINDS = set(['service_key_create', 'service_key_approve', 'ser
'service_key_modify', 'service_key_extend', 'service_key_rotate']) 'service_key_modify', 'service_key_extend', 'service_key_rotate'])
def _validate_logs_arguments(start_time, end_time): def _validate_logs_arguments(start_time, end_time):
if start_time: if start_time:
try: try:

71
endpoints/api/repository.py
View file

@ -10,14 +10,13 @@ from datetime import timedelta, datetime
from flask import request, abort from flask import request, abort
from app import dockerfile_build_queue, tuf_metadata_api from app import dockerfile_build_queue, tuf_metadata_api
from endpoints.api import (format_date, nickname, log_action, validate_json_request, from endpoints.api import (
require_repo_read, require_repo_write, require_repo_admin, format_date, nickname, log_action, validate_json_request, require_repo_read, require_repo_write,
RepositoryParamResource, resource, parse_args, ApiResource, require_repo_admin, RepositoryParamResource, resource, parse_args, ApiResource, request_error,
request_error, require_scope, path_param, page_support, require_scope, path_param, page_support, query_param, truthy_bool, show_if)
query_param, truthy_bool, show_if)
from endpoints.api.repository_models_pre_oci import pre_oci_model as model from endpoints.api.repository_models_pre_oci import pre_oci_model as model
from endpoints.exception import (Unauthorized, NotFound, InvalidRequest, ExceedsLicenseException, from endpoints.exception import (
DownstreamIssue) Unauthorized, NotFound, InvalidRequest, ExceedsLicenseException, DownstreamIssue)
from endpoints.api.billing import lookup_allowed_private_repos, get_namespace_plan from endpoints.api.billing import lookup_allowed_private_repos, get_namespace_plan
from endpoints.api.subscribe import check_repository_usage from endpoints.api.subscribe import check_repository_usage
@ -71,7 +70,8 @@ class RepositoryList(ApiResource):
], ],
}, },
'namespace': { 'namespace': {
'type': 'string', 'type':
'string',
'description': ('Namespace in which the repository should be created. If omitted, the ' 'description': ('Namespace in which the repository should be created. If omitted, the '
'username of the caller is used'), 'username of the caller is used'),
}, },
@ -118,16 +118,17 @@ class RepositoryList(ApiResource):
raise InvalidRequest('Invalid repository name') raise InvalidRequest('Invalid repository name')
kind = req.get('repo_kind', 'image') or 'image' kind = req.get('repo_kind', 'image') or 'image'
model.create_repo(namespace_name, repository_name, owner, req['description'], visibility=visibility, model.create_repo(namespace_name, repository_name, owner, req['description'],
repo_kind=kind) visibility=visibility, repo_kind=kind)
log_action('create_repo', namespace_name, {'repo': repository_name, log_action('create_repo', namespace_name,
'namespace': namespace_name}, repo_name=repository_name) {'repo': repository_name,
'namespace': namespace_name}, repo_name=repository_name)
return { return {
'namespace': namespace_name, 'namespace': namespace_name,
'name': repository_name, 'name': repository_name,
'kind': kind, 'kind': kind,
}, 201 }, 201
raise Unauthorized() raise Unauthorized()
@ -178,9 +179,7 @@ class Repository(RepositoryParamResource):
'RepoUpdate': { 'RepoUpdate': {
'type': 'object', 'type': 'object',
'description': 'Fields which can be updated in a repository.', 'description': 'Fields which can be updated in a repository.',
'required': [ 'required': ['description',],
'description',
],
'properties': { 'properties': {
'description': { 'description': {
'type': 'string', 'type': 'string',
@ -239,12 +238,10 @@ class Repository(RepositoryParamResource):
model.set_description(namespace, repository, values['description']) model.set_description(namespace, repository, values['description'])
log_action('set_repo_description', namespace, log_action('set_repo_description', namespace,
{'repo': repository, 'namespace': namespace, 'description': values['description']}, {'repo': repository,
repo_name=repository) 'namespace': namespace,
return { 'description': values['description']}, repo_name=repository)
'success': True return {'success': True}
}
@require_repo_admin @require_repo_admin
@nickname('deleteRepository') @nickname('deleteRepository')
@ -259,8 +256,7 @@ class Repository(RepositoryParamResource):
# Remove any builds from the queue. # Remove any builds from the queue.
dockerfile_build_queue.delete_namespaced_items(namespace, repository) dockerfile_build_queue.delete_namespaced_items(namespace, repository)
log_action('delete_repo', namespace, log_action('delete_repo', namespace, {'repo': repository, 'namespace': namespace})
{'repo': repository, 'namespace': namespace})
return '', 204 return '', 204
@ -272,9 +268,7 @@ class RepositoryVisibility(RepositoryParamResource):
'ChangeVisibility': { 'ChangeVisibility': {
'type': 'object', 'type': 'object',
'description': 'Change the visibility for the repository.', 'description': 'Change the visibility for the repository.',
'required': [ 'required': ['visibility',],
'visibility',
],
'properties': { 'properties': {
'visibility': { 'visibility': {
'type': 'string', 'type': 'string',
@ -301,8 +295,9 @@ class RepositoryVisibility(RepositoryParamResource):
model.set_repository_visibility(namespace, repository, visibility) model.set_repository_visibility(namespace, repository, visibility)
log_action('change_repo_visibility', namespace, log_action('change_repo_visibility', namespace,
{'repo': repository, 'namespace': namespace, 'visibility': values['visibility']}, {'repo': repository,
repo_name=repository) 'namespace': namespace,
'visibility': values['visibility']}, repo_name=repository)
return {'success': True} return {'success': True}
@ -314,9 +309,7 @@ class RepositoryTrust(RepositoryParamResource):
'ChangeRepoTrust': { 'ChangeRepoTrust': {
'type': 'object', 'type': 'object',
'description': 'Change the trust settings for the repository.', 'description': 'Change the trust settings for the repository.',
'required': [ 'required': ['trust_enabled',],
'trust_enabled',
],
'properties': { 'properties': {
'trust_enabled': { 'trust_enabled': {
'type': 'boolean', 'type': 'boolean',
@ -341,8 +334,10 @@ class RepositoryTrust(RepositoryParamResource):
values = request.get_json() values = request.get_json()
model.set_trust(namespace, repository, values['trust_enabled']) model.set_trust(namespace, repository, values['trust_enabled'])
log_action('change_repo_trust', namespace, log_action(
{'repo': repository, 'namespace': namespace, 'trust_enabled': values['trust_enabled']}, 'change_repo_trust', namespace,
repo_name=repository) {'repo': repository,
'namespace': namespace,
'trust_enabled': values['trust_enabled']}, repo_name=repository)
return {'success': True} return {'success': True}

53
endpoints/api/repository_models_interface.py
View file

@ -8,10 +8,12 @@ import features
from endpoints.api import format_date from endpoints.api import format_date
class RepositoryBaseElement(namedtuple('RepositoryBaseElement', class RepositoryBaseElement(
['namespace_name', 'repository_name', 'is_starred', 'is_public', 'kind_name', namedtuple('RepositoryBaseElement', [
'description', 'namespace_user_organization', 'namespace_name', 'repository_name', 'is_starred', 'is_public', 'kind_name', 'description',
'namespace_user_removed_tag_expiration_s', 'last_modified', 'action_count'])): 'namespace_user_organization', 'namespace_user_removed_tag_expiration_s', 'last_modified',
'action_count', 'should_last_modified', 'should_popularity', 'should_is_starred'
])):
""" """
Repository a single quay repository Repository a single quay repository
:type namespace_name: string :type namespace_name: string
@ -56,31 +58,25 @@ class ApplicationRepository(
:type releases: [Release] :type releases: [Release]
""" """
def to_dict(self, can_write, can_admin): def to_dict(self):
releases_channels_map = defaultdict(list)
for channel in self.channels:
releases_channels_map[channel.linked_tag_name].append(channel.name)
repo_data = { repo_data = {
'namespace': self.repository_base_elements.namespace_name, 'namespace': self.repository_base_elements.namespace_name,
'name': self.repository_base_elements.repository_name, 'name': self.repository_base_elements.repository_name,
'kind': self.repository_base_elements.kind_name, 'kind': self.repository_base_elements.kind_name,
'description': self.repository_base_elements.description, 'description': self.repository_base_elements.description,
'can_write': can_write,
'can_admin': can_admin,
'is_public': self.repository_base_elements.is_public, 'is_public': self.repository_base_elements.is_public,
'is_organization': self.repository_base_elements.namespace_user_organization, 'is_organization': self.repository_base_elements.namespace_user_organization,
'is_starred': self.repository_base_elements.is_starred, 'is_starred': self.repository_base_elements.is_starred,
'channels': [chan.to_dict() for chan in self.channels], 'channels': [chan.to_dict() for chan in self.channels],
'releases': [release.to_dict(releases_channels_map) for release in self.channels], 'releases': [release.to_dict() for release in self.releases],
} }
return repo_data return repo_data
class NonApplicationRepository(namedtuple('NonApplicationRepository', class ImageRepositoryRepository(
['repository_base_elements', 'tags', 'counts', 'badge_token', namedtuple('NonApplicationRepository',
'trust_enabled'])): ['repository_base_elements', 'tags', 'counts', 'badge_token', 'trust_enabled'])):
""" """
Repository a single quay repository Repository a single quay repository
:type repository_base_elements: RepositoryBaseElement :type repository_base_elements: RepositoryBaseElement
@ -90,25 +86,27 @@ class NonApplicationRepository(namedtuple('NonApplicationRepository',
:type trust_enabled: boolean :type trust_enabled: boolean
""" """
def to_dict(self, can_write, can_admin): def to_dict(self):
return { return {
'namespace': self.repository_base_elements.namespace_name, 'namespace': self.repository_base_elements.namespace_name,
'name': self.repository_base_elements.repository_name, 'name': self.repository_base_elements.repository_name,
'kind': self.repository_base_elements.kind_name, 'kind': self.repository_base_elements.kind_name,
'description': self.repository_base_elements.description, 'description': self.repository_base_elements.description,
'can_write': can_write,
'can_admin': can_admin,
'is_public': self.repository_base_elements.is_public, 'is_public': self.repository_base_elements.is_public,
'is_organization': self.repository_base_elements.namespace_user_organization, 'is_organization': self.repository_base_elements.namespace_user_organization,
'is_starred': self.repository_base_elements.is_starred, 'is_starred': self.repository_base_elements.is_starred,
'tags': {tag.name: tag.to_dict() for tag in self.tags}, 'tags': {tag.name: tag.to_dict()
for tag in self.tags},
'status_token': self.badge_token if not self.repository_base_elements.is_public else '', 'status_token': self.badge_token if not self.repository_base_elements.is_public else '',
'trust_enabled': bool(features.SIGNING) and self.trust_enabled, 'trust_enabled': bool(features.SIGNING) and self.trust_enabled,
'tag_expiration_s': self.repository_base_elements.namespace_user_removed_tag_expiration_s, 'tag_expiration_s': self.repository_base_elements.namespace_user_removed_tag_expiration_s,
} }
class Repository(namedtuple('Repository', ['namespace_name', 'repository_name', ])): class Repository(namedtuple('Repository', [
'namespace_name',
'repository_name',
])):
""" """
Repository a single quay repository Repository a single quay repository
:type namespace_name: string :type namespace_name: string
@ -132,24 +130,29 @@ class Channel(namedtuple('Channel', ['name', 'linked_tag_name', 'linked_tag_life
} }
class Release(namedtuple('Channel', ['name', 'released', 'lifetime_start'])): class Release(
namedtuple('Channel', ['name', 'released', 'lifetime_start', 'releases_channels_map'])):
""" """
Repository a single quay repository Repository a single quay repository
:type name: string :type name: string
:type released: string :type released: string
:type last_modified: string :type last_modified: string
:type releases_channels_map: {string -> string}
""" """
def to_dict(self, releases_channels_map): def to_dict(self):
return { return {
'name': self.name, 'name': self.name,
'last_modified': format_date(datetime.fromtimestamp(self.lifetime_start / 1000)), 'last_modified': format_date(datetime.fromtimestamp(self.lifetime_start / 1000)),
'channels': releases_channels_map[self.name], 'channels': self.releases_channels_map[self.name],
} }
class Tag(namedtuple('Tag', ['name', 'image_docker_image_id', 'image_aggregate_size', 'lifetime_start_ts', class Tag(
'tag_manifest_digest'])): namedtuple('Tag', [
'name', 'image_docker_image_id', 'image_aggregate_size', 'lifetime_start_ts',
'tag_manifest_digest'
])):
""" """
:type name: string :type name: string
:type image_docker_image_id: string :type image_docker_image_id: string

37
endpoints/api/repository_models_pre_oci.py
View file

@ -79,11 +79,9 @@ class PreOCIModel(RepositoryDataInterface):
# Also note the +1 on the limit, as paginate_query uses the extra result to determine whether # Also note the +1 on the limit, as paginate_query uses the extra result to determine whether
# there is a next page. # there is a next page.
start_id = model.modelutil.pagination_start(page_token) start_id = model.modelutil.pagination_start(page_token)
repo_query = model.repository.get_visible_repositories(username=username, repo_query = model.repository.get_visible_repositories(
include_public=public, username=username, include_public=public, start_id=start_id, limit=REPOS_PER_PAGE + 1,
start_id=start_id, kind_filter=repo_kind)
limit=REPOS_PER_PAGE + 1,
kind_filter=repo_kind)
repos, next_page_token = model.modelutil.paginate_query(repo_query, limit=REPOS_PER_PAGE, repos, next_page_token = model.modelutil.paginate_query(repo_query, limit=REPOS_PER_PAGE,
id_alias='rid') id_alias='rid')
@ -139,29 +137,30 @@ class PreOCIModel(RepositoryDataInterface):
is_starred = model.repository.repository_is_starred(user, repo) if user else False is_starred = model.repository.repository_is_starred(user, repo) if user else False
is_public = model.repository.is_repository_public(repo) is_public = model.repository.is_repository_public(repo)
base = RepositoryBaseElement(namespace_name, repository_name, is_starred, is_public, repo.kind.name, base = RepositoryBaseElement(
repo.description, repo.namespace_user.organization, namespace_name, repository_name, is_starred, is_public, repo.kind.name, repo.description,
repo.namespace_user.removed_tag_expiration_s, None, None) repo.namespace_user.organization, repo.namespace_user.removed_tag_expiration_s, None, None,
False, False, False)
# Note: This is *temporary* code for the new OCI model stuff. # Note: This is *temporary* code for the new OCI model stuff.
if base.kind_name == 'application': if base.kind_name == 'application':
channels = oci_model.channel.get_repo_channels(repo) channels = oci_model.channel.get_repo_channels(repo)
releases = oci_model.release.get_release_objs(repo) releases = oci_model.release.get_release_objs(repo)
releases_channels_map = defaultdict(list)
return ApplicationRepository(base, return ApplicationRepository(
[Channel(channel.name, channel.linked_tag.name, channel.linked_tag.lifetime_start) base, [create_channel(channel, releases_channels_map) for channel in channels], [
for channel in channels], Release(release.name, release.released, release.lifetime_start, releases_channels_map)
[Release(release.name, release.released, release.lifetime_start) for release in releases
for release in releases]) ])
tags = model.tag.list_active_repo_tags(repo) tags = model.tag.list_active_repo_tags(repo)
start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS) start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
counts = model.log.get_repository_action_counts(repo, start_date) counts = model.log.get_repository_action_counts(repo, start_date)
return NonApplicationRepository(base,
[Tag(tag.name, tag.image.docker_image_id, tag.image.aggregate_size, return ImageRepositoryRepository(base, [
tag.lifetime_start_ts, tag.tagmanifest.digest) for tag in tags], Tag(tag.name, tag.image.docker_image_id, tag.image.aggregate_size, tag.lifetime_start_ts,
[Count(count.date, count.count) for count in counts], tag.tagmanifest.digest) for tag in tags
repo.badge_token, repo.trust_enabled) ], [Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled)
pre_oci_model = PreOCIModel() pre_oci_model = PreOCIModel()

38
endpoints/api/test/test_repository.py
View file

@ -9,7 +9,6 @@ from features import FeatureNameValue
from test.fixtures import * from test.fixtures import *
INVALID_RESPONSE = { INVALID_RESPONSE = {
u'detail': u"u'invalid_req' is not of type 'boolean'", u'detail': u"u'invalid_req' is not of type 'boolean'",
u'error_message': u"u'invalid_req' is not of type 'boolean'", u'error_message': u"u'invalid_req' is not of type 'boolean'",
@ -20,31 +19,44 @@ INVALID_RESPONSE = {
} }
NOT_FOUND_RESPONSE = { NOT_FOUND_RESPONSE = {
u'detail': u'Not Found', u'detail':
u'error_message': u'Not Found', u'Not Found',
u'error_type': u'not_found', u'error_message':
u'message': u'You have requested this URI [/api/v1/repository/devtable/repo/changetrust] but did you mean /api/v1/repository/<apirepopath:repository>/changetrust or /api/v1/repository/<apirepopath:repository>/changevisibility or /api/v1/repository/<apirepopath:repository>/tag/<tag>/images ?', u'Not Found',
u'status': 404, u'error_type':
u'title': u'not_found', u'not_found',
u'type': u'http://localhost/api/v1/error/not_found' u'message':
u'You have requested this URI [/api/v1/repository/devtable/repo/changetrust] but did you mean /api/v1/repository/<apirepopath:repository>/changetrust or /api/v1/repository/<apirepopath:repository>/changevisibility or /api/v1/repository/<apirepopath:repository>/tag/<tag>/images ?',
u'status':
404,
u'title':
u'not_found',
u'type':
u'http://localhost/api/v1/error/not_found'
} }
@pytest.mark.parametrize('trust_enabled,repo_found,expected_body,expected_status', [ @pytest.mark.parametrize('trust_enabled,repo_found,expected_body,expected_status', [
(True, True,{'success': True}, 200), (True, True, {
(False, True, {'success': True}, 200), 'success': True
}, 200),
(False, True, {
'success': True
}, 200),
(False, False, NOT_FOUND_RESPONSE, 404), (False, False, NOT_FOUND_RESPONSE, 404),
('invalid_req', False, INVALID_RESPONSE , 400), ('invalid_req', False, INVALID_RESPONSE, 400),
]) ])
def test_post_changetrust(trust_enabled, repo_found, expected_body, expected_status, client): def test_post_changetrust(trust_enabled, repo_found, expected_body, expected_status, client):
with patch('endpoints.api.repository.tuf_metadata_api') as mock_tuf: with patch('endpoints.api.repository.tuf_metadata_api') as mock_tuf:
with patch('endpoints.api.repository_models_pre_oci.model.repository.get_repository') as mock_model: with patch(
'endpoints.api.repository_models_pre_oci.model.repository.get_repository') as mock_model:
mock_model.return_value = MagicMock() if repo_found else None mock_model.return_value = MagicMock() if repo_found else None
mock_tuf.get_default_tags_with_expiration.return_value = ['tags', 'expiration'] mock_tuf.get_default_tags_with_expiration.return_value = ['tags', 'expiration']
with client_with_identity('devtable', client) as cl: with client_with_identity('devtable', client) as cl:
params = {'repository': 'devtable/repo'} params = {'repository': 'devtable/repo'}
request_body = {'trust_enabled': trust_enabled} request_body = {'trust_enabled': trust_enabled}
assert expected_body == conduct_api_call(cl, RepositoryTrust, 'POST', params, request_body, expected_status).json assert expected_body == conduct_api_call(cl, RepositoryTrust, 'POST', params, request_body,
expected_status).json
def test_signing_disabled(client): def test_signing_disabled(client):

1785
test/test_api_usage.py
View file

File diff suppressed because it is too large Load diff