vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2814 from charltonaustin/create_data_interface_for_subsystem_api/repository_633

Browse source 
refactor(endpoints/api/repository*): added in pre_oci_model abstraction
This commit is contained in:
Charlton Austin 2017-07-25 13:59:52 -04:00 committed by GitHub
commit be206a8b88
6 changed files with 1278 additions and 1487 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

217
data/model/repository.py
View file

@ -7,17 +7,16 @@ from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
from playhouse.shortcuts import case from playhouse.shortcuts import case
from cachetools import ttl_cache from cachetools import ttl_cache
from data.model import (config, DataModelException, tag, db_transaction, storage, permission, from data.model import (
_basequery) config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, from data.database import (
Visibility, RepositoryPermission, RepositoryActionCount, Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage, RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
Label, TagManifestLabel, db_for_update, get_epoch_timestamp, DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore) db_random_func, db_concat_func, RepositorySearchScore)
from data.text import prefix_search from data.text import prefix_search
from util.itertoolrecipes import take from util.itertoolrecipes import take
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
SEARCH_FIELDS = Enum("SearchFields", ["name", "description"]) SEARCH_FIELDS = Enum("SearchFields", ["name", "description"])
@ -87,8 +86,7 @@ def purge_repository(namespace_name, repository_name):
unreferenced_image_q = Image.select(Image.id).where(Image.repository == repo) unreferenced_image_q = Image.select(Image.id).where(Image.repository == repo)
if len(previously_referenced) > 0: if len(previously_referenced) > 0:
unreferenced_image_q = (unreferenced_image_q unreferenced_image_q = (unreferenced_image_q.where(~(Image.id << list(previously_referenced))))
.where(~(Image.id << list(previously_referenced))))
unreferenced_candidates = set(img[0] for img in unreferenced_image_q.tuples()) unreferenced_candidates = set(img[0] for img in unreferenced_image_q.tuples())
@ -116,11 +114,10 @@ def purge_repository(namespace_name, repository_name):
@ttl_cache(maxsize=1, ttl=600) @ttl_cache(maxsize=1, ttl=600)
def _get_gc_expiration_policies(): def _get_gc_expiration_policies():
policy_tuples_query = (Namespace policy_tuples_query = (
.select(Namespace.removed_tag_expiration_s) Namespace.select(Namespace.removed_tag_expiration_s).distinct()
.distinct() .limit(100) # This sucks but it's the only way to limit memory
.limit(100) # This sucks but it's the only way to limit memory .tuples())
.tuples())
return [policy[0] for policy in policy_tuples_query] return [policy[0] for policy in policy_tuples_query]
@ -134,22 +131,15 @@ def find_repository_with_garbage(limit_to_gc_policy_s):
expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s
try: try:
candidates = (RepositoryTag candidates = (RepositoryTag.select(RepositoryTag.repository).join(Repository)
.select(RepositoryTag.repository)
.join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(~(RepositoryTag.lifetime_end_ts >> None), .where(~(RepositoryTag.lifetime_end_ts >> None),
(RepositoryTag.lifetime_end_ts <= expiration_timestamp), (RepositoryTag.lifetime_end_ts <= expiration_timestamp),
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s)) (Namespace.removed_tag_expiration_s == limit_to_gc_policy_s)).limit(500)
.limit(500) .distinct().alias('candidates'))
.distinct()
.alias('candidates'))
found = (RepositoryTag found = (RepositoryTag.select(candidates.c.repository_id).from_(candidates)
.select(candidates.c.repository_id) .order_by(db_random_func()).get())
.from_(candidates)
.order_by(db_random_func())
.get())
if found is None: if found is None:
return return
@ -186,10 +176,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
all_unreferenced_candidates = set() all_unreferenced_candidates = set()
# Remove any images directly referenced by tags, to prune the working set. # Remove any images directly referenced by tags, to prune the working set.
direct_referenced = (RepositoryTag direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
.select(RepositoryTag.image) RepositoryTag.repository == repo.id, RepositoryTag.image << list(candidate_orphan_image_set)))
.where(RepositoryTag.repository == repo.id,
RepositoryTag.image << list(candidate_orphan_image_set)))
candidate_orphan_image_set.difference_update([t.image_id for t in direct_referenced]) candidate_orphan_image_set.difference_update([t.image_id for t in direct_referenced])
# Iteratively try to remove images from the database. The only images we can remove are those # Iteratively try to remove images from the database. The only images we can remove are those
@ -205,26 +193,20 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
with db_transaction(): with db_transaction():
# Any image directly referenced by a tag that still exists, cannot be GCed. # Any image directly referenced by a tag that still exists, cannot be GCed.
direct_referenced = (RepositoryTag direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
.select(RepositoryTag.image) RepositoryTag.repository == repo.id, RepositoryTag.image << candidates_orphans))
.where(RepositoryTag.repository == repo.id,
RepositoryTag.image << candidates_orphans))
# Any image which is the parent of another image, cannot be GCed. # Any image which is the parent of another image, cannot be GCed.
parent_referenced = (Image parent_referenced = (Image.select(Image.parent).where(Image.repository == repo.id,
.select(Image.parent) Image.parent << candidates_orphans))
.where(Image.repository == repo.id,
Image.parent << candidates_orphans))
referenced_candidates = (direct_referenced | parent_referenced) referenced_candidates = (direct_referenced | parent_referenced)
# We desire a few pieces of information from the database from the following # We desire a few pieces of information from the database from the following
# query: all of the image ids which are associated with this repository, # query: all of the image ids which are associated with this repository,
# and the storages which are associated with those images. # and the storages which are associated with those images.
unreferenced_candidates = (Image unreferenced_candidates = (Image.select(Image.id, Image.docker_image_id, ImageStorage.id,
.select(Image.id, Image.docker_image_id, ImageStorage.uuid).join(ImageStorage)
ImageStorage.id, ImageStorage.uuid)
.join(ImageStorage)
.where(Image.id << candidates_orphans, .where(Image.id << candidates_orphans,
~(Image.id << referenced_candidates))) ~(Image.id << referenced_candidates)))
@ -238,8 +220,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
storage_id_whitelist = set([candidate.storage_id for candidate in unreferenced_candidates]) storage_id_whitelist = set([candidate.storage_id for candidate in unreferenced_candidates])
# Lookup any derived images for the images to remove. # Lookup any derived images for the images to remove.
derived = DerivedStorageForImage.select().where( derived = DerivedStorageForImage.select().where(DerivedStorageForImage.source_image <<
DerivedStorageForImage.source_image << image_ids_to_remove) image_ids_to_remove)
has_derived = False has_derived = False
for derived_image in derived: for derived_image in derived:
@ -249,10 +231,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
# Delete any derived images and the images themselves. # Delete any derived images and the images themselves.
if has_derived: if has_derived:
try: try:
(DerivedStorageForImage (DerivedStorageForImage.delete()
.delete() .where(DerivedStorageForImage.source_image << image_ids_to_remove).execute())
.where(DerivedStorageForImage.source_image << image_ids_to_remove)
.execute())
except IntegrityError: except IntegrityError:
logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove) logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove)
return False return False
@ -278,8 +258,10 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
# If any storages were removed and cleanup callbacks are registered, call them with # If any storages were removed and cleanup callbacks are registered, call them with
# the images+storages removed. # the images+storages removed.
if storage_ids_removed and config.image_cleanup_callbacks: if storage_ids_removed and config.image_cleanup_callbacks:
image_storages_removed = [candidate for candidate in all_unreferenced_candidates image_storages_removed = [
if candidate.storage_id in storage_ids_removed] candidate for candidate in all_unreferenced_candidates
if candidate.storage_id in storage_ids_removed
]
for callback in config.image_cleanup_callbacks: for callback in config.image_cleanup_callbacks:
callback(image_storages_removed) callback(image_storages_removed)
@ -295,10 +277,7 @@ def star_repository(user, repository):
def unstar_repository(user, repository): def unstar_repository(user, repository):
""" Unstars a repository. """ """ Unstars a repository. """
try: try:
(Star (Star.delete().where(Star.repository == repository.id, Star.user == user.id).execute())
.delete()
.where(Star.repository == repository.id, Star.user == user.id)
.execute())
except Star.DoesNotExist: except Star.DoesNotExist:
raise DataModelException('Star not found.') raise DataModelException('Star not found.')
@ -308,6 +287,11 @@ def set_trust(repo, trust_enabled):
repo.save() repo.save()
def set_description(repo, description):
repo.description = description
repo.save()
def get_user_starred_repositories(user, kind_filter='image'): def get_user_starred_repositories(user, kind_filter='image'):
""" Retrieves all of the repositories a user has starred. """ """ Retrieves all of the repositories a user has starred. """
try: try:
@ -315,13 +299,8 @@ def get_user_starred_repositories(user, kind_filter='image'):
except RepositoryKind.DoesNotExist: except RepositoryKind.DoesNotExist:
raise DataModelException('Unknown kind of repository') raise DataModelException('Unknown kind of repository')
query = (Repository query = (Repository.select(Repository, User, Visibility, Repository.id.alias('rid')).join(Star)
.select(Repository, User, Visibility, Repository.id.alias('rid')) .switch(Repository).join(User).switch(Repository).join(Visibility)
.join(Star)
.switch(Repository)
.join(User)
.switch(Repository)
.join(Visibility)
.where(Star.user == user, Repository.kind == repo_kind)) .where(Star.user == user, Repository.kind == repo_kind))
return query return query
@ -330,10 +309,7 @@ def get_user_starred_repositories(user, kind_filter='image'):
def repository_is_starred(user, repository): def repository_is_starred(user, repository):
""" Determines whether a user has starred a repository or not. """ """ Determines whether a user has starred a repository or not. """
try: try:
(Star (Star.select().where(Star.repository == repository.id, Star.user == user.id).get())
.select()
.where(Star.repository == repository.id, Star.user == user.id)
.get())
return True return True
except Star.DoesNotExist: except Star.DoesNotExist:
return False return False
@ -346,10 +322,8 @@ def get_when_last_modified(repository_ids):
if not repository_ids: if not repository_ids:
return {} return {}
tuples = (RepositoryTag tuples = (RepositoryTag.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts))
.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts)) .where(RepositoryTag.repository << repository_ids).group_by(RepositoryTag.repository)
.where(RepositoryTag.repository << repository_ids)
.group_by(RepositoryTag.repository)
.tuples()) .tuples())
last_modified_map = {} last_modified_map = {}
@ -366,11 +340,8 @@ def get_stars(repository_ids):
if not repository_ids: if not repository_ids:
return {} return {}
tuples = (Star tuples = (Star.select(Star.repository, fn.Count(Star.id))
.select(Star.repository, fn.Count(Star.id)) .where(Star.repository << repository_ids).group_by(Star.repository).tuples())
.where(Star.repository << repository_ids)
.group_by(Star.repository)
.tuples())
star_map = {} star_map = {}
for record in tuples: for record in tuples:
@ -388,12 +359,10 @@ def get_visible_repositories(username, namespace=None, kind_filter='image', incl
# here, as it will be modified by other queries later on. # here, as it will be modified by other queries later on.
return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1) return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1)
query = (Repository query = (Repository.select(Repository.name,
.select(Repository.name, Repository.id.alias('rid'), Repository.id.alias('rid'), Repository.description,
Repository.description, Namespace.username, Repository.visibility, Namespace.username, Repository.visibility, Repository.kind)
Repository.kind) .switch(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id)))
.switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)))
if username: if username:
# Note: We only need the permissions table if we will filter based on a user's permissions. # Note: We only need the permissions table if we will filter based on a user's permissions.
@ -422,8 +391,8 @@ def get_app_search(lookup, search_fields=None, username=None, limit=50):
search_fields = set([SEARCH_FIELDS.name.name]) search_fields = set([SEARCH_FIELDS.name.name])
return get_filtered_matching_repositories(lookup, filter_username=username, return get_filtered_matching_repositories(lookup, filter_username=username,
search_fields=search_fields, search_fields=search_fields, repo_kind='application',
repo_kind='application', offset=0, limit=limit) offset=0, limit=limit)
def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image', def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image',
@ -460,7 +429,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
unfiltered_page = 0 unfiltered_page = 0
iteration_count = 0 iteration_count = 0
while iteration_count < 10: # Just to be safe while iteration_count < 10: # Just to be safe
# Find the next chunk's worth of repository IDs, paginated by the chunk size. # Find the next chunk's worth of repository IDs, paginated by the chunk size.
unfiltered_page = unfiltered_page + 1 unfiltered_page = unfiltered_page + 1
found_ids = [r.id for r in unfiltered_query.paginate(unfiltered_page, chunk_count)] found_ids = [r.id for r in unfiltered_query.paginate(unfiltered_page, chunk_count)]
@ -476,13 +445,9 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
encountered.update(new_unfiltered_ids) encountered.update(new_unfiltered_ids)
# Filter the repositories found to only those visible to the current user. # Filter the repositories found to only those visible to the current user.
query = (Repository query = (Repository.select(Repository, Namespace).distinct()
.select(Repository, Namespace) .join(Namespace, on=(Namespace.id == Repository.namespace_user)).switch(Repository)
.distinct() .join(RepositoryPermission).where(Repository.id << list(new_unfiltered_ids)))
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.switch(Repository)
.join(RepositoryPermission)
.where(Repository.id << list(new_unfiltered_ids)))
filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind) filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind)
@ -520,16 +485,12 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if SEARCH_FIELDS.description.name in search_fields: if SEARCH_FIELDS.description.name in search_fields:
clause = Repository.description.match(lookup_value) | clause clause = Repository.description.match(lookup_value) | clause
cases = [ cases = [(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),]
(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),
]
computed_score = case(None, cases, RepositorySearchScore.score).alias('score') computed_score = case(None, cases, RepositorySearchScore.score).alias('score')
query = (Repository query = (Repository.select(Repository, Namespace, computed_score)
.select(Repository, Namespace, computed_score) .join(Namespace, on=(Namespace.id == Repository.namespace_user)).where(clause)
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
.where(clause)
.group_by(Repository.id, Namespace.id)) .group_by(Repository.id, Namespace.id))
if repo_kind is not None: if repo_kind is not None:
@ -538,11 +499,8 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
if not include_private: if not include_private:
query = query.where(Repository.visibility == _basequery.get_public_repo_visibility()) query = query.where(Repository.visibility == _basequery.get_public_repo_visibility())
query = (query query = (query.switch(Repository).join(RepositorySearchScore)
.switch(Repository) .group_by(Repository, Namespace, RepositorySearchScore).order_by(SQL('score').desc()))
.join(RepositorySearchScore)
.group_by(Repository, Namespace, RepositorySearchScore)
.order_by(SQL('score').desc()))
return query return query
@ -560,15 +518,10 @@ def is_repository_public(repository):
def repository_is_public(namespace_name, repository_name): def repository_is_public(namespace_name, repository_name):
try: try:
(Repository (Repository.select().join(Namespace, on=(Repository.namespace_user == Namespace.id))
.select() .switch(Repository).join(Visibility).where(Namespace.username == namespace_name,
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) Repository.name == repository_name,
.switch(Repository) Visibility.name == 'public').get())
.join(Visibility)
.where(Namespace.username == namespace_name,
Repository.name == repository_name,
Visibility.name == 'public')
.get())
return True return True
except Repository.DoesNotExist: except Repository.DoesNotExist:
return False return False
@ -585,14 +538,10 @@ def set_repository_visibility(repo, visibility):
def get_email_authorized_for_repo(namespace, repository, email): def get_email_authorized_for_repo(namespace, repository, email):
try: try:
return (RepositoryAuthorizedEmail return (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.select(RepositoryAuthorizedEmail, Repository, Namespace) .join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(Repository) .where(Namespace.username == namespace, Repository.name == repository,
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) RepositoryAuthorizedEmail.email == email).get())
.where(Namespace.username == namespace,
Repository.name == repository,
RepositoryAuthorizedEmail.email == email)
.get())
except RepositoryAuthorizedEmail.DoesNotExist: except RepositoryAuthorizedEmail.DoesNotExist:
return None return None
@ -601,20 +550,16 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email):
try: try:
repo = _basequery.get_existing_repository(namespace_name, repository_name) repo = _basequery.get_existing_repository(namespace_name, repository_name)
except Repository.DoesNotExist: except Repository.DoesNotExist:
raise DataModelException('Invalid repository %s/%s' % raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
(namespace_name, repository_name))
return RepositoryAuthorizedEmail.create(repository=repo, email=email, confirmed=False) return RepositoryAuthorizedEmail.create(repository=repo, email=email, confirmed=False)
def confirm_email_authorization_for_repo(code): def confirm_email_authorization_for_repo(code):
try: try:
found = (RepositoryAuthorizedEmail found = (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
.select(RepositoryAuthorizedEmail, Repository, Namespace) .join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
.join(Repository) .where(RepositoryAuthorizedEmail.code == code).get())
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(RepositoryAuthorizedEmail.code == code)
.get())
except RepositoryAuthorizedEmail.DoesNotExist: except RepositoryAuthorizedEmail.DoesNotExist:
raise DataModelException('Invalid confirmation code.') raise DataModelException('Invalid confirmation code.')
@ -626,17 +571,13 @@ def confirm_email_authorization_for_repo(code):
def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'): def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'):
cutoff = datetime.now() - time_span cutoff = datetime.now() - time_span
return (Repository return (Repository.select(Namespace.username, Repository.name)
.select(Namespace.username, Repository.name) .join(Namespace, on=(Repository.namespace_user == Namespace.id)).switch(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) .join(RepositoryActionCount).where(RepositoryActionCount.date >= cutoff,
.switch(Repository) Repository.visibility == get_public_repo_visibility(),
.join(RepositoryActionCount) Repository.kind == Repository.kind.get_id(repo_kind))
.where(RepositoryActionCount.date >= cutoff,
Repository.visibility == get_public_repo_visibility(),
Repository.kind == Repository.kind.get_id(repo_kind))
.group_by(RepositoryActionCount.repository, Repository.name, Namespace.username) .group_by(RepositoryActionCount.repository, Repository.name, Namespace.username)
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold) .having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold).tuples())
.tuples())
def is_empty(namespace_name, repository_name): def is_empty(namespace_name, repository_name):

298
endpoints/api/repository.py
View file

@ -10,14 +10,13 @@ from datetime import timedelta, datetime
from flask import request, abort from flask import request, abort
from app import dockerfile_build_queue, tuf_metadata_api from app import dockerfile_build_queue, tuf_metadata_api
from data import model, oci_model from endpoints.api import (
from endpoints.api import (format_date, nickname, log_action, validate_json_request, format_date, nickname, log_action, validate_json_request, require_repo_read, require_repo_write,
require_repo_read, require_repo_write, require_repo_admin, require_repo_admin, RepositoryParamResource, resource, parse_args, ApiResource, request_error,
RepositoryParamResource, resource, parse_args, ApiResource, require_scope, path_param, page_support, query_param, truthy_bool, show_if)
request_error, require_scope, path_param, page_support, from endpoints.api.repository_models_pre_oci import pre_oci_model as model
query_param, truthy_bool, show_if) from endpoints.exception import (
from endpoints.exception import (Unauthorized, NotFound, InvalidRequest, ExceedsLicenseException, Unauthorized, NotFound, InvalidRequest, ExceedsLicenseException, DownstreamIssue)
DownstreamIssue)
from endpoints.api.billing import lookup_allowed_private_repos, get_namespace_plan from endpoints.api.billing import lookup_allowed_private_repos, get_namespace_plan
from endpoints.api.subscribe import check_repository_usage from endpoints.api.subscribe import check_repository_usage
@ -27,12 +26,12 @@ from auth.auth_context import get_authenticated_user
from auth import scopes from auth import scopes
from util.names import REPOSITORY_NAME_REGEX from util.names import REPOSITORY_NAME_REGEX
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
REPOS_PER_PAGE = 100 REPOS_PER_PAGE = 100
MAX_DAYS_IN_3_MONTHS = 92 MAX_DAYS_IN_3_MONTHS = 92
def check_allowed_private_repos(namespace): def check_allowed_private_repos(namespace):
""" Checks to see if the given namespace has reached its private repository limit. If so, """ Checks to see if the given namespace has reached its private repository limit. If so,
raises a ExceedsLicenseException. raises a ExceedsLicenseException.
@ -71,7 +70,8 @@ class RepositoryList(ApiResource):
], ],
}, },
'namespace': { 'namespace': {
'type': 'string', 'type':
'string',
'description': ('Namespace in which the repository should be created. If omitted, the ' 'description': ('Namespace in which the repository should be created. If omitted, the '
'username of the caller is used'), 'username of the caller is used'),
}, },
@ -106,8 +106,7 @@ class RepositoryList(ApiResource):
repository_name = req['repository'] repository_name = req['repository']
visibility = req['visibility'] visibility = req['visibility']
existing = model.repository.get_repository(namespace_name, repository_name) if model.repo_exists(namespace_name, repository_name):
if existing:
raise request_error(message='Repository already exists') raise request_error(message='Repository already exists')
visibility = req['visibility'] visibility = req['visibility']
@ -119,13 +118,12 @@ class RepositoryList(ApiResource):
raise InvalidRequest('Invalid repository name') raise InvalidRequest('Invalid repository name')
kind = req.get('repo_kind', 'image') or 'image' kind = req.get('repo_kind', 'image') or 'image'
repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility, model.create_repo(namespace_name, repository_name, owner, req['description'],
repo_kind=kind) visibility=visibility, repo_kind=kind)
repo.description = req['description']
repo.save()
log_action('create_repo', namespace_name, {'repo': repository_name, log_action('create_repo', namespace_name,
'namespace': namespace_name}, repo=repo) {'repo': repository_name,
'namespace': namespace_name}, repo_name=repository_name)
return { return {
'namespace': namespace_name, 'namespace': namespace_name,
'name': repository_name, 'name': repository_name,
@ -134,7 +132,6 @@ class RepositoryList(ApiResource):
raise Unauthorized() raise Unauthorized()
@require_scope(scopes.READ_REPO) @require_scope(scopes.READ_REPO)
@nickname('listRepos') @nickname('listRepos')
@parse_args() @parse_args()
@ -160,89 +157,18 @@ class RepositoryList(ApiResource):
user = get_authenticated_user() user = get_authenticated_user()
username = user.username if user else None username = user.username if user else None
next_page_token = None last_modified = parsed_args['last_modified']
repos = None popularity = parsed_args['popularity']
repo_kind = parsed_args['repo_kind']
# Lookup the requested repositories (either starred or non-starred.) if parsed_args['starred'] and not username:
if parsed_args['starred']: # No repositories should be returned, as there is no user.
if not username: abort(400)
# No repositories should be returned, as there is no user.
abort(400)
# Return the full list of repos starred by the current user that are still visible to them. repos, next_page_token = model.get_repo_list(
def can_view_repo(repo): parsed_args['starred'], user, parsed_args['repo_kind'], parsed_args['namespace'], username,
return ReadRepositoryPermission(repo.namespace_user.username, repo.name).can() parsed_args['public'], page_token, last_modified, popularity)
unfiltered_repos = model.repository.get_user_starred_repositories(user, kind_filter=repo_kind) return {'repositories': [repo.to_dict() for repo in repos]}, next_page_token
repos = [repo for repo in unfiltered_repos if can_view_repo(repo)]
elif parsed_args['namespace']:
# Repositories filtered by namespace do not need pagination (their results are fairly small),
# so we just do the lookup directly.
repos = list(model.repository.get_visible_repositories(username=username,
include_public=parsed_args['public'],
namespace=parsed_args['namespace'],
kind_filter=repo_kind))
else:
# Determine the starting offset for pagination. Note that we don't use the normal
# model.modelutil.paginate method here, as that does not operate over UNION queries, which
# get_visible_repositories will return if there is a logged-in user (for performance reasons).
#
# Also note the +1 on the limit, as paginate_query uses the extra result to determine whether
# there is a next page.
start_id = model.modelutil.pagination_start(page_token)
repo_query = model.repository.get_visible_repositories(username=username,
include_public=parsed_args['public'],
start_id=start_id,
limit=REPOS_PER_PAGE+1,
kind_filter=repo_kind)
repos, next_page_token = model.modelutil.paginate_query(repo_query, limit=REPOS_PER_PAGE,
id_alias='rid')
# Collect the IDs of the repositories found for subequent lookup of popularity
# and/or last modified.
if parsed_args['last_modified'] or parsed_args['popularity']:
repository_ids = [repo.rid for repo in repos]
if parsed_args['last_modified']:
last_modified_map = model.repository.get_when_last_modified(repository_ids)
if parsed_args['popularity']:
action_sum_map = model.log.get_repositories_action_sums(repository_ids)
# Collect the IDs of the repositories that are starred for the user, so we can mark them
# in the returned results.
star_set = set()
if username:
starred_repos = model.repository.get_user_starred_repositories(user)
star_set = {starred.id for starred in starred_repos}
def repo_view(repo_obj):
repo = {
'namespace': repo_obj.namespace_user.username,
'name': repo_obj.name,
'description': repo_obj.description,
'is_public': repo_obj.visibility_id == model.repository.get_public_repo_visibility().id,
'kind': repo_kind,
}
repo_id = repo_obj.rid
if parsed_args['last_modified']:
repo['last_modified'] = last_modified_map.get(repo_id)
if parsed_args['popularity']:
repo['popularity'] = float(action_sum_map.get(repo_id, 0))
if username:
repo['is_starred'] = repo_id in star_set
return repo
return {
'repositories': [repo_view(repo) for repo in repos]
}, next_page_token
@resource('/v1/repository/<apirepopath:repository>') @resource('/v1/repository/<apirepopath:repository>')
@ -253,9 +179,7 @@ class Repository(RepositoryParamResource):
'RepoUpdate': { 'RepoUpdate': {
'type': 'object', 'type': 'object',
'description': 'Fields which can be updated in a repository.', 'description': 'Fields which can be updated in a repository.',
'required': [ 'required': ['description',],
'description',
],
'properties': { 'properties': {
'description': { 'description': {
'type': 'string', 'type': 'string',
@ -273,160 +197,66 @@ class Repository(RepositoryParamResource):
def get(self, namespace, repository, parsed_args): def get(self, namespace, repository, parsed_args):
"""Fetch the specified repository.""" """Fetch the specified repository."""
logger.debug('Get repo: %s/%s' % (namespace, repository)) logger.debug('Get repo: %s/%s' % (namespace, repository))
repo = model.get_repo(namespace, repository, get_authenticated_user())
repo = model.repository.get_repository(namespace, repository)
if repo is None: if repo is None:
raise NotFound() raise NotFound()
can_write = ModifyRepositoryPermission(namespace, repository).can() repo_data = repo.to_dict()
can_admin = AdministerRepositoryPermission(namespace, repository).can() repo_data['can_write'] = ModifyRepositoryPermission(namespace, repository).can()
repo_data['can_admin'] = AdministerRepositoryPermission(namespace, repository).can()
is_starred = (model.repository.repository_is_starred(get_authenticated_user(), repo) if parsed_args['includeStats'] and repo.repository_base_elements.kind_name != 'application':
if get_authenticated_user() else False)
is_public = model.repository.is_repository_public(repo)
# Note: This is *temporary* code for the new OCI model stuff.
if repo.kind.name == 'application':
def channel_view(channel):
return {
'name': channel.name,
'release': channel.linked_tag.name,
'last_modified': format_date(datetime.fromtimestamp(channel.linked_tag.lifetime_start / 1000)),
}
def release_view(release):
return {
'name': release.name,
'last_modified': format_date(datetime.fromtimestamp(release.lifetime_start / 1000)),
'channels': releases_channels_map[release.name],
}
channels = oci_model.channel.get_repo_channels(repo)
releases_channels_map = defaultdict(list)
for channel in channels:
releases_channels_map[channel.linked_tag.name].append(channel.name)
repo_data = {
'namespace': namespace,
'name': repository,
'kind': repo.kind.name,
'description': repo.description,
'can_write': can_write,
'can_admin': can_admin,
'is_public': is_public,
'is_organization': repo.namespace_user.organization,
'is_starred': is_starred,
'channels': [channel_view(chan) for chan in channels],
'releases': [release_view(release) for release in oci_model.release.get_release_objs(repo)],
}
return repo_data
# Older image-only repo code.
def tag_view(tag):
tag_info = {
'name': tag.name,
'image_id': tag.image.docker_image_id,
'size': tag.image.aggregate_size
}
if tag.lifetime_start_ts > 0:
last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts))
tag_info['last_modified'] = last_modified
if tag.lifetime_end_ts:
expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts))
tag_info['expiration'] = expiration
if tag.tagmanifest is not None:
tag_info['manifest_digest'] = tag.tagmanifest.digest
return tag_info
stats = None
tags = model.tag.list_active_repo_tags(repo)
tag_dict = {tag.name: tag_view(tag) for tag in tags}
if parsed_args['includeStats']:
stats = [] stats = []
found_dates = {} found_dates = {}
start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS) for count in repo.counts:
counts = model.log.get_repository_action_counts(repo, start_date) stats.append(count.to_dict())
for count in counts:
stats.append({
'date': count.date.isoformat(),
'count': count.count,
})
found_dates['%s/%s' % (count.date.month, count.date.day)] = True found_dates['%s/%s' % (count.date.month, count.date.day)] = True
# Fill in any missing stats with zeros. # Fill in any missing stats with zeros.
for day in range(1, MAX_DAYS_IN_3_MONTHS): for day in range(1, MAX_DAYS_IN_3_MONTHS):
day_date = datetime.now() - timedelta(days=day) day_date = datetime.now() - timedelta(days=day)
key = '%s/%s' % (day_date.month, day_date.day) key = '%s/%s' % (day_date.month, day_date.day)
if not key in found_dates: if key not in found_dates:
stats.append({ stats.append({
'date': day_date.date().isoformat(), 'date': day_date.date().isoformat(),
'count': 0, 'count': 0,
}) })
repo_data = {
'namespace': namespace,
'name': repository,
'kind': repo.kind.name,
'description': repo.description,
'tags': tag_dict,
'can_write': can_write,
'can_admin': can_admin,
'is_public': is_public,
'is_organization': repo.namespace_user.organization,
'is_starred': is_starred,
'status_token': repo.badge_token if not is_public else '',
'trust_enabled': bool(features.SIGNING) and repo.trust_enabled,
'tag_expiration_s': repo.namespace_user.removed_tag_expiration_s,
}
if stats is not None:
repo_data['stats'] = stats repo_data['stats'] = stats
return repo_data return repo_data
@require_repo_write @require_repo_write
@nickname('updateRepo') @nickname('updateRepo')
@validate_json_request('RepoUpdate') @validate_json_request('RepoUpdate')
def put(self, namespace, repository): def put(self, namespace, repository):
""" Update the description in the specified repository. """ """ Update the description in the specified repository. """
repo = model.repository.get_repository(namespace, repository) if not model.repo_exists(namespace, repository):
if repo: raise NotFound()
values = request.get_json()
repo.description = values['description']
repo.save()
log_action('set_repo_description', namespace, values = request.get_json()
{'repo': repository, 'namespace': namespace, 'description': values['description']}, model.set_description(namespace, repository, values['description'])
repo=repo)
return { log_action('set_repo_description', namespace,
'success': True {'repo': repository,
} 'namespace': namespace,
raise NotFound() 'description': values['description']}, repo_name=repository)
return {'success': True}
@require_repo_admin @require_repo_admin
@nickname('deleteRepository') @nickname('deleteRepository')
def delete(self, namespace, repository): def delete(self, namespace, repository):
""" Delete a repository. """ """ Delete a repository. """
model.repository.purge_repository(namespace, repository) username = model.purge_repository(namespace, repository)
user = model.user.get_namespace_user(namespace)
if features.BILLING: if features.BILLING:
plan = get_namespace_plan(namespace) plan = get_namespace_plan(namespace)
check_repository_usage(user, plan) model.check_repository_usage(username, plan)
# Remove any builds from the queue. # Remove any builds from the queue.
dockerfile_build_queue.delete_namespaced_items(namespace, repository) dockerfile_build_queue.delete_namespaced_items(namespace, repository)
log_action('delete_repo', namespace, log_action('delete_repo', namespace, {'repo': repository, 'namespace': namespace})
{'repo': repository, 'namespace': namespace})
return '', 204 return '', 204
@ -438,9 +268,7 @@ class RepositoryVisibility(RepositoryParamResource):
'ChangeVisibility': { 'ChangeVisibility': {
'type': 'object', 'type': 'object',
'description': 'Change the visibility for the repository.', 'description': 'Change the visibility for the repository.',
'required': [ 'required': ['visibility',],
'visibility',
],
'properties': { 'properties': {
'visibility': { 'visibility': {
'type': 'string', 'type': 'string',
@ -459,17 +287,17 @@ class RepositoryVisibility(RepositoryParamResource):
@validate_json_request('ChangeVisibility') @validate_json_request('ChangeVisibility')
def post(self, namespace, repository): def post(self, namespace, repository):
""" Change the visibility of a repository. """ """ Change the visibility of a repository. """
repo = model.repository.get_repository(namespace, repository) if model.repo_exists(namespace, repository):
if repo:
values = request.get_json() values = request.get_json()
visibility = values['visibility'] visibility = values['visibility']
if visibility == 'private': if visibility == 'private':
check_allowed_private_repos(namespace) check_allowed_private_repos(namespace)
model.repository.set_repository_visibility(repo, visibility) model.set_repository_visibility(namespace, repository, visibility)
log_action('change_repo_visibility', namespace, log_action('change_repo_visibility', namespace,
{'repo': repository, 'namespace': namespace, 'visibility': values['visibility']}, {'repo': repository,
repo=repo) 'namespace': namespace,
'visibility': values['visibility']}, repo_name=repository)
return {'success': True} return {'success': True}
@ -481,9 +309,7 @@ class RepositoryTrust(RepositoryParamResource):
'ChangeRepoTrust': { 'ChangeRepoTrust': {
'type': 'object', 'type': 'object',
'description': 'Change the trust settings for the repository.', 'description': 'Change the trust settings for the repository.',
'required': [ 'required': ['trust_enabled',],
'trust_enabled',
],
'properties': { 'properties': {
'trust_enabled': { 'trust_enabled': {
'type': 'boolean', 'type': 'boolean',
@ -499,19 +325,19 @@ class RepositoryTrust(RepositoryParamResource):
@validate_json_request('ChangeRepoTrust') @validate_json_request('ChangeRepoTrust')
def post(self, namespace, repository): def post(self, namespace, repository):
""" Change the visibility of a repository. """ """ Change the visibility of a repository. """
repo = model.repository.get_repository(namespace, repository) if not model.repo_exists(namespace, repository):
if not repo:
raise NotFound() raise NotFound()
tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository) tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository)
if tags and not tuf_metadata_api.delete_metadata(namespace, repository): if tags and not tuf_metadata_api.delete_metadata(namespace, repository):
raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'}) raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'})
values = request.get_json() values = request.get_json()
model.repository.set_trust(repo, values['trust_enabled']) model.set_trust(namespace, repository, values['trust_enabled'])
log_action('change_repo_trust', namespace, log_action(
{'repo': repository, 'namespace': namespace, 'trust_enabled': values['trust_enabled']}, 'change_repo_trust', namespace,
repo=repo) {'repo': repository,
'namespace': namespace,
'trust_enabled': values['trust_enabled']}, repo_name=repository)
return {'success': True} return {'success': True}

255
endpoints/api/repository_models_interface.py Normal file
View file

@ -0,0 +1,255 @@
from abc import ABCMeta, abstractmethod
from collections import namedtuple, defaultdict
from datetime import datetime
from six import add_metaclass
import features
from endpoints.api import format_date
class RepositoryBaseElement(
namedtuple('RepositoryBaseElement', [
'namespace_name', 'repository_name', 'is_starred', 'is_public', 'kind_name', 'description',
'namespace_user_organization', 'namespace_user_removed_tag_expiration_s', 'last_modified',
'action_count', 'should_last_modified', 'should_popularity', 'should_is_starred'
])):
"""
Repository a single quay repository
:type namespace_name: string
:type repository_name: string
:type is_starred: boolean
:type is_public: boolean
:type kind_name: string
:type description: string
:type namespace_user_organization: boolean
:type should_last_modified: boolean
:type should_popularity: boolean
:type should_is_starred: boolean
"""
def to_dict(self):
repo = {
'namespace': self.namespace_name,
'name': self.repository_name,
'description': self.description,
'is_public': self.is_public,
'kind': self.kind_name,
}
if self.should_last_modified:
repo['last_modified'] = self.last_modified
if self.should_popularity:
repo['popularity'] = float(self.action_count if self.action_count else 0)
if self.should_is_starred:
repo['is_starred'] = self.is_starred
return repo
class ApplicationRepository(
namedtuple('ApplicationRepository', ['repository_base_elements', 'channels', 'releases'])):
"""
Repository a single quay repository
:type repository_base_elements: RepositoryBaseElement
:type channels: [Channel]
:type releases: [Release]
"""
def to_dict(self):
repo_data = {
'namespace': self.repository_base_elements.namespace_name,
'name': self.repository_base_elements.repository_name,
'kind': self.repository_base_elements.kind_name,
'description': self.repository_base_elements.description,
'is_public': self.repository_base_elements.is_public,
'is_organization': self.repository_base_elements.namespace_user_organization,
'is_starred': self.repository_base_elements.is_starred,
'channels': [chan.to_dict() for chan in self.channels],
'releases': [release.to_dict() for release in self.releases],
}
return repo_data
class ImageRepositoryRepository(
namedtuple('NonApplicationRepository',
['repository_base_elements', 'tags', 'counts', 'badge_token', 'trust_enabled'])):
"""
Repository a single quay repository
:type repository_base_elements: RepositoryBaseElement
:type tags: [Tag]
:type counts: [count]
:type badge_token: string
:type trust_enabled: boolean
"""
def to_dict(self):
return {
'namespace': self.repository_base_elements.namespace_name,
'name': self.repository_base_elements.repository_name,
'kind': self.repository_base_elements.kind_name,
'description': self.repository_base_elements.description,
'is_public': self.repository_base_elements.is_public,
'is_organization': self.repository_base_elements.namespace_user_organization,
'is_starred': self.repository_base_elements.is_starred,
'tags': {tag.name: tag.to_dict()
for tag in self.tags},
'status_token': self.badge_token if not self.repository_base_elements.is_public else '',
'trust_enabled': bool(features.SIGNING) and self.trust_enabled,
'tag_expiration_s': self.repository_base_elements.namespace_user_removed_tag_expiration_s,
}
class Repository(namedtuple('Repository', [
'namespace_name',
'repository_name',
])):
"""
Repository a single quay repository
:type namespace_name: string
:type repository_name: string
"""
class Channel(namedtuple('Channel', ['name', 'linked_tag_name', 'linked_tag_lifetime_start'])):
"""
Repository a single quay repository
:type name: string
:type linked_tag_name: string
:type linked_tag_lifetime_start: string
"""
def to_dict(self):
return {
'name': self.name,
'release': self.linked_tag_name,
'last_modified': format_date(datetime.fromtimestamp(self.linked_tag_lifetime_start / 1000)),
}
class Release(
namedtuple('Channel', ['name', 'released', 'lifetime_start', 'releases_channels_map'])):
"""
Repository a single quay repository
:type name: string
:type released: string
:type last_modified: string
:type releases_channels_map: {string -> string}
"""
def to_dict(self):
return {
'name': self.name,
'last_modified': format_date(datetime.fromtimestamp(self.lifetime_start / 1000)),
'channels': self.releases_channels_map[self.name],
}
class Tag(
namedtuple('Tag', [
'name', 'image_docker_image_id', 'image_aggregate_size', 'lifetime_start_ts',
'tag_manifest_digest'
])):
"""
:type name: string
:type image_docker_image_id: string
:type image_aggregate_size: int
:type lifetime_start_ts: int
:type tag_manifest_digest: string
"""
def to_dict(self):
tag_info = {
'name': self.name,
'image_id': self.image_docker_image_id,
'size': self.image_aggregate_size
}
if self.lifetime_start_ts > 0:
last_modified = format_date(datetime.fromtimestamp(self.lifetime_start_ts))
tag_info['last_modified'] = last_modified
if self.tag_manifest_digest is not None:
tag_info['manifest_digest'] = self.tag_manifest_digest
return tag_info
class Count(namedtuple('Count', ['date', 'count'])):
"""
date: DateTime
count: int
"""
def to_dict(self):
return {
'date': self.date.isoformat(),
'count': self.count,
}
@add_metaclass(ABCMeta)
class RepositoryDataInterface(object):
"""
Interface that represents all data store interactions required by a Repository.
"""
@abstractmethod
def get_repo(self, namespace_name, repository_name, user):
"""
Returns a repository
"""
@abstractmethod
def repo_exists(self, namespace_name, repository_name):
"""
Returns true if a repo exists and false if not
"""
@abstractmethod
def create_repo(self, namespace, name, creating_user, description, visibility='private',
repo_kind='image'):
"""
Returns creates a new repo
"""
@abstractmethod
def get_repo_list(self, starred, user, repo_kind, namespace, username, public, page_token,
last_modified, popularity):
"""
Returns a RepositoryBaseElement
"""
@abstractmethod
def set_repository_visibility(self, namespace_name, repository_name, visibility):
"""
Sets a repository's visibility if it is found
"""
@abstractmethod
def set_trust(self, namespace_name, repository_name, trust):
"""
Sets a repository's trust_enabled field if it is found
"""
@abstractmethod
def set_description(self, namespace_name, repository_name, description):
"""
Sets a repository's description if it is found.
"""
@abstractmethod
def purge_repository(self, namespace_name, repository_name):
"""
Removes a repository
"""
@abstractmethod
def check_repository_usage(self, user_name, plan_found):
"""
Creates a notification for a user if they are over or under on their repository usage
"""

166
endpoints/api/repository_models_pre_oci.py Normal file
View file

@ -0,0 +1,166 @@
from collections import defaultdict
from datetime import datetime, timedelta
from auth.permissions import ReadRepositoryPermission
from data import model, oci_model
from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \
ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count
MAX_DAYS_IN_3_MONTHS = 92
REPOS_PER_PAGE = 100
def create_channel(channel, releases_channels_map):
releases_channels_map[channel.linked_tag_name].append(channel.name)
return Channel(channel.name, channel.linked_tag.name, channel.linked_tag.lifetime_start)
class PreOCIModel(RepositoryDataInterface):
"""
PreOCIModel implements the data model for the Repo Email using a database schema
before it was changed to support the OCI specification.
"""
def check_repository_usage(self, username, plan_found):
private_repos = model.user.get_private_repo_count(username)
if plan_found is None:
repos_allowed = 0
else:
repos_allowed = plan_found['privateRepos']
user_or_org = model.user.get_namespace_user(username)
if private_repos > repos_allowed:
model.notification.create_unique_notification('over_private_usage', user_or_org,
{'namespace': username})
else:
model.notification.delete_notifications_by_kind(user_or_org, 'over_private_usage')
def purge_repository(self, namespace_name, repository_name):
model.repository.purge_repository(namespace_name, repository_name)
user = model.user.get_namespace_user(namespace_name)
return user.username
def set_description(self, namespace_name, repository_name, description):
repo = model.repository.get_repository(namespace_name, repository_name)
model.repository.set_description(repo, description)
def set_trust(self, namespace_name, repository_name, trust):
repo = model.repository.get_repository(namespace_name, repository_name)
model.repository.set_trust(repo, trust)
def set_repository_visibility(self, namespace_name, repository_name, visibility):
repo = model.repository.get_repository(namespace_name, repository_name)
model.repository.set_repository_visibility(repo, visibility)
def get_repo_list(self, starred, user, repo_kind, namespace, username, public, page_token,
last_modified, popularity):
next_page_token = None
# Lookup the requested repositories (either starred or non-starred.)
if starred:
# Return the full list of repos starred by the current user that are still visible to them.
def can_view_repo(repo):
return ReadRepositoryPermission(repo.namespace_user.username, repo.name).can()
unfiltered_repos = model.repository.get_user_starred_repositories(user,
kind_filter=repo_kind)
repos = [repo for repo in unfiltered_repos if can_view_repo(repo)]
elif namespace:
# Repositories filtered by namespace do not need pagination (their results are fairly small),
# so we just do the lookup directly.
repos = list(
model.repository.get_visible_repositories(username=username, include_public=public,
namespace=namespace, kind_filter=repo_kind))
else:
# Determine the starting offset for pagination. Note that we don't use the normal
# model.modelutil.paginate method here, as that does not operate over UNION queries, which
# get_visible_repositories will return if there is a logged-in user (for performance reasons).
#
# Also note the +1 on the limit, as paginate_query uses the extra result to determine whether
# there is a next page.
start_id = model.modelutil.pagination_start(page_token)
repo_query = model.repository.get_visible_repositories(
username=username, include_public=public, start_id=start_id, limit=REPOS_PER_PAGE + 1,
kind_filter=repo_kind)
repos, next_page_token = model.modelutil.paginate_query(repo_query, limit=REPOS_PER_PAGE,
id_alias='rid')
# Collect the IDs of the repositories found for subequent lookup of popularity
# and/or last modified.
last_modified_map = {}
action_sum_map = {}
if last_modified or popularity:
repository_ids = [repo.rid for repo in repos]
if last_modified:
last_modified_map = model.repository.get_when_last_modified(repository_ids)
if popularity:
action_sum_map = model.log.get_repositories_action_sums(repository_ids)
# Collect the IDs of the repositories that are starred for the user, so we can mark them
# in the returned results.
star_set = set()
if username:
starred_repos = model.repository.get_user_starred_repositories(user)
star_set = {starred.id for starred in starred_repos}
return [
RepositoryBaseElement(repo.namespace_user.username, repo.name, repo.id in star_set,
repo.visibility_id == model.repository.get_public_repo_visibility().id,
repo_kind, repo.description, repo.namespace_user.organization,
repo.namespace_user.removed_tag_expiration_s,
last_modified_map.get(repo.rid),
action_sum_map.get(repo.rid), last_modified, popularity, username)
for repo in repos
], next_page_token
def repo_exists(self, namespace_name, repository_name):
repo = model.repository.get_repository(namespace_name, repository_name)
if repo is None:
return False
return True
def create_repo(self, namespace_name, repository_name, owner, description, visibility='private',
repo_kind='image'):
repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility,
repo_kind=repo_kind)
model.repository.set_description(repo, description)
return Repository(namespace_name, repository_name)
def get_repo(self, namespace_name, repository_name, user):
repo = model.repository.get_repository(namespace_name, repository_name)
if repo is None:
return None
is_starred = model.repository.repository_is_starred(user, repo) if user else False
is_public = model.repository.is_repository_public(repo)
base = RepositoryBaseElement(
namespace_name, repository_name, is_starred, is_public, repo.kind.name, repo.description,
repo.namespace_user.organization, repo.namespace_user.removed_tag_expiration_s, None, None,
False, False, False)
# Note: This is *temporary* code for the new OCI model stuff.
if base.kind_name == 'application':
channels = oci_model.channel.get_repo_channels(repo)
releases = oci_model.release.get_release_objs(repo)
releases_channels_map = defaultdict(list)
return ApplicationRepository(
base, [create_channel(channel, releases_channels_map) for channel in channels], [
Release(release.name, release.released, release.lifetime_start, releases_channels_map)
for release in releases
])
tags = model.tag.list_active_repo_tags(repo)
start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
counts = model.log.get_repository_action_counts(repo, start_date)
return ImageRepositoryRepository(base, [
Tag(tag.name, tag.image.docker_image_id, tag.image.aggregate_size, tag.lifetime_start_ts,
tag.tagmanifest.digest) for tag in tags
], [Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled)
pre_oci_model = PreOCIModel()

40
endpoints/api/test/test_repository.py
View file

@ -9,7 +9,6 @@ from features import FeatureNameValue
from test.fixtures import * from test.fixtures import *
INVALID_RESPONSE = { INVALID_RESPONSE = {
u'detail': u"u'invalid_req' is not of type 'boolean'", u'detail': u"u'invalid_req' is not of type 'boolean'",
u'error_message': u"u'invalid_req' is not of type 'boolean'", u'error_message': u"u'invalid_req' is not of type 'boolean'",
@ -20,31 +19,44 @@ INVALID_RESPONSE = {
} }
NOT_FOUND_RESPONSE = { NOT_FOUND_RESPONSE = {
u'detail': u'Not Found', u'detail':
u'error_message': u'Not Found', u'Not Found',
u'error_type': u'not_found', u'error_message':
u'message': u'You have requested this URI [/api/v1/repository/devtable/repo/changetrust] but did you mean /api/v1/repository/<apirepopath:repository>/changetrust or /api/v1/repository/<apirepopath:repository>/changevisibility or /api/v1/repository/<apirepopath:repository>/tag/<tag>/images ?', u'Not Found',
u'status': 404, u'error_type':
u'title': u'not_found', u'not_found',
u'type': u'http://localhost/api/v1/error/not_found' u'message':
u'You have requested this URI [/api/v1/repository/devtable/repo/changetrust] but did you mean /api/v1/repository/<apirepopath:repository>/changetrust or /api/v1/repository/<apirepopath:repository>/changevisibility or /api/v1/repository/<apirepopath:repository>/tag/<tag>/images ?',
u'status':
404,
u'title':
u'not_found',
u'type':
u'http://localhost/api/v1/error/not_found'
} }
@pytest.mark.parametrize('trust_enabled,repo_found,expected_body,expected_status', [ @pytest.mark.parametrize('trust_enabled,repo_found,expected_body,expected_status', [
(True, True,{'success': True}, 200), (True, True, {
(False, True, {'success': True}, 200), 'success': True
}, 200),
(False, True, {
'success': True
}, 200),
(False, False, NOT_FOUND_RESPONSE, 404), (False, False, NOT_FOUND_RESPONSE, 404),
('invalid_req', False, INVALID_RESPONSE , 400), ('invalid_req', False, INVALID_RESPONSE, 400),
]) ])
def test_post_changetrust(trust_enabled, repo_found, expected_body, expected_status, client): def test_post_changetrust(trust_enabled, repo_found, expected_body, expected_status, client):
with patch('endpoints.api.repository.tuf_metadata_api') as mock_tuf: with patch('endpoints.api.repository.tuf_metadata_api') as mock_tuf:
with patch('endpoints.api.repository.model') as mock_model: with patch(
mock_model.repository.get_repository.return_value = MagicMock() if repo_found else None 'endpoints.api.repository_models_pre_oci.model.repository.get_repository') as mock_model:
mock_model.return_value = MagicMock() if repo_found else None
mock_tuf.get_default_tags_with_expiration.return_value = ['tags', 'expiration'] mock_tuf.get_default_tags_with_expiration.return_value = ['tags', 'expiration']
with client_with_identity('devtable', client) as cl: with client_with_identity('devtable', client) as cl:
params = {'repository': 'devtable/repo'} params = {'repository': 'devtable/repo'}
request_body = {'trust_enabled': trust_enabled} request_body = {'trust_enabled': trust_enabled}
assert expected_body == conduct_api_call(cl, RepositoryTrust, 'POST', params, request_body, expected_status).json assert expected_body == conduct_api_call(cl, RepositoryTrust, 'POST', params, request_body,
expected_status).json
def test_signing_disabled(client): def test_signing_disabled(client):

1789
test/test_api_usage.py
View file

File diff suppressed because it is too large Load diff