Merge pull request #2814 from charltonaustin/create_data_interface_for_subsystem_api/repository_633
refactor(endpoints/api/repository*): added in pre_oci_model abstraction
This commit is contained in:
Charlton Austin
GitHub
commit
be206a8b88
6 changed files with 1278 additions and 1487 deletions
|
|
@ -7,17 +7,16 @@ from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
|
|||
from playhouse.shortcuts import case
|
||||
from cachetools import ttl_cache
|
||||
|
||||
from data.model import (config, DataModelException, tag, db_transaction, storage, permission,
|
||||
_basequery)
|
||||
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User,
|
||||
Visibility, RepositoryPermission, RepositoryActionCount,
|
||||
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
|
||||
Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
|
||||
db_random_func, db_concat_func, RepositorySearchScore)
|
||||
from data.model import (
|
||||
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
|
||||
from data.database import (
|
||||
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
|
||||
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
|
||||
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
|
||||
db_random_func, db_concat_func, RepositorySearchScore)
|
||||
from data.text import prefix_search
|
||||
from util.itertoolrecipes import take
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
SEARCH_FIELDS = Enum("SearchFields", ["name", "description"])
|
||||
|
||||
|
|
@ -87,8 +86,7 @@ def purge_repository(namespace_name, repository_name):
|
|||
unreferenced_image_q = Image.select(Image.id).where(Image.repository == repo)
|
||||
|
||||
if len(previously_referenced) > 0:
|
||||
unreferenced_image_q = (unreferenced_image_q
|
||||
.where(~(Image.id << list(previously_referenced))))
|
||||
unreferenced_image_q = (unreferenced_image_q.where(~(Image.id << list(previously_referenced))))
|
||||
|
||||
unreferenced_candidates = set(img[0] for img in unreferenced_image_q.tuples())
|
||||
|
||||
|
|
@ -116,11 +114,10 @@ def purge_repository(namespace_name, repository_name):
|
|||
|
||||
@ttl_cache(maxsize=1, ttl=600)
|
||||
def _get_gc_expiration_policies():
|
||||
policy_tuples_query = (Namespace
|
||||
.select(Namespace.removed_tag_expiration_s)
|
||||
.distinct()
|
||||
.limit(100) # This sucks but it's the only way to limit memory
|
||||
.tuples())
|
||||
policy_tuples_query = (
|
||||
Namespace.select(Namespace.removed_tag_expiration_s).distinct()
|
||||
.limit(100) # This sucks but it's the only way to limit memory
|
||||
.tuples())
|
||||
return [policy[0] for policy in policy_tuples_query]
|
||||
|
||||
|
||||
|
|
@ -134,22 +131,15 @@ def find_repository_with_garbage(limit_to_gc_policy_s):
|
|||
expiration_timestamp = get_epoch_timestamp() - limit_to_gc_policy_s
|
||||
|
||||
try:
|
||||
candidates = (RepositoryTag
|
||||
.select(RepositoryTag.repository)
|
||||
.join(Repository)
|
||||
candidates = (RepositoryTag.select(RepositoryTag.repository).join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(~(RepositoryTag.lifetime_end_ts >> None),
|
||||
(RepositoryTag.lifetime_end_ts <= expiration_timestamp),
|
||||
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s))
|
||||
.limit(500)
|
||||
.distinct()
|
||||
.alias('candidates'))
|
||||
(Namespace.removed_tag_expiration_s == limit_to_gc_policy_s)).limit(500)
|
||||
.distinct().alias('candidates'))
|
||||
|
||||
found = (RepositoryTag
|
||||
.select(candidates.c.repository_id)
|
||||
.from_(candidates)
|
||||
.order_by(db_random_func())
|
||||
.get())
|
||||
found = (RepositoryTag.select(candidates.c.repository_id).from_(candidates)
|
||||
.order_by(db_random_func()).get())
|
||||
|
||||
if found is None:
|
||||
return
|
||||
|
|
@ -186,10 +176,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
|
|||
all_unreferenced_candidates = set()
|
||||
|
||||
# Remove any images directly referenced by tags, to prune the working set.
|
||||
direct_referenced = (RepositoryTag
|
||||
.select(RepositoryTag.image)
|
||||
.where(RepositoryTag.repository == repo.id,
|
||||
RepositoryTag.image << list(candidate_orphan_image_set)))
|
||||
direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
|
||||
RepositoryTag.repository == repo.id, RepositoryTag.image << list(candidate_orphan_image_set)))
|
||||
candidate_orphan_image_set.difference_update([t.image_id for t in direct_referenced])
|
||||
|
||||
# Iteratively try to remove images from the database. The only images we can remove are those
|
||||
|
|
@ -205,26 +193,20 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
|
|||
|
||||
with db_transaction():
|
||||
# Any image directly referenced by a tag that still exists, cannot be GCed.
|
||||
direct_referenced = (RepositoryTag
|
||||
.select(RepositoryTag.image)
|
||||
.where(RepositoryTag.repository == repo.id,
|
||||
RepositoryTag.image << candidates_orphans))
|
||||
direct_referenced = (RepositoryTag.select(RepositoryTag.image).where(
|
||||
RepositoryTag.repository == repo.id, RepositoryTag.image << candidates_orphans))
|
||||
|
||||
# Any image which is the parent of another image, cannot be GCed.
|
||||
parent_referenced = (Image
|
||||
.select(Image.parent)
|
||||
.where(Image.repository == repo.id,
|
||||
Image.parent << candidates_orphans))
|
||||
parent_referenced = (Image.select(Image.parent).where(Image.repository == repo.id,
|
||||
Image.parent << candidates_orphans))
|
||||
|
||||
referenced_candidates = (direct_referenced | parent_referenced)
|
||||
|
||||
# We desire a few pieces of information from the database from the following
|
||||
# query: all of the image ids which are associated with this repository,
|
||||
# and the storages which are associated with those images.
|
||||
unreferenced_candidates = (Image
|
||||
.select(Image.id, Image.docker_image_id,
|
||||
ImageStorage.id, ImageStorage.uuid)
|
||||
.join(ImageStorage)
|
||||
unreferenced_candidates = (Image.select(Image.id, Image.docker_image_id, ImageStorage.id,
|
||||
ImageStorage.uuid).join(ImageStorage)
|
||||
.where(Image.id << candidates_orphans,
|
||||
~(Image.id << referenced_candidates)))
|
||||
|
||||
|
|
@ -238,8 +220,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
|
|||
storage_id_whitelist = set([candidate.storage_id for candidate in unreferenced_candidates])
|
||||
|
||||
# Lookup any derived images for the images to remove.
|
||||
derived = DerivedStorageForImage.select().where(
|
||||
DerivedStorageForImage.source_image << image_ids_to_remove)
|
||||
derived = DerivedStorageForImage.select().where(DerivedStorageForImage.source_image <<
|
||||
image_ids_to_remove)
|
||||
|
||||
has_derived = False
|
||||
for derived_image in derived:
|
||||
|
|
@ -249,10 +231,8 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
|
|||
# Delete any derived images and the images themselves.
|
||||
if has_derived:
|
||||
try:
|
||||
(DerivedStorageForImage
|
||||
.delete()
|
||||
.where(DerivedStorageForImage.source_image << image_ids_to_remove)
|
||||
.execute())
|
||||
(DerivedStorageForImage.delete()
|
||||
.where(DerivedStorageForImage.source_image << image_ids_to_remove).execute())
|
||||
except IntegrityError:
|
||||
logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove)
|
||||
return False
|
||||
|
|
@ -278,8 +258,10 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
|
|||
# If any storages were removed and cleanup callbacks are registered, call them with
|
||||
# the images+storages removed.
|
||||
if storage_ids_removed and config.image_cleanup_callbacks:
|
||||
image_storages_removed = [candidate for candidate in all_unreferenced_candidates
|
||||
if candidate.storage_id in storage_ids_removed]
|
||||
image_storages_removed = [
|
||||
candidate for candidate in all_unreferenced_candidates
|
||||
if candidate.storage_id in storage_ids_removed
|
||||
]
|
||||
for callback in config.image_cleanup_callbacks:
|
||||
callback(image_storages_removed)
|
||||
|
||||
|
|
@ -295,10 +277,7 @@ def star_repository(user, repository):
|
|||
def unstar_repository(user, repository):
|
||||
""" Unstars a repository. """
|
||||
try:
|
||||
(Star
|
||||
.delete()
|
||||
.where(Star.repository == repository.id, Star.user == user.id)
|
||||
.execute())
|
||||
(Star.delete().where(Star.repository == repository.id, Star.user == user.id).execute())
|
||||
except Star.DoesNotExist:
|
||||
raise DataModelException('Star not found.')
|
||||
|
||||
|
|
@ -308,6 +287,11 @@ def set_trust(repo, trust_enabled):
|
|||
repo.save()
|
||||
|
||||
|
||||
def set_description(repo, description):
|
||||
repo.description = description
|
||||
repo.save()
|
||||
|
||||
|
||||
def get_user_starred_repositories(user, kind_filter='image'):
|
||||
""" Retrieves all of the repositories a user has starred. """
|
||||
try:
|
||||
|
|
@ -315,13 +299,8 @@ def get_user_starred_repositories(user, kind_filter='image'):
|
|||
except RepositoryKind.DoesNotExist:
|
||||
raise DataModelException('Unknown kind of repository')
|
||||
|
||||
query = (Repository
|
||||
.select(Repository, User, Visibility, Repository.id.alias('rid'))
|
||||
.join(Star)
|
||||
.switch(Repository)
|
||||
.join(User)
|
||||
.switch(Repository)
|
||||
.join(Visibility)
|
||||
query = (Repository.select(Repository, User, Visibility, Repository.id.alias('rid')).join(Star)
|
||||
.switch(Repository).join(User).switch(Repository).join(Visibility)
|
||||
.where(Star.user == user, Repository.kind == repo_kind))
|
||||
|
||||
return query
|
||||
|
|
@ -330,10 +309,7 @@ def get_user_starred_repositories(user, kind_filter='image'):
|
|||
def repository_is_starred(user, repository):
|
||||
""" Determines whether a user has starred a repository or not. """
|
||||
try:
|
||||
(Star
|
||||
.select()
|
||||
.where(Star.repository == repository.id, Star.user == user.id)
|
||||
.get())
|
||||
(Star.select().where(Star.repository == repository.id, Star.user == user.id).get())
|
||||
return True
|
||||
except Star.DoesNotExist:
|
||||
return False
|
||||
|
|
@ -346,10 +322,8 @@ def get_when_last_modified(repository_ids):
|
|||
if not repository_ids:
|
||||
return {}
|
||||
|
||||
tuples = (RepositoryTag
|
||||
.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts))
|
||||
.where(RepositoryTag.repository << repository_ids)
|
||||
.group_by(RepositoryTag.repository)
|
||||
tuples = (RepositoryTag.select(RepositoryTag.repository, fn.Max(RepositoryTag.lifetime_start_ts))
|
||||
.where(RepositoryTag.repository << repository_ids).group_by(RepositoryTag.repository)
|
||||
.tuples())
|
||||
|
||||
last_modified_map = {}
|
||||
|
|
@ -366,11 +340,8 @@ def get_stars(repository_ids):
|
|||
if not repository_ids:
|
||||
return {}
|
||||
|
||||
tuples = (Star
|
||||
.select(Star.repository, fn.Count(Star.id))
|
||||
.where(Star.repository << repository_ids)
|
||||
.group_by(Star.repository)
|
||||
.tuples())
|
||||
tuples = (Star.select(Star.repository, fn.Count(Star.id))
|
||||
.where(Star.repository << repository_ids).group_by(Star.repository).tuples())
|
||||
|
||||
star_map = {}
|
||||
for record in tuples:
|
||||
|
|
@ -388,12 +359,10 @@ def get_visible_repositories(username, namespace=None, kind_filter='image', incl
|
|||
# here, as it will be modified by other queries later on.
|
||||
return Repository.select(Repository.id.alias('rid')).where(Repository.id == -1)
|
||||
|
||||
query = (Repository
|
||||
.select(Repository.name, Repository.id.alias('rid'),
|
||||
Repository.description, Namespace.username, Repository.visibility,
|
||||
Repository.kind)
|
||||
.switch(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id)))
|
||||
query = (Repository.select(Repository.name,
|
||||
Repository.id.alias('rid'), Repository.description,
|
||||
Namespace.username, Repository.visibility, Repository.kind)
|
||||
.switch(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id)))
|
||||
|
||||
if username:
|
||||
# Note: We only need the permissions table if we will filter based on a user's permissions.
|
||||
|
|
@ -422,8 +391,8 @@ def get_app_search(lookup, search_fields=None, username=None, limit=50):
|
|||
search_fields = set([SEARCH_FIELDS.name.name])
|
||||
|
||||
return get_filtered_matching_repositories(lookup, filter_username=username,
|
||||
search_fields=search_fields,
|
||||
repo_kind='application', offset=0, limit=limit)
|
||||
search_fields=search_fields, repo_kind='application',
|
||||
offset=0, limit=limit)
|
||||
|
||||
|
||||
def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image',
|
||||
|
|
@ -460,7 +429,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
|
|||
unfiltered_page = 0
|
||||
iteration_count = 0
|
||||
|
||||
while iteration_count < 10: # Just to be safe
|
||||
while iteration_count < 10: # Just to be safe
|
||||
# Find the next chunk's worth of repository IDs, paginated by the chunk size.
|
||||
unfiltered_page = unfiltered_page + 1
|
||||
found_ids = [r.id for r in unfiltered_query.paginate(unfiltered_page, chunk_count)]
|
||||
|
|
@ -476,13 +445,9 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
|
|||
encountered.update(new_unfiltered_ids)
|
||||
|
||||
# Filter the repositories found to only those visible to the current user.
|
||||
query = (Repository
|
||||
.select(Repository, Namespace)
|
||||
.distinct()
|
||||
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
|
||||
.switch(Repository)
|
||||
.join(RepositoryPermission)
|
||||
.where(Repository.id << list(new_unfiltered_ids)))
|
||||
query = (Repository.select(Repository, Namespace).distinct()
|
||||
.join(Namespace, on=(Namespace.id == Repository.namespace_user)).switch(Repository)
|
||||
.join(RepositoryPermission).where(Repository.id << list(new_unfiltered_ids)))
|
||||
|
||||
filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind)
|
||||
|
||||
|
|
@ -520,16 +485,12 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
|
|||
if SEARCH_FIELDS.description.name in search_fields:
|
||||
clause = Repository.description.match(lookup_value) | clause
|
||||
|
||||
cases = [
|
||||
(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),
|
||||
]
|
||||
cases = [(Repository.name.match(lookup_value), 100 * RepositorySearchScore.score),]
|
||||
|
||||
computed_score = case(None, cases, RepositorySearchScore.score).alias('score')
|
||||
|
||||
query = (Repository
|
||||
.select(Repository, Namespace, computed_score)
|
||||
.join(Namespace, on=(Namespace.id == Repository.namespace_user))
|
||||
.where(clause)
|
||||
query = (Repository.select(Repository, Namespace, computed_score)
|
||||
.join(Namespace, on=(Namespace.id == Repository.namespace_user)).where(clause)
|
||||
.group_by(Repository.id, Namespace.id))
|
||||
|
||||
if repo_kind is not None:
|
||||
|
|
@ -538,11 +499,8 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p
|
|||
if not include_private:
|
||||
query = query.where(Repository.visibility == _basequery.get_public_repo_visibility())
|
||||
|
||||
query = (query
|
||||
.switch(Repository)
|
||||
.join(RepositorySearchScore)
|
||||
.group_by(Repository, Namespace, RepositorySearchScore)
|
||||
.order_by(SQL('score').desc()))
|
||||
query = (query.switch(Repository).join(RepositorySearchScore)
|
||||
.group_by(Repository, Namespace, RepositorySearchScore).order_by(SQL('score').desc()))
|
||||
|
||||
return query
|
||||
|
||||
|
|
@ -560,15 +518,10 @@ def is_repository_public(repository):
|
|||
|
||||
def repository_is_public(namespace_name, repository_name):
|
||||
try:
|
||||
(Repository
|
||||
.select()
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.switch(Repository)
|
||||
.join(Visibility)
|
||||
.where(Namespace.username == namespace_name,
|
||||
Repository.name == repository_name,
|
||||
Visibility.name == 'public')
|
||||
.get())
|
||||
(Repository.select().join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.switch(Repository).join(Visibility).where(Namespace.username == namespace_name,
|
||||
Repository.name == repository_name,
|
||||
Visibility.name == 'public').get())
|
||||
return True
|
||||
except Repository.DoesNotExist:
|
||||
return False
|
||||
|
|
@ -585,14 +538,10 @@ def set_repository_visibility(repo, visibility):
|
|||
|
||||
def get_email_authorized_for_repo(namespace, repository, email):
|
||||
try:
|
||||
return (RepositoryAuthorizedEmail
|
||||
.select(RepositoryAuthorizedEmail, Repository, Namespace)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Namespace.username == namespace,
|
||||
Repository.name == repository,
|
||||
RepositoryAuthorizedEmail.email == email)
|
||||
.get())
|
||||
return (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
|
||||
.join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(Namespace.username == namespace, Repository.name == repository,
|
||||
RepositoryAuthorizedEmail.email == email).get())
|
||||
except RepositoryAuthorizedEmail.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
|
@ -601,20 +550,16 @@ def create_email_authorization_for_repo(namespace_name, repository_name, email):
|
|||
try:
|
||||
repo = _basequery.get_existing_repository(namespace_name, repository_name)
|
||||
except Repository.DoesNotExist:
|
||||
raise DataModelException('Invalid repository %s/%s' %
|
||||
(namespace_name, repository_name))
|
||||
raise DataModelException('Invalid repository %s/%s' % (namespace_name, repository_name))
|
||||
|
||||
return RepositoryAuthorizedEmail.create(repository=repo, email=email, confirmed=False)
|
||||
|
||||
|
||||
def confirm_email_authorization_for_repo(code):
|
||||
try:
|
||||
found = (RepositoryAuthorizedEmail
|
||||
.select(RepositoryAuthorizedEmail, Repository, Namespace)
|
||||
.join(Repository)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(RepositoryAuthorizedEmail.code == code)
|
||||
.get())
|
||||
found = (RepositoryAuthorizedEmail.select(RepositoryAuthorizedEmail, Repository, Namespace)
|
||||
.join(Repository).join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.where(RepositoryAuthorizedEmail.code == code).get())
|
||||
except RepositoryAuthorizedEmail.DoesNotExist:
|
||||
raise DataModelException('Invalid confirmation code.')
|
||||
|
||||
|
|
@ -626,17 +571,13 @@ def confirm_email_authorization_for_repo(code):
|
|||
|
||||
def list_popular_public_repos(action_count_threshold, time_span, repo_kind='image'):
|
||||
cutoff = datetime.now() - time_span
|
||||
return (Repository
|
||||
.select(Namespace.username, Repository.name)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id))
|
||||
.switch(Repository)
|
||||
.join(RepositoryActionCount)
|
||||
.where(RepositoryActionCount.date >= cutoff,
|
||||
Repository.visibility == get_public_repo_visibility(),
|
||||
Repository.kind == Repository.kind.get_id(repo_kind))
|
||||
return (Repository.select(Namespace.username, Repository.name)
|
||||
.join(Namespace, on=(Repository.namespace_user == Namespace.id)).switch(Repository)
|
||||
.join(RepositoryActionCount).where(RepositoryActionCount.date >= cutoff,
|
||||
Repository.visibility == get_public_repo_visibility(),
|
||||
Repository.kind == Repository.kind.get_id(repo_kind))
|
||||
.group_by(RepositoryActionCount.repository, Repository.name, Namespace.username)
|
||||
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold)
|
||||
.tuples())
|
||||
.having(fn.Sum(RepositoryActionCount.count) >= action_count_threshold).tuples())
|
||||
|
||||
|
||||
def is_empty(namespace_name, repository_name):
|
||||
|
|
|
|||
|
|
@ -10,14 +10,13 @@ from datetime import timedelta, datetime
|
|||
from flask import request, abort
|
||||
|
||||
from app import dockerfile_build_queue, tuf_metadata_api
|
||||
from data import model, oci_model
|
||||
from endpoints.api import (format_date, nickname, log_action, validate_json_request,
|
||||
require_repo_read, require_repo_write, require_repo_admin,
|
||||
RepositoryParamResource, resource, parse_args, ApiResource,
|
||||
request_error, require_scope, path_param, page_support,
|
||||
query_param, truthy_bool, show_if)
|
||||
from endpoints.exception import (Unauthorized, NotFound, InvalidRequest, ExceedsLicenseException,
|
||||
DownstreamIssue)
|
||||
from endpoints.api import (
|
||||
format_date, nickname, log_action, validate_json_request, require_repo_read, require_repo_write,
|
||||
require_repo_admin, RepositoryParamResource, resource, parse_args, ApiResource, request_error,
|
||||
require_scope, path_param, page_support, query_param, truthy_bool, show_if)
|
||||
from endpoints.api.repository_models_pre_oci import pre_oci_model as model
|
||||
from endpoints.exception import (
|
||||
Unauthorized, NotFound, InvalidRequest, ExceedsLicenseException, DownstreamIssue)
|
||||
from endpoints.api.billing import lookup_allowed_private_repos, get_namespace_plan
|
||||
from endpoints.api.subscribe import check_repository_usage
|
||||
|
||||
|
|
@ -27,12 +26,12 @@ from auth.auth_context import get_authenticated_user
|
|||
from auth import scopes
|
||||
from util.names import REPOSITORY_NAME_REGEX
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPOS_PER_PAGE = 100
|
||||
MAX_DAYS_IN_3_MONTHS = 92
|
||||
|
||||
|
||||
def check_allowed_private_repos(namespace):
|
||||
""" Checks to see if the given namespace has reached its private repository limit. If so,
|
||||
raises a ExceedsLicenseException.
|
||||
|
|
@ -71,7 +70,8 @@ class RepositoryList(ApiResource):
|
|||
],
|
||||
},
|
||||
'namespace': {
|
||||
'type': 'string',
|
||||
'type':
|
||||
'string',
|
||||
'description': ('Namespace in which the repository should be created. If omitted, the '
|
||||
'username of the caller is used'),
|
||||
},
|
||||
|
|
@ -106,8 +106,7 @@ class RepositoryList(ApiResource):
|
|||
repository_name = req['repository']
|
||||
visibility = req['visibility']
|
||||
|
||||
existing = model.repository.get_repository(namespace_name, repository_name)
|
||||
if existing:
|
||||
if model.repo_exists(namespace_name, repository_name):
|
||||
raise request_error(message='Repository already exists')
|
||||
|
||||
visibility = req['visibility']
|
||||
|
|
@ -119,13 +118,12 @@ class RepositoryList(ApiResource):
|
|||
raise InvalidRequest('Invalid repository name')
|
||||
|
||||
kind = req.get('repo_kind', 'image') or 'image'
|
||||
repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility,
|
||||
repo_kind=kind)
|
||||
repo.description = req['description']
|
||||
repo.save()
|
||||
model.create_repo(namespace_name, repository_name, owner, req['description'],
|
||||
visibility=visibility, repo_kind=kind)
|
||||
|
||||
log_action('create_repo', namespace_name, {'repo': repository_name,
|
||||
'namespace': namespace_name}, repo=repo)
|
||||
log_action('create_repo', namespace_name,
|
||||
{'repo': repository_name,
|
||||
'namespace': namespace_name}, repo_name=repository_name)
|
||||
return {
|
||||
'namespace': namespace_name,
|
||||
'name': repository_name,
|
||||
|
|
@ -134,7 +132,6 @@ class RepositoryList(ApiResource):
|
|||
|
||||
raise Unauthorized()
|
||||
|
||||
|
||||
@require_scope(scopes.READ_REPO)
|
||||
@nickname('listRepos')
|
||||
@parse_args()
|
||||
|
|
@ -160,89 +157,18 @@ class RepositoryList(ApiResource):
|
|||
|
||||
user = get_authenticated_user()
|
||||
username = user.username if user else None
|
||||
next_page_token = None
|
||||
repos = None
|
||||
repo_kind = parsed_args['repo_kind']
|
||||
last_modified = parsed_args['last_modified']
|
||||
popularity = parsed_args['popularity']
|
||||
|
||||
# Lookup the requested repositories (either starred or non-starred.)
|
||||
if parsed_args['starred']:
|
||||
if not username:
|
||||
# No repositories should be returned, as there is no user.
|
||||
abort(400)
|
||||
if parsed_args['starred'] and not username:
|
||||
# No repositories should be returned, as there is no user.
|
||||
abort(400)
|
||||
|
||||
# Return the full list of repos starred by the current user that are still visible to them.
|
||||
def can_view_repo(repo):
|
||||
return ReadRepositoryPermission(repo.namespace_user.username, repo.name).can()
|
||||
repos, next_page_token = model.get_repo_list(
|
||||
parsed_args['starred'], user, parsed_args['repo_kind'], parsed_args['namespace'], username,
|
||||
parsed_args['public'], page_token, last_modified, popularity)
|
||||
|
||||
unfiltered_repos = model.repository.get_user_starred_repositories(user, kind_filter=repo_kind)
|
||||
repos = [repo for repo in unfiltered_repos if can_view_repo(repo)]
|
||||
elif parsed_args['namespace']:
|
||||
# Repositories filtered by namespace do not need pagination (their results are fairly small),
|
||||
# so we just do the lookup directly.
|
||||
repos = list(model.repository.get_visible_repositories(username=username,
|
||||
include_public=parsed_args['public'],
|
||||
namespace=parsed_args['namespace'],
|
||||
kind_filter=repo_kind))
|
||||
else:
|
||||
# Determine the starting offset for pagination. Note that we don't use the normal
|
||||
# model.modelutil.paginate method here, as that does not operate over UNION queries, which
|
||||
# get_visible_repositories will return if there is a logged-in user (for performance reasons).
|
||||
#
|
||||
# Also note the +1 on the limit, as paginate_query uses the extra result to determine whether
|
||||
# there is a next page.
|
||||
start_id = model.modelutil.pagination_start(page_token)
|
||||
repo_query = model.repository.get_visible_repositories(username=username,
|
||||
include_public=parsed_args['public'],
|
||||
start_id=start_id,
|
||||
limit=REPOS_PER_PAGE+1,
|
||||
kind_filter=repo_kind)
|
||||
|
||||
repos, next_page_token = model.modelutil.paginate_query(repo_query, limit=REPOS_PER_PAGE,
|
||||
id_alias='rid')
|
||||
|
||||
# Collect the IDs of the repositories found for subequent lookup of popularity
|
||||
# and/or last modified.
|
||||
if parsed_args['last_modified'] or parsed_args['popularity']:
|
||||
repository_ids = [repo.rid for repo in repos]
|
||||
|
||||
if parsed_args['last_modified']:
|
||||
last_modified_map = model.repository.get_when_last_modified(repository_ids)
|
||||
|
||||
if parsed_args['popularity']:
|
||||
action_sum_map = model.log.get_repositories_action_sums(repository_ids)
|
||||
|
||||
# Collect the IDs of the repositories that are starred for the user, so we can mark them
|
||||
# in the returned results.
|
||||
star_set = set()
|
||||
if username:
|
||||
starred_repos = model.repository.get_user_starred_repositories(user)
|
||||
star_set = {starred.id for starred in starred_repos}
|
||||
|
||||
def repo_view(repo_obj):
|
||||
repo = {
|
||||
'namespace': repo_obj.namespace_user.username,
|
||||
'name': repo_obj.name,
|
||||
'description': repo_obj.description,
|
||||
'is_public': repo_obj.visibility_id == model.repository.get_public_repo_visibility().id,
|
||||
'kind': repo_kind,
|
||||
}
|
||||
|
||||
repo_id = repo_obj.rid
|
||||
|
||||
if parsed_args['last_modified']:
|
||||
repo['last_modified'] = last_modified_map.get(repo_id)
|
||||
|
||||
if parsed_args['popularity']:
|
||||
repo['popularity'] = float(action_sum_map.get(repo_id, 0))
|
||||
|
||||
if username:
|
||||
repo['is_starred'] = repo_id in star_set
|
||||
|
||||
return repo
|
||||
|
||||
return {
|
||||
'repositories': [repo_view(repo) for repo in repos]
|
||||
}, next_page_token
|
||||
return {'repositories': [repo.to_dict() for repo in repos]}, next_page_token
|
||||
|
||||
|
||||
@resource('/v1/repository/<apirepopath:repository>')
|
||||
|
|
@ -253,9 +179,7 @@ class Repository(RepositoryParamResource):
|
|||
'RepoUpdate': {
|
||||
'type': 'object',
|
||||
'description': 'Fields which can be updated in a repository.',
|
||||
'required': [
|
||||
'description',
|
||||
],
|
||||
'required': ['description',],
|
||||
'properties': {
|
||||
'description': {
|
||||
'type': 'string',
|
||||
|
|
@ -273,160 +197,66 @@ class Repository(RepositoryParamResource):
|
|||
def get(self, namespace, repository, parsed_args):
|
||||
"""Fetch the specified repository."""
|
||||
logger.debug('Get repo: %s/%s' % (namespace, repository))
|
||||
|
||||
repo = model.repository.get_repository(namespace, repository)
|
||||
repo = model.get_repo(namespace, repository, get_authenticated_user())
|
||||
if repo is None:
|
||||
raise NotFound()
|
||||
|
||||
can_write = ModifyRepositoryPermission(namespace, repository).can()
|
||||
can_admin = AdministerRepositoryPermission(namespace, repository).can()
|
||||
repo_data = repo.to_dict()
|
||||
repo_data['can_write'] = ModifyRepositoryPermission(namespace, repository).can()
|
||||
repo_data['can_admin'] = AdministerRepositoryPermission(namespace, repository).can()
|
||||
|
||||
is_starred = (model.repository.repository_is_starred(get_authenticated_user(), repo)
|
||||
if get_authenticated_user() else False)
|
||||
is_public = model.repository.is_repository_public(repo)
|
||||
|
||||
# Note: This is *temporary* code for the new OCI model stuff.
|
||||
if repo.kind.name == 'application':
|
||||
def channel_view(channel):
|
||||
return {
|
||||
'name': channel.name,
|
||||
'release': channel.linked_tag.name,
|
||||
'last_modified': format_date(datetime.fromtimestamp(channel.linked_tag.lifetime_start / 1000)),
|
||||
}
|
||||
|
||||
def release_view(release):
|
||||
return {
|
||||
'name': release.name,
|
||||
'last_modified': format_date(datetime.fromtimestamp(release.lifetime_start / 1000)),
|
||||
'channels': releases_channels_map[release.name],
|
||||
}
|
||||
|
||||
channels = oci_model.channel.get_repo_channels(repo)
|
||||
releases_channels_map = defaultdict(list)
|
||||
for channel in channels:
|
||||
releases_channels_map[channel.linked_tag.name].append(channel.name)
|
||||
|
||||
repo_data = {
|
||||
'namespace': namespace,
|
||||
'name': repository,
|
||||
'kind': repo.kind.name,
|
||||
'description': repo.description,
|
||||
'can_write': can_write,
|
||||
'can_admin': can_admin,
|
||||
'is_public': is_public,
|
||||
'is_organization': repo.namespace_user.organization,
|
||||
'is_starred': is_starred,
|
||||
'channels': [channel_view(chan) for chan in channels],
|
||||
'releases': [release_view(release) for release in oci_model.release.get_release_objs(repo)],
|
||||
}
|
||||
|
||||
return repo_data
|
||||
|
||||
# Older image-only repo code.
|
||||
def tag_view(tag):
|
||||
tag_info = {
|
||||
'name': tag.name,
|
||||
'image_id': tag.image.docker_image_id,
|
||||
'size': tag.image.aggregate_size
|
||||
}
|
||||
|
||||
if tag.lifetime_start_ts > 0:
|
||||
last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts))
|
||||
tag_info['last_modified'] = last_modified
|
||||
|
||||
if tag.lifetime_end_ts:
|
||||
expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts))
|
||||
tag_info['expiration'] = expiration
|
||||
|
||||
if tag.tagmanifest is not None:
|
||||
tag_info['manifest_digest'] = tag.tagmanifest.digest
|
||||
|
||||
return tag_info
|
||||
|
||||
stats = None
|
||||
tags = model.tag.list_active_repo_tags(repo)
|
||||
tag_dict = {tag.name: tag_view(tag) for tag in tags}
|
||||
if parsed_args['includeStats']:
|
||||
if parsed_args['includeStats'] and repo.repository_base_elements.kind_name != 'application':
|
||||
stats = []
|
||||
found_dates = {}
|
||||
|
||||
start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
|
||||
counts = model.log.get_repository_action_counts(repo, start_date)
|
||||
for count in counts:
|
||||
stats.append({
|
||||
'date': count.date.isoformat(),
|
||||
'count': count.count,
|
||||
})
|
||||
|
||||
for count in repo.counts:
|
||||
stats.append(count.to_dict())
|
||||
found_dates['%s/%s' % (count.date.month, count.date.day)] = True
|
||||
|
||||
# Fill in any missing stats with zeros.
|
||||
for day in range(1, MAX_DAYS_IN_3_MONTHS):
|
||||
day_date = datetime.now() - timedelta(days=day)
|
||||
key = '%s/%s' % (day_date.month, day_date.day)
|
||||
if not key in found_dates:
|
||||
if key not in found_dates:
|
||||
stats.append({
|
||||
'date': day_date.date().isoformat(),
|
||||
'count': 0,
|
||||
})
|
||||
|
||||
repo_data = {
|
||||
'namespace': namespace,
|
||||
'name': repository,
|
||||
'kind': repo.kind.name,
|
||||
'description': repo.description,
|
||||
'tags': tag_dict,
|
||||
'can_write': can_write,
|
||||
'can_admin': can_admin,
|
||||
'is_public': is_public,
|
||||
'is_organization': repo.namespace_user.organization,
|
||||
'is_starred': is_starred,
|
||||
'status_token': repo.badge_token if not is_public else '',
|
||||
'trust_enabled': bool(features.SIGNING) and repo.trust_enabled,
|
||||
'tag_expiration_s': repo.namespace_user.removed_tag_expiration_s,
|
||||
}
|
||||
|
||||
if stats is not None:
|
||||
repo_data['stats'] = stats
|
||||
|
||||
return repo_data
|
||||
|
||||
|
||||
@require_repo_write
|
||||
@nickname('updateRepo')
|
||||
@validate_json_request('RepoUpdate')
|
||||
def put(self, namespace, repository):
|
||||
""" Update the description in the specified repository. """
|
||||
repo = model.repository.get_repository(namespace, repository)
|
||||
if repo:
|
||||
values = request.get_json()
|
||||
repo.description = values['description']
|
||||
repo.save()
|
||||
if not model.repo_exists(namespace, repository):
|
||||
raise NotFound()
|
||||
|
||||
log_action('set_repo_description', namespace,
|
||||
{'repo': repository, 'namespace': namespace, 'description': values['description']},
|
||||
repo=repo)
|
||||
return {
|
||||
'success': True
|
||||
}
|
||||
raise NotFound()
|
||||
values = request.get_json()
|
||||
model.set_description(namespace, repository, values['description'])
|
||||
|
||||
log_action('set_repo_description', namespace,
|
||||
{'repo': repository,
|
||||
'namespace': namespace,
|
||||
'description': values['description']}, repo_name=repository)
|
||||
return {'success': True}
|
||||
|
||||
@require_repo_admin
|
||||
@nickname('deleteRepository')
|
||||
def delete(self, namespace, repository):
|
||||
""" Delete a repository. """
|
||||
model.repository.purge_repository(namespace, repository)
|
||||
user = model.user.get_namespace_user(namespace)
|
||||
username = model.purge_repository(namespace, repository)
|
||||
|
||||
if features.BILLING:
|
||||
plan = get_namespace_plan(namespace)
|
||||
check_repository_usage(user, plan)
|
||||
model.check_repository_usage(username, plan)
|
||||
|
||||
# Remove any builds from the queue.
|
||||
dockerfile_build_queue.delete_namespaced_items(namespace, repository)
|
||||
|
||||
log_action('delete_repo', namespace,
|
||||
{'repo': repository, 'namespace': namespace})
|
||||
log_action('delete_repo', namespace, {'repo': repository, 'namespace': namespace})
|
||||
return '', 204
|
||||
|
||||
|
||||
|
|
@ -438,9 +268,7 @@ class RepositoryVisibility(RepositoryParamResource):
|
|||
'ChangeVisibility': {
|
||||
'type': 'object',
|
||||
'description': 'Change the visibility for the repository.',
|
||||
'required': [
|
||||
'visibility',
|
||||
],
|
||||
'required': ['visibility',],
|
||||
'properties': {
|
||||
'visibility': {
|
||||
'type': 'string',
|
||||
|
|
@ -459,17 +287,17 @@ class RepositoryVisibility(RepositoryParamResource):
|
|||
@validate_json_request('ChangeVisibility')
|
||||
def post(self, namespace, repository):
|
||||
""" Change the visibility of a repository. """
|
||||
repo = model.repository.get_repository(namespace, repository)
|
||||
if repo:
|
||||
if model.repo_exists(namespace, repository):
|
||||
values = request.get_json()
|
||||
visibility = values['visibility']
|
||||
if visibility == 'private':
|
||||
check_allowed_private_repos(namespace)
|
||||
|
||||
model.repository.set_repository_visibility(repo, visibility)
|
||||
model.set_repository_visibility(namespace, repository, visibility)
|
||||
log_action('change_repo_visibility', namespace,
|
||||
{'repo': repository, 'namespace': namespace, 'visibility': values['visibility']},
|
||||
repo=repo)
|
||||
{'repo': repository,
|
||||
'namespace': namespace,
|
||||
'visibility': values['visibility']}, repo_name=repository)
|
||||
return {'success': True}
|
||||
|
||||
|
||||
|
|
@ -481,9 +309,7 @@ class RepositoryTrust(RepositoryParamResource):
|
|||
'ChangeRepoTrust': {
|
||||
'type': 'object',
|
||||
'description': 'Change the trust settings for the repository.',
|
||||
'required': [
|
||||
'trust_enabled',
|
||||
],
|
||||
'required': ['trust_enabled',],
|
||||
'properties': {
|
||||
'trust_enabled': {
|
||||
'type': 'boolean',
|
||||
|
|
@ -499,19 +325,19 @@ class RepositoryTrust(RepositoryParamResource):
|
|||
@validate_json_request('ChangeRepoTrust')
|
||||
def post(self, namespace, repository):
|
||||
""" Change the visibility of a repository. """
|
||||
repo = model.repository.get_repository(namespace, repository)
|
||||
if not repo:
|
||||
if not model.repo_exists(namespace, repository):
|
||||
raise NotFound()
|
||||
|
||||
tags, _ = tuf_metadata_api.get_default_tags_with_expiration(namespace, repository)
|
||||
if tags and not tuf_metadata_api.delete_metadata(namespace, repository):
|
||||
raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'})
|
||||
|
||||
raise DownstreamIssue({'message': 'Unable to delete downstream trust metadata'})
|
||||
values = request.get_json()
|
||||
model.repository.set_trust(repo, values['trust_enabled'])
|
||||
model.set_trust(namespace, repository, values['trust_enabled'])
|
||||
|
||||
log_action('change_repo_trust', namespace,
|
||||
{'repo': repository, 'namespace': namespace, 'trust_enabled': values['trust_enabled']},
|
||||
repo=repo)
|
||||
log_action(
|
||||
'change_repo_trust', namespace,
|
||||
{'repo': repository,
|
||||
'namespace': namespace,
|
||||
'trust_enabled': values['trust_enabled']}, repo_name=repository)
|
||||
|
||||
return {'success': True}
|
||||
|
|
|
|||
255
endpoints/api/repository_models_interface.py
Normal file
255
endpoints/api/repository_models_interface.py
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
from abc import ABCMeta, abstractmethod
|
||||
from collections import namedtuple, defaultdict
|
||||
|
||||
from datetime import datetime
|
||||
from six import add_metaclass
|
||||
|
||||
import features
|
||||
from endpoints.api import format_date
|
||||
|
||||
|
||||
class RepositoryBaseElement(
|
||||
namedtuple('RepositoryBaseElement', [
|
||||
'namespace_name', 'repository_name', 'is_starred', 'is_public', 'kind_name', 'description',
|
||||
'namespace_user_organization', 'namespace_user_removed_tag_expiration_s', 'last_modified',
|
||||
'action_count', 'should_last_modified', 'should_popularity', 'should_is_starred'
|
||||
])):
|
||||
"""
|
||||
Repository a single quay repository
|
||||
:type namespace_name: string
|
||||
:type repository_name: string
|
||||
:type is_starred: boolean
|
||||
:type is_public: boolean
|
||||
:type kind_name: string
|
||||
:type description: string
|
||||
:type namespace_user_organization: boolean
|
||||
:type should_last_modified: boolean
|
||||
:type should_popularity: boolean
|
||||
:type should_is_starred: boolean
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
repo = {
|
||||
'namespace': self.namespace_name,
|
||||
'name': self.repository_name,
|
||||
'description': self.description,
|
||||
'is_public': self.is_public,
|
||||
'kind': self.kind_name,
|
||||
}
|
||||
|
||||
if self.should_last_modified:
|
||||
repo['last_modified'] = self.last_modified
|
||||
|
||||
if self.should_popularity:
|
||||
repo['popularity'] = float(self.action_count if self.action_count else 0)
|
||||
|
||||
if self.should_is_starred:
|
||||
repo['is_starred'] = self.is_starred
|
||||
|
||||
return repo
|
||||
|
||||
|
||||
class ApplicationRepository(
|
||||
namedtuple('ApplicationRepository', ['repository_base_elements', 'channels', 'releases'])):
|
||||
"""
|
||||
Repository a single quay repository
|
||||
:type repository_base_elements: RepositoryBaseElement
|
||||
:type channels: [Channel]
|
||||
:type releases: [Release]
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
repo_data = {
|
||||
'namespace': self.repository_base_elements.namespace_name,
|
||||
'name': self.repository_base_elements.repository_name,
|
||||
'kind': self.repository_base_elements.kind_name,
|
||||
'description': self.repository_base_elements.description,
|
||||
'is_public': self.repository_base_elements.is_public,
|
||||
'is_organization': self.repository_base_elements.namespace_user_organization,
|
||||
'is_starred': self.repository_base_elements.is_starred,
|
||||
'channels': [chan.to_dict() for chan in self.channels],
|
||||
'releases': [release.to_dict() for release in self.releases],
|
||||
}
|
||||
|
||||
return repo_data
|
||||
|
||||
|
||||
class ImageRepositoryRepository(
|
||||
namedtuple('NonApplicationRepository',
|
||||
['repository_base_elements', 'tags', 'counts', 'badge_token', 'trust_enabled'])):
|
||||
"""
|
||||
Repository a single quay repository
|
||||
:type repository_base_elements: RepositoryBaseElement
|
||||
:type tags: [Tag]
|
||||
:type counts: [count]
|
||||
:type badge_token: string
|
||||
:type trust_enabled: boolean
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'namespace': self.repository_base_elements.namespace_name,
|
||||
'name': self.repository_base_elements.repository_name,
|
||||
'kind': self.repository_base_elements.kind_name,
|
||||
'description': self.repository_base_elements.description,
|
||||
'is_public': self.repository_base_elements.is_public,
|
||||
'is_organization': self.repository_base_elements.namespace_user_organization,
|
||||
'is_starred': self.repository_base_elements.is_starred,
|
||||
'tags': {tag.name: tag.to_dict()
|
||||
for tag in self.tags},
|
||||
'status_token': self.badge_token if not self.repository_base_elements.is_public else '',
|
||||
'trust_enabled': bool(features.SIGNING) and self.trust_enabled,
|
||||
'tag_expiration_s': self.repository_base_elements.namespace_user_removed_tag_expiration_s,
|
||||
}
|
||||
|
||||
|
||||
class Repository(namedtuple('Repository', [
|
||||
'namespace_name',
|
||||
'repository_name',
|
||||
])):
|
||||
"""
|
||||
Repository a single quay repository
|
||||
:type namespace_name: string
|
||||
:type repository_name: string
|
||||
"""
|
||||
|
||||
|
||||
class Channel(namedtuple('Channel', ['name', 'linked_tag_name', 'linked_tag_lifetime_start'])):
|
||||
"""
|
||||
Repository a single quay repository
|
||||
:type name: string
|
||||
:type linked_tag_name: string
|
||||
:type linked_tag_lifetime_start: string
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'name': self.name,
|
||||
'release': self.linked_tag_name,
|
||||
'last_modified': format_date(datetime.fromtimestamp(self.linked_tag_lifetime_start / 1000)),
|
||||
}
|
||||
|
||||
|
||||
class Release(
|
||||
namedtuple('Channel', ['name', 'released', 'lifetime_start', 'releases_channels_map'])):
|
||||
"""
|
||||
Repository a single quay repository
|
||||
:type name: string
|
||||
:type released: string
|
||||
:type last_modified: string
|
||||
:type releases_channels_map: {string -> string}
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'name': self.name,
|
||||
'last_modified': format_date(datetime.fromtimestamp(self.lifetime_start / 1000)),
|
||||
'channels': self.releases_channels_map[self.name],
|
||||
}
|
||||
|
||||
|
||||
class Tag(
|
||||
namedtuple('Tag', [
|
||||
'name', 'image_docker_image_id', 'image_aggregate_size', 'lifetime_start_ts',
|
||||
'tag_manifest_digest'
|
||||
])):
|
||||
"""
|
||||
:type name: string
|
||||
:type image_docker_image_id: string
|
||||
:type image_aggregate_size: int
|
||||
:type lifetime_start_ts: int
|
||||
:type tag_manifest_digest: string
|
||||
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
tag_info = {
|
||||
'name': self.name,
|
||||
'image_id': self.image_docker_image_id,
|
||||
'size': self.image_aggregate_size
|
||||
}
|
||||
|
||||
if self.lifetime_start_ts > 0:
|
||||
last_modified = format_date(datetime.fromtimestamp(self.lifetime_start_ts))
|
||||
tag_info['last_modified'] = last_modified
|
||||
|
||||
if self.tag_manifest_digest is not None:
|
||||
tag_info['manifest_digest'] = self.tag_manifest_digest
|
||||
|
||||
return tag_info
|
||||
|
||||
|
||||
class Count(namedtuple('Count', ['date', 'count'])):
|
||||
"""
|
||||
date: DateTime
|
||||
count: int
|
||||
"""
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'date': self.date.isoformat(),
|
||||
'count': self.count,
|
||||
}
|
||||
|
||||
|
||||
@add_metaclass(ABCMeta)
|
||||
class RepositoryDataInterface(object):
|
||||
"""
|
||||
Interface that represents all data store interactions required by a Repository.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_repo(self, namespace_name, repository_name, user):
|
||||
"""
|
||||
Returns a repository
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def repo_exists(self, namespace_name, repository_name):
|
||||
"""
|
||||
Returns true if a repo exists and false if not
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_repo(self, namespace, name, creating_user, description, visibility='private',
|
||||
repo_kind='image'):
|
||||
"""
|
||||
Returns creates a new repo
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_repo_list(self, starred, user, repo_kind, namespace, username, public, page_token,
|
||||
last_modified, popularity):
|
||||
"""
|
||||
Returns a RepositoryBaseElement
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def set_repository_visibility(self, namespace_name, repository_name, visibility):
|
||||
"""
|
||||
Sets a repository's visibility if it is found
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def set_trust(self, namespace_name, repository_name, trust):
|
||||
"""
|
||||
Sets a repository's trust_enabled field if it is found
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def set_description(self, namespace_name, repository_name, description):
|
||||
"""
|
||||
Sets a repository's description if it is found.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def purge_repository(self, namespace_name, repository_name):
|
||||
"""
|
||||
Removes a repository
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def check_repository_usage(self, user_name, plan_found):
|
||||
"""
|
||||
Creates a notification for a user if they are over or under on their repository usage
|
||||
"""
|
||||
166
endpoints/api/repository_models_pre_oci.py
Normal file
166
endpoints/api/repository_models_pre_oci.py
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
from collections import defaultdict
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from auth.permissions import ReadRepositoryPermission
|
||||
from data import model, oci_model
|
||||
from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \
|
||||
ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count
|
||||
|
||||
MAX_DAYS_IN_3_MONTHS = 92
|
||||
REPOS_PER_PAGE = 100
|
||||
|
||||
|
||||
def create_channel(channel, releases_channels_map):
|
||||
releases_channels_map[channel.linked_tag_name].append(channel.name)
|
||||
return Channel(channel.name, channel.linked_tag.name, channel.linked_tag.lifetime_start)
|
||||
|
||||
|
||||
class PreOCIModel(RepositoryDataInterface):
|
||||
"""
|
||||
PreOCIModel implements the data model for the Repo Email using a database schema
|
||||
before it was changed to support the OCI specification.
|
||||
"""
|
||||
|
||||
def check_repository_usage(self, username, plan_found):
|
||||
private_repos = model.user.get_private_repo_count(username)
|
||||
if plan_found is None:
|
||||
repos_allowed = 0
|
||||
else:
|
||||
repos_allowed = plan_found['privateRepos']
|
||||
|
||||
user_or_org = model.user.get_namespace_user(username)
|
||||
if private_repos > repos_allowed:
|
||||
model.notification.create_unique_notification('over_private_usage', user_or_org,
|
||||
{'namespace': username})
|
||||
else:
|
||||
model.notification.delete_notifications_by_kind(user_or_org, 'over_private_usage')
|
||||
|
||||
def purge_repository(self, namespace_name, repository_name):
|
||||
model.repository.purge_repository(namespace_name, repository_name)
|
||||
user = model.user.get_namespace_user(namespace_name)
|
||||
return user.username
|
||||
|
||||
def set_description(self, namespace_name, repository_name, description):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
model.repository.set_description(repo, description)
|
||||
|
||||
def set_trust(self, namespace_name, repository_name, trust):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
model.repository.set_trust(repo, trust)
|
||||
|
||||
def set_repository_visibility(self, namespace_name, repository_name, visibility):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
model.repository.set_repository_visibility(repo, visibility)
|
||||
|
||||
def get_repo_list(self, starred, user, repo_kind, namespace, username, public, page_token,
|
||||
last_modified, popularity):
|
||||
next_page_token = None
|
||||
# Lookup the requested repositories (either starred or non-starred.)
|
||||
if starred:
|
||||
# Return the full list of repos starred by the current user that are still visible to them.
|
||||
def can_view_repo(repo):
|
||||
return ReadRepositoryPermission(repo.namespace_user.username, repo.name).can()
|
||||
|
||||
unfiltered_repos = model.repository.get_user_starred_repositories(user,
|
||||
kind_filter=repo_kind)
|
||||
repos = [repo for repo in unfiltered_repos if can_view_repo(repo)]
|
||||
elif namespace:
|
||||
# Repositories filtered by namespace do not need pagination (their results are fairly small),
|
||||
# so we just do the lookup directly.
|
||||
repos = list(
|
||||
model.repository.get_visible_repositories(username=username, include_public=public,
|
||||
namespace=namespace, kind_filter=repo_kind))
|
||||
else:
|
||||
# Determine the starting offset for pagination. Note that we don't use the normal
|
||||
# model.modelutil.paginate method here, as that does not operate over UNION queries, which
|
||||
# get_visible_repositories will return if there is a logged-in user (for performance reasons).
|
||||
#
|
||||
# Also note the +1 on the limit, as paginate_query uses the extra result to determine whether
|
||||
# there is a next page.
|
||||
start_id = model.modelutil.pagination_start(page_token)
|
||||
repo_query = model.repository.get_visible_repositories(
|
||||
username=username, include_public=public, start_id=start_id, limit=REPOS_PER_PAGE + 1,
|
||||
kind_filter=repo_kind)
|
||||
|
||||
repos, next_page_token = model.modelutil.paginate_query(repo_query, limit=REPOS_PER_PAGE,
|
||||
id_alias='rid')
|
||||
|
||||
# Collect the IDs of the repositories found for subequent lookup of popularity
|
||||
# and/or last modified.
|
||||
last_modified_map = {}
|
||||
action_sum_map = {}
|
||||
if last_modified or popularity:
|
||||
repository_ids = [repo.rid for repo in repos]
|
||||
|
||||
if last_modified:
|
||||
last_modified_map = model.repository.get_when_last_modified(repository_ids)
|
||||
|
||||
if popularity:
|
||||
action_sum_map = model.log.get_repositories_action_sums(repository_ids)
|
||||
|
||||
# Collect the IDs of the repositories that are starred for the user, so we can mark them
|
||||
# in the returned results.
|
||||
star_set = set()
|
||||
if username:
|
||||
starred_repos = model.repository.get_user_starred_repositories(user)
|
||||
star_set = {starred.id for starred in starred_repos}
|
||||
|
||||
return [
|
||||
RepositoryBaseElement(repo.namespace_user.username, repo.name, repo.id in star_set,
|
||||
repo.visibility_id == model.repository.get_public_repo_visibility().id,
|
||||
repo_kind, repo.description, repo.namespace_user.organization,
|
||||
repo.namespace_user.removed_tag_expiration_s,
|
||||
last_modified_map.get(repo.rid),
|
||||
action_sum_map.get(repo.rid), last_modified, popularity, username)
|
||||
for repo in repos
|
||||
], next_page_token
|
||||
|
||||
def repo_exists(self, namespace_name, repository_name):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
if repo is None:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def create_repo(self, namespace_name, repository_name, owner, description, visibility='private',
|
||||
repo_kind='image'):
|
||||
repo = model.repository.create_repository(namespace_name, repository_name, owner, visibility,
|
||||
repo_kind=repo_kind)
|
||||
model.repository.set_description(repo, description)
|
||||
return Repository(namespace_name, repository_name)
|
||||
|
||||
def get_repo(self, namespace_name, repository_name, user):
|
||||
repo = model.repository.get_repository(namespace_name, repository_name)
|
||||
if repo is None:
|
||||
return None
|
||||
|
||||
is_starred = model.repository.repository_is_starred(user, repo) if user else False
|
||||
is_public = model.repository.is_repository_public(repo)
|
||||
base = RepositoryBaseElement(
|
||||
namespace_name, repository_name, is_starred, is_public, repo.kind.name, repo.description,
|
||||
repo.namespace_user.organization, repo.namespace_user.removed_tag_expiration_s, None, None,
|
||||
False, False, False)
|
||||
|
||||
# Note: This is *temporary* code for the new OCI model stuff.
|
||||
if base.kind_name == 'application':
|
||||
channels = oci_model.channel.get_repo_channels(repo)
|
||||
releases = oci_model.release.get_release_objs(repo)
|
||||
releases_channels_map = defaultdict(list)
|
||||
return ApplicationRepository(
|
||||
base, [create_channel(channel, releases_channels_map) for channel in channels], [
|
||||
Release(release.name, release.released, release.lifetime_start, releases_channels_map)
|
||||
for release in releases
|
||||
])
|
||||
|
||||
tags = model.tag.list_active_repo_tags(repo)
|
||||
start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
|
||||
counts = model.log.get_repository_action_counts(repo, start_date)
|
||||
|
||||
return ImageRepositoryRepository(base, [
|
||||
Tag(tag.name, tag.image.docker_image_id, tag.image.aggregate_size, tag.lifetime_start_ts,
|
||||
tag.tagmanifest.digest) for tag in tags
|
||||
], [Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled)
|
||||
|
||||
|
||||
pre_oci_model = PreOCIModel()
|
||||
|
|
@ -9,7 +9,6 @@ from features import FeatureNameValue
|
|||
|
||||
from test.fixtures import *
|
||||
|
||||
|
||||
INVALID_RESPONSE = {
|
||||
u'detail': u"u'invalid_req' is not of type 'boolean'",
|
||||
u'error_message': u"u'invalid_req' is not of type 'boolean'",
|
||||
|
|
@ -20,31 +19,44 @@ INVALID_RESPONSE = {
|
|||
}
|
||||
|
||||
NOT_FOUND_RESPONSE = {
|
||||
u'detail': u'Not Found',
|
||||
u'error_message': u'Not Found',
|
||||
u'error_type': u'not_found',
|
||||
u'message': u'You have requested this URI [/api/v1/repository/devtable/repo/changetrust] but did you mean /api/v1/repository/<apirepopath:repository>/changetrust or /api/v1/repository/<apirepopath:repository>/changevisibility or /api/v1/repository/<apirepopath:repository>/tag/<tag>/images ?',
|
||||
u'status': 404,
|
||||
u'title': u'not_found',
|
||||
u'type': u'http://localhost/api/v1/error/not_found'
|
||||
u'detail':
|
||||
u'Not Found',
|
||||
u'error_message':
|
||||
u'Not Found',
|
||||
u'error_type':
|
||||
u'not_found',
|
||||
u'message':
|
||||
u'You have requested this URI [/api/v1/repository/devtable/repo/changetrust] but did you mean /api/v1/repository/<apirepopath:repository>/changetrust or /api/v1/repository/<apirepopath:repository>/changevisibility or /api/v1/repository/<apirepopath:repository>/tag/<tag>/images ?',
|
||||
u'status':
|
||||
404,
|
||||
u'title':
|
||||
u'not_found',
|
||||
u'type':
|
||||
u'http://localhost/api/v1/error/not_found'
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('trust_enabled,repo_found,expected_body,expected_status', [
|
||||
(True, True,{'success': True}, 200),
|
||||
(False, True, {'success': True}, 200),
|
||||
(True, True, {
|
||||
'success': True
|
||||
}, 200),
|
||||
(False, True, {
|
||||
'success': True
|
||||
}, 200),
|
||||
(False, False, NOT_FOUND_RESPONSE, 404),
|
||||
('invalid_req', False, INVALID_RESPONSE , 400),
|
||||
('invalid_req', False, INVALID_RESPONSE, 400),
|
||||
])
|
||||
def test_post_changetrust(trust_enabled, repo_found, expected_body, expected_status, client):
|
||||
with patch('endpoints.api.repository.tuf_metadata_api') as mock_tuf:
|
||||
with patch('endpoints.api.repository.model') as mock_model:
|
||||
mock_model.repository.get_repository.return_value = MagicMock() if repo_found else None
|
||||
with patch(
|
||||
'endpoints.api.repository_models_pre_oci.model.repository.get_repository') as mock_model:
|
||||
mock_model.return_value = MagicMock() if repo_found else None
|
||||
mock_tuf.get_default_tags_with_expiration.return_value = ['tags', 'expiration']
|
||||
with client_with_identity('devtable', client) as cl:
|
||||
params = {'repository': 'devtable/repo'}
|
||||
request_body = {'trust_enabled': trust_enabled}
|
||||
assert expected_body == conduct_api_call(cl, RepositoryTrust, 'POST', params, request_body, expected_status).json
|
||||
assert expected_body == conduct_api_call(cl, RepositoryTrust, 'POST', params, request_body,
|
||||
expected_status).json
|
||||
|
||||
|
||||
def test_signing_disabled(client):
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
Reference in a new issue