vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Repository endpoint tags pagination (#3238)

Browse source 
* endpoint/api/repository: limit the number of tags returned

- Limit the number of tags returned by /api/v1/repository/<ns:repo> to 500.
- Uses the tag history endpoint instead, with an active tag filte.
- Update UI to use tag history endpoint instead.
This commit is contained in:
Kenny Lee Sin Cheong 2018-09-14 15:30:54 -04:00 committed by GitHub
parent 6d5489b254
commit 8e643ce5d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 99 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
data/model/tag.py
View file

@ -491,7 +491,7 @@ def get_tag_image(namespace_name, repository_name, tag_name, include_storage=Fal
return _get_repo_tag_image(tag_name, include_storage, modifier)
def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None):
def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None, active_tags_only=False):
query = (RepositoryTag
.select(RepositoryTag, Image, ImageStorage)
.join(Image)
@ -503,6 +503,9 @@ def list_repository_tag_history(repo_obj, page=1, size=100, specific_tag=None):
.limit(size + 1)
.offset(size * (page - 1)))
if active_tags_only:
query = _tag_alive(query)
if specific_tag:
query = query.where(RepositoryTag.name == specific_tag)

2
data/registry_model/interface.py
View file

@ -81,7 +81,7 @@ class RegistryDataInterface(object):
"""
@abstractmethod
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None):
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None, active_tags_only=False):
"""
Returns the history of all tags in the repository (unless filtered). This includes tags that
have been made in-active due to newer versions of those tags coming into service.

5
data/registry_model/registry_pre_oci_model.py
View file

@ -164,14 +164,15 @@ class PreOCIModel(RegistryDataInterface):
else None))
for tag in tags]
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None):
def list_repository_tag_history(self, repository_ref, page=1, size=100, specific_tag_name=None, active_tags_only=False):
"""
Returns the history of all tags in the repository (unless filtered). This includes tags that
have been made in-active due to newer versions of those tags coming into service.
"""
tags, manifest_map, has_more = model.tag.list_repository_tag_history(repository_ref._db_id,
page, size,
specific_tag_name)
specific_tag_name,
active_tags_only)
return [Tag.for_repository_tag(tag, manifest_map.get(tag.id),
legacy_image=LegacyImage.for_image(tag.image))
for tag in tags], has_more

6
endpoints/api/repository.py
View file

@ -192,12 +192,16 @@ class Repository(RepositoryParamResource):
@parse_args()
@query_param('includeStats', 'Whether to include action statistics', type=truthy_bool,
default=False)
@query_param('includeTags', 'Whether to include repository tags', type=truthy_bool,
default=True)
@require_repo_read
@nickname('getRepo')
def get(self, namespace, repository, parsed_args):
"""Fetch the specified repository."""
logger.debug('Get repo: %s/%s' % (namespace, repository))
repo = model.get_repo(namespace, repository, get_authenticated_user())
include_tags = parsed_args['includeTags']
max_tags = 500;
repo = model.get_repo(namespace, repository, get_authenticated_user(), include_tags, max_tags)
if repo is None:
raise NotFound()

9
endpoints/api/repository_models_interface.py
View file

@ -87,7 +87,7 @@ class ImageRepositoryRepository(
"""
def to_dict(self):
return {
img_repo = {
'namespace': self.repository_base_elements.namespace_name,
'name': self.repository_base_elements.repository_name,
'kind': self.repository_base_elements.kind_name,
@ -95,12 +95,13 @@ class ImageRepositoryRepository(
'is_public': self.repository_base_elements.is_public,
'is_organization': self.repository_base_elements.namespace_user_organization,
'is_starred': self.repository_base_elements.is_starred,
'tags': {tag.name: tag.to_dict()
for tag in self.tags},
'status_token': self.badge_token if not self.repository_base_elements.is_public else '',
'trust_enabled': bool(features.SIGNING) and self.trust_enabled,
'tag_expiration_s': self.repository_base_elements.namespace_user_removed_tag_expiration_s,
}
if self.tags is not None:
img_repo['tags'] = {tag.name: tag.to_dict() for tag in self.tags}
return img_repo
class Repository(namedtuple('Repository', [
@ -203,7 +204,7 @@ class RepositoryDataInterface(object):
"""
@abstractmethod
def get_repo(self, namespace_name, repository_name, user):
def get_repo(self, namespace_name, repository_name, user, include_tags=True, max_tags=500):
"""
Returns a repository
"""

20
endpoints/api/repository_models_pre_oci.py
View file

@ -134,7 +134,7 @@ class PreOCIModel(RepositoryDataInterface):
repo_kind=repo_kind, description=description)
return Repository(namespace_name, repository_name)
def get_repo(self, namespace_name, repository_name, user):
def get_repo(self, namespace_name, repository_name, user, include_tags=True, max_tags=500):
repo = model.repository.get_repository(namespace_name, repository_name)
if repo is None:
return None
@ -156,18 +156,22 @@ class PreOCIModel(RepositoryDataInterface):
for release in releases
])
tags = None
repo_ref = RepositoryReference.for_repo_obj(repo)
tags = registry_model.list_repository_tags(repo_ref, include_legacy_images=True)
if include_tags:
tags, _ = registry_model.list_repository_tag_history(repo_ref, page=1, size=max_tags, active_tags_only=True)
tags = [
Tag(tag.name, tag.legacy_image.docker_image_id, tag.legacy_image.aggregate_size,
tag.lifetime_start_ts,
tag.manifest_digest,
tag.lifetime_end_ts) for tag in tags
]
start_date = datetime.now() - timedelta(days=MAX_DAYS_IN_3_MONTHS)
counts = model.log.get_repository_action_counts(repo, start_date)
return ImageRepositoryRepository(base, [
Tag(tag.name, tag.legacy_image.docker_image_id, tag.legacy_image.aggregate_size,
tag.lifetime_start_ts,
tag.manifest_digest,
tag.lifetime_end_ts) for tag in tags
], [Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled)
return ImageRepositoryRepository(base, tags,
[Count(count.date, count.count) for count in counts], repo.badge_token, repo.trust_enabled)
pre_oci_model = PreOCIModel()

18
endpoints/api/tag.py
View file

@ -7,7 +7,8 @@ from auth.auth_context import get_authenticated_user
from data.registry_model import registry_model
from endpoints.api import (resource, nickname, require_repo_read, require_repo_write,
RepositoryParamResource, log_action, validate_json_request, path_param,
parse_args, query_param, truthy_bool, disallow_for_app_repositories)
parse_args, query_param, truthy_bool, disallow_for_app_repositories,
format_date)
from endpoints.api.image import image_dict
from endpoints.exception import NotFound, InvalidRequest
from util.names import TAG_ERROR, TAG_REGEX
@ -30,6 +31,16 @@ def _tag_dict(tag):
if tag.legacy_image:
tag_info['docker_image_id'] = tag.legacy_image.docker_image_id
tag_info['image_id'] = tag.legacy_image.docker_image_id
tag_info['size'] = tag.legacy_image.aggregate_size
if tag.lifetime_start_ts > 0:
last_modified = format_date(datetime.fromtimestamp(tag.lifetime_start_ts))
tag_info['last_modified'] = last_modified
if tag.lifetime_end_ts is not None:
expiration = format_date(datetime.fromtimestamp(tag.lifetime_end_ts))
tag_info['expiration'] = expiration
return tag_info
@ -46,11 +57,13 @@ class ListRepositoryTags(RepositoryParamResource):
@query_param('limit', 'Limit to the number of results to return per page. Max 100.', type=int,
default=50)
@query_param('page', 'Page index for the results. Default 1.', type=int, default=1)
@query_param('onlyActiveTags', 'Filter to only active tags.', type=truthy_bool, default=False)
@nickname('listRepoTags')
def get(self, namespace, repository, parsed_args):
specific_tag = parsed_args.get('specificTag') or None
page = max(1, parsed_args.get('page', 1))
limit = min(100, max(1, parsed_args.get('limit', 50)))
active_tags_only = parsed_args.get('onlyActiveTags')
repo_ref = registry_model.lookup_repository(namespace, repository)
if repo_ref is None:
@ -58,7 +71,8 @@ class ListRepositoryTags(RepositoryParamResource):
history, has_more = registry_model.list_repository_tag_history(repo_ref, page=page,
size=limit,
specific_tag_name=specific_tag)
specific_tag_name=specific_tag,
active_tags_only=active_tags_only)
return {
'tags': [_tag_dict(tag) for tag in history],
'page': page,

6
static/js/directives/repo-view/repo-panel-tags.js
View file

@ -237,13 +237,13 @@ angular.module('quay').directive('repoPanelTags', function () {
}));
}, true);
$scope.$watch('repository', function(repository) {
if (!repository) { return; }
$scope.$watch('repository', function(updatedRepoObject, previousRepoObject) {
if (updatedRepoObject.tags === previousRepoObject.tags) { return; }
// Process each of the tags.
setTagState();
loadRepoSignatures();
});
}, true);
$scope.loadImageVulnerabilities = function(image_id, imageData) {
VulnerabilityService.loadImageVulnerabilities($scope.repository, image_id, function(resp) {

4
static/js/directives/ui/tag-operations-dialog.js
View file

@ -46,7 +46,7 @@ angular.module('quay').directive('tagOperationsDialog', function () {
};
$scope.isAnotherImageTag = function(image, tag) {
if (!$scope.repository) { return; }
if (!$scope.repository.tags) { return; }
var found = $scope.repository.tags[tag];
if (found == null) { return false; }
@ -54,7 +54,7 @@ angular.module('quay').directive('tagOperationsDialog', function () {
};
$scope.isOwnedTag = function(image, tag) {
if (!$scope.repository) { return; }
if (!$scope.repository.tags) { return; }
var found = $scope.repository.tags[tag];
if (found == null) { return false; }

3
static/js/pages/app-view.js
View file

@ -25,7 +25,8 @@
var params = {
'repository': $scope.namespace + '/' + $scope.name,
'repo_kind': 'application',
'includeStats': true
'includeStats': true,
'includeTags': false
};
$scope.repositoryResource = ApiService.getRepoAsResource(params).get(function(repo) {

3
static/js/pages/build-view.js
View file

@ -35,7 +35,8 @@
var loadRepository = function() {
var params = {
'repository': $scope.namespace + '/' + $scope.name
'repository': $scope.namespace + '/' + $scope.name,
'includeTags': false
};
$scope.repoResource = ApiService.getRepoAsResource(params).get(function(repo) {

3
static/js/pages/create-repository-notification.js
View file

@ -16,7 +16,8 @@
var loadRepository = function() {
var params = {
'repository': $scope.namespace + '/' + $scope.name
'repository': $scope.namespace + '/' + $scope.name,
'includeTags': false
};
$scope.repositoryResource = ApiService.getRepoAsResource(params).get(function(repo) {

3
static/js/pages/manifest-view.js
View file

@ -36,7 +36,8 @@
var loadRepository = function() {
var params = {
'repository': namespace + '/' + name
'repository': namespace + '/' + name,
'includeTags': false
};
$scope.repositoryResource = ApiService.getRepoAsResource(params).get(function(repo) {

35
static/js/pages/repo-view.js
View file

@ -33,6 +33,8 @@
'historyFilter': ''
};
$scope.repositoryTags = {};
var buildPollChannel = null;
// Make sure we track the current user.
@ -50,17 +52,48 @@
});
};
var loadRepositoryTags = function() {
loadPaginatedRepositoryTags(1);
};
var loadPaginatedRepositoryTags = function(page) {
var params = {
'repository': $scope.namespace + '/' + $scope.name,
'limit': 100,
'page': page,
'onlyActiveTags': true
};
ApiService.listRepoTags(null, params).then(function(resp) {
var newTags = resp.tags.reduce(function(result, item, index, array) {
var tag_name = item['name'];
result[tag_name] = item;
return result;
}, {});
$.extend($scope.repositoryTags, newTags);
if (resp.has_additional) {
loadPaginatedRepositoryTags(page + 1);
}
});
};
var loadRepository = function() {
// Mark the images to be reloaded.
$scope.viewScope.images = null;
var params = {
'repository': $scope.namespace + '/' + $scope.name,
'includeStats': true
'includeStats': true,
'includeTags': false
};
$scope.repositoryResource = ApiService.getRepoAsResource(params).get(function(repo) {
if (repo != undefined) {
loadRepositoryTags();
repo.tags = $scope.repositoryTags;
$scope.repository = repo;
$scope.viewScope.repository = repo;

3
static/js/pages/trigger-setup.js
View file

@ -17,7 +17,8 @@
var loadRepository = function() {
var params = {
'repository': namespace + '/' + name
'repository': namespace + '/' + name,
'includeTags': false
};
$scope.repositoryResource = ApiService.getRepoAsResource(params).get(function(repo) {

4
test/test_api_usage.py
View file

@ -2219,11 +2219,11 @@ class TestGetRepository(ApiTestCase):
self.login(ADMIN_ACCESS_USER)
# base + repo + is_starred + tags
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 5):
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 6):
self.getJsonResponse(Repository, params=dict(repository=ADMIN_ACCESS_USER + '/simple'))
# base + repo + is_starred + tags
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 5):
with assert_query_count(BASE_LOGGEDIN_QUERY_COUNT + 6):
json = self.getJsonResponse(Repository,
params=dict(repository=ADMIN_ACCESS_USER + '/gargantuan'))