vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add ID-based pagination to logs using new decorators and an encrypted token

Browse source 
Fixes #599
This commit is contained in:
Joseph Schorr 2015-12-22 09:05:17 -05:00
parent af77b92bcf
commit bd0a098282
8 changed files with 110 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
config.py
View file

@ -289,3 +289,8 @@ class DefaultConfig(object):
BITTORRENT_PIECE_SIZE = 512 * 1024
BITTORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce'
BITTORRENT_FILENAME_PEPPER = '3ae93fef-c30a-427e-9ba0-eea0fd710419'
# "Secret" key for generating encrypted paging tokens. Only needed to be secret to
# hide the ID range for production (in which this value is overridden). Should *not*
# be relied upon for secure encryption otherwise.
PAGE_TOKEN_KEY = 'GdE4<~8&9LHDPM++'

2
data/model/__init__.py
View file

@ -95,4 +95,4 @@ config = Config()
# moving the minimal number of things to _basequery
# TODO document the methods and modules for each one of the submodules below.
from data.model import (blob, build, image, log, notification, oauth, organization, permission,
repository, storage, tag, team, token, user, release)
repository, storage, tag, team, token, user, release, modelutil)

9
data/model/log.py
View file

@ -41,9 +41,7 @@ def get_aggregated_logs(start_time, end_time, performer=None, repository=None, n
return query.group_by(date, LogEntry.kind)
def list_logs(start_time, end_time, performer=None, repository=None, namespace=None, page=None,
count=None):
def get_logs_query(start_time, end_time, performer=None, repository=None, namespace=None):
Performer = User.alias()
selections = [LogEntry, Performer]
@ -52,10 +50,7 @@ def list_logs(start_time, end_time, performer=None, repository=None, namespace=N
.join(Performer, JOIN_LEFT_OUTER,
on=(LogEntry.performer == Performer.id).alias('performer')))
if page and count:
query = query.paginate(page, count)
return list(query.order_by(LogEntry.datetime.desc()))
return query
def log_action(kind_name, user_or_organization_name, performer=None, repository=None,

29
data/model/modelutil.py Normal file
View file

@ -0,0 +1,29 @@
def paginate(query, model, descending=False, page_token=None, limit=50):
""" Paginates the given query using an ID range, starting at the optional page_token.
Returns a *list* of matching results along with an unencrypted page_token for the
next page, if any. If descending is set to True, orders by the ID descending rather
than ascending.
"""
query = query.limit(limit + 1)
if descending:
query = query.order_by(model.id.desc())
else:
query = query.order_by(model.id)
if page_token is not None:
start_id = page_token.get('start_id')
if start_id is not None:
if descending:
query = query.where(model.id <= start_id)
else:
query = query.where(model.id >= start_id)
results = list(query)
page_token = None
if len(results) > limit:
page_token = {
'start_id': results[limit].id
}
return results[0:limit], page_token

37
endpoints/api/__init__.py
View file

@ -1,5 +1,6 @@
import logging
import datetime
import json
from app import app, metric_queue
from flask import Blueprint, request, make_response, jsonify, session
@ -21,6 +22,7 @@ from auth.auth import process_oauth
from endpoints.csrf import csrf_protect
from endpoints.decorators import check_anon_protection
from util.saas.metricqueue import time_decorator
from util.security.aes import AESCipher
logger = logging.getLogger(__name__)
@ -209,6 +211,41 @@ def query_param(name, help_str, type=reqparse.text_type, default=None,
return add_param
def page_support(func):
""" Adds pagination support to an API endpoint. The decorated API will have an
added query parameter named 'next_page'. Works in tandem with the
modelutil paginate method.
"""
@wraps(func)
@query_param('next_page', 'The page token for the next page', type=str)
def wrapper(self, query_args, *args, **kwargs):
page_token = None
unecrypted = None
if query_args['next_page']:
# Decrypt the page token.
cipher = AESCipher(app.config['PAGE_TOKEN_KEY'])
try:
unecrypted = cipher.decrypt(query_args['next_page'])
except TypeError:
pass
if unecrypted is not None:
try:
page_token = json.loads(unecrypted)
except ValueError:
pass
(result, next_page_token) = func(self, query_args, page_token, *args, **kwargs)
if next_page_token is not None:
cipher = AESCipher(app.config['PAGE_TOKEN_KEY'])
result['next_page'] = cipher.encrypt(json.dumps(next_page_token))
return result
return wrapper
def parse_args(func):
@wraps(func)
def wrapper(self, *args, **kwargs):

41
endpoints/api/logs.py
View file

@ -8,15 +8,14 @@ from dateutil.relativedelta import relativedelta
from endpoints.api import (resource, nickname, ApiResource, query_param, parse_args,
RepositoryParamResource, require_repo_admin, related_user_resource,
format_date, Unauthorized, NotFound, require_user_admin,
internal_only, path_param, require_scope)
internal_only, path_param, require_scope, page_support)
from auth.permissions import AdministerOrganizationPermission, AdministerOrganizationPermission
from auth.auth_context import get_authenticated_user
from data import model
from data import model, database
from auth import scopes
from app import avatar
LOGS_PER_PAGE = 50
MAX_PAGES = 20
LOGS_PER_PAGE = 20
def log_view(log, kinds):
view = {
@ -79,20 +78,22 @@ def _validate_logs_arguments(start_time, end_time, performer_name):
return (start_time, end_time, performer)
def get_logs(start_time, end_time, performer_name=None, repository=None, namespace=None, page=None):
def get_logs(start_time, end_time, performer_name=None, repository=None, namespace=None,
page_token=None):
(start_time, end_time, performer) = _validate_logs_arguments(start_time, end_time, performer_name)
page = min(MAX_PAGES, page if page else 1)
kinds = model.log.get_log_entry_kinds()
logs = model.log.list_logs(start_time, end_time, performer=performer, repository=repository,
namespace=namespace, page=page, count=LOGS_PER_PAGE + 1)
logs_query = model.log.get_logs_query(start_time, end_time, performer=performer,
repository=repository, namespace=namespace)
logs, next_page_token = model.modelutil.paginate(logs_query, database.LogEntry, descending=True,
page_token=page_token, limit=LOGS_PER_PAGE)
return {
'start_time': format_date(start_time),
'end_time': format_date(end_time),
'logs': [log_view(log, kinds) for log in logs[0:LOGS_PER_PAGE]],
'page': page,
'has_additional': len(logs) > LOGS_PER_PAGE,
}
'logs': [log_view(log, kinds) for log in logs],
}, next_page_token
def get_aggregate_logs(start_time, end_time, performer_name=None, repository=None, namespace=None):
(start_time, end_time, performer) = _validate_logs_arguments(start_time, end_time, performer_name)
@ -116,7 +117,8 @@ class RepositoryLogs(RepositoryParamResource):
@query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
@query_param('page', 'The page number for the logs', type=int, default=1)
def get(self, args, namespace, repository):
@page_support
def get(self, args, page_token, namespace, repository):
""" List the logs for the specified repository. """
repo = model.repository.get_repository(namespace, repository)
if not repo:
@ -124,7 +126,7 @@ class RepositoryLogs(RepositoryParamResource):
start_time = args['starttime']
end_time = args['endtime']
return get_logs(start_time, end_time, repository=repo, page=args['page'])
return get_logs(start_time, end_time, repository=repo, page_token=page_token)
@resource('/v1/user/logs')
@ -137,8 +139,8 @@ class UserLogs(ApiResource):
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('performer', 'Username for which to filter logs.', type=str)
@query_param('page', 'The page number for the logs', type=int, default=1)
def get(self, args):
@page_support
def get(self, args, page_token):
""" List the logs for the current user. """
performer_name = args['performer']
start_time = args['starttime']
@ -146,7 +148,7 @@ class UserLogs(ApiResource):
user = get_authenticated_user()
return get_logs(start_time, end_time, performer_name=performer_name, namespace=user.username,
page=args['page'])
page_token=page_token)
@resource('/v1/organization/<orgname>/logs')
@ -160,8 +162,9 @@ class OrgLogs(ApiResource):
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
@query_param('performer', 'Username for which to filter logs.', type=str)
@query_param('page', 'The page number for the logs', type=int, default=1)
@page_support
@require_scope(scopes.ORG_ADMIN)
def get(self, args, orgname):
def get(self, args, page_token, orgname):
""" List the logs for the specified organization. """
permission = AdministerOrganizationPermission(orgname)
if permission.can():
@ -170,7 +173,7 @@ class OrgLogs(ApiResource):
end_time = args['endtime']
return get_logs(start_time, end_time, namespace=orgname, performer_name=performer_name,
page=args['page'])
page_token=page_token)
raise Unauthorized()

8
endpoints/api/superuser.py
View file

@ -12,7 +12,8 @@ import features
from app import app, avatar, superusers, authentication, config_provider
from endpoints.api import (ApiResource, nickname, resource, validate_json_request,
internal_only, require_scope, show_if, parse_args,
query_param, abort, require_fresh_login, path_param, verify_not_prod)
query_param, abort, require_fresh_login, path_param, verify_not_prod,
page_support)
from endpoints.api.logs import get_logs, get_aggregate_logs
from data import model
from auth.permissions import SuperUserPermission
@ -116,14 +117,15 @@ class SuperUserLogs(ApiResource):
@query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
@query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
@query_param('page', 'The page number for the logs', type=int, default=1)
@page_support
@require_scope(scopes.SUPERUSER)
def get(self, args):
def get(self, args, page_token):
""" List the usage logs for the current system. """
if SuperUserPermission().can():
start_time = args['starttime']
end_time = args['endtime']
return get_logs(start_time, end_time, page=args['page'])
return get_logs(start_time, end_time, page_token=page_token)
abort(403)

15
static/js/directives/ui/logs-view.js
View file

@ -23,7 +23,6 @@ angular.module('quay').directive('logsView', function () {
$scope.kindsAllowed = null;
$scope.chartVisible = true;
$scope.chartLoading = true;
$scope.currentPage = 1;
$scope.options = {};
@ -309,7 +308,7 @@ angular.module('quay').directive('logsView', function () {
$scope.chartLoading = false;
});
$scope.currentPage = 0;
$scope.nextPageToken = null;
$scope.hasAdditional = true;
$scope.loading = false;
$scope.logs = [];
@ -317,11 +316,15 @@ angular.module('quay').directive('logsView', function () {
};
$scope.nextPage = function() {
if ($scope.loading) { return; }
if ($scope.loading || !$scope.hasAdditional) { return; }
$scope.loading = true;
var logsUrl = getUrl('logs') + '&page=' + ($scope.currentPage + 1);
var logsUrl = getUrl('logs');
if ($scope.nextPageToken) {
logsUrl = logsUrl + '&next_page=' + encodeURIComponent($scope.nextPageToken);
}
var loadLogs = Restangular.one(logsUrl);
loadLogs.customGET().then(function(resp) {
resp.logs.forEach(function(log) {
@ -329,8 +332,8 @@ angular.module('quay').directive('logsView', function () {
});
$scope.loading = false;
$scope.currentPage = resp.page;
$scope.hasAdditional = resp.has_additional;
$scope.nextPageToken = resp.next_page;
$scope.hasAdditional = !!resp.next_page;
});
};