Add ID-based pagination to logs using new decorators and an encrypted token

Fixes #599

config.py

@@ -289,3 +289,8 @@ class DefaultConfig(object):
   BITTORRENT_PIECE_SIZE = 512 * 1024
   BITTORRENT_ANNOUNCE_URL = 'https://localhost:6881/announce'
   BITTORRENT_FILENAME_PEPPER = '3ae93fef-c30a-427e-9ba0-eea0fd710419'
+
+  # "Secret" key for generating encrypted paging tokens. Only needed to be secret to
+  # hide the ID range for production (in which this value is overridden). Should *not*
+  # be relied upon for secure encryption otherwise.
+  PAGE_TOKEN_KEY = 'GdE4<~8&9LHDPM++'

data/model/__init__.py

@@ -95,4 +95,4 @@ config = Config()
 # moving the minimal number of things to _basequery
 # TODO document the methods and modules for each one of the submodules below.
 from data.model import (blob, build, image, log, notification, oauth, organization, permission,
-                        repository, storage, tag, team, token, user, release)
+                        repository, storage, tag, team, token, user, release, modelutil)

data/model/log.py

@@ -41,9 +41,7 @@ def get_aggregated_logs(start_time, end_time, performer=None, repository=None, n
   return query.group_by(date, LogEntry.kind)
 
 
-def list_logs(start_time, end_time, performer=None, repository=None, namespace=None, page=None,
+def get_logs_query(start_time, end_time, performer=None, repository=None, namespace=None):
-              count=None):
-
   Performer = User.alias()
   selections = [LogEntry, Performer]
 
@@ -52,10 +50,7 @@ def list_logs(start_time, end_time, performer=None, repository=None, namespace=N
                 .join(Performer, JOIN_LEFT_OUTER,
                   on=(LogEntry.performer == Performer.id).alias('performer')))
 
-  if page and count:
+  return query
-    query = query.paginate(page, count)
-
-  return list(query.order_by(LogEntry.datetime.desc()))
 
 
 def log_action(kind_name, user_or_organization_name, performer=None, repository=None,

data/model/modelutil.py

@@ -0,0 +1,29 @@
+def paginate(query, model, descending=False, page_token=None, limit=50):
+  """ Paginates the given query using an ID range, starting at the optional page_token.
+      Returns a *list* of matching results along with an unencrypted page_token for the
+      next page, if any. If descending is set to True, orders by the ID descending rather
+      than ascending.
+  """
+  query = query.limit(limit + 1)
+
+  if descending:
+    query = query.order_by(model.id.desc())
+  else:
+    query = query.order_by(model.id)
+
+  if page_token is not None:
+    start_id = page_token.get('start_id')
+    if start_id is not None:
+      if descending:
+        query = query.where(model.id <= start_id)
+      else:
+        query = query.where(model.id >= start_id)
+
+  results = list(query)
+  page_token = None
+  if len(results) > limit:
+    page_token = {
+      'start_id': results[limit].id
+    }
+
+  return results[0:limit], page_token

endpoints/api/__init__.py

@@ -1,5 +1,6 @@
 import logging
 import datetime
+import json
 
 from app import app, metric_queue
 from flask import Blueprint, request, make_response, jsonify, session
@@ -21,6 +22,7 @@ from auth.auth import process_oauth
 from endpoints.csrf import csrf_protect
 from endpoints.decorators import check_anon_protection
 from util.saas.metricqueue import time_decorator
+from util.security.aes import AESCipher
 
 
 logger = logging.getLogger(__name__)
@@ -209,6 +211,41 @@ def query_param(name, help_str, type=reqparse.text_type, default=None,
   return add_param
 
 
+def page_support(func):
+  """ Adds pagination support to an API endpoint. The decorated API will have an
+      added query parameter named 'next_page'. Works in tandem with the
+      modelutil paginate method.
+  """
+  @wraps(func)
+  @query_param('next_page', 'The page token for the next page', type=str)
+  def wrapper(self, query_args, *args, **kwargs):
+    page_token = None
+    unecrypted = None
+
+    if query_args['next_page']:
+      # Decrypt the page token.
+      cipher = AESCipher(app.config['PAGE_TOKEN_KEY'])
+      try:
+        unecrypted = cipher.decrypt(query_args['next_page'])
+      except TypeError:
+        pass
+
+      if unecrypted is not None:
+        try:
+          page_token = json.loads(unecrypted)
+        except ValueError:
+          pass
+
+    (result, next_page_token) = func(self, query_args, page_token, *args, **kwargs)
+    if next_page_token is not None:
+      cipher = AESCipher(app.config['PAGE_TOKEN_KEY'])
+      result['next_page'] = cipher.encrypt(json.dumps(next_page_token))
+
+    return result
+
+  return wrapper
+
+
 def parse_args(func):
   @wraps(func)
   def wrapper(self, *args, **kwargs):

endpoints/api/logs.py

@@ -8,15 +8,14 @@ from dateutil.relativedelta import relativedelta
 from endpoints.api import (resource, nickname, ApiResource, query_param, parse_args,
                            RepositoryParamResource, require_repo_admin, related_user_resource,
                            format_date, Unauthorized, NotFound, require_user_admin,
-                           internal_only, path_param, require_scope)
+                           internal_only, path_param, require_scope, page_support)
 from auth.permissions import AdministerOrganizationPermission, AdministerOrganizationPermission
 from auth.auth_context import get_authenticated_user
-from data import model
+from data import model, database
 from auth import scopes
 from app import avatar
 
-LOGS_PER_PAGE = 50
+LOGS_PER_PAGE = 20
-MAX_PAGES = 20
 
 def log_view(log, kinds):
   view = {
@@ -79,20 +78,22 @@ def _validate_logs_arguments(start_time, end_time, performer_name):
   return (start_time, end_time, performer)
 
 
-def get_logs(start_time, end_time, performer_name=None, repository=None, namespace=None, page=None):
+def get_logs(start_time, end_time, performer_name=None, repository=None, namespace=None,
+             page_token=None):
   (start_time, end_time, performer) = _validate_logs_arguments(start_time, end_time, performer_name)
-  page = min(MAX_PAGES, page if page else 1)
   kinds = model.log.get_log_entry_kinds()
-  logs = model.log.list_logs(start_time, end_time, performer=performer, repository=repository,
+  logs_query = model.log.get_logs_query(start_time, end_time, performer=performer,
-                             namespace=namespace, page=page, count=LOGS_PER_PAGE + 1)
+                                        repository=repository, namespace=namespace)
+
+  logs, next_page_token = model.modelutil.paginate(logs_query, database.LogEntry, descending=True,
+                                                   page_token=page_token, limit=LOGS_PER_PAGE)
 
   return {
     'start_time': format_date(start_time),
     'end_time': format_date(end_time),
-    'logs': [log_view(log, kinds) for log in logs[0:LOGS_PER_PAGE]],
+    'logs': [log_view(log, kinds) for log in logs],
-    'page': page,
+  }, next_page_token
-    'has_additional': len(logs) > LOGS_PER_PAGE,
+
-  }
 
 def get_aggregate_logs(start_time, end_time, performer_name=None, repository=None, namespace=None):
   (start_time, end_time, performer) = _validate_logs_arguments(start_time, end_time, performer_name)
@@ -116,7 +117,8 @@ class RepositoryLogs(RepositoryParamResource):
   @query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
   @query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
   @query_param('page', 'The page number for the logs', type=int, default=1)
-  def get(self, args, namespace, repository):
+  @page_support
+  def get(self, args, page_token, namespace, repository):
     """ List the logs for the specified repository. """
     repo = model.repository.get_repository(namespace, repository)
     if not repo:
@@ -124,7 +126,7 @@ class RepositoryLogs(RepositoryParamResource):
 
     start_time = args['starttime']
     end_time = args['endtime']
-    return get_logs(start_time, end_time, repository=repo, page=args['page'])
+    return get_logs(start_time, end_time, repository=repo, page_token=page_token)
 
 
 @resource('/v1/user/logs')
@@ -137,8 +139,8 @@ class UserLogs(ApiResource):
   @query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
   @query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
   @query_param('performer', 'Username for which to filter logs.', type=str)
-  @query_param('page', 'The page number for the logs', type=int, default=1)
+  @page_support
-  def get(self, args):
+  def get(self, args, page_token):
     """ List the logs for the current user. """
     performer_name = args['performer']
     start_time = args['starttime']
@@ -146,7 +148,7 @@ class UserLogs(ApiResource):
 
     user = get_authenticated_user()
     return get_logs(start_time, end_time, performer_name=performer_name, namespace=user.username,
-                    page=args['page'])
+                    page_token=page_token)
 
 
 @resource('/v1/organization/<orgname>/logs')
@@ -160,8 +162,9 @@ class OrgLogs(ApiResource):
   @query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
   @query_param('performer', 'Username for which to filter logs.', type=str)
   @query_param('page', 'The page number for the logs', type=int, default=1)
+  @page_support
   @require_scope(scopes.ORG_ADMIN)
-  def get(self, args, orgname):
+  def get(self, args, page_token, orgname):
     """ List the logs for the specified organization. """
     permission = AdministerOrganizationPermission(orgname)
     if permission.can():
@@ -170,7 +173,7 @@ class OrgLogs(ApiResource):
       end_time = args['endtime']
 
       return get_logs(start_time, end_time, namespace=orgname, performer_name=performer_name,
-                      page=args['page'])
+                      page_token=page_token)
 
     raise Unauthorized()
 

endpoints/api/superuser.py

@@ -12,7 +12,8 @@ import features
 from app import app, avatar, superusers, authentication, config_provider
 from endpoints.api import (ApiResource, nickname, resource, validate_json_request,
                            internal_only, require_scope, show_if, parse_args,
-                           query_param, abort, require_fresh_login, path_param, verify_not_prod)
+                           query_param, abort, require_fresh_login, path_param, verify_not_prod,
+                           page_support)
 from endpoints.api.logs import get_logs, get_aggregate_logs
 from data import model
 from auth.permissions import SuperUserPermission
@@ -116,14 +117,15 @@ class SuperUserLogs(ApiResource):
   @query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
   @query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
   @query_param('page', 'The page number for the logs', type=int, default=1)
+  @page_support
   @require_scope(scopes.SUPERUSER)
-  def get(self, args):
+  def get(self, args, page_token):
     """ List the usage logs for the current system. """
     if SuperUserPermission().can():
       start_time = args['starttime']
       end_time = args['endtime']
 
-      return get_logs(start_time, end_time, page=args['page'])
+      return get_logs(start_time, end_time, page_token=page_token)
 
     abort(403)
 

static/js/directives/ui/logs-view.js

@@ -23,7 +23,6 @@ angular.module('quay').directive('logsView', function () {
       $scope.kindsAllowed = null;
       $scope.chartVisible = true;
       $scope.chartLoading = true;
-      $scope.currentPage = 1;
 
       $scope.options = {};
 
@@ -309,7 +308,7 @@ angular.module('quay').directive('logsView', function () {
           $scope.chartLoading = false;
         });
 
-        $scope.currentPage = 0;
+        $scope.nextPageToken = null;
         $scope.hasAdditional = true;
         $scope.loading = false;
         $scope.logs = [];
@@ -317,11 +316,15 @@ angular.module('quay').directive('logsView', function () {
       };
 
       $scope.nextPage = function() {
-        if ($scope.loading) { return; }
+        if ($scope.loading || !$scope.hasAdditional) { return; }
 
         $scope.loading = true;
 
-        var logsUrl = getUrl('logs') + '&page=' + ($scope.currentPage + 1);
+        var logsUrl = getUrl('logs');
+        if ($scope.nextPageToken) {
+          logsUrl = logsUrl + '&next_page=' + encodeURIComponent($scope.nextPageToken);
+        }
+
         var loadLogs = Restangular.one(logsUrl);
         loadLogs.customGET().then(function(resp) {
           resp.logs.forEach(function(log) {
@@ -329,8 +332,8 @@ angular.module('quay').directive('logsView', function () {
           });
 
           $scope.loading = false;
-          $scope.currentPage = resp.page;
+          $scope.nextPageToken = resp.next_page;
-          $scope.hasAdditional = resp.has_additional;
+          $scope.hasAdditional = !!resp.next_page;
         });
       };