Fix pagination of repositories

Fixes #1725

data/model/_basequery.py

@@ -34,10 +34,15 @@ def get_public_repo_visibility():
   return Visibility.get(name='public')
 
 
-def filter_to_repos_for_user(query, username=None, namespace=None, include_public=True):
+def filter_to_repos_for_user(query, username=None, namespace=None, include_public=True,
+                             start_id=None):
   if not include_public and not username:
     return Repository.select().where(Repository.id == '-1')
 
+  # Add the start ID if necessary.
+  if start_id is not None:
+    query = query.where(Repository.id >= start_id)
+
   # Build a set of queries that, when unioned together, return the full set of visible repositories
   # for the filters specified.
   queries = []

data/model/modelutil.py

@@ -19,16 +19,30 @@ def paginate(query, model, descending=False, page_token=None, limit=50, id_alias
   else:
     query = query.order_by(id_field)
 
-  if page_token is not None:
-    start_id = page_token.get('start_id')
-    if start_id is not None:
-      if descending:
-        query = query.where(model.id <= start_id)
-      else:
-        query = query.where(model.id >= start_id)
+  start_id = pagination_start(page_token)
+  if start_id is not None:
+    if descending:
+      query = query.where(model.id <= start_id)
+    else:
+      query = query.where(model.id >= start_id)
   else:
     query = query.limit(limit + 1)
 
+  return paginate_query(query, limit=limit, id_alias=id_alias)
+
+
+def pagination_start(page_token=None):
+  """ Returns the start ID for pagination for the given page token. Will return None if None. """
+  if page_token is not None:
+    return page_token.get('start_id')
+
+  return None
+
+
+def paginate_query(query, limit=50, id_alias=None):
+  """ Executes the given query and returns a page's worth of results, as well as the page token
+      for the next page (if any).
+  """
   results = list(query)
   page_token = None
   if len(results) > limit:

data/model/repository.py

@@ -2,7 +2,7 @@ import logging
 import random
 
 from datetime import timedelta, datetime
-from peewee import JOIN_LEFT_OUTER, fn
+from peewee import JOIN_LEFT_OUTER, fn, SQL
 from cachetools import ttl_cache
 
 from data.model import (DataModelException, tag, db_transaction, storage, permission,
@@ -245,7 +245,8 @@ def get_when_last_modified(repository_ids):
   return last_modified_map
 
 
-def get_visible_repositories(username, namespace=None, include_public=False):
+def get_visible_repositories(username, namespace=None, include_public=False, start_id=None,
+                             limit=None):
   """ Returns the repositories visible to the given user (if any).
   """
   if not include_public and not username:
@@ -263,7 +264,12 @@ def get_visible_repositories(username, namespace=None, include_public=False):
     # Note: We only need the permissions table if we will filter based on a user's permissions.
     query = query.switch(Repository).distinct().join(RepositoryPermission, JOIN_LEFT_OUTER)
 
-  query = _basequery.filter_to_repos_for_user(query, username, namespace, include_public)
+  query = _basequery.filter_to_repos_for_user(query, username, namespace, include_public,
+                                              start_id=start_id)
+
+  if limit is not None:
+    query = query.limit(limit).order_by(SQL('rid'))
+
   return query