v2: add pagination decorator

data/model/v2.py

@@ -100,3 +100,11 @@ def synthesize_v1_image(repo, storage, image_id, created, comment, command, comp
 def save_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest_digest, manifest_bytes):
   model.tag.store_tag_manifest(namespace_name, repo_name, tag_name, leaf_layer_id, manifest_digest,
                                manifest_bytes)
+
+
+def repository_tags(namespace_name, repo_name, limit, offset):
+  return [Tag()]
+
+
+def get_visible_repositories(username, limit, offset):
+  return [Repository()]

endpoints/v2/__init__.py

@@ -2,13 +2,14 @@ import logging
 
 from functools import wraps
 from urlparse import urlparse
+from urllib import urlencode
 
 from flask import Blueprint, make_response, url_for, request, jsonify
 from semantic_version import Spec
 
 import features
 
-from app import app, metric_queue
+from app import app, metric_queue, get_app_url
 from auth.auth_context import get_grant_context
 from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission,
                               AdministerRepositoryPermission)
@@ -19,12 +20,53 @@ from endpoints.v2.errors import V2RegistryException, Unauthorized
 from util.http import abort
 from util.registry.dockerver import docker_version
 from util.metrics.metricqueue import time_blueprint
+from util.pagination import encrypt_page_token, decrypt_page_token
 
 logger = logging.getLogger(__name__)
 v2_bp = Blueprint('v2', __name__)
 
 time_blueprint(v2_bp, metric_queue)
 
+
+_MAX_RESULTS_PER_PAGE = 50
+
+
+def _paginate(limit_kwarg_name='limit', offset_kwarg_name='offset',
+              callback_kwarg_name='pagination_callback'):
+  def wrapper(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+      try:
+        requested_limit = int(request.args.get('n', _MAX_RESULTS_PER_PAGE))
+      except ValueError:
+        requested_limit = 0
+
+      limit = max(min(requested_limit, _MAX_RESULTS_PER_PAGE), 1)
+      next_page_token = request.args.get('next_page', None)
+
+      # Decrypt the next page token, if any.
+      offset = 0
+      page_info = decrypt_page_token(next_page_token)
+      if page_info is not None:
+        # Note: we use offset here instead of ID >= n because one of the V2 queries is a UNION.
+        offset = page_info.get('offset', 0)
+
+      def callback(num_results, response):
+        if num_results <= limit:
+          return
+        next_page_token = encrypt_page_token({'offset': limit+offset})
+        link = get_app_url() + url_for(request.endpoint, **request.view_args)
+        link += '?%s; rel="next"' % urlencode({'n': limit, 'next_page': next_page_token})
+        response.headers['Link'] = link
+
+      kwargs[limit_kwarg_name] = limit
+      kwargs[offset_kwarg_name] = offset
+      kwargs[callback_kwarg_name] = callback
+      func(*args, **kwargs)
+    return wrapped
+  return wrapper
+
+
 @v2_bp.app_errorhandler(V2RegistryException)
 def handle_registry_v2_exception(error):
   response = jsonify({
@@ -104,8 +146,10 @@ def v2_support_enabled():
   return response
 
 
-from endpoints.v2 import v2auth
-from endpoints.v2 import manifest
-from endpoints.v2 import blob
-from endpoints.v2 import tag
-from endpoints.v2 import catalog
+from endpoints.v2 import (
+  blob,
+  catalog,
+  manifest,
+  tag,
+  v2auth,
+)

endpoints/v2/catalog.py

@@ -1,30 +1,24 @@
-from flask import jsonify, url_for
+from flask import jsonify
 
-from endpoints.v2 import v2_bp
 from auth.registry_jwt_auth import process_registry_jwt_auth, get_granted_entity
 from endpoints.decorators import anon_protect
-from data import model
-from endpoints.v2.v2util import add_pagination
+from endpoints.v2 import v2_bp, _paginate
 
 @v2_bp.route('/_catalog', methods=['GET'])
 @process_registry_jwt_auth()
 @anon_protect
-def catalog_search():
-  url = url_for('v2.catalog_search')
-
+@_paginate()
+def catalog_search(limit, offset, pagination_callback):
   username = None
   entity = get_granted_entity()
   if entity:
     username = entity.user.username
 
-  query = model.repository.get_visible_repositories(username, include_public=(username is None))
-  link, query = add_pagination(query, url)
-
+  visible_repositories = v2.get_visible_repositories(username, limit, offset)
   response = jsonify({
-    'repositories': ['%s/%s' % (repo.namespace_user.username, repo.name) for repo in query],
+    'repositories': ['%s/%s' % (repo.namespace_name, repo.name)
+                     for repo in visible_repositories],
   })
 
-  if link is not None:
-    response.headers['Link'] = link
-
+  pagination_callback(len(visible_repositories), response)
   return response

endpoints/v2/tag.py

@@ -1,33 +1,27 @@
-from flask import jsonify, url_for
+from flask import jsonify
 
 from auth.registry_jwt_auth import process_registry_jwt_auth
 from endpoints.common import parse_repository_name
-from endpoints.v2 import v2_bp, require_repo_read
+from endpoints.v2 import v2_bp, require_repo_read, _paginate
 from endpoints.v2.errors import NameUnknown
-from endpoints.v2.v2util import add_pagination
 from endpoints.decorators import anon_protect
-from data import model
 
 @v2_bp.route('/<repopath:repository>/tags/list', methods=['GET'])
 @parse_repository_name()
 @process_registry_jwt_auth(scopes=['pull'])
 @require_repo_read
 @anon_protect
-def list_all_tags(namespace_name, repo_name):
-  repository = model.repository.get_repository(namespace_name, repo_name)
-  if repository is None:
+@_paginate()
+def list_all_tags(namespace_name, repo_name, limit, offset, pagination_callback):
+  repo = v2.get_repository(namespace_name, repo_name)
+  if repo is None:
     raise NameUnknown()
 
-  query = model.tag.list_repository_tags(namespace_name, repo_name)
-  url = url_for('v2.list_all_tags', repository='%s/%s' % (namespace_name, repo_name))
-  link, query = add_pagination(query, url)
-
+  tags = v2.repository_tags(namespace_name, repo_name, limit, offset)
   response = jsonify({
     'name': '{0}/{1}'.format(namespace_name, repo_name),
-    'tags': [tag.name for tag in query],
+    'tags': [tag.name for tag in tags],
   })
 
-  if link is not None:
-    response.headers['Link'] = link
-
+  pagination_callback(len(tags), response)
   return response

endpoints/v2/v2auth.py

@@ -1,6 +1,7 @@
 import logging
 import re
 
+from cachetools import lru_cache
 from flask import request, jsonify, abort
 
 from app import app, userevents, instance_keys
@@ -9,7 +10,6 @@ from auth.auth import process_auth
 from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
 from auth.permissions import (ModifyRepositoryPermission, ReadRepositoryPermission,
                               CreateRepositoryPermission)
-from cachetools import lru_cache
 from endpoints.v2 import v2_bp
 from endpoints.decorators import anon_protect
 from util.cache import no_cache
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 TOKEN_VALIDITY_LIFETIME_S = 60 * 60  # 1 hour
 SCOPE_REGEX_TEMPLATE = (
-    r'^repository:((?:{}\/)?((?:[\.a-zA-Z0-9_\-]+\/)?[\.a-zA-Z0-9_\-]+)):((?:push|pull|\*)(?:,(?:push|pull|\*))*)$'
+  r'^repository:((?:{}\/)?((?:[\.a-zA-Z0-9_\-]+\/)?[\.a-zA-Z0-9_\-]+)):((?:push|pull|\*)(?:,(?:push|pull|\*))*)$'
 )
 
 

endpoints/v2/v2util.py

@@ -1,42 +0,0 @@
-from flask import request
-from app import get_app_url
-from util.pagination import encrypt_page_token, decrypt_page_token
-import urllib
-import logging
-
-_MAX_RESULTS_PER_PAGE = 50
-
-def add_pagination(query, url):
-  """ Adds optional pagination to the given query by looking for the Docker V2 pagination request
-      args.
-  """
-  try:
-    requested_limit = int(request.args.get('n', _MAX_RESULTS_PER_PAGE))
-  except ValueError:
-    requested_limit = 0
-
-  limit = max(min(requested_limit, _MAX_RESULTS_PER_PAGE), 1)
-  next_page_token = request.args.get('next_page', None)
-
-  # Decrypt the next page token, if any.
-  offset = 0
-  page_info = decrypt_page_token(next_page_token)
-  if page_info is not None:
-    # Note: we use offset here instead of ID >= n because one of the V2 queries is a UNION.
-    offset = page_info.get('offset', 0)
-    query = query.offset(offset)
-
-  query = query.limit(limit + 1)
-  url = get_app_url() + url
-
-  results = list(query)
-  if len(results) <= limit:
-    return None, results
-
-  # Add a link to the next page of results.
-  page_info = dict(offset=limit + offset)
-  next_page_token = encrypt_page_token(page_info)
-
-  link = url + '?' + urllib.urlencode(dict(n=limit, next_page=next_page_token))
-  link = link + '; rel="next"'
-  return link, results[0:-1]