From 2caaf84f31f4a507bf28561646bb590c12bfe5d1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 19 Jun 2018 13:27:16 -0400 Subject: [PATCH] Add caching support to catalog We will now cache the results of the catalog for 60s and not hit the database at all if cached --- data/cache/cache_key.py | 8 ++++++++ endpoints/v2/catalog.py | 28 +++++++++++++++++++--------- test/registry/protocol_v2.py | 7 ++++++- test/registry/registry_tests.py | 20 ++++++++++++++++++++ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/data/cache/cache_key.py b/data/cache/cache_key.py index 3b10558e5..f30d6d345 100644 --- a/data/cache/cache_key.py +++ b/data/cache/cache_key.py @@ -4,5 +4,13 @@ class CacheKey(namedtuple('CacheKey', ['key', 'expiration'])): """ Defines a key into the data model cache. """ pass + def for_repository_blob(namespace_name, repo_name, digest): + """ Returns a cache key for a blob in a repository. """ return CacheKey('repository_blob__%s_%s_%s' % (namespace_name, repo_name, digest), '60s') + + +def for_catalog_page(auth_context_key, start_id, limit): + """ Returns a cache key for a single page of a catalog lookup for an authed context. """ + params = (auth_context_key or '(anon)', start_id or 0, limit or 0) + return CacheKey('catalog_page__%s_%s_%s' % params, '60s') diff --git a/endpoints/v2/catalog.py b/endpoints/v2/catalog.py index ff87f22e2..6e1d09854 100644 --- a/endpoints/v2/catalog.py +++ b/endpoints/v2/catalog.py @@ -2,10 +2,13 @@ import features from flask import jsonify -from auth.auth_context import get_authenticated_user +from app import model_cache +from auth.auth_context import get_authenticated_user, get_authenticated_context from auth.registry_jwt_auth import process_registry_jwt_auth +from data.cache import cache_key from endpoints.decorators import anon_protect from endpoints.v2 import v2_bp, paginate +from endpoints.v2.models_interface import Repository from endpoints.v2.models_pre_oci import data_model as model @@ -14,16 +17,23 @@ from endpoints.v2.models_pre_oci import data_model as model @anon_protect @paginate() def catalog_search(start_id, limit, pagination_callback): - include_public = bool(features.PUBLIC_CATALOG) - if not include_public and not get_authenticated_user(): - return jsonify({'repositories': []}) + def _load_catalog(): + include_public = bool(features.PUBLIC_CATALOG) + if not include_public and not get_authenticated_user(): + return [] - username = get_authenticated_user().username if get_authenticated_user() else None - if username and not get_authenticated_user().enabled: - return jsonify({'repositories': []}) + username = get_authenticated_user().username if get_authenticated_user() else None + if username and not get_authenticated_user().enabled: + return [] + + repos = model.get_visible_repositories(username, start_id, limit, include_public=include_public) + return [repo._asdict() for repo in repos] + + context_key = get_authenticated_context().unique_key if get_authenticated_context() else None + catalog_cache_key = cache_key.for_catalog_page(context_key, start_id, limit) + visible_repositories = [Repository(**repo_dict) for repo_dict + in model_cache.retrieve(catalog_cache_key, _load_catalog)] - visible_repositories = model.get_visible_repositories(username, start_id, limit, - include_public=include_public) response = jsonify({ 'repositories': ['%s/%s' % (repo.namespace_name, repo.name) for repo in visible_repositories][0:limit], diff --git a/test/registry/protocol_v2.py b/test/registry/protocol_v2.py index 3603df865..ec0b24923 100644 --- a/test/registry/protocol_v2.py +++ b/test/registry/protocol_v2.py @@ -390,7 +390,7 @@ class V2Protocol(RegistryProtocol): return results def catalog(self, session, page_size=2, credentials=None, options=None, expected_failure=None, - namespace=None, repo_name=None): + namespace=None, repo_name=None, bearer_token=None): options = options or ProtocolOptions() scopes = options.scopes or [] @@ -409,6 +409,11 @@ class V2Protocol(RegistryProtocol): 'Authorization': 'Bearer ' + token, } + if bearer_token is not None: + headers = { + 'Authorization': 'Bearer ' + bearer_token, + } + results = [] url = '/v2/_catalog' params = {} diff --git a/test/registry/registry_tests.py b/test/registry/registry_tests.py index 3962534c4..9eb74fa53 100644 --- a/test/registry/registry_tests.py +++ b/test/registry/registry_tests.py @@ -719,6 +719,26 @@ def test_catalog(public_catalog, credentials, expected_repos, page_size, v2_prot assert set(expected_repos).issubset(set(results)) +def test_catalog_caching(v2_protocol, basic_images, liveserver_session, app_reloader, + liveserver, registry_server_executor): + """ Test: Calling the catalog after initially pulled will result in the catalog being cached. """ + credentials = ('devtable', 'password') + + # Conduct the initial catalog call to prime the cache. + results = v2_protocol.catalog(liveserver_session, credentials=credentials, + namespace='devtable', repo_name='simple') + + token, _ = v2_protocol.auth(liveserver_session, credentials, 'devtable', 'simple') + + # Disconnect the server from the database. + registry_server_executor.on(liveserver).break_database() + + # Call the catalog again, which should now be cached. + cached_results = v2_protocol.catalog(liveserver_session, bearer_token=token) + assert len(cached_results) == len(results) + assert set(cached_results) == set(results) + + @pytest.mark.parametrize('username, namespace, repository', [ ('devtable', 'devtable', 'simple'), ('devtable', 'devtable', 'gargantuan'),