util.secscan.api: init read-only failover

config.py

@@ -310,6 +310,9 @@ class DefaultConfig(object):
   # If specified, the endpoint to be used for all POST calls to the security scanner.
   SECURITY_SCANNER_ENDPOINT_BATCH = None
 
+  # If specified, GET requests that return non-200 will be retried at the following instances.
+  SECURITY_SCANNER_READONLY_FAILOVER_ENDPOINTS = []
+
   # The indexing engine version running inside the security scanner.
   SECURITY_SCANNER_ENGINE_VERSION_TARGET = 2
 

util/secscan/api.py

@@ -9,6 +9,7 @@ from flask import url_for
 from data.database import CloseForLongOperation
 from data import model
 from data.model.storage import get_storage_locations
+from util.failover import failover, FailoverException
 from util.secscan.validator import SecurityConfigValidator
 from util.security.instancekeys import InstanceKeys
 from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
@@ -19,6 +20,9 @@ TOKEN_VALIDITY_LIFETIME_S = 60  # Amount of time the security scanner has to cal
 
 UNKNOWN_PARENT_LAYER_ERROR_MSG = 'worker: parent layer is unknown, it must be processed first'
 
+MITM_CERT_PATH = '/conf/mitm.cert'
+DEFAULT_HTTP_HEADERS = {'Connection': 'close'}
+
 logger = logging.getLogger(__name__)
 
 
@@ -309,32 +313,56 @@ class SecurityScannerAPI(object):
 
     return json_response
 
-  def _call(self, method, relative_url, params=None, body=None):
-    """ Issues an HTTP call to the sec API at the given relative URL.
-        This function disconnects from the database while awaiting a response
-        from the API server.
+  def _request(self, method, endpoint, path, body, params, timeout):
+    """ Issues an HTTP request to the security endpoint. """
+    if self._config is None:
+      raise Exception('Cannot call unconfigured security system')
+
+    url = _join_api_url(endpoint, self._config.get('SECURITY_SCANNER_API_VERSION', 'v1'), path)
+    signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
+
+    logger.debug('%sing security URL %s', method.upper(), url)
+    return self._client.request(method, url, json=body, params=params, timeout=timeout,
+                                verify=MITM_CERT_PATH, headers=DEFAULT_HTTP_HEADERS,
+                                proxies={'https': 'https://' + signer_proxy_url,
+                                         'http': 'http://' + signer_proxy_url})
+
+  def _call(self, method, path, params=None, body=None):
+    """ Issues an HTTP request to the security endpoint handling the logic of using an alternative
+        BATCH endpoint for non-GET requests and failover for GET requests.
     """
     if self._config is None:
       raise Exception('Cannot call unconfigured security system')
 
-    client = self._client
-    headers = {'Connection': 'close'}
-
-    timeout = self._config.get('SECURITY_SCANNER_API_TIMEOUT_SECONDS', 10)
+    timeout = self._config['SECURITY_SCANNER_API_TIMEOUT_SECONDS']
     endpoint = self._config['SECURITY_SCANNER_ENDPOINT']
-    if method != 'GET':
-      timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS', timeout)
-      endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') or endpoint
-
-    api_url = urljoin(endpoint, '/' + self._config.get('SECURITY_SCANNER_API_VERSION', 'v1')) + '/'
-    url = urljoin(api_url, relative_url)
-    signer_proxy_url = self._config.get('JWTPROXY_SIGNER', 'localhost:8080')
 
     with CloseForLongOperation(self._config):
-      logger.debug('%sing security URL %s', method.upper(), url)
-      return client.request(method, url, json=body, params=params, timeout=timeout,
-                            verify='/conf/mitm.cert', headers=headers,
-                            proxies={
-                              'https': 'https://' + signer_proxy_url,
-                              'http': 'http://' + signer_proxy_url
-                            })
+      # If the request isn't a read, attempt to use a batch stack and do not fail over.
+      if method != 'GET':
+        if self._config.get('SECURITY_SCANNER_ENDPOINT_BATCH') is not None:
+          endpoint = self._config['SECURITY_SCANNER_ENDPOINT_BATCH']
+          timeout = self._config.get('SECURITY_SCANNER_API_BATCH_TIMEOUT_SECONDS') or timeout
+        return self._request(method, endpoint, path, body, params, timeout)
+
+      # The request is read-only and can failover.
+      all_endpoints = [endpoint] + self._config['SECURITY_SCANNER_READONLY_FAILOVER_ENDPOINTS']
+      try:
+        return  _failover_read_request(*[((self._request, endpoint, path, body, params, timeout), {})
+                                        for endpoint in all_endpoints])
+      except FailoverException:
+        raise APIRequestFailure()
+
+
+def _join_api_url(endpoint, api_version, path):
+  pathless_url = urljoin(endpoint, '/' + api_version) + '/'
+  return urljoin(pathless_url, path)
+
+
+@failover
+def _failover_read_request(request_fn, endpoint, path, body, params, timeout):
+  """ This function auto-retries read-only requests until they return a 2xx status code. """
+  resp = request_fn('GET', endpoint, path, body, params, timeout)
+  if resp.status_code / 100 != 2:
+    raise FailoverException('status code was not 2xx')
+  return resp