Raise an APIRequestFailure exception when security scanner is unavailable

Put worker to sleep for the duration of the default indexing interval
when an APIRequestFailure occurs, when the API request fails due to a
connection error, timeout, or other ambiguous errors, from
analyze_layer or get_layer_data .

test/test_secscan.py

@@ -8,7 +8,7 @@ from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
 from endpoints.notificationevent import VulnerabilityFoundEvent
 from endpoints.v2 import v2_bp
 from initdb import setup_database_for_testing, finished_database_for_testing
-from util.secscan.api import SecurityScannerAPI
+from util.secscan.api import SecurityScannerAPI, APIRequestFailure
 from util.secscan.analyzer import LayerAnalyzer
 from util.secscan.fake import fake_security_scanner
 from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult
@@ -160,6 +160,7 @@ class TestSecurityScanner(unittest.TestCase):
       security_scanner.set_internal_error_layer_id(security_scanner.layer_id(layer))
 
       analyzer = LayerAnalyzer(app.config, self.api)
+      with self.assertRaises(APIRequestFailure) as ctx:
         analyzer.analyze_recursively(layer)
 
       layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')

util/secscan/analyzer.py

@@ -57,7 +57,7 @@ class LayerAnalyzer(object):
     except AnalyzeLayerRetryException:
       # Something went wrong when trying to analyze the layer, but we should retry, so leave
       # the layer unindexed. Another worker will come along and handle it.
-      pass
+      raise APIRequestFailure
     except MissingParentLayerException:
       # Pass upward, as missing parent is handled in the analyze_recursively method.
       raise
@@ -145,7 +145,7 @@ class LayerAnalyzer(object):
         try:
           layer_data = self._api.get_layer_data(layer, include_vulnerabilities=True)
         except APIRequestFailure:
-          layer_data = None
+          raise
 
         if layer_data is not None:
           # Dispatch events for any detected vulnerabilities

workers/securityworker.py

@@ -10,7 +10,7 @@ from workers.worker import Worker
 from data.database import UseThenDisconnect
 from data.model.image import (get_images_eligible_for_scan, get_image_pk_field,
                               get_max_id_for_sec_scan, get_min_id_for_sec_scan)
-from util.secscan.api import SecurityConfigValidator
+from util.secscan.api import SecurityConfigValidator, APIRequestFailure
 from util.secscan.analyzer import LayerAnalyzer, PreemptedException
 from util.migrate.allocator import yield_random_entries
 from endpoints.v2 import v2_bp
@@ -73,6 +73,8 @@ class SecurityWorker(Worker):
         except PreemptedException:
           logger.info('Another worker pre-empted us for layer: %s', candidate.id)
           abt.set()
+        except APIRequestFailure:
+          time.sleep(DEFAULT_INDEXING_INTERVAL)
 
         unscanned_images_gauge.Set(num_remaining)