Implement helper classes for tracking streaming diffs, both indexed and non-indexed

These classes will be used to handle the Layer ID paginated diffs from Clair.

util/secscan/notifier.py

@@ -1,6 +1,8 @@
 import logging
 import sys
 
+from enum import Enum
+
 from collections import defaultdict
 
 from app import secscan_api
@@ -10,21 +12,27 @@ from data.database import (Image, ImageStorage, ExternalNotificationEvent, Repos
 from endpoints.notificationhelper import notification_batch
 from util.secscan import PRIORITY_LEVELS
 from util.secscan.api import APIRequestFailure
-from util.morecollections import AttrDict
+from util.morecollections import AttrDict, StreamingDiffTracker
 
 logger = logging.getLogger(__name__)
 
 
-def process_notification_data(notification_data):
-  """ Processes the given notification data to spawn vulnerability notifications as necessary.
-      Returns whether the processing succeeded.
-  """
-  if not 'New' in notification_data:
-    # Nothing to do.
-    return True
+class ProcessNotificationPageResult(Enum):
+  FINISHED_PAGE = 'Finished Page'
+  FINISHED_PROCESSING = 'Finished Processing'
+  FAILED = 'Failed'
 
-  new_data = notification_data['New']
-  old_data = notification_data.get('Old', {})
+
+def process_notification_page_data(notification_page_data):
+  """ Processes the given notification page data to spawn vulnerability notifications as necessary.
+      Returns the status of the processing.
+  """
+  if not 'New' in notification_page_data:
+    # Nothing more to do.
+    return ProcessNotificationPageResult.FINISHED_PROCESSING
+
+  new_data = notification_page_data['New']
+  old_data = notification_page_data.get('Old', {})
 
   new_vuln = new_data['Vulnerability']
   old_vuln = old_data.get('Vulnerability', {})
@@ -44,7 +52,7 @@ def process_notification_data(notification_data):
 
   if not notify_layers:
     # Nothing more to do.
-    return True
+    return ProcessNotificationPageResult.FINISHED_PAGE
 
   # Lookup the external event for when we have vulnerabilities.
   event = ExternalNotificationEvent.get(name='vulnerability_found')
@@ -76,7 +84,7 @@ def process_notification_data(notification_data):
         try:
           is_vulerable = secscan_api.check_layer_vulnerable(tag_layer_id, cve_id)
         except APIRequestFailure:
-          return False
+          return ProcessNotificationPageResult.FAILED
 
         check_map[tag_layer_id] = is_vulerable
 
@@ -110,5 +118,5 @@ def process_notification_data(notification_data):
       })
       spawn_notification(repository, 'vulnerability_found', event_data)
 
-  return True
+  return ProcessNotificationPageResult.FINISHED_PAGE