Implement helper classes for tracking streaming diffs, both indexed and non-indexed

These classes will be used to handle the Layer ID paginated diffs from Clair.

test/test_secscan.py

@@ -650,5 +650,109 @@ class TestSecurityScanner(unittest.TestCase):
       self.assertIsNotNone(notification_queue.get())
 
 
+  def test_notification_worker_offset_pages(self):
+
+    def get_layer_id(repo_name, tag):
+      # Create a repository notification for the repo, if it doesn't exist.
+      has_notification = model.notification.list_repo_notifications(ADMIN_ACCESS_USER, repo_name,
+                                                                    'vulnerability_found')
+      if not list(has_notification):
+        repo = model.repository.get_repository(ADMIN_ACCESS_USER, repo_name)
+        model.notification.create_repo_notification(repo, 'vulnerability_found',
+                                                    'quay_notification', {}, {'level': 100})
+
+      layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, repo_name, tag, include_storage=True)
+      return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
+
+    # Define offsetting sets of layer IDs, to test cross-pagination support. In this test, we
+    # will only serve 2 layer IDs per page: the first page will serve both of the 'New' layer IDs,
+    # but since the first 2 'Old' layer IDs are "earlier" than the shared ID of
+    # `devtable/simple:latest`, they won't get served in the 'New' list until the *second* page. The
+    # notification handling system should correctly not notify for this layer, even though it is
+    # marked 'New' on page 1 and marked 'Old' on page 2. In practice, Clair will served these IDs
+    # sorted in the same manner.
+    new_layer_ids = [get_layer_id('simple', 'latest'), get_layer_id('complex', 'prod')]
+    old_layer_ids = ['someid1', 'someid2', get_layer_id('simple', 'latest')]
+
+    apis_called = []
+
+    @urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
+    def get_matching_layer_vulnerable(url, request):
+      apis_called.append('VULN')
+      return json.dumps({
+        "Layer": {
+          "Name": 'somelayerid',
+          "Namespace": "debian:8",
+          "IndexedByVersion": 1,
+          "Features": [
+            {
+              "Name": "coreutils",
+              "Namespace": "debian:8",
+              "Version": "8.23-4",
+              "Vulnerabilities": [
+                {
+                  "Name": "CVE-TEST",
+                  "Namespace": "debian:8",
+                  "Severity": "Low",
+                }
+              ],
+            }
+          ]
+        }
+      })
+
+    @urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/notifications/somenotification$', method='DELETE')
+    def delete_notification(url, request):
+      apis_called.append('DELETE')
+      return {'status_code': 201, 'content': ''}
+
+    @urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/notifications/somenotification$', method='GET')
+    def get_notification(url, request):
+      if url.query.find('page=nextpage') >= 0:
+        apis_called.append('GET-2')
+
+        data = {
+          'Notification': self._get_notification_data(new_layer_ids[2:], old_layer_ids[2:]),
+        }
+
+        return json.dumps(data)
+      else:
+        apis_called.append('GET-1')
+
+        notification_data = self._get_notification_data(new_layer_ids[0:2], old_layer_ids[0:2])
+        notification_data['NextPage'] = 'nextpage'
+
+        data = {
+          'Notification': notification_data,
+        }
+
+        return json.dumps(data)
+
+    # Ensure that there are no event queue items for any layers.
+    self.assertIsNone(notification_queue.get())
+
+    # Test with a known notification with pages.
+    data = {
+      'Name': 'somenotification'
+    }
+
+    with HTTMock(get_notification, delete_notification, get_matching_layer_vulnerable):
+      worker = SecurityNotificationWorker(None)
+      self.assertTrue(worker.perform_notification_work(data))
+
+    # Verify each of the expected API calls were made.
+    self.assertEquals(set(['GET-1', 'GET-2', 'DELETE', 'VULN']), set(apis_called))
+
+    # Verify that we have notifications *just* for the New layer.
+    expected_item = notification_queue.get()
+    self.assertIsNotNone(expected_item)
+    item_body = json.loads(expected_item['body'])
+    self.assertEquals('devtable/complex', item_body['event_data']['repository'])
+    self.assertEquals(['prod'], item_body['event_data']['tags'])
+
+    # Make sure we have no additional notifications.
+    self.assertIsNone(notification_queue.get())
+
+
 if __name__ == '__main__':
   unittest.main()