Implement helper classes for tracking streaming diffs, both indexed and non-indexed
These classes will be used to handle the Layer ID paginated diffs from Clair.
This commit is contained in:
Joseph Schorr
parent
a2ac62f5ce
commit
ced0149520
4 changed files with 624 additions and 13 deletions
285
test/test_morecollections.py
Normal file
285
test/test_morecollections.py
Normal file
|
|
@ -0,0 +1,285 @@
|
|||
import unittest
|
||||
|
||||
from util.morecollections import (FastIndexList, StreamingDiffTracker,
|
||||
IndexedStreamingDiffTracker)
|
||||
|
||||
class FastIndexListTests(unittest.TestCase):
|
||||
def test_basic_usage(self):
|
||||
indexlist = FastIndexList()
|
||||
|
||||
# Add 1
|
||||
indexlist.add(1)
|
||||
self.assertEquals([1], indexlist.values())
|
||||
self.assertEquals(0, indexlist.index(1))
|
||||
|
||||
# Add 2
|
||||
indexlist.add(2)
|
||||
self.assertEquals([1, 2], indexlist.values())
|
||||
self.assertEquals(0, indexlist.index(1))
|
||||
self.assertEquals(1, indexlist.index(2))
|
||||
|
||||
# Pop nothing.
|
||||
indexlist.pop_until(-1)
|
||||
self.assertEquals([1, 2], indexlist.values())
|
||||
self.assertEquals(0, indexlist.index(1))
|
||||
self.assertEquals(1, indexlist.index(2))
|
||||
|
||||
# Pop 1.
|
||||
self.assertEquals([1], indexlist.pop_until(0))
|
||||
self.assertEquals([2], indexlist.values())
|
||||
self.assertIsNone(indexlist.index(1))
|
||||
self.assertEquals(0, indexlist.index(2))
|
||||
|
||||
# Add 3.
|
||||
indexlist.add(3)
|
||||
self.assertEquals([2, 3], indexlist.values())
|
||||
self.assertEquals(0, indexlist.index(2))
|
||||
self.assertEquals(1, indexlist.index(3))
|
||||
|
||||
# Pop 2, 3.
|
||||
self.assertEquals([2, 3], indexlist.pop_until(1))
|
||||
self.assertEquals([], indexlist.values())
|
||||
self.assertIsNone(indexlist.index(1))
|
||||
self.assertIsNone(indexlist.index(2))
|
||||
self.assertIsNone(indexlist.index(3))
|
||||
|
||||
def test_popping(self):
|
||||
indexlist = FastIndexList()
|
||||
indexlist.add('hello')
|
||||
indexlist.add('world')
|
||||
indexlist.add('you')
|
||||
indexlist.add('rock')
|
||||
|
||||
self.assertEquals(0, indexlist.index('hello'))
|
||||
self.assertEquals(1, indexlist.index('world'))
|
||||
self.assertEquals(2, indexlist.index('you'))
|
||||
self.assertEquals(3, indexlist.index('rock'))
|
||||
|
||||
indexlist.pop_until(1)
|
||||
self.assertEquals(0, indexlist.index('you'))
|
||||
self.assertEquals(1, indexlist.index('rock'))
|
||||
|
||||
|
||||
class IndexedStreamingDiffTrackerTests(unittest.TestCase):
|
||||
def test_basic(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
|
||||
tracker.push_old([('b', 1)])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'c'], added)
|
||||
|
||||
def test_same_streams(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
|
||||
tracker.push_old([('a', 0), ('b', 1), ('c', 2)])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals([], added)
|
||||
|
||||
def test_only_new(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new([('a', 0), ('b', 1), ('c', 2)])
|
||||
tracker.push_old([])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'b', 'c'], added)
|
||||
|
||||
def test_pagination(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 2)
|
||||
tracker.push_new([('a', 0), ('b', 1)])
|
||||
tracker.push_old([])
|
||||
|
||||
tracker.push_new([('c', 2)])
|
||||
tracker.push_old([])
|
||||
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'b', 'c'], added)
|
||||
|
||||
def test_old_pagination(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 2)
|
||||
tracker.push_new([('a', 10), ('b', 11)])
|
||||
tracker.push_old([('z', 1), ('y', 2)])
|
||||
|
||||
tracker.push_new([('c', 12)])
|
||||
tracker.push_old([('a', 10)])
|
||||
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['b', 'c'], added)
|
||||
|
||||
def test_very_offset(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 2)
|
||||
tracker.push_new([('a', 10), ('b', 11)])
|
||||
tracker.push_old([('z', 1), ('y', 2)])
|
||||
|
||||
tracker.push_new([('c', 12), ('d', 13)])
|
||||
tracker.push_old([('x', 3), ('w', 4)])
|
||||
|
||||
tracker.push_new([('e', 14)])
|
||||
tracker.push_old([('a', 10), ('d', 13)])
|
||||
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['b', 'c', 'e'], added)
|
||||
|
||||
def test_many_old(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 2)
|
||||
tracker.push_new([('z', 26), ('hello', 100)])
|
||||
tracker.push_old([('a', 1), ('b', 2)])
|
||||
|
||||
tracker.push_new([])
|
||||
tracker.push_old([('c', 1), ('d', 2)])
|
||||
|
||||
tracker.push_new([])
|
||||
tracker.push_old([('e', 3), ('f', 4)])
|
||||
|
||||
tracker.push_new([])
|
||||
tracker.push_old([('g', 5), ('z', 26)])
|
||||
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['hello'], added)
|
||||
|
||||
def test_high_old_bound(self):
|
||||
added = []
|
||||
|
||||
tracker = IndexedStreamingDiffTracker(added.append, 2)
|
||||
tracker.push_new([('z', 26), ('hello', 100)])
|
||||
tracker.push_old([('end1', 999), ('end2', 1000)])
|
||||
|
||||
tracker.push_new([])
|
||||
tracker.push_old([])
|
||||
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['z', 'hello'], added)
|
||||
|
||||
|
||||
class StreamingDiffTrackerTests(unittest.TestCase):
|
||||
def test_basic(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new(['a', 'b', 'c'])
|
||||
tracker.push_old(['b'])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'c'], added)
|
||||
|
||||
def test_same_streams(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new(['a', 'b', 'c'])
|
||||
tracker.push_old(['a', 'b', 'c'])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals([], added)
|
||||
|
||||
def test_some_new(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 5)
|
||||
tracker.push_new(['a', 'b', 'c', 'd', 'e'])
|
||||
tracker.push_old(['a', 'b', 'c'])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['d', 'e'], added)
|
||||
|
||||
def test_offset_new(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 5)
|
||||
tracker.push_new(['b', 'c', 'd', 'e'])
|
||||
tracker.push_old(['a', 'b', 'c'])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['d', 'e'], added)
|
||||
|
||||
def test_multiple_calls(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new(['a', 'b', 'c'])
|
||||
tracker.push_old(['b', 'd', 'e'])
|
||||
|
||||
tracker.push_new(['f', 'g', 'h'])
|
||||
tracker.push_old(['g', 'h'])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'c', 'f'], added)
|
||||
|
||||
def test_empty_old(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 3)
|
||||
tracker.push_new(['a', 'b', 'c'])
|
||||
tracker.push_old([])
|
||||
|
||||
tracker.push_new(['f', 'g', 'h'])
|
||||
tracker.push_old([])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'b', 'c', 'f', 'g', 'h'], added)
|
||||
|
||||
def test_more_old(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 2)
|
||||
tracker.push_new(['c', 'd'])
|
||||
tracker.push_old(['a', 'b'])
|
||||
|
||||
tracker.push_new([])
|
||||
tracker.push_old(['c'])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['d'], added)
|
||||
|
||||
def test_more_new(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 4)
|
||||
tracker.push_new(['a', 'b', 'c', 'd'])
|
||||
tracker.push_old(['r'])
|
||||
|
||||
tracker.push_new(['e', 'f', 'r', 'z'])
|
||||
tracker.push_old([])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'b', 'c', 'd', 'e', 'f', 'z'], added)
|
||||
|
||||
def test_more_new2(self):
|
||||
added = []
|
||||
|
||||
tracker = StreamingDiffTracker(added.append, 4)
|
||||
tracker.push_new(['a', 'b', 'c', 'd'])
|
||||
tracker.push_old(['r'])
|
||||
|
||||
tracker.push_new(['e', 'f', 'g', 'h'])
|
||||
tracker.push_old([])
|
||||
|
||||
tracker.push_new(['i', 'j', 'r', 'z'])
|
||||
tracker.push_old([])
|
||||
tracker.done()
|
||||
|
||||
self.assertEquals(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'z'], added)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -650,5 +650,109 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
self.assertIsNotNone(notification_queue.get())
|
||||
|
||||
|
||||
def test_notification_worker_offset_pages(self):
|
||||
|
||||
def get_layer_id(repo_name, tag):
|
||||
# Create a repository notification for the repo, if it doesn't exist.
|
||||
has_notification = model.notification.list_repo_notifications(ADMIN_ACCESS_USER, repo_name,
|
||||
'vulnerability_found')
|
||||
if not list(has_notification):
|
||||
repo = model.repository.get_repository(ADMIN_ACCESS_USER, repo_name)
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found',
|
||||
'quay_notification', {}, {'level': 100})
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, repo_name, tag, include_storage=True)
|
||||
return '%s.%s' % (layer.docker_image_id, layer.storage.uuid)
|
||||
|
||||
# Define offsetting sets of layer IDs, to test cross-pagination support. In this test, we
|
||||
# will only serve 2 layer IDs per page: the first page will serve both of the 'New' layer IDs,
|
||||
# but since the first 2 'Old' layer IDs are "earlier" than the shared ID of
|
||||
# `devtable/simple:latest`, they won't get served in the 'New' list until the *second* page. The
|
||||
# notification handling system should correctly not notify for this layer, even though it is
|
||||
# marked 'New' on page 1 and marked 'Old' on page 2. In practice, Clair will served these IDs
|
||||
# sorted in the same manner.
|
||||
new_layer_ids = [get_layer_id('simple', 'latest'), get_layer_id('complex', 'prod')]
|
||||
old_layer_ids = ['someid1', 'someid2', get_layer_id('simple', 'latest')]
|
||||
|
||||
apis_called = []
|
||||
|
||||
@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/layers/(.+)')
|
||||
def get_matching_layer_vulnerable(url, request):
|
||||
apis_called.append('VULN')
|
||||
return json.dumps({
|
||||
"Layer": {
|
||||
"Name": 'somelayerid',
|
||||
"Namespace": "debian:8",
|
||||
"IndexedByVersion": 1,
|
||||
"Features": [
|
||||
{
|
||||
"Name": "coreutils",
|
||||
"Namespace": "debian:8",
|
||||
"Version": "8.23-4",
|
||||
"Vulnerabilities": [
|
||||
{
|
||||
"Name": "CVE-TEST",
|
||||
"Namespace": "debian:8",
|
||||
"Severity": "Low",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
|
||||
@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/notifications/somenotification$', method='DELETE')
|
||||
def delete_notification(url, request):
|
||||
apis_called.append('DELETE')
|
||||
return {'status_code': 201, 'content': ''}
|
||||
|
||||
@urlmatch(netloc=r'(.*\.)?mockclairservice', path=r'/v1/notifications/somenotification$', method='GET')
|
||||
def get_notification(url, request):
|
||||
if url.query.find('page=nextpage') >= 0:
|
||||
apis_called.append('GET-2')
|
||||
|
||||
data = {
|
||||
'Notification': self._get_notification_data(new_layer_ids[2:], old_layer_ids[2:]),
|
||||
}
|
||||
|
||||
return json.dumps(data)
|
||||
else:
|
||||
apis_called.append('GET-1')
|
||||
|
||||
notification_data = self._get_notification_data(new_layer_ids[0:2], old_layer_ids[0:2])
|
||||
notification_data['NextPage'] = 'nextpage'
|
||||
|
||||
data = {
|
||||
'Notification': notification_data,
|
||||
}
|
||||
|
||||
return json.dumps(data)
|
||||
|
||||
# Ensure that there are no event queue items for any layers.
|
||||
self.assertIsNone(notification_queue.get())
|
||||
|
||||
# Test with a known notification with pages.
|
||||
data = {
|
||||
'Name': 'somenotification'
|
||||
}
|
||||
|
||||
with HTTMock(get_notification, delete_notification, get_matching_layer_vulnerable):
|
||||
worker = SecurityNotificationWorker(None)
|
||||
self.assertTrue(worker.perform_notification_work(data))
|
||||
|
||||
# Verify each of the expected API calls were made.
|
||||
self.assertEquals(set(['GET-1', 'GET-2', 'DELETE', 'VULN']), set(apis_called))
|
||||
|
||||
# Verify that we have notifications *just* for the New layer.
|
||||
expected_item = notification_queue.get()
|
||||
self.assertIsNotNone(expected_item)
|
||||
item_body = json.loads(expected_item['body'])
|
||||
self.assertEquals('devtable/complex', item_body['event_data']['repository'])
|
||||
self.assertEquals(['prod'], item_body['event_data']['tags'])
|
||||
|
||||
# Make sure we have no additional notifications.
|
||||
self.assertIsNone(notification_queue.get())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Reference in a new issue