Security scanner flow changes and auto-retry
Changes the security scanner code to raise exceptions now for non-successful operations. One of the new exceptions raised is MissingParentLayerException, which, when raised, will cause the security worker to perform a full rescan of all parent images for the current layer, before trying once more to scan the current layer. This should allow the system to be "self-healing" in the case where the security scanner engine somehow loses or corrupts a parent layer.
This commit is contained in:
Joseph Schorr
parent
9fa16679f8
commit
405eca074c
5 changed files with 228 additions and 82 deletions
|
|
@ -8,7 +8,7 @@ from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
|
|||
from endpoints.notificationevent import VulnerabilityFoundEvent
|
||||
from endpoints.v2 import v2_bp
|
||||
from initdb import setup_database_for_testing, finished_database_for_testing
|
||||
from util.secscan.api import SecurityScannerAPI, AnalyzeLayerException
|
||||
from util.secscan.api import SecurityScannerAPI
|
||||
from util.secscan.analyzer import LayerAnalyzer
|
||||
from util.secscan.fake import fake_security_scanner
|
||||
from util.secscan.notifier import process_notification_data
|
||||
|
|
@ -131,7 +131,7 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
|
||||
def test_analyze_layer_failure(self):
|
||||
""" Tests that failing to analyze a layer marks it as not analyzed. """
|
||||
""" Tests that failing to analyze a layer (because it 422s) marks it as analyzed but failed. """
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
|
|
@ -148,6 +148,8 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
|
||||
def test_analyze_layer_internal_error(self):
|
||||
""" Tests that failing to analyze a layer (because it 500s) marks it as not analyzed. """
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
|
@ -162,7 +164,29 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
self.assertAnalyzed(layer, security_scanner, False, -1)
|
||||
|
||||
|
||||
def test_analyze_layer_missing_parent(self):
|
||||
def test_analyze_layer_error(self):
|
||||
""" Tests that failing to analyze a layer (because it 400s) marks it as analyzed but failed. """
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
||||
with fake_security_scanner() as security_scanner:
|
||||
# Make is so trying to analyze the parent will fail with an error.
|
||||
security_scanner.set_error_layer_id(security_scanner.layer_id(layer.parent))
|
||||
|
||||
# Try to the layer and its parents, but with one request causing an error.
|
||||
analyzer = LayerAnalyzer(app.config, self.api)
|
||||
analyzer.analyze_recursively(layer)
|
||||
|
||||
# Make sure it is marked as analyzed, but in a failed state.
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertAnalyzed(layer, security_scanner, False, 1)
|
||||
|
||||
|
||||
def test_analyze_layer_missing_parent_handled(self):
|
||||
""" Tests that a missing parent causes an automatic reanalysis, which succeeds. """
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
|
@ -176,19 +200,81 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertAnalyzed(layer, security_scanner, True, 1)
|
||||
|
||||
# Mark the layer as not analyzed and delete its parent layer from the security scanner.
|
||||
# Mark the layer as not yet scanned.
|
||||
layer.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
|
||||
layer.security_indexed = False
|
||||
layer.save()
|
||||
|
||||
# Remove the layer's parent entirely from the security scanner.
|
||||
security_scanner.remove_layer(security_scanner.layer_id(layer.parent))
|
||||
|
||||
# Try to analyze again; this should fail because the parent is missing.
|
||||
with self.assertRaisesRegexp(AnalyzeLayerException, 'Bad request to security scanner'):
|
||||
analyzer.analyze_recursively(layer)
|
||||
# Analyze again, which should properly re-analyze the missing parent and this layer.
|
||||
analyzer.analyze_recursively(layer)
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertAnalyzed(layer, security_scanner, True, 1)
|
||||
|
||||
|
||||
def test_analyze_layer_invalid_parent(self):
|
||||
""" Tests that trying to reanalyze a parent that is invalid causes the layer to be marked
|
||||
as analyzed, but failed.
|
||||
"""
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
||||
with fake_security_scanner() as security_scanner:
|
||||
# Analyze the layer and its parents.
|
||||
analyzer = LayerAnalyzer(app.config, self.api)
|
||||
analyzer.analyze_recursively(layer)
|
||||
|
||||
# Make sure it was analyzed.
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertAnalyzed(layer, security_scanner, True, 1)
|
||||
|
||||
# Mark the layer as not yet scanned.
|
||||
layer.security_indexed_engine = IMAGE_NOT_SCANNED_ENGINE_VERSION
|
||||
layer.security_indexed = False
|
||||
layer.save()
|
||||
|
||||
# Remove the layer's parent entirely from the security scanner.
|
||||
security_scanner.remove_layer(security_scanner.layer_id(layer.parent))
|
||||
|
||||
# Make is so trying to analyze the parent will fail.
|
||||
security_scanner.set_error_layer_id(security_scanner.layer_id(layer.parent))
|
||||
|
||||
# Try to analyze again, which should try to reindex the parent and fail.
|
||||
analyzer.analyze_recursively(layer)
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertAnalyzed(layer, security_scanner, False, 1)
|
||||
|
||||
|
||||
def test_analyze_layer_unsupported_parent(self):
|
||||
""" Tests that attempting to analyze a layer whose parent is unanalyzable, results in the layer
|
||||
being marked as analyzed, but failed.
|
||||
"""
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
||||
with fake_security_scanner() as security_scanner:
|
||||
# Make is so trying to analyze the parent will fail.
|
||||
security_scanner.set_fail_layer_id(security_scanner.layer_id(layer.parent))
|
||||
|
||||
# Attempt to the layer and its parents. This should mark the layer itself as unanalyzable.
|
||||
analyzer = LayerAnalyzer(app.config, self.api)
|
||||
analyzer.analyze_recursively(layer)
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertAnalyzed(layer, security_scanner, False, 1)
|
||||
|
||||
|
||||
def test_analyze_layer_missing_storage(self):
|
||||
""" Tests trying to analyze a layer with missing storage. """
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
|
@ -199,16 +285,16 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
storage.remove(locations, path)
|
||||
storage.remove(locations, 'all_files_exist')
|
||||
|
||||
with fake_security_scanner():
|
||||
with fake_security_scanner() as security_scanner:
|
||||
analyzer = LayerAnalyzer(app.config, self.api)
|
||||
analyzer.analyze_recursively(layer)
|
||||
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest')
|
||||
self.assertEquals(False, layer.security_indexed)
|
||||
self.assertEquals(1, layer.security_indexed_engine)
|
||||
self.assertAnalyzed(layer, security_scanner, False, 1)
|
||||
|
||||
|
||||
def assert_analyze_layer_notify(self, security_indexed_engine, security_indexed, expect_notification):
|
||||
def assert_analyze_layer_notify(self, security_indexed_engine, security_indexed,
|
||||
expect_notification):
|
||||
layer = model.tag.get_tag_image(ADMIN_ACCESS_USER, SIMPLE_REPO, 'latest', include_storage=True)
|
||||
self.assertFalse(layer.security_indexed)
|
||||
self.assertEquals(-1, layer.security_indexed_engine)
|
||||
|
|
@ -218,7 +304,8 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
# Add a repo event for the layer.
|
||||
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found',
|
||||
'quay_notification', {}, {'level': 100})
|
||||
|
||||
# Update the layer's state before analyzing.
|
||||
layer.security_indexed_engine = security_indexed_engine
|
||||
|
|
@ -285,7 +372,8 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
# Add a repo event for the layer.
|
||||
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification',
|
||||
{}, {'level': 100})
|
||||
|
||||
# Ensure that there are no event queue items for the layer.
|
||||
self.assertIsNone(notification_queue.get())
|
||||
|
|
@ -314,7 +402,8 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
# Add a repo event for the layer.
|
||||
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification',
|
||||
{}, {'level': 100})
|
||||
|
||||
# Ensure that there are no event queue items for the layer.
|
||||
self.assertIsNone(notification_queue.get())
|
||||
|
|
@ -343,7 +432,8 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
# Add a repo event for the layer.
|
||||
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification',
|
||||
{}, {'level': 100})
|
||||
|
||||
# Ensure that there are no event queue items for the layer.
|
||||
self.assertIsNone(notification_queue.get())
|
||||
|
|
@ -389,7 +479,8 @@ class TestSecurityScanner(unittest.TestCase):
|
|||
|
||||
# Add a repo event for the layer.
|
||||
repo = model.repository.get_repository(ADMIN_ACCESS_USER, SIMPLE_REPO)
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification', {}, {'level': 100})
|
||||
model.notification.create_repo_notification(repo, 'vulnerability_found', 'quay_notification',
|
||||
{}, {'level': 100})
|
||||
|
||||
# Ensure that there are no event queue items for the layer.
|
||||
self.assertIsNone(notification_queue.get())
|
||||
|
|
|
|||
Reference in a new issue