vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Extend processing before processing security notifications

Browse source 
Makes sure queue items don't expire during processing

Fixes #1776
This commit is contained in:
Joseph Schorr 2016-08-29 13:08:38 -04:00
parent 3afe5a1c12
commit 08a3b70b56
1 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
workers/security_notification_worker.py
View file

@ -10,7 +10,8 @@ from util.secscan.notifier import process_notification_data
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_EXTENDED_SECONDS = 600 _READING_SECONDS = 120 # 2 minutes
_PROCESSING_SECONDS = 60 * 60 # 1 hour
_LAYER_LIMIT = 100 # The number of layers to request on each page. _LAYER_LIMIT = 100 # The number of layers to request on each page.
class SecurityNotificationWorker(QueueWorker): class SecurityNotificationWorker(QueueWorker):
@ -27,6 +28,7 @@ class SecurityNotificationWorker(QueueWorker):
current_page = data.get('page', None) current_page = data.get('page', None)
while True: while True:
self.extend_processing(_READING_SECONDS, json.dumps(data))
(response_data, should_retry) = secscan_api.get_notification(notification_name, (response_data, should_retry) = secscan_api.get_notification(notification_name,
layer_limit=_LAYER_LIMIT, layer_limit=_LAYER_LIMIT,
page=current_page) page=current_page)
@ -38,6 +40,7 @@ class SecurityNotificationWorker(QueueWorker):
logger.error('Failed to handle security notification %s', notification_name) logger.error('Failed to handle security notification %s', notification_name)
return False return False
self.extend_processing(_PROCESSING_SECONDS, json.dumps(data))
notification_data = response_data['Notification'] notification_data = response_data['Notification']
if not process_notification_data(notification_data): if not process_notification_data(notification_data):
raise JobException() raise JobException()
@ -56,7 +59,6 @@ class SecurityNotificationWorker(QueueWorker):
# something goes wrong in the next loop iteration), and continue. # something goes wrong in the next loop iteration), and continue.
current_page = notification_data['NextPage'] current_page = notification_data['NextPage']
data['page'] = current_page data['page'] = current_page
self.extend_processing(_EXTENDED_SECONDS, json.dumps(data))
if __name__ == '__main__': if __name__ == '__main__':