Improve the builder response to being terminated or dying.

data/queue.py

@@ -7,6 +7,9 @@ from app import app
 transaction_factory = app.config['DB_TRANSACTION_FACTORY']
 
 
+MINIMUM_EXTENSION = timedelta(seconds=20)
+
+
 class WorkQueue(object):
   def __init__(self, queue_name, canonical_name_match_list=None):
     self.queue_name = queue_name
@@ -80,17 +83,24 @@ class WorkQueue(object):
     completed_item.delete_instance()
 
   @staticmethod
-  def incomplete(incomplete_item, retry_after=300):
+  def incomplete(incomplete_item, retry_after=300, restore_retry=False):
     retry_date = datetime.now() + timedelta(seconds=retry_after)
     incomplete_item.available_after = retry_date
     incomplete_item.available = True
+
+    if restore_retry:
+      incomplete_item.retries_remaining += 1
+
     incomplete_item.save()
 
   @staticmethod
   def extend_processing(queue_item, seconds_from_now):
     new_expiration = datetime.now() + timedelta(seconds=seconds_from_now)
-    queue_item.processing_expires = new_expiration
-    queue_item.save()
+
+    # Only actually write the new expiration to the db if it moves the expiration some minimum
+    if new_expiration - queue_item.processing_expires > MINIMUM_EXTENSION:
+      queue_item.processing_expires = new_expiration
+      queue_item.save()
 
 
 image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'])