Split the app into separate backends, which can use different worker types and different timeouts.

util/queueprocess.py

@@ -3,7 +3,6 @@ import logging
 import multiprocessing
 import os
 import time
-import gipc
 import sys
 import traceback
 
@@ -31,7 +30,7 @@ class QueueProcess(object):
 
   @staticmethod
   def run_process(target, args):
-    gipc.start_process(target=target, args=args)
+    Process(target=target, args=args).start()
 
   def run(self):
     # Important! gipc is used here because normal multiprocessing does not work
@@ -50,9 +49,9 @@ def _run(get_producer, queues, chunk_size, args):
 
     for queue in queues:
       try:
-        queue.put(data, block=True, timeout=10)
+        queue.put(data, block=True)
       except Exception as ex:
-        # One of the listeners stopped listening.
+        logger.exception('Exception writing to queue.')
         return
             
     if data is None or isinstance(data, Exception):