Split the app into separate backends, which can use different worker types and different timeouts.

app.py

@@ -3,7 +3,7 @@ import os
 import json
 import yaml
 
-from flask import Flask as BaseFlask, Config as BaseConfig
+from flask import Flask as BaseFlask, Config as BaseConfig, request, Request
 from flask.ext.principal import Principal
 from flask.ext.login import LoginManager
 from flask.ext.mail import Mail
@@ -18,12 +18,12 @@ from data.users import UserAuthentication
 from util.analytics import Analytics
 from util.exceptionlog import Sentry
 from util.queuemetrics import QueueMetrics
+from util.names import urn_generator
 from data.billing import Billing
 from data.buildlogs import BuildLogs
 from data.archivedlogs import LogArchive
 from data.queue import WorkQueue
 from data.userevent import UserEventsBuilderModule
-from datetime import datetime
 
 
 class Config(BaseConfig):
@@ -60,6 +60,7 @@ LICENSE_FILENAME = 'conf/stack/license.enc'
 
 app = Flask(__name__)
 logger = logging.getLogger(__name__)
+profile = logging.getLogger('profile')
 
 
 if 'TEST' in os.environ:
@@ -82,6 +83,37 @@ else:
   environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}'))
   app.config.update(environ_config)
 
+  app.teardown_request(database.close_db_filter)
+
+
+class RequestWithId(Request):
+  request_gen = staticmethod(urn_generator(['request']))
+
+  def __init__(self, *args, **kwargs):
+    super(RequestWithId, self).__init__(*args, **kwargs)
+    self.request_id = self.request_gen()
+
+
+@app.before_request
+def _request_start():
+  profile.debug('Starting request: %s', request.path)
+
+
+@app.after_request
+def _request_end(r):
+  profile.debug('Ending request: %s', request.path)
+  return r
+
+
+class InjectingFilter(logging.Filter):
+  def filter(self, record):
+    record.msg = '[%s] %s' % (request.request_id, record.msg)
+    return True
+
+profile.addFilter(InjectingFilter())
+
+app.request_class = RequestWithId
+
 features.import_features(app.config)
 
 Principal(app, use_sessions=False)
@@ -105,9 +137,6 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf
                                    reporter=queue_metrics.report)
 notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
 
-# TODO: Remove this in the prod push following the notifications change.
-webhook_queue = WorkQueue(app.config['WEBHOOK_QUEUE_NAME'], tf)
-
 database.configure(app.config)
 model.config.app_config = app.config
 model.config.store = storage