From 328db8b6608dc89ba73f4c8155454f1e19c4926b Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Tue, 14 Oct 2014 13:58:08 -0400 Subject: [PATCH] Split the app into separate backends, which can use different worker types and different timeouts. --- Dockerfile.web | 4 +- app.py | 39 +++++++-- application.py | 84 +------------------ ...unicorn_config.py => gunicorn_registry.py} | 2 +- conf/gunicorn_verbs.py | 6 ++ conf/gunicorn_web.py | 7 ++ conf/http-base.conf | 12 ++- conf/init/gunicorn/log/run | 2 - conf/init/gunicorn/run | 8 -- conf/init/gunicorn_registry/log/run | 2 + conf/init/gunicorn_registry/run | 8 ++ conf/init/gunicorn_verbs/log/run | 2 + conf/init/gunicorn_verbs/run | 8 ++ conf/init/gunicorn_web/log/run | 2 + conf/init/gunicorn_web/run | 8 ++ conf/logging.conf | 8 ++ conf/server-base.conf | 31 +++++-- config.py | 3 - data/database.py | 12 ++- registry.py | 13 +++ test/testconfig.py | 1 - util/queueprocess.py | 7 +- verbs.py | 9 ++ web.py | 17 ++++ 24 files changed, 178 insertions(+), 117 deletions(-) rename conf/{gunicorn_config.py => gunicorn_registry.py} (73%) create mode 100644 conf/gunicorn_verbs.py create mode 100644 conf/gunicorn_web.py delete mode 100755 conf/init/gunicorn/log/run delete mode 100755 conf/init/gunicorn/run create mode 100755 conf/init/gunicorn_registry/log/run create mode 100755 conf/init/gunicorn_registry/run create mode 100755 conf/init/gunicorn_verbs/log/run create mode 100755 conf/init/gunicorn_verbs/run create mode 100755 conf/init/gunicorn_web/log/run create mode 100755 conf/init/gunicorn_web/run create mode 100644 registry.py create mode 100644 verbs.py create mode 100644 web.py diff --git a/Dockerfile.web b/Dockerfile.web index 8579a2923..f00d619bd 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -34,7 +34,9 @@ ADD conf/init/doupdatelimits.sh /etc/my_init.d/ ADD conf/init/preplogsdir.sh /etc/my_init.d/ ADD conf/init/runmigration.sh /etc/my_init.d/ -ADD conf/init/gunicorn /etc/service/gunicorn +ADD conf/init/gunicorn_web /etc/service/gunicorn_web +ADD conf/init/gunicorn_registry /etc/service/gunicorn_registry +ADD conf/init/gunicorn_verbs /etc/service/gunicorn_verbs ADD conf/init/nginx /etc/service/nginx ADD conf/init/diffsworker /etc/service/diffsworker ADD conf/init/notificationworker /etc/service/notificationworker diff --git a/app.py b/app.py index 8f0a57d62..3bca06cd4 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,7 @@ import os import json import yaml -from flask import Flask as BaseFlask, Config as BaseConfig +from flask import Flask as BaseFlask, Config as BaseConfig, request, Request from flask.ext.principal import Principal from flask.ext.login import LoginManager from flask.ext.mail import Mail @@ -18,12 +18,12 @@ from data.users import UserAuthentication from util.analytics import Analytics from util.exceptionlog import Sentry from util.queuemetrics import QueueMetrics +from util.names import urn_generator from data.billing import Billing from data.buildlogs import BuildLogs from data.archivedlogs import LogArchive from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule -from datetime import datetime class Config(BaseConfig): @@ -60,6 +60,7 @@ LICENSE_FILENAME = 'conf/stack/license.enc' app = Flask(__name__) logger = logging.getLogger(__name__) +profile = logging.getLogger('profile') if 'TEST' in os.environ: @@ -82,6 +83,37 @@ else: environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) app.config.update(environ_config) + app.teardown_request(database.close_db_filter) + + +class RequestWithId(Request): + request_gen = staticmethod(urn_generator(['request'])) + + def __init__(self, *args, **kwargs): + super(RequestWithId, self).__init__(*args, **kwargs) + self.request_id = self.request_gen() + + +@app.before_request +def _request_start(): + profile.debug('Starting request: %s', request.path) + + +@app.after_request +def _request_end(r): + profile.debug('Ending request: %s', request.path) + return r + + +class InjectingFilter(logging.Filter): + def filter(self, record): + record.msg = '[%s] %s' % (request.request_id, record.msg) + return True + +profile.addFilter(InjectingFilter()) + +app.request_class = RequestWithId + features.import_features(app.config) Principal(app, use_sessions=False) @@ -105,9 +137,6 @@ dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf reporter=queue_metrics.report) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf) -# TODO: Remove this in the prod push following the notifications change. -webhook_queue = WorkQueue(app.config['WEBHOOK_QUEUE_NAME'], tf) - database.configure(app.config) model.config.app_config = app.config model.config.store = storage diff --git a/application.py b/application.py index 4c0adb9b9..a9bd0df6e 100644 --- a/application.py +++ b/application.py @@ -1,90 +1,14 @@ import logging import logging.config -import uuid - -from peewee import Proxy from app import app as application -from flask import request, Request -from util.names import urn_generator -from data.database import db as model_db, read_slave - -# Turn off debug logging for boto -logging.getLogger('boto').setLevel(logging.CRITICAL) - -from endpoints.api import api_bp -from endpoints.index import index -from endpoints.web import web -from endpoints.tags import tags -from endpoints.registry import registry -from endpoints.verbs import verbs -from endpoints.webhooks import webhooks -from endpoints.realtime import realtime -from endpoints.callbacks import callback - -from logentries import LogentriesHandler -logger = logging.getLogger(__name__) +# Bind all of the blueprints +import web +import verbs +import registry -werkzeug = logging.getLogger('werkzeug') -werkzeug.setLevel(logging.DEBUG) - -profile = logging.getLogger('profile') -profile.setLevel(logging.DEBUG) - -logentries_key = application.config.get('LOGENTRIES_KEY', None) -if logentries_key: - logger.debug('Initializing logentries with key: %s' % logentries_key) - werkzeug.addHandler(LogentriesHandler(logentries_key)) - profile.addHandler(LogentriesHandler(logentries_key)) - -application.register_blueprint(web) -application.register_blueprint(callback, url_prefix='/oauth2') -application.register_blueprint(index, url_prefix='/v1') -application.register_blueprint(tags, url_prefix='/v1') -application.register_blueprint(registry, url_prefix='/v1') -application.register_blueprint(verbs, url_prefix='/c1') -application.register_blueprint(api_bp, url_prefix='/api') -application.register_blueprint(webhooks, url_prefix='/webhooks') -application.register_blueprint(realtime, url_prefix='/realtime') - -class RequestWithId(Request): - request_gen = staticmethod(urn_generator(['request'])) - - def __init__(self, *args, **kwargs): - super(RequestWithId, self).__init__(*args, **kwargs) - self.request_id = self.request_gen() - -@application.before_request -def _request_start(): - profile.debug('Starting request: %s', request.path) - - -@application.after_request -def _request_end(r): - profile.debug('Ending request: %s', request.path) - return r - -class InjectingFilter(logging.Filter): - def filter(self, record): - record.msg = '[%s] %s' % (request.request_id, record.msg) - return True - -profile.addFilter(InjectingFilter()) - -def close_db(exc): - db = model_db - if not db.is_closed(): - logger.debug('Disconnecting from database.') - db.close() - - if read_slave.obj is not None and not read_slave.is_closed(): - logger.debug('Disconnecting from read slave.') - read_slave.close() - -application.teardown_request(close_db) -application.request_class = RequestWithId if __name__ == '__main__': logging.config.fileConfig('conf/logging.conf', disable_existing_loggers=False) diff --git a/conf/gunicorn_config.py b/conf/gunicorn_registry.py similarity index 73% rename from conf/gunicorn_config.py rename to conf/gunicorn_registry.py index ca8ad5363..c154a4711 100644 --- a/conf/gunicorn_config.py +++ b/conf/gunicorn_registry.py @@ -1,4 +1,4 @@ -bind = 'unix:/tmp/gunicorn.sock' +bind = 'unix:/tmp/gunicorn_registry.sock' workers = 16 worker_class = 'gevent' timeout = 2000 diff --git a/conf/gunicorn_verbs.py b/conf/gunicorn_verbs.py new file mode 100644 index 000000000..30f0bafd6 --- /dev/null +++ b/conf/gunicorn_verbs.py @@ -0,0 +1,6 @@ +bind = 'unix:/tmp/gunicorn_verbs.sock' +workers = 8 +timeout = 2000 +logconfig = 'conf/logging.conf' +pythonpath = '.' +preload_app = True \ No newline at end of file diff --git a/conf/gunicorn_web.py b/conf/gunicorn_web.py new file mode 100644 index 000000000..919dfc88d --- /dev/null +++ b/conf/gunicorn_web.py @@ -0,0 +1,7 @@ +bind = 'unix:/tmp/gunicorn_web.sock' +workers = 2 +worker_class = 'gevent' +timeout = 30 +logconfig = 'conf/logging.conf' +pythonpath = '.' +preload_app = True \ No newline at end of file diff --git a/conf/http-base.conf b/conf/http-base.conf index bfa1a85f2..ad3d9f178 100644 --- a/conf/http-base.conf +++ b/conf/http-base.conf @@ -14,8 +14,12 @@ gzip_types text/plain text/xml text/css text/javascript application/x-javascript application/octet-stream; -upstream app_server { - server unix:/tmp/gunicorn.sock fail_timeout=0; - # For a TCP configuration: - # server 192.168.0.7:8000 fail_timeout=0; +upstream web_app_server { + server unix:/tmp/gunicorn_web.sock fail_timeout=0; +} +upstream verbs_app_server { + server unix:/tmp/gunicorn_verbs.sock fail_timeout=0; +} +upstream registry_app_server { + server unix:/tmp/gunicorn_registry.sock fail_timeout=0; } diff --git a/conf/init/gunicorn/log/run b/conf/init/gunicorn/log/run deleted file mode 100755 index 106d6c4f8..000000000 --- a/conf/init/gunicorn/log/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec svlogd /var/log/gunicorn/ \ No newline at end of file diff --git a/conf/init/gunicorn/run b/conf/init/gunicorn/run deleted file mode 100755 index a61e7c651..000000000 --- a/conf/init/gunicorn/run +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/bash - -echo 'Starting gunicon' - -cd / -venv/bin/gunicorn -c conf/gunicorn_config.py application:application - -echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/gunicorn_registry/log/run b/conf/init/gunicorn_registry/log/run new file mode 100755 index 000000000..1896ef533 --- /dev/null +++ b/conf/init/gunicorn_registry/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/gunicorn_registry/ \ No newline at end of file diff --git a/conf/init/gunicorn_registry/run b/conf/init/gunicorn_registry/run new file mode 100755 index 000000000..a0a09f5a2 --- /dev/null +++ b/conf/init/gunicorn_registry/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting gunicon' + +cd / +venv/bin/gunicorn -c conf/gunicorn_registry.py registry:application + +echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/gunicorn_verbs/log/run b/conf/init/gunicorn_verbs/log/run new file mode 100755 index 000000000..2b061e193 --- /dev/null +++ b/conf/init/gunicorn_verbs/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/gunicorn_verbs/ \ No newline at end of file diff --git a/conf/init/gunicorn_verbs/run b/conf/init/gunicorn_verbs/run new file mode 100755 index 000000000..1cf2ee51c --- /dev/null +++ b/conf/init/gunicorn_verbs/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting gunicon' + +cd / +nice -10 venv/bin/gunicorn -c conf/gunicorn_verbs.py verbs:application + +echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/init/gunicorn_web/log/run b/conf/init/gunicorn_web/log/run new file mode 100755 index 000000000..de17cdf61 --- /dev/null +++ b/conf/init/gunicorn_web/log/run @@ -0,0 +1,2 @@ +#!/bin/sh +exec svlogd /var/log/gunicorn_web/ \ No newline at end of file diff --git a/conf/init/gunicorn_web/run b/conf/init/gunicorn_web/run new file mode 100755 index 000000000..86d107618 --- /dev/null +++ b/conf/init/gunicorn_web/run @@ -0,0 +1,8 @@ +#! /bin/bash + +echo 'Starting gunicon' + +cd / +venv/bin/gunicorn -c conf/gunicorn_web.py web:application + +echo 'Gunicorn exited' \ No newline at end of file diff --git a/conf/logging.conf b/conf/logging.conf index 4023e7743..6dadc997b 100644 --- a/conf/logging.conf +++ b/conf/logging.conf @@ -17,6 +17,14 @@ qualname=application.profiler level=DEBUG handlers=console +[logger_boto] +level=INFO +handlers=console + +[logger_werkzeug] +level=DEBUG +handlers=console + [logger_gunicorn.error] level=INFO handlers=console diff --git a/conf/server-base.conf b/conf/server-base.conf index 4636afdde..f3cb3076b 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -1,4 +1,3 @@ -client_max_body_size 20G; client_body_temp_path /var/log/nginx/client_body 1 2; server_name _; @@ -11,17 +10,35 @@ if ($args ~ "_escaped_fragment_") { rewrite ^ /snapshot$uri; } +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header Host $http_host; +proxy_redirect off; + +proxy_set_header Transfer-Encoding $http_transfer_encoding; + location / { - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Host $http_host; - proxy_redirect off; + proxy_pass http://web_app_server; +} + +location /v1/ { proxy_buffering off; proxy_request_buffering off; - proxy_set_header Transfer-Encoding $http_transfer_encoding; - proxy_pass http://app_server; + proxy_pass http://registry_app_server; + proxy_read_timeout 2000; + proxy_temp_path /var/log/nginx/proxy_temp 1 2; + + client_max_body_size 20G; +} + +location /c1/ { + proxy_buffering off; + + proxy_request_buffering off; + + proxy_pass http://verbs_app_server; proxy_read_timeout 2000; proxy_temp_path /var/log/nginx/proxy_temp 1 2; } diff --git a/config.py b/config.py index 6742d1a43..c51bf9cb5 100644 --- a/config.py +++ b/config.py @@ -132,9 +132,6 @@ class DefaultConfig(object): DIFFS_QUEUE_NAME = 'imagediff' DOCKERFILE_BUILD_QUEUE_NAME = 'dockerfilebuild' - # TODO: Remove this in the prod push following the notifications change. - WEBHOOK_QUEUE_NAME = 'webhook' - # Super user config. Note: This MUST BE an empty list for the default config. SUPER_USERS = [] diff --git a/data/database.py b/data/database.py index a57b6cfb9..1914a954c 100644 --- a/data/database.py +++ b/data/database.py @@ -7,9 +7,9 @@ from datetime import datetime from peewee import * from data.read_slave import ReadSlaveModel from sqlalchemy.engine.url import make_url -from urlparse import urlparse from util.names import urn_generator + logger = logging.getLogger(__name__) @@ -80,6 +80,16 @@ def uuid_generator(): return str(uuid.uuid4()) +def close_db_filter(_): + if not db.is_closed(): + logger.debug('Disconnecting from database.') + db.close() + + if read_slave.obj is not None and not read_slave.is_closed(): + logger.debug('Disconnecting from read slave.') + read_slave.close() + + class BaseModel(ReadSlaveModel): class Meta: database = db diff --git a/registry.py b/registry.py new file mode 100644 index 000000000..2a356e1ec --- /dev/null +++ b/registry.py @@ -0,0 +1,13 @@ +import logging +import logging.config + +from app import app as application + +from endpoints.index import index +from endpoints.tags import tags +from endpoints.registry import registry + + +application.register_blueprint(index, url_prefix='/v1') +application.register_blueprint(tags, url_prefix='/v1') +application.register_blueprint(registry, url_prefix='/v1') diff --git a/test/testconfig.py b/test/testconfig.py index 46288de7e..eb11f270a 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -1,7 +1,6 @@ from datetime import datetime, timedelta from config import DefaultConfig -from test.testlogs import TestBuildLogs class FakeTransaction(object): diff --git a/util/queueprocess.py b/util/queueprocess.py index d8e00418f..f6522589a 100644 --- a/util/queueprocess.py +++ b/util/queueprocess.py @@ -3,7 +3,6 @@ import logging import multiprocessing import os import time -import gipc import sys import traceback @@ -31,7 +30,7 @@ class QueueProcess(object): @staticmethod def run_process(target, args): - gipc.start_process(target=target, args=args) + Process(target=target, args=args).start() def run(self): # Important! gipc is used here because normal multiprocessing does not work @@ -50,9 +49,9 @@ def _run(get_producer, queues, chunk_size, args): for queue in queues: try: - queue.put(data, block=True, timeout=10) + queue.put(data, block=True) except Exception as ex: - # One of the listeners stopped listening. + logger.exception('Exception writing to queue.') return if data is None or isinstance(data, Exception): diff --git a/verbs.py b/verbs.py new file mode 100644 index 000000000..fcde6ad47 --- /dev/null +++ b/verbs.py @@ -0,0 +1,9 @@ +import logging +import logging.config + +from app import app as application + +from endpoints.verbs import verbs + + +application.register_blueprint(verbs, url_prefix='/c1') diff --git a/web.py b/web.py new file mode 100644 index 000000000..6db09bec7 --- /dev/null +++ b/web.py @@ -0,0 +1,17 @@ +import logging +import logging.config + +from app import app as application + +from endpoints.api import api_bp +from endpoints.web import web +from endpoints.webhooks import webhooks +from endpoints.realtime import realtime +from endpoints.callbacks import callback + + +application.register_blueprint(web) +application.register_blueprint(callback, url_prefix='/oauth2') +application.register_blueprint(api_bp, url_prefix='/api') +application.register_blueprint(webhooks, url_prefix='/webhooks') +application.register_blueprint(realtime, url_prefix='/realtime')