diff --git a/app.py b/app.py index 6ce4fbf90..c3c5163a5 100644 --- a/app.py +++ b/app.py @@ -4,30 +4,31 @@ import json from flask import Flask, Config, request, Request from flask.ext.principal import Principal -from flask.ext.login import LoginManager +from flask.ext.login import LoginManager, UserMixin from flask.ext.mail import Mail import features from storage import Storage + +from avatars.avatars import Avatar + from data import model from data import database from data.userfiles import Userfiles from data.users import UserAuthentication +from data.billing import Billing from data.buildlogs import BuildLogs from data.archivedlogs import LogArchive -from data.queue import WorkQueue from data.userevent import UserEventsBuilderModule - -from avatars.avatars import Avatar +from data.queue import WorkQueue from util.analytics import Analytics -from data.billing import Billing -from util.config.provider import FileConfigProvider, TestConfigProvider from util.exceptionlog import Sentry from util.names import urn_generator from util.oauth import GoogleOAuthConfig, GithubOAuthConfig from util.queuemetrics import QueueMetrics +from util.config.provider import FileConfigProvider, TestConfigProvider from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager @@ -103,17 +104,18 @@ analytics = Analytics(app) billing = Billing(app) sentry = Sentry(app) build_logs = BuildLogs(app) -queue_metrics = QueueMetrics(app) authentication = UserAuthentication(app) userevents = UserEventsBuilderModule(app) superusers = SuperUserManager(app) +queue_metrics = QueueMetrics(app) + +tf = app.config['DB_TRANSACTION_FACTORY'] github_login = GithubOAuthConfig(app.config, 'GITHUB_LOGIN_CONFIG') github_trigger = GithubOAuthConfig(app.config, 'GITHUB_TRIGGER_CONFIG') google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG') oauth_apps = [github_login, github_trigger, google_login] -tf = app.config['DB_TRANSACTION_FACTORY'] image_diff_queue = WorkQueue(app.config['DIFFS_QUEUE_NAME'], tf) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, reporter=queue_metrics.report) @@ -128,5 +130,29 @@ if app.config['SECRET_KEY'] is None: logger.debug('Generating in-memory secret key') app.config['SECRET_KEY'] = generate_secret_key() +@login_manager.user_loader +def load_user(user_uuid): + logger.debug('User loader loading deferred user with uuid: %s' % user_uuid) + return LoginWrappedDBUser(user_uuid) + +class LoginWrappedDBUser(UserMixin): + def __init__(self, user_uuid, db_user=None): + self._uuid = user_uuid + self._db_user = db_user + + def db_user(self): + if not self._db_user: + self._db_user = model.get_user_by_uuid(self._uuid) + return self._db_user + + def is_authenticated(self): + return self.db_user() is not None + + def is_active(self): + return self.db_user().verified + + def get_id(self): + return unicode(self._uuid) + def get_app_url(): return '%s://%s' % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) diff --git a/auth/permissions.py b/auth/permissions.py index 47a41c257..4ee73bdb3 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -89,6 +89,8 @@ class QuayDeferredPermissionUser(Identity): if not self._permissions_loaded: logger.debug('Loading user permissions after deferring.') user_object = model.get_user_by_uuid(self.id) + if user_object is None: + return super(QuayDeferredPermissionUser, self).can(permission) if user_object is None: return super(QuayDeferredPermissionUser, self).can(permission) diff --git a/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb b/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb deleted file mode 100644 index 8b3fb3fb6..000000000 Binary files a/binary_dependencies/nginx_1.4.2-nobuffer-3_amd64.deb and /dev/null differ diff --git a/binary_dependencies/tengine_2.1.0-1_amd64.deb b/binary_dependencies/tengine_2.1.0-1_amd64.deb new file mode 100644 index 000000000..17541acc8 Binary files /dev/null and b/binary_dependencies/tengine_2.1.0-1_amd64.deb differ diff --git a/conf/hosted-http-base.conf b/conf/hosted-http-base.conf index c3e910e8f..fa5994e6f 100644 --- a/conf/hosted-http-base.conf +++ b/conf/hosted-http-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + server { listen 80 default_server; server_name _; diff --git a/conf/http-base.conf b/conf/http-base.conf index 1eb0b6170..d525b3dd3 100644 --- a/conf/http-base.conf +++ b/conf/http-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + types_hash_max_size 2048; include /usr/local/nginx/conf/mime.types.default; @@ -30,4 +32,4 @@ upstream build_manager_controller_server { upstream build_manager_websocket_server { server localhost:8787; -} \ No newline at end of file +} diff --git a/conf/nginx-nossl.conf b/conf/nginx-nossl.conf index cc985906a..13c5d73b2 100644 --- a/conf/nginx-nossl.conf +++ b/conf/nginx-nossl.conf @@ -1,8 +1,12 @@ +# vim: ft=nginx + include root-base.conf; http { include http-base.conf; + include rate-limiting.conf; + server { include server-base.conf; diff --git a/conf/nginx.conf b/conf/nginx.conf index 01d554ae2..234839e53 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + include root-base.conf; http { @@ -5,6 +7,8 @@ http { include hosted-http-base.conf; + include rate-limiting.conf; + server { include server-base.conf; diff --git a/conf/rate-limiting.conf b/conf/rate-limiting.conf new file mode 100644 index 000000000..937397ec9 --- /dev/null +++ b/conf/rate-limiting.conf @@ -0,0 +1,6 @@ +# vim: ft=nginx + +limit_req_zone $binary_remote_addr zone=webapp:10m rate=10r/s; +limit_req_zone $binary_remote_addr zone=api:10m rate=1r/s; +limit_req_status 429; +limit_req_log_level warn; diff --git a/conf/root-base.conf b/conf/root-base.conf index dc8685c34..02c004564 100644 --- a/conf/root-base.conf +++ b/conf/root-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + pid /tmp/nginx.pid; error_log /var/log/nginx/nginx.error.log; diff --git a/conf/server-base.conf b/conf/server-base.conf index 5b06b76c5..9284fe1cf 100644 --- a/conf/server-base.conf +++ b/conf/server-base.conf @@ -1,3 +1,5 @@ +# vim: ft=nginx + client_body_temp_path /var/log/nginx/client_body 1 2; server_name _; @@ -19,6 +21,8 @@ proxy_set_header Transfer-Encoding $http_transfer_encoding; location / { proxy_pass http://web_app_server; + + #limit_req zone=webapp burst=10 nodelay; } location /realtime { @@ -37,6 +41,8 @@ location /v1/ { proxy_temp_path /var/log/nginx/proxy_temp 1 2; client_max_body_size 20G; + + #limit_req zone=api burst=5 nodelay; } location /c1/ { @@ -47,6 +53,8 @@ location /c1/ { proxy_pass http://verbs_app_server; proxy_read_timeout 2000; proxy_temp_path /var/log/nginx/proxy_temp 1 2; + + #limit_req zone=api burst=5 nodelay; } location /static/ { diff --git a/data/database.py b/data/database.py index 1953b4a14..aba8a578d 100644 --- a/data/database.py +++ b/data/database.py @@ -70,13 +70,14 @@ read_slave = Proxy() db_random_func = CallableProxy() -def validate_database_url(url): +def validate_database_url(url, connect_timeout=5): driver = _db_from_url(url, { - 'connect_timeout': 5 + 'connect_timeout': connect_timeout }) driver.connect() driver.close() + def _db_from_url(url, db_kwargs): parsed_url = make_url(url) @@ -89,6 +90,10 @@ def _db_from_url(url, db_kwargs): if parsed_url.password: db_kwargs['password'] = parsed_url.password + # Note: sqlite does not support connect_timeout. + if parsed_url.drivername == 'sqlite' and 'connect_timeout' in db_kwargs: + del db_kwargs['connect_timeout'] + return SCHEME_DRIVERS[parsed_url.drivername](parsed_url.database, **db_kwargs) @@ -129,8 +134,9 @@ def close_db_filter(_): class QuayUserField(ForeignKeyField): - def __init__(self, allows_robots=False, *args, **kwargs): + def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs): self.allows_robots = allows_robots + self.robot_null_delete = robot_null_delete if not 'rel_model' in kwargs: kwargs['rel_model'] = User @@ -164,7 +170,11 @@ class User(BaseModel): for query, fk in self.dependencies(search_nullable=True): if isinstance(fk, QuayUserField) and fk.allows_robots: model = fk.model_class - model.delete().where(query).execute() + + if fk.robot_null_delete: + model.update(**{fk.name: None}).where(query).execute() + else: + model.delete().where(query).execute() # Delete the instance itself. super(User, self).delete_instance(recursive=False, delete_nullable=False) @@ -466,7 +476,7 @@ class LogEntry(BaseModel): kind = ForeignKeyField(LogEntryKind, index=True) account = QuayUserField(index=True, related_name='account') performer = QuayUserField(allows_robots=True, index=True, null=True, - related_name='performer') + related_name='performer', robot_null_delete=True) repository = ForeignKeyField(Repository, index=True, null=True) datetime = DateTimeField(default=datetime.now, index=True) ip = CharField(null=True) diff --git a/data/model/legacy.py b/data/model/legacy.py index a5c779871..f8c04e04c 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -14,7 +14,7 @@ from data.database import (User, Repository, Image, AccessToken, Role, Repositor ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite, DerivedImageStorage, ImageStorageTransformation, random_string_generator, - db, BUILD_PHASE, QuayUserField) + db, BUILD_PHASE, QuayUserField, validate_database_url) from peewee import JOIN_LEFT_OUTER, fn from util.validation import (validate_username, validate_email, validate_password, INVALID_PASSWORD_MESSAGE) @@ -2257,11 +2257,20 @@ def delete_user(user): # TODO: also delete any repository data associated -def check_health(): +def check_health(app_config): + # Attempt to connect to the database first. If the DB is not responding, + # using the validate_database_url will timeout quickly, as opposed to + # making a normal connect which will just hang (thus breaking the health + # check). + try: + validate_database_url(app_config['DB_URI'], connect_timeout=3) + except Exception: + logger.exception('Could not connect to the database') + return False + # We will connect to the db, check that it contains some log entry kinds try: - found_count = LogEntryKind.select().count() - return found_count > 0 + return bool(list(LogEntryKind.select().limit(1))) except: return False diff --git a/emails/base.html b/emails/base.html index e01343d53..00a89e51d 100644 --- a/emails/base.html +++ b/emails/base.html @@ -4,6 +4,12 @@ {{ subject }} + + {% if action_metadata %} + + {% endif %}