import json import logging import os from functools import partial from Crypto.PublicKey import RSA from flask import Flask, request, Request, _request_ctx_stack from flask_login import LoginManager, UserMixin from flask_mail import Mail from flask_principal import Principal from jwkest.jwk import RSAKey from werkzeug.routing import BaseConverter import features from avatars.avatars import Avatar from buildman.manager.buildcanceller import BuildCanceller from data import database from data import model from data.archivedlogs import LogArchive from data.billing import Billing from data.buildlogs import BuildLogs from data.queue import WorkQueue, BuildMetricQueueReporter from data.userevent import UserEventsBuilderModule from data.userfiles import Userfiles from data.users import UserAuthentication from storage import Storage from util import get_app_url from util.saas.analytics import Analytics from util.saas.useranalytics import UserAnalytics from util.saas.exceptionlog import Sentry from util.names import urn_generator from util.oauth.services import GoogleOAuthService, GithubOAuthService, GitLabOAuthService from util.config.configutil import generate_secret_key from util.config.provider import get_config_provider from util.config.superusermanager import SuperUserManager from util.label_validator import LabelValidator from util.license import LicenseValidator from util.metrics.metricqueue import MetricQueue from util.metrics.prometheus import PrometheusPlugin from util.saas.cloudwatch import start_cloudwatch_sender from util.secscan.api import SecurityScannerAPI from util.security.instancekeys import InstanceKeys from util.security.signing import Signer OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py' OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG' DOCKER_V2_SIGNINGKEY_FILENAME = 'docker_v2.pem' app = Flask(__name__) logger = logging.getLogger(__name__) # Instantiate the configuration. is_testing = 'TEST' in os.environ is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', testing=is_testing, kubernetes=is_kubernetes) if is_testing: from test.testconfig import TestConfig logger.debug('Loading test config.') app.config.from_object(TestConfig()) else: from config import DefaultConfig logger.debug('Loading default config.') app.config.from_object(DefaultConfig()) app.teardown_request(database.close_db_filter) # Load the override config via the provider. config_provider.update_app_config(app.config) # Update any configuration found in the override environment variable. environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}')) app.config.update(environ_config) # Allow user to define a custom storage preference for the local instance. _distributed_storage_preference = os.environ.get('QUAY_DISTRIBUTED_STORAGE_PREFERENCE', '').split() if _distributed_storage_preference: app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = _distributed_storage_preference # Generate a secret key if none was specified. if app.config['SECRET_KEY'] is None: logger.debug('Generating in-memory secret key') app.config['SECRET_KEY'] = generate_secret_key() # If the "preferred" scheme is https, then http is not allowed. Therefore, ensure we have a secure # session cookie. if (app.config['PREFERRED_URL_SCHEME'] == 'https' and not app.config.get('FORCE_NONSECURE_SESSION_COOKIE', False)): app.config['SESSION_COOKIE_SECURE'] = True # Load features from config. features.import_features(app.config) class RequestWithId(Request): request_gen = staticmethod(urn_generator(['request'])) def __init__(self, *args, **kwargs): super(RequestWithId, self).__init__(*args, **kwargs) self.request_id = self.request_gen() @app.before_request def _request_start(): logger.debug('Starting request: %s', request.path) @app.after_request def _request_end(r): logger.debug('Ending request: %s', request.path) return r class InjectingFilter(logging.Filter): def filter(self, record): if _request_ctx_stack.top is not None: record.msg = '[%s] %s' % (request.request_id, record.msg) return True root_logger = logging.getLogger() # Add the request id filter to all handlers of the root logger for handler in root_logger.handlers: handler.addFilter(InjectingFilter()) app.request_class = RequestWithId # Register custom converters. class RegexConverter(BaseConverter): """ Converter for handling custom regular expression patterns in paths. """ def __init__(self, url_map, regex_value): super(RegexConverter, self).__init__(url_map) self.regex = regex_value class RepositoryPathConverter(BaseConverter): """ Converter for handling repository paths. Handles both library and non-library paths (if configured). """ def __init__(self, url_map): super(RepositoryPathConverter, self).__init__(url_map) self.weight = 200 if features.LIBRARY_SUPPORT: # Allow names without namespaces. self.regex = r'[^/]+(/[^/]+)?' else: self.regex = r'([^/]+/[^/]+)' class APIRepositoryPathConverter(BaseConverter): """ Converter for handling repository paths. Does not handle library paths. """ def __init__(self, url_map): super(APIRepositoryPathConverter, self).__init__(url_map) self.weight = 200 self.regex = r'([^/]+/[^/]+)' app.url_map.converters['regex'] = RegexConverter app.url_map.converters['repopath'] = RepositoryPathConverter app.url_map.converters['apirepopath'] = APIRepositoryPathConverter Principal(app, use_sessions=False) tf = app.config['DB_TRANSACTION_FACTORY'] chunk_cleanup_queue = WorkQueue(app.config['CHUNK_CLEANUP_QUEUE_NAME'], tf) avatar = Avatar(app) login_manager = LoginManager(app) mail = Mail(app) prometheus = PrometheusPlugin(app) metric_queue = MetricQueue(prometheus) instance_keys = InstanceKeys(app) storage = Storage(app, metric_queue, chunk_cleanup_queue, instance_keys) userfiles = Userfiles(app, storage) log_archive = LogArchive(app, storage) analytics = Analytics(app) user_analytics = UserAnalytics(app) billing = Billing(app) sentry = Sentry(app) build_logs = BuildLogs(app) authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECTORY) userevents = UserEventsBuilderModule(app) superusers = SuperUserManager(app) signer = Signer(app, config_provider) instance_keys = InstanceKeys(app) label_validator = LabelValidator(app) build_canceller = BuildCanceller(app) license_validator = LicenseValidator(config_provider) license_validator.start() start_cloudwatch_sender(metric_queue, app) github_login = GithubOAuthService(app.config, 'GITHUB_LOGIN_CONFIG') github_trigger = GithubOAuthService(app.config, 'GITHUB_TRIGGER_CONFIG') gitlab_trigger = GitLabOAuthService(app.config, 'GITLAB_TRIGGER_CONFIG') google_login = GoogleOAuthService(app.config, 'GOOGLE_LOGIN_CONFIG') oauth_apps = [github_login, github_trigger, gitlab_trigger, google_login] image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf, has_namespace=False) dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf, reporter=BuildMetricQueueReporter(metric_queue), has_namespace=True) notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf, has_namespace=True) secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf, has_namespace=False) all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue, secscan_notification_queue, chunk_cleanup_queue] secscan_api = SecurityScannerAPI(app, app.config, storage) # Check for a key in config. If none found, generate a new signing key for Docker V2 manifests. _v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME) if os.path.exists(_v2_key_path): docker_v2_signing_key = RSAKey().load(_v2_key_path) else: docker_v2_signing_key = RSAKey(key=RSA.generate(2048)) database.configure(app.config) model.config.app_config = app.config model.config.store = storage model.config.register_image_cleanup_callback(secscan_api.cleanup_layers) @login_manager.user_loader def load_user(user_uuid): logger.debug('User loader loading deferred user with uuid: %s', user_uuid) return LoginWrappedDBUser(user_uuid) class LoginWrappedDBUser(UserMixin): def __init__(self, user_uuid, db_user=None): self._uuid = user_uuid self._db_user = db_user def db_user(self): if not self._db_user: self._db_user = model.user.get_user_by_uuid(self._uuid) return self._db_user @property def is_authenticated(self): return self.db_user() is not None @property def is_active(self): return self.db_user().verified def get_id(self): return unicode(self._uuid) get_app_url = partial(get_app_url, app.config)