quay/app.py

import logging
import os
import json

from functools import partial
from flask import Flask, request, Request, _request_ctx_stack
from flask.ext.principal import Principal
from flask.ext.login import LoginManager, UserMixin
from flask.ext.mail import Mail
from werkzeug.routing import BaseConverter
from jwkest.jwk import RSAKey
from Crypto.PublicKey import RSA

import features

from avatars.avatars import Avatar
from storage import Storage
from data import model
from data import database
from data.userfiles import Userfiles
from data.users import UserAuthentication
from data.billing import Billing
from data.buildlogs import BuildLogs
from data.archivedlogs import LogArchive
from data.userevent import UserEventsBuilderModule
from data.queue import WorkQueue, BuildMetricQueueReporter
from util import get_app_url
from util.saas.analytics import Analytics
from util.saas.exceptionlog import Sentry
from util.names import urn_generator
from util.config.oauth import (GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig,
                               DexOAuthConfig)

from util.security.signing import Signer
from util.security.instancekeys import InstanceKeys
from util.saas.cloudwatch import start_cloudwatch_sender
from util.config.provider import get_config_provider
from util.config.configutil import generate_secret_key
from util.config.superusermanager import SuperUserManager
from util.secscan.api import SecurityScannerAPI
from util.metrics.metricqueue import MetricQueue
from util.metrics.prometheus import PrometheusPlugin

OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/'
OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml'
OVERRIDE_CONFIG_PY_FILENAME = 'conf/stack/config.py'

OVERRIDE_CONFIG_KEY = 'QUAY_OVERRIDE_CONFIG'

DOCKER_V2_SIGNINGKEY_FILENAME = 'docker_v2.pem'

app = Flask(__name__)
logger = logging.getLogger(__name__)

# Instantiate the configuration.
is_testing = 'TEST' in os.environ
is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ
config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py',
                                      testing=is_testing, kubernetes=is_kubernetes)

if is_testing:
  from test.testconfig import TestConfig
  logger.debug('Loading test config.')
  app.config.from_object(TestConfig())
else:
  from config import DefaultConfig
  logger.debug('Loading default config.')
  app.config.from_object(DefaultConfig())
  app.teardown_request(database.close_db_filter)

# Load the override config via the provider.
config_provider.update_app_config(app.config)

# Update any configuration found in the override environment variable.
environ_config = json.loads(os.environ.get(OVERRIDE_CONFIG_KEY, '{}'))
app.config.update(environ_config)

# Allow user to define a custom storage preference for the local instance.
_distributed_storage_preference = os.environ.get('QUAY_DISTRIBUTED_STORAGE_PREFERENCE', '').split()
if _distributed_storage_preference:
  app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = _distributed_storage_preference

# Generate a secret key if none was specified.
if app.config['SECRET_KEY'] is None:
  logger.debug('Generating in-memory secret key')
  app.config['SECRET_KEY'] = generate_secret_key()

# If the "preferred" scheme is https, then http is not allowed. Therefore, ensure we have a secure
# session cookie.
if app.config['PREFERRED_URL_SCHEME'] == 'https':
  app.config['SESSION_COOKIE_SECURE'] = True

# Load features from config.
features.import_features(app.config)


class RequestWithId(Request):
  request_gen = staticmethod(urn_generator(['request']))

  def __init__(self, *args, **kwargs):
    super(RequestWithId, self).__init__(*args, **kwargs)
    self.request_id = self.request_gen()


@app.before_request
def _request_start():
  logger.debug('Starting request: %s', request.path)


@app.after_request
def _request_end(r):
  logger.debug('Ending request: %s', request.path)
  return r


class InjectingFilter(logging.Filter):
  def filter(self, record):
    if _request_ctx_stack.top is not None:
      record.msg = '[%s] %s' % (request.request_id, record.msg)
    return True

root_logger = logging.getLogger()

# Add the request id filter to all handlers of the root logger
for handler in root_logger.handlers:
  handler.addFilter(InjectingFilter())

app.request_class = RequestWithId

# Register custom converters.
class RegexConverter(BaseConverter):
  """ Converter for handling custom regular expression patterns in paths. """
  def __init__(self, url_map, regex_value):
    super(RegexConverter, self).__init__(url_map)
    self.regex = regex_value


class RepositoryPathConverter(BaseConverter):
  """ Converter for handling repository paths. Handles both library and non-library paths (if
      configured).
  """
  def __init__(self, url_map):
    super(RepositoryPathConverter, self).__init__(url_map)
    self.weight = 200

    if features.LIBRARY_SUPPORT:
      # Allow names without namespaces.
      self.regex = r'[^/]+(/[^/]+)?'
    else:
      self.regex = r'([^/]+/[^/]+)'


class APIRepositoryPathConverter(BaseConverter):
  """ Converter for handling repository paths. Does not handle library paths.
  """
  def __init__(self, url_map):
    super(APIRepositoryPathConverter, self).__init__(url_map)
    self.weight = 200
    self.regex = r'([^/]+/[^/]+)'


app.url_map.converters['regex'] = RegexConverter
app.url_map.converters['repopath'] = RepositoryPathConverter
app.url_map.converters['apirepopath'] = APIRepositoryPathConverter

Principal(app, use_sessions=False)

avatar = Avatar(app)
login_manager = LoginManager(app)
mail = Mail(app)
prometheus = PrometheusPlugin(app)
metric_queue = MetricQueue(prometheus)
storage = Storage(app, metric_queue)
userfiles = Userfiles(app, storage)
log_archive = LogArchive(app, storage)
analytics = Analytics(app)
billing = Billing(app)
sentry = Sentry(app)
build_logs = BuildLogs(app)
authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECTORY)
userevents = UserEventsBuilderModule(app)
superusers = SuperUserManager(app)
signer = Signer(app, config_provider)
instance_keys = InstanceKeys(app)

start_cloudwatch_sender(metric_queue, app)

tf = app.config['DB_TRANSACTION_FACTORY']

github_login = GithubOAuthConfig(app.config, 'GITHUB_LOGIN_CONFIG')
github_trigger = GithubOAuthConfig(app.config, 'GITHUB_TRIGGER_CONFIG')
gitlab_trigger = GitLabOAuthConfig(app.config, 'GITLAB_TRIGGER_CONFIG')
google_login = GoogleOAuthConfig(app.config, 'GOOGLE_LOGIN_CONFIG')
dex_login = DexOAuthConfig(app.config, 'DEX_LOGIN_CONFIG')

oauth_apps = [github_login, github_trigger, gitlab_trigger, google_login, dex_login]

image_replication_queue = WorkQueue(app.config['REPLICATION_QUEUE_NAME'], tf)
dockerfile_build_queue = WorkQueue(app.config['DOCKERFILE_BUILD_QUEUE_NAME'], tf,
                                   reporter=BuildMetricQueueReporter(metric_queue))
notification_queue = WorkQueue(app.config['NOTIFICATION_QUEUE_NAME'], tf)
secscan_notification_queue = WorkQueue(app.config['SECSCAN_NOTIFICATION_QUEUE_NAME'], tf)
secscan_api = SecurityScannerAPI(app, app.config, storage)

# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.
_v2_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, DOCKER_V2_SIGNINGKEY_FILENAME)
if os.path.exists(_v2_key_path):
  docker_v2_signing_key = RSAKey().load(_v2_key_path)
else:
  docker_v2_signing_key = RSAKey(key=RSA.generate(2048))


database.configure(app.config)
model.config.app_config = app.config
model.config.store = storage

@login_manager.user_loader
def load_user(user_uuid):
  logger.debug('User loader loading deferred user with uuid: %s' % user_uuid)
  return LoginWrappedDBUser(user_uuid)

class LoginWrappedDBUser(UserMixin):
  def __init__(self, user_uuid, db_user=None):
    self._uuid = user_uuid
    self._db_user = db_user

  def db_user(self):
    if not self._db_user:
      self._db_user = model.user.get_user_by_uuid(self._uuid)
    return self._db_user

  @property
  def is_authenticated(self):
    return self.db_user() is not None

  @property
  def is_active(self):
    return self.db_user().verified

  def get_id(self):
    return unicode(self._uuid)

get_app_url = partial(get_app_url, app.config)