quay/util/config/validator.py

import logging
import subprocess
import time

from StringIO import StringIO
from hashlib import sha1

import ldap
import peewee
import redis

from flask import Flask
from flask_mail import Mail, Message

from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
from auth.auth_context import get_authenticated_user
from bitbucket import BitBucket
from boot import setup_jwt_proxy
from data.database import validate_database_url
from data.users import LDAP_CERT_FILENAME
from data.users.externaljwt import ExternalJWTAuthN
from data.users.externalldap import LDAPConnection, LDAPUsers
from data.users.keystone import get_keystone_users
from storage import get_storage_driver
from oauth.services.github import GithubOAuthService
from oauth.services.google import GoogleOAuthService
from oauth.services.gitlab import GitLabOAuthService
from util.secscan.api import SecurityScannerAPI
from util.registry.torrent import torrent_jwt
from util.security.signing import SIGNING_ENGINES
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException

logger = logging.getLogger(__name__)

class ConfigValidationException(Exception):
  """ Exception raised when the configuration fails to validate for a known reason. """
  pass


# Note: Only add files required for HTTPS to the SSL_FILESNAMES list.
SSL_FILENAMES = ['ssl.cert', 'ssl.key']
DB_SSL_FILENAMES = ['database.pem']
JWT_FILENAMES = ['jwt-authn.cert']
ACI_CERT_FILENAMES = ['signing-public.gpg', 'signing-private.gpg']
LDAP_FILENAMES = [LDAP_CERT_FILENAME]
CONFIG_FILENAMES = (SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES + ACI_CERT_FILENAMES +
                    LDAP_FILENAMES)
EXTRA_CA_DIRECTORY = 'extra_ca_certs'

def get_storage_providers(config):
  storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})

  drivers = {}

  try:
    for name, parameters in storage_config.items():
      drivers[name] = (parameters[0], get_storage_driver(None, None, None, parameters))
  except TypeError:
    logger.exception('Missing required storage configuration provider')
    raise ConfigValidationException('Missing required parameter(s) for storage %s' % name)

  return drivers

def validate_service_for_config(service, config, password=None):
  """ Attempts to validate the configuration for the given service. """
  if not service in VALIDATORS:
    return {
      'status': False
    }

  try:
    VALIDATORS[service](config, get_authenticated_user(), password)
    return {
      'status': True
    }
  except Exception as ex:
    logger.exception('Validation exception')
    return {
      'status': False,
      'reason': str(ex)
    }


def _validate_database(config, user_obj, _):
  """ Validates connecting to the database. """
  try:
    validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
  except peewee.OperationalError as ex:
    if ex.args and len(ex.args) > 1:
      raise ConfigValidationException(ex.args[1])
    else:
      raise ex


def _validate_redis(config, user_obj, _):
  """ Validates connecting to redis. """
  redis_config = config.get('BUILDLOGS_REDIS', {})
  if not 'host' in  redis_config:
    raise ConfigValidationException('Missing redis hostname')

  client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
  client.ping()


def _validate_registry_storage(config, user_obj, _):
  """ Validates registry storage. """
  replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)

  providers = get_storage_providers(config).items()
  if not providers:
    raise ConfigValidationException('Storage configuration required')

  for name, (storage_type, driver) in providers:
    try:
      if replication_enabled and storage_type == 'LocalStorage':
        raise ConfigValidationException('Locally mounted directory not supported ' +
                                        'with storage replication')

      # Run validation on the driver.
      driver.validate(app.config['HTTPCLIENT'])

      # Run setup on the driver if the read/write succeeded.
      driver.setup()
    except Exception as ex:
      raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, str(ex)))


def _validate_mailing(config, user_obj, _):
  """ Validates sending email. """
  test_app = Flask("mail-test-app")
  test_app.config.update(config)
  test_app.config.update({
    'MAIL_FAIL_SILENTLY': False,
    'TESTING': False
  })

  test_mail = Mail(test_app)
  test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
                     sender=config.get('MAIL_DEFAULT_SENDER'))
  test_msg.add_recipient(user_obj.email)
  test_mail.send(test_msg)


def _validate_gitlab(config, user_obj, _):
  """ Validates the OAuth credentials and API endpoint for a GitLab service. """
  github_config = config.get('GITLAB_TRIGGER_CONFIG')
  if not github_config:
    raise ConfigValidationException('Missing GitLab client id and client secret')

  endpoint = github_config.get('GITLAB_ENDPOINT')
  if not endpoint:
    raise ConfigValidationException('Missing GitLab Endpoint')

  if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
    raise ConfigValidationException('GitLab Endpoint must start with http:// or https://')

  if not github_config.get('CLIENT_ID'):
    raise ConfigValidationException('Missing Client ID')

  if not github_config.get('CLIENT_SECRET'):
    raise ConfigValidationException('Missing Client Secret')

  client = app.config['HTTPCLIENT']
  oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise ConfigValidationException('Invalid client id or client secret')


def _validate_github(config_key):
  return lambda config, user_obj, _: _validate_github_with_key(config_key, config)


def _validate_github_with_key(config_key, config):
  """ Validates the OAuth credentials and API endpoint for a Github service. """
  github_config = config.get(config_key)
  if not github_config:
    raise ConfigValidationException('Missing GitHub client id and client secret')

  endpoint = github_config.get('GITHUB_ENDPOINT')
  if not endpoint:
    raise ConfigValidationException('Missing GitHub Endpoint')

  if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
    raise ConfigValidationException('Github Endpoint must start with http:// or https://')

  if not github_config.get('CLIENT_ID'):
    raise ConfigValidationException('Missing Client ID')

  if not github_config.get('CLIENT_SECRET'):
    raise ConfigValidationException('Missing Client Secret')

  if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
    raise ConfigValidationException('Organization restriction must have at least one allowed ' +
                                    'organization')

  client = app.config['HTTPCLIENT']
  oauth = GithubOAuthService(config, config_key)
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise ConfigValidationException('Invalid client id or client secret')

  if github_config.get('ALLOWED_ORGANIZATIONS'):
    for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
      if not oauth.validate_organization(org_id, client):
        raise ConfigValidationException('Invalid organization: %s' % org_id)


def _validate_bitbucket(config, user_obj, _):
  """ Validates the config for BitBucket. """
  trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
  if not trigger_config:
    raise ConfigValidationException('Missing client ID and client secret')

  if not trigger_config.get('CONSUMER_KEY'):
    raise ConfigValidationException('Missing Consumer Key')

  if not trigger_config.get('CONSUMER_SECRET'):
    raise ConfigValidationException('Missing Consumer Secret')

  key = trigger_config['CONSUMER_KEY']
  secret = trigger_config['CONSUMER_SECRET']
  callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())

  bitbucket_client = BitBucket(key, secret, callback_url)
  (result, _, _) = bitbucket_client.get_authorization_url()
  if not result:
    raise ConfigValidationException('Invalid consumer key or secret')


def _validate_google_login(config, user_obj, _):
  """ Validates the Google Login client ID and secret. """
  google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
  if not google_login_config:
    raise ConfigValidationException('Missing client ID and client secret')

  if not google_login_config.get('CLIENT_ID'):
    raise ConfigValidationException('Missing Client ID')

  if not google_login_config.get('CLIENT_SECRET'):
    raise ConfigValidationException('Missing Client Secret')

  client = app.config['HTTPCLIENT']
  oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise ConfigValidationException('Invalid client id or client secret')


def _validate_ssl(config, user_obj, _):
  """ Validates the SSL configuration (if enabled). """

  # Skip if non-SSL.
  if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
    return

  # Skip if externally terminated.
  if config.get('EXTERNAL_TLS_TERMINATION', False) is True:
    return

  # Verify that we have all the required SSL files.
  for filename in SSL_FILENAMES:
    if not config_provider.volume_file_exists(filename):
      raise ConfigValidationException('Missing required SSL file: %s' % filename)

  # Read the contents of the SSL certificate.
  with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
    cert_contents = f.read()

  # Validate the certificate.
  try:
    certificate = load_certificate(cert_contents)
  except CertInvalidException as cie:
    raise ConfigValidationException('Could not load SSL certificate: %s' % cie.message)

  # Verify the certificate has not expired.
  if certificate.expired:
    raise ConfigValidationException('The specified SSL certificate has expired.')

  # Verify the hostname matches the name in the certificate.
  if not certificate.matches_name(config['SERVER_HOSTNAME']):
    msg = ('Supported names "%s" in SSL cert do not match server hostname "%s"' %
           (', '.join(list(certificate.names)), config['SERVER_HOSTNAME']))
    raise ConfigValidationException(msg)

  # Verify the private key against the certificate.
  private_key_path = None
  with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
    private_key_path = f.name

  if not private_key_path:
    # Only in testing.
    return

  try:
    certificate.validate_private_key(private_key_path)
  except KeyInvalidException as kie:
    raise ConfigValidationException('SSL private key failed to validate: %s' % kie.message)


def _validate_ldap(config, user_obj, password):
  """ Validates the LDAP connection. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
    return

  # If there is a custom LDAP certificate, then reinstall the certificates for the container.
  if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
    subprocess.check_call(['/conf/init/certs_install.sh'])

  # Note: raises ldap.INVALID_CREDENTIALS on failure
  admin_dn = config.get('LDAP_ADMIN_DN')
  admin_passwd = config.get('LDAP_ADMIN_PASSWD')

  if not admin_dn:
    raise ConfigValidationException('Missing Admin DN for LDAP configuration')

  if not admin_passwd:
    raise ConfigValidationException('Missing Admin Password for LDAP configuration')

  ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
  if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
    raise ConfigValidationException('LDAP URI must start with ldap:// or ldaps://')

  allow_tls_fallback = config.get('LDAP_ALLOW_INSECURE_FALLBACK', False)

  try:
    with LDAPConnection(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback):
      pass
  except ldap.LDAPError as ex:
    values = ex.args[0] if ex.args else {}
    if not isinstance(values, dict):
      raise ConfigValidationException(str(ex.args))

    raise ConfigValidationException(values.get('desc', 'Unknown error'))

  # Verify that the superuser exists. If not, raise an exception.
  base_dn = config.get('LDAP_BASE_DN')
  user_rdn = config.get('LDAP_USER_RDN', [])
  uid_attr = config.get('LDAP_UID_ATTR', 'uid')
  email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')
  requires_email = config.get('FEATURE_MAILING', True)

  users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
                    allow_tls_fallback, requires_email=requires_email)

  username = user_obj.username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    msg = ('Verification of superuser %s failed: %s. \n\nThe user either does not exist ' +
           'in the remote authentication system ' +
           'OR LDAP auth is misconfigured.') % (username, err_msg)
    raise ConfigValidationException(msg)


def _validate_jwt(config, user_obj, password):
  """ Validates the JWT authentication system. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
    return

  verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
  query_endpoint = config.get('JWT_QUERY_ENDPOINT', None)
  getuser_endpoint = config.get('JWT_GETUSER_ENDPOINT', None)

  issuer = config.get('JWT_AUTH_ISSUER')

  if not verify_endpoint:
    raise ConfigValidationException('Missing JWT Verification endpoint')

  if not issuer:
    raise ConfigValidationException('Missing JWT Issuer ID')

  # Try to instatiate the JWT authentication mechanism. This will raise an exception if
  # the key cannot be found.
  users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
                           OVERRIDE_CONFIG_DIRECTORY,
                           app.config['HTTPCLIENT'],
                           app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
                           requires_email=config.get('FEATURE_MAILING', True))

  # Verify that the superuser exists. If not, raise an exception.
  username = user_obj.username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    msg = ('Verification of superuser %s failed: %s. \n\nThe user either does not ' +
           'exist in the remote authentication system ' +
           'OR JWT auth is misconfigured') % (username, err_msg)
    raise ConfigValidationException(msg)

  # If the query endpoint exists, ensure we can query to find the current user and that we can
  # look up users directly.
  if query_endpoint:
    (results, err_msg) = users.query_users(username)
    if not results:
      err_msg = err_msg or ('Could not find users matching query: %s' % username)
      raise ConfigValidationException('Query endpoint is misconfigured or not returning ' +
                                      'proper users: %s' % err_msg)

    # Make sure the get user endpoint is also configured.
    if not getuser_endpoint:
      raise ConfigValidationException('The lookup user endpoint must be configured if the ' +
                                      'query endpoint is set')

    (result, err_msg) = users.get_user(username)
    if not result:
      err_msg = err_msg or ('Could not find user %s' % username)
      raise ConfigValidationException('Lookup endpoint is misconfigured or not returning ' +
                                      'properly: %s' % err_msg)


def _validate_keystone(config, user_obj, password):
  """ Validates the Keystone authentication system. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
    return

  auth_url = config.get('KEYSTONE_AUTH_URL')
  auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
  admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
  admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
  admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')

  if not auth_url:
    raise ConfigValidationException('Missing authentication URL')

  if not admin_username:
    raise ConfigValidationException('Missing admin username')

  if not admin_password:
    raise ConfigValidationException('Missing admin password')

  if not admin_tenant:
    raise ConfigValidationException('Missing admin tenant')

  requires_email = config.get('FEATURE_MAILING', True)
  users = get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
                             requires_email)

  # Verify that the superuser exists. If not, raise an exception.
  username = user_obj.username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    msg = ('Verification of superuser %s failed: %s \n\nThe user either does not ' +
           'exist in the remote authentication system ' +
           'OR Keystone auth is misconfigured.') % (username, err_msg)
    raise ConfigValidationException(msg)


def _validate_signer(config, user_obj, _):
  """ Validates the GPG public+private key pair used for signing converted ACIs. """
  if config.get('SIGNING_ENGINE') is None:
    return

  if config['SIGNING_ENGINE'] not in SIGNING_ENGINES:
    raise ConfigValidationException('Unknown signing engine: %s' % config['SIGNING_ENGINE'])

  engine = SIGNING_ENGINES[config['SIGNING_ENGINE']](config, config_provider)
  engine.detached_sign(StringIO('test string'))


def _validate_security_scanner(config, user_obj, _):
  """ Validates the configuration for talking to a Quay Security Scanner. """
  client = app.config['HTTPCLIENT']
  api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)

  if not config.get('TESTING', False):
    # Generate a temporary Quay key to use for signing the outgoing requests.
    setup_jwt_proxy()

  # We have to wait for JWT proxy to restart with the newly generated key.
  max_tries = 5
  response = None
  while max_tries > 0:
    response = api.ping()
    if response.status_code == 200:
      return

    time.sleep(1)
    max_tries = max_tries - 1

  message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
  raise ConfigValidationException('Could not ping security scanner: %s' % message)


def _validate_bittorrent(config, user_obj, _):
  """ Validates the configuration for using BitTorrent for downloads. """
  announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
  if not announce_url:
    raise ConfigValidationException('Missing announce URL')

  # Ensure that the tracker is reachable and accepts requests signed with a registry key.
  client = app.config['HTTPCLIENT']

  params = {
    'info_hash': sha1('somedata').digest(),
    'peer_id': '-QUAY00-6wfG2wk6wWLc',
    'uploaded': 0,
    'downloaded': 0,
    'left': 0,
    'numwant': 0,
    'port': 80,
  }

  encoded_jwt = torrent_jwt(params)
  params['jwt'] = encoded_jwt

  resp = client.get(announce_url, timeout=5, params=params)
  logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)

  if resp.status_code == 404:
    raise ConfigValidationException('Announce path not found; did you forget `/announce`?')

  if resp.status_code == 500:
    raise ConfigValidationException('Did not get expected response from Tracker; ' +
                                    'please check your settings')

  if resp.status_code == 200:
    if 'invalid jwt' in resp.text:
      raise ConfigValidationException('Could not authorize to Tracker; is your Tracker ' +
                                      'properly configured?')

    if 'failure reason' in resp.text:
      raise ConfigValidationException('Could not validate signed announce request: ' + resp.text)


VALIDATORS = {
  'database': _validate_database,
  'redis': _validate_redis,
  'registry-storage': _validate_registry_storage,
  'mail': _validate_mailing,
  'github-login': _validate_github('GITHUB_LOGIN_CONFIG'),
  'github-trigger': _validate_github('GITHUB_TRIGGER_CONFIG'),
  'gitlab-trigger': _validate_gitlab,
  'bitbucket-trigger': _validate_bitbucket,
  'google-login': _validate_google_login,
  'ssl': _validate_ssl,
  'ldap': _validate_ldap,
  'jwt': _validate_jwt,
  'keystone': _validate_keystone,
  'signer': _validate_signer,
  'security-scanner': _validate_security_scanner,
  'bittorrent': _validate_bittorrent,
}