quay/util/config/validator.py

import logging
import subprocess
import time

from StringIO import StringIO
from fnmatch import fnmatch
from hashlib import sha1

import OpenSSL
import ldap
import peewee
import redis

from flask import Flask
from flask_mail import Mail, Message

from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
from auth.auth_context import get_authenticated_user
from bitbucket import BitBucket
from boot import setup_jwt_proxy
from data.database import validate_database_url
from data.users import LDAP_CERT_FILENAME
from data.users.externaljwt import ExternalJWTAuthN
from data.users.externalldap import LDAPConnection, LDAPUsers
from data.users.keystone import get_keystone_users
from storage import get_storage_driver
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
from util.secscan.api import SecurityScannerAPI
from util.registry.torrent import torrent_jwt
from util.security.signing import SIGNING_ENGINES

logger = logging.getLogger(__name__)

class ConfigValidationException(Exception):
  """ Exception raised when the configuration fails to validate for a known reason. """
  pass


# Note: Only add files required for HTTPS to the SSL_FILESNAMES list.
SSL_FILENAMES = ['ssl.cert', 'ssl.key']
DB_SSL_FILENAMES = ['database.pem']
JWT_FILENAMES = ['jwt-authn.cert']
ACI_CERT_FILENAMES = ['signing-public.gpg', 'signing-private.gpg']
LDAP_FILENAMES = [LDAP_CERT_FILENAME]
CONFIG_FILENAMES = (SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES + ACI_CERT_FILENAMES +
                    LDAP_FILENAMES)


def get_storage_providers(config):
  storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})

  drivers = {}

  try:
    for name, parameters in storage_config.items():
      drivers[name] = (parameters[0], get_storage_driver(None, None, None, parameters))
  except TypeError:
    logger.exception('Missing required storage configuration provider')
    raise ConfigValidationException('Missing required parameter(s) for storage %s' % name)

  return drivers

def validate_service_for_config(service, config, password=None):
  """ Attempts to validate the configuration for the given service. """
  if not service in VALIDATORS:
    return {
      'status': False
    }

  try:
    VALIDATORS[service](config, get_authenticated_user(), password)
    return {
      'status': True
    }
  except Exception as ex:
    logger.exception('Validation exception')
    return {
      'status': False,
      'reason': str(ex)
    }


def _validate_database(config, user_obj, _):
  """ Validates connecting to the database. """
  try:
    validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
  except peewee.OperationalError as ex:
    if ex.args and len(ex.args) > 1:
      raise ConfigValidationException(ex.args[1])
    else:
      raise ex


def _validate_redis(config, user_obj, _):
  """ Validates connecting to redis. """
  redis_config = config.get('BUILDLOGS_REDIS', {})
  if not 'host' in  redis_config:
    raise ConfigValidationException('Missing redis hostname')

  client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
  client.ping()


def _validate_registry_storage(config, user_obj, _):
  """ Validates registry storage. """
  replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)

  providers = get_storage_providers(config).items()
  if not providers:
    raise ConfigValidationException('Storage configuration required')

  for name, (storage_type, driver) in providers:
    try:
      if replication_enabled and storage_type == 'LocalStorage':
        raise ConfigValidationException('Locally mounted directory not supported ' +
                                        'with storage replication')

      # Run validation on the driver.
      driver.validate(app.config['HTTPCLIENT'])

      # Run setup on the driver if the read/write succeeded.
      driver.setup()
    except Exception as ex:
      raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, str(ex)))


def _validate_mailing(config, user_obj, _):
  """ Validates sending email. """
  test_app = Flask("mail-test-app")
  test_app.config.update(config)
  test_app.config.update({
    'MAIL_FAIL_SILENTLY': False,
    'TESTING': False
  })

  test_mail = Mail(test_app)
  test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
                     sender=config.get('MAIL_DEFAULT_SENDER'))
  test_msg.add_recipient(user_obj.email)
  test_mail.send(test_msg)


def _validate_gitlab(config, user_obj, _):
  """ Validates the OAuth credentials and API endpoint for a GitLab service. """
  github_config = config.get('GITLAB_TRIGGER_CONFIG')
  if not github_config:
    raise ConfigValidationException('Missing GitLab client id and client secret')

  endpoint = github_config.get('GITLAB_ENDPOINT')
  if not endpoint:
    raise ConfigValidationException('Missing GitLab Endpoint')

  if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
    raise ConfigValidationException('GitLab Endpoint must start with http:// or https://')

  if not github_config.get('CLIENT_ID'):
    raise ConfigValidationException('Missing Client ID')

  if not github_config.get('CLIENT_SECRET'):
    raise ConfigValidationException('Missing Client Secret')

  client = app.config['HTTPCLIENT']
  oauth = GitLabOAuthConfig(config, 'GITLAB_TRIGGER_CONFIG')
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise ConfigValidationException('Invalid client id or client secret')


def _validate_github(config_key):
  return lambda config, user_obj, _: _validate_github_with_key(config_key, config)


def _validate_github_with_key(config_key, config):
  """ Validates the OAuth credentials and API endpoint for a Github service. """
  github_config = config.get(config_key)
  if not github_config:
    raise ConfigValidationException('Missing GitHub client id and client secret')

  endpoint = github_config.get('GITHUB_ENDPOINT')
  if not endpoint:
    raise ConfigValidationException('Missing GitHub Endpoint')

  if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
    raise ConfigValidationException('Github Endpoint must start with http:// or https://')

  if not github_config.get('CLIENT_ID'):
    raise ConfigValidationException('Missing Client ID')

  if not github_config.get('CLIENT_SECRET'):
    raise ConfigValidationException('Missing Client Secret')

  if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
    raise ConfigValidationException('Organization restriction must have at least one allowed ' +
                                    'organization')

  client = app.config['HTTPCLIENT']
  oauth = GithubOAuthConfig(config, config_key)
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise ConfigValidationException('Invalid client id or client secret')

  if github_config.get('ALLOWED_ORGANIZATIONS'):
    for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
      if not oauth.validate_organization(org_id, client):
        raise ConfigValidationException('Invalid organization: %s' % org_id)


def _validate_bitbucket(config, user_obj, _):
  """ Validates the config for BitBucket. """
  trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
  if not trigger_config:
    raise ConfigValidationException('Missing client ID and client secret')

  if not trigger_config.get('CONSUMER_KEY'):
    raise ConfigValidationException('Missing Consumer Key')

  if not trigger_config.get('CONSUMER_SECRET'):
    raise ConfigValidationException('Missing Consumer Secret')

  key = trigger_config['CONSUMER_KEY']
  secret = trigger_config['CONSUMER_SECRET']
  callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())

  bitbucket_client = BitBucket(key, secret, callback_url)
  (result, _, _) = bitbucket_client.get_authorization_url()
  if not result:
    raise ConfigValidationException('Invalid consumer key or secret')


def _validate_google_login(config, user_obj, _):
  """ Validates the Google Login client ID and secret. """
  google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
  if not google_login_config:
    raise ConfigValidationException('Missing client ID and client secret')

  if not google_login_config.get('CLIENT_ID'):
    raise ConfigValidationException('Missing Client ID')

  if not google_login_config.get('CLIENT_SECRET'):
    raise ConfigValidationException('Missing Client Secret')

  client = app.config['HTTPCLIENT']
  oauth = GoogleOAuthConfig(config, 'GOOGLE_LOGIN_CONFIG')
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise ConfigValidationException('Invalid client id or client secret')


def _validate_ssl(config, user_obj, _):
  """ Validates the SSL configuration (if enabled). """

  # Skip if non-SSL.
  if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
    return

  # Skip if externally terminated.
  if config.get('EXTERNAL_TLS_TERMINATION', False) is True:
    return

  for filename in SSL_FILENAMES:
    if not config_provider.volume_file_exists(filename):
      raise ConfigValidationException('Missing required SSL file: %s' % filename)

  with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
    cert_contents = f.read()

  # Validate the certificate.
  try:
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_contents)
  except:
    raise ConfigValidationException('Could not parse certificate file. Is it a valid PEM certificate?')

  if cert.has_expired():
    raise ConfigValidationException('The specified SSL certificate has expired.')

  private_key_path = None
  with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
    private_key_path = f.name

  if not private_key_path:
    # Only in testing.
    return

  # Validate the private key with the certificate.
  context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
  context.use_certificate(cert)

  try:
    context.use_privatekey_file(private_key_path)
  except:
    raise ConfigValidationException('Could not parse key file. Is it a valid PEM private key?')

  try:
    context.check_privatekey()
  except OpenSSL.SSL.Error as e:
    raise ConfigValidationException('SSL key failed to validate: %s' % str(e))

  # Verify the hostname matches the name in the certificate.
  common_name = cert.get_subject().commonName
  if common_name is None:
    raise ConfigValidationException('Missing CommonName (CN) from SSL certificate')

  # Build the list of allowed host patterns.
  hosts = set([common_name])

  # Find the DNS extension, if any.
  for i in range(0, cert.get_extension_count()):
    ext = cert.get_extension(i)
    if ext.get_short_name() == 'subjectAltName':
      value = str(ext)
      hosts.update([host.strip()[4:] for host in value.split(',')])

  # Check each host.
  for host in hosts:
    if fnmatch(config['SERVER_HOSTNAME'], host):
      return

  msg = ('Supported names "%s" in SSL cert do not match server hostname "%s"' %
         (', '.join(list(hosts)), config['SERVER_HOSTNAME']))
  raise ConfigValidationException(msg)



def _validate_ldap(config, user_obj, password):
  """ Validates the LDAP connection. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
    return

  # If there is a custom LDAP certificate, then reinstall the certificates for the container.
  if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
    subprocess.check_call(['/conf/init/certs_install.sh'])

  # Note: raises ldap.INVALID_CREDENTIALS on failure
  admin_dn = config.get('LDAP_ADMIN_DN')
  admin_passwd = config.get('LDAP_ADMIN_PASSWD')

  if not admin_dn:
    raise ConfigValidationException('Missing Admin DN for LDAP configuration')

  if not admin_passwd:
    raise ConfigValidationException('Missing Admin Password for LDAP configuration')

  ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
  if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
    raise ConfigValidationException('LDAP URI must start with ldap:// or ldaps://')

  allow_tls_fallback = config.get('LDAP_ALLOW_INSECURE_FALLBACK', False)

  try:
    with LDAPConnection(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback):
      pass
  except ldap.LDAPError as ex:
    values = ex.args[0] if ex.args else {}
    if not isinstance(values, dict):
      raise ConfigValidationException(str(ex.args))

    raise ConfigValidationException(values.get('desc', 'Unknown error'))

  # Verify that the superuser exists. If not, raise an exception.
  base_dn = config.get('LDAP_BASE_DN')
  user_rdn = config.get('LDAP_USER_RDN', [])
  uid_attr = config.get('LDAP_UID_ATTR', 'uid')
  email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')
  requires_email = config.get('FEATURE_MAILING', True)

  users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
                    allow_tls_fallback, requires_email=requires_email)

  username = user_obj.username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    msg = ('Verification of superuser %s failed: %s. \n\nThe user either does not exist ' +
           'in the remote authentication system ' +
           'OR LDAP auth is misconfigured.') % (username, err_msg)
    raise ConfigValidationException(msg)


def _validate_jwt(config, user_obj, password):
  """ Validates the JWT authentication system. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
    return

  verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
  query_endpoint = config.get('JWT_QUERY_ENDPOINT', None)
  getuser_endpoint = config.get('JWT_GETUSER_ENDPOINT', None)

  issuer = config.get('JWT_AUTH_ISSUER')

  if not verify_endpoint:
    raise ConfigValidationException('Missing JWT Verification endpoint')

  if not issuer:
    raise ConfigValidationException('Missing JWT Issuer ID')

  # Try to instatiate the JWT authentication mechanism. This will raise an exception if
  # the key cannot be found.
  users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
                           OVERRIDE_CONFIG_DIRECTORY,
                           app.config['HTTPCLIENT'],
                           app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
                           requires_email=config.get('FEATURE_MAILING', True))

  # Verify that the superuser exists. If not, raise an exception.
  username = user_obj.username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    msg = ('Verification of superuser %s failed: %s. \n\nThe user either does not ' +
           'exist in the remote authentication system ' +
           'OR JWT auth is misconfigured') % (username, err_msg)
    raise ConfigValidationException(msg)

  # If the query endpoint exists, ensure we can query to find the current user and that we can
  # look up users directly.
  if query_endpoint:
    (results, err_msg) = users.query_users(username)
    if not results:
      err_msg = err_msg or ('Could not find users matching query: %s' % username)
      raise ConfigValidationException('Query endpoint is misconfigured or not returning ' +
                                      'proper users: %s' % err_msg)

    # Make sure the get user endpoint is also configured.
    if not getuser_endpoint:
      raise ConfigValidationException('The lookup user endpoint must be configured if the ' +
                                      'query endpoint is set')

    (result, err_msg) = users.get_user(username)
    if not result:
      err_msg = err_msg or ('Could not find user %s' % username)
      raise ConfigValidationException('Lookup endpoint is misconfigured or not returning ' +
                                      'properly: %s' % err_msg)


def _validate_keystone(config, user_obj, password):
  """ Validates the Keystone authentication system. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
    return

  auth_url = config.get('KEYSTONE_AUTH_URL')
  auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
  admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
  admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
  admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')

  if not auth_url:
    raise ConfigValidationException('Missing authentication URL')

  if not admin_username:
    raise ConfigValidationException('Missing admin username')

  if not admin_password:
    raise ConfigValidationException('Missing admin password')

  if not admin_tenant:
    raise ConfigValidationException('Missing admin tenant')

  requires_email = config.get('FEATURE_MAILING', True)
  users = get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
                             requires_email)

  # Verify that the superuser exists. If not, raise an exception.
  username = user_obj.username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    msg = ('Verification of superuser %s failed: %s \n\nThe user either does not ' +
           'exist in the remote authentication system ' +
           'OR Keystone auth is misconfigured.') % (username, err_msg)
    raise ConfigValidationException(msg)


def _validate_signer(config, user_obj, _):
  """ Validates the GPG public+private key pair used for signing converted ACIs. """
  if config.get('SIGNING_ENGINE') is None:
    return

  if config['SIGNING_ENGINE'] not in SIGNING_ENGINES:
    raise ConfigValidationException('Unknown signing engine: %s' % config['SIGNING_ENGINE'])

  engine = SIGNING_ENGINES[config['SIGNING_ENGINE']](config, config_provider)
  engine.detached_sign(StringIO('test string'))


def _validate_security_scanner(config, user_obj, _):
  """ Validates the configuration for talking to a Quay Security Scanner. """

  if not config.get('TESTING', False):
    # Generate a temporary Quay key to use for signing the outgoing requests.
    setup_jwt_proxy()

    # Wait a few seconds for the JWT proxy to startup.
    time.sleep(2)

  # Make a ping request to the security service.
  client = app.config['HTTPCLIENT']
  api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
  response = api.ping()
  if response.status_code != 200:
    message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
    raise ConfigValidationException('Could not ping security scanner: %s' % message)


def _validate_bittorrent(config, user_obj, _):
  """ Validates the configuration for using BitTorrent for downloads. """
  announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
  if not announce_url:
    raise ConfigValidationException('Missing announce URL')

  # Ensure that the tracker is reachable and accepts requests signed with a registry key.
  client = app.config['HTTPCLIENT']

  params = {
    'info_hash': sha1('somedata').digest(),
    'peer_id': '-QUAY00-6wfG2wk6wWLc',
    'uploaded': 0,
    'downloaded': 0,
    'left': 0,
    'numwant': 0,
    'port': 80,
  }

  encoded_jwt = torrent_jwt(params)
  params['jwt'] = encoded_jwt

  resp = client.get(announce_url, timeout=5, params=params)
  logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)

  if resp.status_code == 404:
    raise ConfigValidationException('Announce path not found; did you forget `/announce`?')

  if resp.status_code == 500:
    raise ConfigValidationException('Did not get expected response from Tracker; ' +
                                    'please check your settings')

  if resp.status_code == 200:
    if 'invalid jwt' in resp.text:
      raise ConfigValidationException('Could not authorize to Tracker; is your Tracker ' +
                                      'properly configured?')

    if 'failure reason' in resp.text:
      raise ConfigValidationException('Could not validate signed announce request: ' + resp.text)


VALIDATORS = {
  'database': _validate_database,
  'redis': _validate_redis,
  'registry-storage': _validate_registry_storage,
  'mail': _validate_mailing,
  'github-login': _validate_github('GITHUB_LOGIN_CONFIG'),
  'github-trigger': _validate_github('GITHUB_TRIGGER_CONFIG'),
  'gitlab-trigger': _validate_gitlab,
  'bitbucket-trigger': _validate_bitbucket,
  'google-login': _validate_google_login,
  'ssl': _validate_ssl,
  'ldap': _validate_ldap,
  'jwt': _validate_jwt,
  'keystone': _validate_keystone,
  'signer': _validate_signer,
  'security-scanner': _validate_security_scanner,
  'bittorrent': _validate_bittorrent,
}