quay/util/config/validator.py

import redis
import os
import json
import ldap
import peewee
import OpenSSL
import logging

from fnmatch import fnmatch
from data.users.keystone import KeystoneUsers
from data.users.externaljwt import ExternalJWTAuthN
from data.users.externalldap import LDAPConnection, LDAPUsers

from flask import Flask
from flask.ext.mail import Mail, Message
from data.database import validate_database_url, User
from storage import get_storage_driver
from auth.auth_context import get_authenticated_user
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
from bitbucket import BitBucket

from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY

logger = logging.getLogger(__name__)

# Note: Only add files required for HTTPS to the SSL_FILESNAMES list.
SSL_FILENAMES = ['ssl.cert', 'ssl.key']
DB_SSL_FILENAMES = ['database.pem']
JWT_FILENAMES = ['jwt-authn.cert']

CONFIG_FILENAMES = SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES

def get_storage_provider(config):
  parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}])
  try:
    return get_storage_driver(parameters)
  except TypeError:
    raise Exception('Missing required storage configuration parameter(s)')

def validate_service_for_config(service, config, password=None):
  """ Attempts to validate the configuration for the given service. """
  if not service in _VALIDATORS:
    return {
      'status': False
    }

  try:
    _VALIDATORS[service](config, password)
    return {
      'status': True
    }
  except Exception as ex:
    logger.exception('Validation exception')
    return {
      'status': False,
      'reason': str(ex)
    }


def _validate_database(config, _):
  """ Validates connecting to the database. """
  try:
    validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
  except peewee.OperationalError as ex:
    if ex.args and len(ex.args) > 1:
      raise Exception(ex.args[1])
    else:
      raise ex


def _validate_redis(config, _):
  """ Validates connecting to redis. """
  redis_config = config.get('BUILDLOGS_REDIS', {})
  if not 'host' in  redis_config:
    raise Exception('Missing redis hostname')

  client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
  client.ping()


def _validate_registry_storage(config, _):
  """ Validates registry storage. """
  driver = get_storage_provider(config)

  # Run custom validation on the driver.
  driver.validate(app.config['HTTPCLIENT'])

  # Put and remove a temporary file to make sure the normal storage paths work.
  driver.put_content('_verify', 'testing 123')
  driver.remove('_verify')

  # Run setup on the driver if the read/write succeeded.
  try:
    driver.setup()
  except Exception as ex:
    raise Exception('Could not prepare storage: %s' % str(ex))


def _validate_mailing(config, _):
  """ Validates sending email. """
  test_app = Flask("mail-test-app")
  test_app.config.update(config)
  test_app.config.update({
    'MAIL_FAIL_SILENTLY': False,
    'TESTING': False
  })

  test_mail = Mail(test_app)
  test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
                     sender=config.get('MAIL_DEFAULT_SENDER'))
  test_msg.add_recipient(get_authenticated_user().email)
  test_mail.send(test_msg)


def _validate_gitlab(config, _):
  """ Validates the OAuth credentials and API endpoint for a GitLab service. """
  github_config = config.get('GITLAB_TRIGGER_CONFIG')
  if not github_config:
    raise Exception('Missing GitLab client id and client secret')

  endpoint = github_config.get('GITLAB_ENDPOINT')
  if not endpoint:
    raise Exception('Missing GitLab Endpoint')

  if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
    raise Exception('GitLab Endpoint must start with http:// or https://')

  if not github_config.get('CLIENT_ID'):
    raise Exception('Missing Client ID')

  if not github_config.get('CLIENT_SECRET'):
    raise Exception('Missing Client Secret')

  client = app.config['HTTPCLIENT']
  oauth = GitLabOAuthConfig(config, 'GITLAB_TRIGGER_CONFIG')
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise Exception('Invalid client id or client secret')


def _validate_github(config_key):
  return lambda config, _: _validate_github_with_key(config_key, config)


def _validate_github_with_key(config_key, config):
  """ Validates the OAuth credentials and API endpoint for a Github service. """
  github_config = config.get(config_key)
  if not github_config:
    raise Exception('Missing GitHub client id and client secret')

  endpoint = github_config.get('GITHUB_ENDPOINT')
  if not endpoint:
    raise Exception('Missing GitHub Endpoint')

  if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
    raise Exception('Github Endpoint must start with http:// or https://')

  if not github_config.get('CLIENT_ID'):
    raise Exception('Missing Client ID')

  if not github_config.get('CLIENT_SECRET'):
    raise Exception('Missing Client Secret')

  if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
    raise Exception('Organization restriction must have at least one allowed organization')

  client = app.config['HTTPCLIENT']
  oauth = GithubOAuthConfig(config, config_key)
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise Exception('Invalid client id or client secret')

  if github_config.get('ALLOWED_ORGANIZATIONS'):
    for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
      if not oauth.validate_organization(org_id, client):
        raise Exception('Invalid organization: %s' % org_id)


def _validate_bitbucket(config, _):
  """ Validates the config for BitBucket. """
  trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
  if not trigger_config:
    raise Exception('Missing client ID and client secret')

  if not trigger_config.get('CONSUMER_KEY'):
    raise Exception('Missing Consumer Key')

  if not trigger_config.get('CONSUMER_SECRET'):
    raise Exception('Missing Consumer Secret')

  key = trigger_config['CONSUMER_KEY']
  secret = trigger_config['CONSUMER_SECRET']
  callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())

  bitbucket_client = BitBucket(key, secret, callback_url)
  (result, _, _) = bitbucket_client.get_authorization_url()
  if not result:
    raise Exception('Invaid consumer key or secret')


def _validate_google_login(config, _):
  """ Validates the Google Login client ID and secret. """
  google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
  if not google_login_config:
    raise Exception('Missing client ID and client secret')

  if not google_login_config.get('CLIENT_ID'):
    raise Exception('Missing Client ID')

  if not google_login_config.get('CLIENT_SECRET'):
    raise Exception('Missing Client Secret')

  client = app.config['HTTPCLIENT']
  oauth = GoogleOAuthConfig(config, 'GOOGLE_LOGIN_CONFIG')
  result = oauth.validate_client_id_and_secret(client, app.config)
  if not result:
    raise Exception('Invalid client id or client secret')


def _validate_ssl(config, _):
  """ Validates the SSL configuration (if enabled). """
  if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
    return

  for filename in SSL_FILENAMES:
    if not config_provider.volume_file_exists(filename):
      raise Exception('Missing required SSL file: %s' % filename)

  with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
    cert_contents = f.read()

  # Validate the certificate.
  try:
    cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_contents)
  except:
    raise Exception('Could not parse certificate file. Is it a valid PEM certificate?')

  if cert.has_expired():
    raise Exception('The specified SSL certificate has expired.')

  private_key_path = None
  with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
    private_key_path = f.name

  if not private_key_path:
    # Only in testing.
    return

  # Validate the private key with the certificate.
  context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
  context.use_certificate(cert)

  try:
    context.use_privatekey_file(private_key_path)
  except:
    raise Exception('Could not parse key file. Is it a valid PEM private key?')

  try:
    context.check_privatekey()
  except OpenSSL.SSL.Error as e:
    raise Exception('SSL key failed to validate: %s' % str(e))

  # Verify the hostname matches the name in the certificate.
  common_name = cert.get_subject().commonName
  if common_name is None:
    raise Exception('Missing CommonName (CN) from SSL certificate')

  # Build the list of allowed host patterns.
  hosts = set([common_name])

  # Find the DNS extension, if any.
  for i in range(0, cert.get_extension_count()):
    ext = cert.get_extension(i)
    if ext.get_short_name() == 'subjectAltName':
      value = str(ext)
      hosts.update([host.strip()[4:] for host in value.split(',')])

  # Check each host.
  for host in hosts:
    if fnmatch(config['SERVER_HOSTNAME'], host):
      return

  raise Exception('Supported names "%s" in SSL cert do not match server hostname "%s"' %
                  (', '.join(list(hosts)), config['SERVER_HOSTNAME']))



def _validate_ldap(config, password):
  """ Validates the LDAP connection. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
    return

  # Note: raises ldap.INVALID_CREDENTIALS on failure
  admin_dn = config.get('LDAP_ADMIN_DN')
  admin_passwd = config.get('LDAP_ADMIN_PASSWD')

  if not admin_dn:
    raise Exception('Missing Admin DN for LDAP configuration')

  if not admin_passwd:
    raise Exception('Missing Admin Password for LDAP configuration')

  ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
  if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
    raise Exception('LDAP URI must start with ldap:// or ldaps://')

  try:
    with LDAPConnection(ldap_uri, admin_dn, admin_passwd):
      pass
  except ldap.LDAPError as ex:
    values = ex.args[0] if ex.args else {}
    if not isinstance(values, dict):
      raise Exception(str(ex.args))

    raise Exception(values.get('desc', 'Unknown error'))

  # Verify that the superuser exists. If not, raise an exception.
  base_dn = config.get('LDAP_BASE_DN')
  user_rdn = config.get('LDAP_USER_RDN', [])
  uid_attr = config.get('LDAP_UID_ATTR', 'uid')
  email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')

  users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr)

  username = get_authenticated_user().username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    raise Exception(('Verification of superuser %s failed: %s. \n\nThe user either does not exist ' +
                     'in the remote authentication system ' +
                     'OR LDAP auth is misconfigured.') % (username, err_msg))


def _validate_jwt(config, password):
  """ Validates the JWT authentication system. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
    return

  verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
  issuer = config.get('JWT_AUTH_ISSUER')

  if not verify_endpoint:
    raise Exception('Missing JWT Verification endpoint')

  if not issuer:
    raise Exception('Missing JWT Issuer ID')

  # Try to instatiate the JWT authentication mechanism. This will raise an exception if
  # the key cannot be found.
  users = ExternalJWTAuthN(verify_endpoint, issuer, OVERRIDE_CONFIG_DIRECTORY,
                           app.config['HTTPCLIENT'],
                           app.config.get('JWT_AUTH_MAX_FRESH_S', 300))

  # Verify that the superuser exists. If not, raise an exception.
  username = get_authenticated_user().username
  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    raise Exception(('Verification of superuser %s failed: %s. \n\nThe user either does not ' +
                     'exist in the remote authentication system ' +
                     'OR JWT auth is misconfigured.') % (username, err_msg))


def _validate_keystone(config, password):
  """ Validates the Keystone authentication system. """
  if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
    return

  auth_url = config.get('KEYSTONE_AUTH_URL')
  admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
  admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
  admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')

  if not auth_url:
    raise Exception('Missing authentication URL')

  if not admin_username:
    raise Exception('Missing admin username')

  if not admin_password:
    raise Exception('Missing admin password')

  if not admin_tenant:
    raise Exception('Missing admin tenant')

  users = KeystoneUsers(auth_url, admin_username, admin_password, admin_tenant)

  # Verify that the superuser exists. If not, raise an exception.
  username = get_authenticated_user().username

  (result, err_msg) = users.verify_credentials(username, password)
  if not result:
    raise Exception(('Verification of superuser %s failed: %s \n\nThe user either does not ' +
                     'exist in the remote authentication system ' +
                     'OR Keystone auth is misconfigured.') % (username, err_msg))


_VALIDATORS = {
  'database': _validate_database,
  'redis': _validate_redis,
  'registry-storage': _validate_registry_storage,
  'mail': _validate_mailing,
  'github-login': _validate_github('GITHUB_LOGIN_CONFIG'),
  'github-trigger': _validate_github('GITHUB_TRIGGER_CONFIG'),
  'gitlab-trigger': _validate_gitlab,
  'bitbucket-trigger': _validate_bitbucket,
  'google-login': _validate_google_login,
  'ssl': _validate_ssl,
  'ldap': _validate_ldap,
  'jwt': _validate_jwt,
  'keystone': _validate_keystone,
}