542 lines
18 KiB
Python
542 lines
18 KiB
Python
import logging
|
|
import subprocess
|
|
import time
|
|
|
|
from StringIO import StringIO
|
|
from fnmatch import fnmatch
|
|
|
|
import OpenSSL
|
|
import ldap
|
|
import peewee
|
|
import redis
|
|
|
|
from flask import Flask
|
|
from flask_mail import Mail, Message
|
|
from hashlib import sha1
|
|
|
|
from app import app, config_provider, get_app_url, OVERRIDE_CONFIG_DIRECTORY
|
|
from auth.auth_context import get_authenticated_user
|
|
from bitbucket import BitBucket
|
|
from boot import setup_jwt_proxy
|
|
from data.database import validate_database_url
|
|
from data.users import LDAP_CERT_FILENAME
|
|
from data.users.externaljwt import ExternalJWTAuthN
|
|
from data.users.externalldap import LDAPConnection, LDAPUsers
|
|
from data.users.keystone import get_keystone_users
|
|
from storage import get_storage_driver
|
|
from util.config.oauth import GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig
|
|
from util.secscan.api import SecurityScannerAPI
|
|
from util.registry.torrent import torrent_jwt
|
|
from util.security.signing import SIGNING_ENGINES
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Note: Only add files required for HTTPS to the SSL_FILESNAMES list.
|
|
SSL_FILENAMES = ['ssl.cert', 'ssl.key']
|
|
DB_SSL_FILENAMES = ['database.pem']
|
|
JWT_FILENAMES = ['jwt-authn.cert']
|
|
ACI_CERT_FILENAMES = ['signing-public.gpg', 'signing-private.gpg']
|
|
LDAP_FILENAMES = [LDAP_CERT_FILENAME]
|
|
CONFIG_FILENAMES = (SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES + ACI_CERT_FILENAMES +
|
|
LDAP_FILENAMES)
|
|
|
|
|
|
def get_storage_providers(config):
|
|
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
|
|
|
|
drivers = {}
|
|
|
|
try:
|
|
for name, parameters in storage_config.items():
|
|
drivers[name] = (parameters[0], get_storage_driver(None, parameters))
|
|
except TypeError:
|
|
logger.exception('Missing required storage configuration provider')
|
|
raise Exception('Missing required storage configuration parameter(s): %s' % name)
|
|
|
|
return drivers
|
|
|
|
def validate_service_for_config(service, config, password=None):
|
|
""" Attempts to validate the configuration for the given service. """
|
|
if not service in _VALIDATORS:
|
|
return {
|
|
'status': False
|
|
}
|
|
|
|
try:
|
|
_VALIDATORS[service](config, password)
|
|
return {
|
|
'status': True
|
|
}
|
|
except Exception as ex:
|
|
logger.exception('Validation exception')
|
|
return {
|
|
'status': False,
|
|
'reason': str(ex)
|
|
}
|
|
|
|
|
|
def _validate_database(config, _):
|
|
""" Validates connecting to the database. """
|
|
try:
|
|
validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
|
|
except peewee.OperationalError as ex:
|
|
if ex.args and len(ex.args) > 1:
|
|
raise Exception(ex.args[1])
|
|
else:
|
|
raise ex
|
|
|
|
|
|
def _validate_redis(config, _):
|
|
""" Validates connecting to redis. """
|
|
redis_config = config.get('BUILDLOGS_REDIS', {})
|
|
if not 'host' in redis_config:
|
|
raise Exception('Missing redis hostname')
|
|
|
|
client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
|
|
client.ping()
|
|
|
|
|
|
def _validate_registry_storage(config, _):
|
|
""" Validates registry storage. """
|
|
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
|
|
|
|
providers = get_storage_providers(config).items()
|
|
|
|
if not providers:
|
|
raise Exception('Storage configuration required')
|
|
|
|
for name, (storage_type, driver) in providers:
|
|
try:
|
|
if replication_enabled and storage_type == 'LocalStorage':
|
|
raise Exception('Locally mounted directory not supported with storage replication')
|
|
|
|
# Run validation on the driver.
|
|
driver.validate(app.config['HTTPCLIENT'])
|
|
|
|
# Run setup on the driver if the read/write succeeded.
|
|
driver.setup()
|
|
except Exception as ex:
|
|
raise Exception('Invalid storage configuration: %s: %s' % (name, str(ex)))
|
|
|
|
|
|
def _validate_mailing(config, _):
|
|
""" Validates sending email. """
|
|
test_app = Flask("mail-test-app")
|
|
test_app.config.update(config)
|
|
test_app.config.update({
|
|
'MAIL_FAIL_SILENTLY': False,
|
|
'TESTING': False
|
|
})
|
|
|
|
test_mail = Mail(test_app)
|
|
test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'],
|
|
sender=config.get('MAIL_DEFAULT_SENDER'))
|
|
test_msg.add_recipient(get_authenticated_user().email)
|
|
test_mail.send(test_msg)
|
|
|
|
|
|
def _validate_gitlab(config, _):
|
|
""" Validates the OAuth credentials and API endpoint for a GitLab service. """
|
|
github_config = config.get('GITLAB_TRIGGER_CONFIG')
|
|
if not github_config:
|
|
raise Exception('Missing GitLab client id and client secret')
|
|
|
|
endpoint = github_config.get('GITLAB_ENDPOINT')
|
|
if not endpoint:
|
|
raise Exception('Missing GitLab Endpoint')
|
|
|
|
if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
|
|
raise Exception('GitLab Endpoint must start with http:// or https://')
|
|
|
|
if not github_config.get('CLIENT_ID'):
|
|
raise Exception('Missing Client ID')
|
|
|
|
if not github_config.get('CLIENT_SECRET'):
|
|
raise Exception('Missing Client Secret')
|
|
|
|
client = app.config['HTTPCLIENT']
|
|
oauth = GitLabOAuthConfig(config, 'GITLAB_TRIGGER_CONFIG')
|
|
result = oauth.validate_client_id_and_secret(client, app.config)
|
|
if not result:
|
|
raise Exception('Invalid client id or client secret')
|
|
|
|
|
|
def _validate_github(config_key):
|
|
return lambda config, _: _validate_github_with_key(config_key, config)
|
|
|
|
|
|
def _validate_github_with_key(config_key, config):
|
|
""" Validates the OAuth credentials and API endpoint for a Github service. """
|
|
github_config = config.get(config_key)
|
|
if not github_config:
|
|
raise Exception('Missing GitHub client id and client secret')
|
|
|
|
endpoint = github_config.get('GITHUB_ENDPOINT')
|
|
if not endpoint:
|
|
raise Exception('Missing GitHub Endpoint')
|
|
|
|
if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
|
|
raise Exception('Github Endpoint must start with http:// or https://')
|
|
|
|
if not github_config.get('CLIENT_ID'):
|
|
raise Exception('Missing Client ID')
|
|
|
|
if not github_config.get('CLIENT_SECRET'):
|
|
raise Exception('Missing Client Secret')
|
|
|
|
if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
|
|
raise Exception('Organization restriction must have at least one allowed organization')
|
|
|
|
client = app.config['HTTPCLIENT']
|
|
oauth = GithubOAuthConfig(config, config_key)
|
|
result = oauth.validate_client_id_and_secret(client, app.config)
|
|
if not result:
|
|
raise Exception('Invalid client id or client secret')
|
|
|
|
if github_config.get('ALLOWED_ORGANIZATIONS'):
|
|
for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
|
|
if not oauth.validate_organization(org_id, client):
|
|
raise Exception('Invalid organization: %s' % org_id)
|
|
|
|
|
|
def _validate_bitbucket(config, _):
|
|
""" Validates the config for BitBucket. """
|
|
trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
|
|
if not trigger_config:
|
|
raise Exception('Missing client ID and client secret')
|
|
|
|
if not trigger_config.get('CONSUMER_KEY'):
|
|
raise Exception('Missing Consumer Key')
|
|
|
|
if not trigger_config.get('CONSUMER_SECRET'):
|
|
raise Exception('Missing Consumer Secret')
|
|
|
|
key = trigger_config['CONSUMER_KEY']
|
|
secret = trigger_config['CONSUMER_SECRET']
|
|
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url())
|
|
|
|
bitbucket_client = BitBucket(key, secret, callback_url)
|
|
(result, _, _) = bitbucket_client.get_authorization_url()
|
|
if not result:
|
|
raise Exception('Invaid consumer key or secret')
|
|
|
|
|
|
def _validate_google_login(config, _):
|
|
""" Validates the Google Login client ID and secret. """
|
|
google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
|
|
if not google_login_config:
|
|
raise Exception('Missing client ID and client secret')
|
|
|
|
if not google_login_config.get('CLIENT_ID'):
|
|
raise Exception('Missing Client ID')
|
|
|
|
if not google_login_config.get('CLIENT_SECRET'):
|
|
raise Exception('Missing Client Secret')
|
|
|
|
client = app.config['HTTPCLIENT']
|
|
oauth = GoogleOAuthConfig(config, 'GOOGLE_LOGIN_CONFIG')
|
|
result = oauth.validate_client_id_and_secret(client, app.config)
|
|
if not result:
|
|
raise Exception('Invalid client id or client secret')
|
|
|
|
|
|
def _validate_ssl(config, _):
|
|
""" Validates the SSL configuration (if enabled). """
|
|
|
|
# Skip if non-SSL.
|
|
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
|
|
return
|
|
|
|
# Skip if externally terminated.
|
|
if config.get('EXTERNAL_TLS_TERMINATION', False) == True:
|
|
return
|
|
|
|
for filename in SSL_FILENAMES:
|
|
if not config_provider.volume_file_exists(filename):
|
|
raise Exception('Missing required SSL file: %s' % filename)
|
|
|
|
with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
|
|
cert_contents = f.read()
|
|
|
|
# Validate the certificate.
|
|
try:
|
|
cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, cert_contents)
|
|
except:
|
|
raise Exception('Could not parse certificate file. Is it a valid PEM certificate?')
|
|
|
|
if cert.has_expired():
|
|
raise Exception('The specified SSL certificate has expired.')
|
|
|
|
private_key_path = None
|
|
with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
|
|
private_key_path = f.name
|
|
|
|
if not private_key_path:
|
|
# Only in testing.
|
|
return
|
|
|
|
# Validate the private key with the certificate.
|
|
context = OpenSSL.SSL.Context(OpenSSL.SSL.TLSv1_METHOD)
|
|
context.use_certificate(cert)
|
|
|
|
try:
|
|
context.use_privatekey_file(private_key_path)
|
|
except:
|
|
raise Exception('Could not parse key file. Is it a valid PEM private key?')
|
|
|
|
try:
|
|
context.check_privatekey()
|
|
except OpenSSL.SSL.Error as e:
|
|
raise Exception('SSL key failed to validate: %s' % str(e))
|
|
|
|
# Verify the hostname matches the name in the certificate.
|
|
common_name = cert.get_subject().commonName
|
|
if common_name is None:
|
|
raise Exception('Missing CommonName (CN) from SSL certificate')
|
|
|
|
# Build the list of allowed host patterns.
|
|
hosts = set([common_name])
|
|
|
|
# Find the DNS extension, if any.
|
|
for i in range(0, cert.get_extension_count()):
|
|
ext = cert.get_extension(i)
|
|
if ext.get_short_name() == 'subjectAltName':
|
|
value = str(ext)
|
|
hosts.update([host.strip()[4:] for host in value.split(',')])
|
|
|
|
# Check each host.
|
|
for host in hosts:
|
|
if fnmatch(config['SERVER_HOSTNAME'], host):
|
|
return
|
|
|
|
raise Exception('Supported names "%s" in SSL cert do not match server hostname "%s"' %
|
|
(', '.join(list(hosts)), config['SERVER_HOSTNAME']))
|
|
|
|
|
|
|
|
def _validate_ldap(config, password):
|
|
""" Validates the LDAP connection. """
|
|
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
|
|
return
|
|
|
|
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
|
|
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
|
|
subprocess.check_call(['/conf/init/certs_install.sh'])
|
|
|
|
# Note: raises ldap.INVALID_CREDENTIALS on failure
|
|
admin_dn = config.get('LDAP_ADMIN_DN')
|
|
admin_passwd = config.get('LDAP_ADMIN_PASSWD')
|
|
|
|
if not admin_dn:
|
|
raise Exception('Missing Admin DN for LDAP configuration')
|
|
|
|
if not admin_passwd:
|
|
raise Exception('Missing Admin Password for LDAP configuration')
|
|
|
|
ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
|
|
if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
|
|
raise Exception('LDAP URI must start with ldap:// or ldaps://')
|
|
|
|
allow_tls_fallback = config.get('LDAP_ALLOW_INSECURE_FALLBACK', False)
|
|
|
|
try:
|
|
with LDAPConnection(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback):
|
|
pass
|
|
except ldap.LDAPError as ex:
|
|
values = ex.args[0] if ex.args else {}
|
|
if not isinstance(values, dict):
|
|
raise Exception(str(ex.args))
|
|
|
|
raise Exception(values.get('desc', 'Unknown error'))
|
|
|
|
# Verify that the superuser exists. If not, raise an exception.
|
|
base_dn = config.get('LDAP_BASE_DN')
|
|
user_rdn = config.get('LDAP_USER_RDN', [])
|
|
uid_attr = config.get('LDAP_UID_ATTR', 'uid')
|
|
email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')
|
|
requires_email = config.get('FEATURE_MAILING', True)
|
|
|
|
users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
|
|
allow_tls_fallback, requires_email=requires_email)
|
|
|
|
username = get_authenticated_user().username
|
|
(result, err_msg) = users.verify_credentials(username, password)
|
|
if not result:
|
|
raise Exception(('Verification of superuser %s failed: %s. \n\nThe user either does not exist ' +
|
|
'in the remote authentication system ' +
|
|
'OR LDAP auth is misconfigured.') % (username, err_msg))
|
|
|
|
|
|
def _validate_jwt(config, password):
|
|
""" Validates the JWT authentication system. """
|
|
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
|
|
return
|
|
|
|
verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
|
|
query_endpoint = config.get('JWT_QUERY_ENDPOINT', None)
|
|
getuser_endpoint = config.get('JWT_GETUSER_ENDPOINT', None)
|
|
|
|
issuer = config.get('JWT_AUTH_ISSUER')
|
|
|
|
if not verify_endpoint:
|
|
raise Exception('Missing JWT Verification endpoint')
|
|
|
|
if not issuer:
|
|
raise Exception('Missing JWT Issuer ID')
|
|
|
|
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
|
|
# the key cannot be found.
|
|
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
|
|
OVERRIDE_CONFIG_DIRECTORY,
|
|
app.config['HTTPCLIENT'],
|
|
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
|
|
requires_email=config.get('FEATURE_MAILING', True))
|
|
|
|
# Verify that the superuser exists. If not, raise an exception.
|
|
username = get_authenticated_user().username
|
|
(result, err_msg) = users.verify_credentials(username, password)
|
|
if not result:
|
|
raise Exception(('Verification of superuser %s failed: %s. \n\nThe user either does not ' +
|
|
'exist in the remote authentication system ' +
|
|
'OR JWT auth is misconfigured') % (username, err_msg))
|
|
|
|
# If the query endpoint exists, ensure we can query to find the current user and that we can
|
|
# look up users directly.
|
|
if query_endpoint:
|
|
(results, err_msg) = users.query_users(username)
|
|
if not results:
|
|
err_msg = err_msg or ('Could not find users matching query: %s' % username)
|
|
raise Exception('Query endpoint is misconfigured or not returning proper users: %s' % err_msg)
|
|
|
|
# Make sure the get user endpoint is also configured.
|
|
if not getuser_endpoint:
|
|
raise Exception('The lookup user endpoint must be configured if the query endpoint is set')
|
|
|
|
(result, err_msg) = users.get_user(username)
|
|
if not result:
|
|
err_msg = err_msg or ('Could not find user %s' % username)
|
|
raise Exception('Lookup endpoint is misconfigured or not returning properly: %s' % err_msg)
|
|
|
|
|
|
def _validate_keystone(config, password):
|
|
""" Validates the Keystone authentication system. """
|
|
if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
|
|
return
|
|
|
|
auth_url = config.get('KEYSTONE_AUTH_URL')
|
|
auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
|
|
admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
|
|
admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
|
|
admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')
|
|
|
|
if not auth_url:
|
|
raise Exception('Missing authentication URL')
|
|
|
|
if not admin_username:
|
|
raise Exception('Missing admin username')
|
|
|
|
if not admin_password:
|
|
raise Exception('Missing admin password')
|
|
|
|
if not admin_tenant:
|
|
raise Exception('Missing admin tenant')
|
|
|
|
requires_email = config.get('FEATURE_MAILING', True)
|
|
users = get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
|
|
requires_email)
|
|
|
|
# Verify that the superuser exists. If not, raise an exception.
|
|
username = get_authenticated_user().username
|
|
|
|
(result, err_msg) = users.verify_credentials(username, password)
|
|
if not result:
|
|
raise Exception(('Verification of superuser %s failed: %s \n\nThe user either does not ' +
|
|
'exist in the remote authentication system ' +
|
|
'OR Keystone auth is misconfigured.') % (username, err_msg))
|
|
|
|
|
|
def _validate_signer(config, _):
|
|
""" Validates the GPG public+private key pair used for signing converted ACIs. """
|
|
if config.get('SIGNING_ENGINE') is None:
|
|
return
|
|
|
|
if config['SIGNING_ENGINE'] not in SIGNING_ENGINES:
|
|
raise Exception('Unknown signing engine: %s' % config['SIGNING_ENGINE'])
|
|
|
|
engine = SIGNING_ENGINES[config['SIGNING_ENGINE']](config, config_provider)
|
|
engine.detached_sign(StringIO('test string'))
|
|
|
|
|
|
def _validate_security_scanner(config, _):
|
|
""" Validates the configuration for talking to a Quay Security Scanner. """
|
|
# Generate a temporary Quay key to use for signing the outgoing requests.
|
|
setup_jwt_proxy()
|
|
|
|
# Wait a few seconds for the JWT proxy to startup.
|
|
time.sleep(2)
|
|
|
|
# Make a ping request to the security service.
|
|
client = app.config['HTTPCLIENT']
|
|
api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True)
|
|
response = api.ping()
|
|
if response.status_code != 200:
|
|
message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
|
|
raise Exception('Could not ping security scanner: %s' % message)
|
|
|
|
|
|
def _validate_bittorrent(config, _):
|
|
""" Validates the configuration for using BitTorrent for downloads. """
|
|
|
|
# Ensure that the tracker is reachable and accepts requests signed with a registry key.
|
|
client = app.config['HTTPCLIENT']
|
|
|
|
params = {
|
|
'info_hash': sha1('somedata').digest(),
|
|
'peer_id': '-QUAY00-6wfG2wk6wWLc',
|
|
'uploaded': 0,
|
|
'downloaded': 0,
|
|
'left': 0,
|
|
'numwant': 0,
|
|
'port': 80,
|
|
}
|
|
|
|
encoded_jwt = torrent_jwt(params)
|
|
params['jwt'] = encoded_jwt
|
|
|
|
resp = client.get(config['BITTORRENT_ANNOUNCE_URL'], timeout=5, params=params)
|
|
logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)
|
|
|
|
if resp.status_code == 404:
|
|
raise Exception('Announce path not found; did you forget `/announce`?')
|
|
|
|
if resp.status_code == 500:
|
|
raise Exception('Did not get expected response from Tracker; please check your settings')
|
|
|
|
if resp.status_code == 200:
|
|
if 'invalid jwt' in resp.text:
|
|
raise Exception('Could not authorize to Tracker; is your Tracker properly configured?')
|
|
|
|
if 'failure reason' in resp.text:
|
|
raise Exception('Could not validate signed announce request: ' + resp.text)
|
|
|
|
|
|
_VALIDATORS = {
|
|
'database': _validate_database,
|
|
'redis': _validate_redis,
|
|
'registry-storage': _validate_registry_storage,
|
|
'mail': _validate_mailing,
|
|
'github-login': _validate_github('GITHUB_LOGIN_CONFIG'),
|
|
'github-trigger': _validate_github('GITHUB_TRIGGER_CONFIG'),
|
|
'gitlab-trigger': _validate_gitlab,
|
|
'bitbucket-trigger': _validate_bitbucket,
|
|
'google-login': _validate_google_login,
|
|
'ssl': _validate_ssl,
|
|
'ldap': _validate_ldap,
|
|
'jwt': _validate_jwt,
|
|
'keystone': _validate_keystone,
|
|
'signer': _validate_signer,
|
|
'security-scanner': _validate_security_scanner,
|
|
'bittorrent': _validate_bittorrent,
|
|
}
|