initial import for Open Source 🎉

util/config/validators/__init__.py

@@ -0,0 +1,20 @@
+from abc import ABCMeta, abstractmethod, abstractproperty
+from six import add_metaclass
+
+class ConfigValidationException(Exception):
+  """ Exception raised when the configuration fails to validate for a known reason. """
+  pass
+
+
+@add_metaclass(ABCMeta)
+class BaseValidator(object):
+  @abstractproperty
+  def name(self):
+    """ The key for the validation API. """
+    pass
+
+  @classmethod
+  @abstractmethod
+  def validate(cls, validator_context):
+    """ Raises Exception if failure to validate. """
+    pass

util/config/validators/test/test_validate_access.py

@@ -0,0 +1,29 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_access import AccessSettingsValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config, expected_exception', [
+  ({}, None),
+  ({'FEATURE_DIRECT_LOGIN': False}, ConfigValidationException),
+  ({'FEATURE_DIRECT_LOGIN': False, 'SOMETHING_LOGIN_CONFIG': {}}, None),
+  ({'FEATURE_DIRECT_LOGIN': False, 'FEATURE_GITHUB_LOGIN': True}, None),
+  ({'FEATURE_DIRECT_LOGIN': False, 'FEATURE_GOOGLE_LOGIN': True}, None),
+  ({'FEATURE_USER_CREATION': True, 'FEATURE_INVITE_ONLY_USER_CREATION': False}, None),
+  ({'FEATURE_USER_CREATION': True, 'FEATURE_INVITE_ONLY_USER_CREATION': True}, None),
+  ({'FEATURE_INVITE_ONLY_USER_CREATION': True}, None),
+  ({'FEATURE_USER_CREATION': False, 'FEATURE_INVITE_ONLY_USER_CREATION': True},
+   ConfigValidationException),
+  ({'FEATURE_USER_CREATION': False, 'FEATURE_INVITE_ONLY_USER_CREATION': False}, None),
+])
+def test_validate_invalid_oidc_login_config(unvalidated_config, expected_exception, app):
+  validator = AccessSettingsValidator()
+
+  if expected_exception is not None:
+    with pytest.raises(expected_exception):
+      validator.validate(ValidatorContext(unvalidated_config))
+  else:
+    validator.validate(ValidatorContext(unvalidated_config))

util/config/validators/test/test_validate_actionlog_archiving.py

@@ -0,0 +1,52 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'ACTION_LOG_ARCHIVE_PATH': 'foo'}),
+  ({'ACTION_LOG_ARCHIVE_LOCATION': ''}),
+])
+def test_skip_validate_actionlog(unvalidated_config, app):
+  validator = ActionLogArchivingValidator()
+  validator.validate(ValidatorContext(unvalidated_config))
+
+
+@pytest.mark.parametrize('config, expected_error', [
+  ({'FEATURE_ACTION_LOG_ROTATION': True}, 'Missing action log archive path'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': ''}, 'Missing action log archive path'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'foo'}, 'Missing action log archive storage location'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'foo',
+    'ACTION_LOG_ARCHIVE_LOCATION': ''}, 'Missing action log archive storage location'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'foo',
+    'ACTION_LOG_ARCHIVE_LOCATION': 'invalid'},
+    'Action log archive storage location `invalid` not found in storage config'),
+])
+def test_invalid_config(config, expected_error, app):
+  validator = ActionLogArchivingValidator()
+
+  with pytest.raises(ConfigValidationException) as ipe:
+    validator.validate(ValidatorContext(config))
+
+  assert str(ipe.value) == expected_error
+
+def test_valid_config(app):
+  config = ValidatorContext({
+    'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'somepath',
+    'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
+    'DISTRIBUTED_STORAGE_CONFIG': {
+      'somelocation': {},
+    },
+  })
+
+  validator = ActionLogArchivingValidator()
+  validator.validate(config)

util/config/validators/test/test_validate_apptokenauth.py

@@ -0,0 +1,30 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({'AUTHENTICATION_TYPE': 'AppToken'}),
+  ({'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': False}),
+  ({'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': True, 
+    'FEATURE_DIRECT_LOGIN': True}),
+])
+def test_validate_invalid_auth_config(unvalidated_config, app):
+  validator = AppTokenAuthValidator()
+
+  with pytest.raises(ConfigValidationException):
+    validator.validate(ValidatorContext(unvalidated_config))
+
+
+def test_validate_auth(app):
+  config = ValidatorContext({
+    'AUTHENTICATION_TYPE': 'AppToken',
+    'FEATURE_APP_SPECIFIC_TOKENS': True,
+    'FEATURE_DIRECT_LOGIN': False,
+  })
+
+  validator = AppTokenAuthValidator()
+  validator.validate(config)

util/config/validators/test/test_validate_bitbucket_trigger.py

@@ -0,0 +1,48 @@
+import pytest
+
+from httmock import urlmatch, HTTMock
+
+from util.config import URLSchemeAndHostname
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  (ValidatorContext({})),
+  (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
+  (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
+  (ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
+])
+def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
+  validator = BitbucketTriggerValidator()
+
+  with pytest.raises(ConfigValidationException):
+    validator.validate(unvalidated_config)
+
+def test_validate_bitbucket_trigger(app):
+  url_hit = [False]
+
+  @urlmatch(netloc=r'bitbucket.org')
+  def handler(url, request):
+    url_hit[0] = True
+    return {
+      'status_code': 200,
+      'content': 'oauth_token=foo&oauth_token_secret=bar',
+    }
+
+  with HTTMock(handler):
+    validator = BitbucketTriggerValidator()
+
+    url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
+    unvalidated_config = ValidatorContext({
+      'BITBUCKET_TRIGGER_CONFIG': {
+        'CONSUMER_KEY': 'foo',
+        'CONSUMER_SECRET': 'bar',
+      },
+    }, url_scheme_and_hostname=url_scheme_and_hostname)
+
+    validator.validate(unvalidated_config)
+
+    assert url_hit[0]

util/config/validators/test/test_validate_database.py

@@ -0,0 +1,23 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_database import DatabaseValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
+  (ValidatorContext(None), None, None, TypeError),
+  (ValidatorContext({}), None, None, KeyError),
+  (ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
+  (ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
+  (ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
+])
+def test_validate_database(unvalidated_config, user, user_password, expected, app):
+  validator = DatabaseValidator()
+
+  if expected is not None:
+    with pytest.raises(expected):
+      validator.validate(unvalidated_config)
+  else:
+    validator.validate(unvalidated_config)

util/config/validators/test/test_validate_github.py

@@ -0,0 +1,69 @@
+import pytest
+
+from httmock import urlmatch, HTTMock
+
+from config import build_requests_session
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
+
+from test.fixtures import *
+
+@pytest.fixture(params=[GitHubLoginValidator, GitHubTriggerValidator])
+def github_validator(request):
+  return request.param
+
+
+@pytest.mark.parametrize('github_config', [
+  ({}),
+  ({'GITHUB_ENDPOINT': 'foo'}),
+  ({'GITHUB_ENDPOINT': 'http://github.com'}),
+  ({'GITHUB_ENDPOINT': 'http://github.com', 'CLIENT_ID': 'foo'}),
+  ({'GITHUB_ENDPOINT': 'http://github.com', 'CLIENT_SECRET': 'foo'}),
+  ({
+    'GITHUB_ENDPOINT': 'http://github.com',
+    'CLIENT_ID': 'foo',
+    'CLIENT_SECRET': 'foo',
+    'ORG_RESTRICT': True
+  }),
+  ({
+    'GITHUB_ENDPOINT': 'http://github.com',
+    'CLIENT_ID': 'foo',
+    'CLIENT_SECRET': 'foo',
+    'ORG_RESTRICT': True,
+    'ALLOWED_ORGANIZATIONS': [],
+  }),
+])
+def test_validate_invalid_github_config(github_config, github_validator, app):
+  with pytest.raises(ConfigValidationException):
+    unvalidated_config = {}
+    unvalidated_config[github_validator.config_key] = github_config
+    github_validator.validate(ValidatorContext(unvalidated_config))
+
+def test_validate_github(github_validator, app):
+  url_hit = [False, False]
+
+  @urlmatch(netloc=r'somehost')
+  def handler(url, request):
+    url_hit[0] = True
+    return {'status_code': 200, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
+
+  @urlmatch(netloc=r'somehost', path=r'/api/v3/applications/foo/tokens/foo')
+  def app_handler(url, request):
+    url_hit[1] = True
+    return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
+
+  with HTTMock(app_handler, handler):
+    unvalidated_config = ValidatorContext({
+      github_validator.config_key: {
+        'GITHUB_ENDPOINT': 'http://somehost',
+        'CLIENT_ID': 'foo',
+        'CLIENT_SECRET': 'bar',
+      },
+    })
+
+    unvalidated_config.http_client = build_requests_session()
+    github_validator.validate(unvalidated_config)
+
+  assert url_hit[0]
+  assert url_hit[1]

util/config/validators/test/test_validate_gitlab_trigger.py

@@ -0,0 +1,49 @@
+import json
+import pytest
+
+from httmock import urlmatch, HTTMock
+
+from config import build_requests_session
+from util.config import URLSchemeAndHostname
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'foo'}}),
+  ({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'http://someendpoint', 'CLIENT_ID': 'foo'}}),
+  ({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'http://someendpoint', 'CLIENT_SECRET': 'foo'}}),
+])
+def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
+  validator = GitLabTriggerValidator()
+
+  with pytest.raises(ConfigValidationException):
+    validator.validate(ValidatorContext(unvalidated_config))
+
+def test_validate_gitlab_enterprise_trigger(app):
+  url_hit = [False]
+
+  @urlmatch(netloc=r'somegitlab', path='/oauth/token')
+  def handler(_, __):
+    url_hit[0] = True
+    return {'status_code': 400, 'content': json.dumps({'error': 'invalid code'})}
+
+  with HTTMock(handler):
+    validator = GitLabTriggerValidator()
+
+    url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
+
+    unvalidated_config = ValidatorContext({
+      'GITLAB_TRIGGER_CONFIG': {
+        'GITLAB_ENDPOINT': 'http://somegitlab',
+        'CLIENT_ID': 'foo',
+        'CLIENT_SECRET': 'bar',
+      },
+    }, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname)
+
+    validator.validate(unvalidated_config)
+
+  assert url_hit[0]

util/config/validators/test/test_validate_google_login.py

@@ -0,0 +1,45 @@
+import pytest
+
+from httmock import urlmatch, HTTMock
+
+from config import build_requests_session
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_google_login import GoogleLoginValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'GOOGLE_LOGIN_CONFIG': {}}),
+  ({'GOOGLE_LOGIN_CONFIG': {'CLIENT_ID': 'foo'}}),
+  ({'GOOGLE_LOGIN_CONFIG': {'CLIENT_SECRET': 'foo'}}),
+])
+def test_validate_invalid_google_login_config(unvalidated_config, app):
+  validator = GoogleLoginValidator()
+
+  with pytest.raises(ConfigValidationException):
+    validator.validate(ValidatorContext(unvalidated_config))
+
+def test_validate_google_login(app):
+  url_hit = [False]
+  @urlmatch(netloc=r'www.googleapis.com', path='/oauth2/v3/token')
+  def handler(_, __):
+    url_hit[0] = True
+    return {'status_code': 200, 'content': ''}
+
+  validator = GoogleLoginValidator()
+
+  with HTTMock(handler):
+    unvalidated_config = ValidatorContext({
+      'GOOGLE_LOGIN_CONFIG': {
+        'CLIENT_ID': 'foo',
+        'CLIENT_SECRET': 'bar',
+      },
+    })
+
+    unvalidated_config.http_client = build_requests_session()
+
+    validator.validate(unvalidated_config)
+
+  assert url_hit[0]

util/config/validators/test/test_validate_jwt.py

@@ -0,0 +1,65 @@
+import pytest
+
+from config import build_requests_session
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_jwt import JWTAuthValidator
+from util.morecollections import AttrDict
+
+from test.test_external_jwt_authn import fake_jwt
+
+from test.fixtures import *
+from app import config_provider
+
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'AUTHENTICATION_TYPE': 'Database'}),
+])
+def test_validate_noop(unvalidated_config, app):
+  config = ValidatorContext(unvalidated_config)
+  config.config_provider = config_provider
+  JWTAuthValidator.validate(config)
+
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({'AUTHENTICATION_TYPE': 'JWT'}),
+  ({'AUTHENTICATION_TYPE': 'JWT', 'JWT_AUTH_ISSUER': 'foo'}),
+  ({'AUTHENTICATION_TYPE': 'JWT', 'JWT_VERIFY_ENDPOINT': 'foo'}),
+])
+def test_invalid_config(unvalidated_config, app):
+  with pytest.raises(ConfigValidationException):
+    config = ValidatorContext(unvalidated_config)
+    config.config_provider = config_provider
+    JWTAuthValidator.validate(config)
+
+
+# TODO: fix these when re-adding jwt auth mechanism to jwt validators
+@pytest.mark.skip(reason='No way of currently testing this')
+@pytest.mark.parametrize('username, password, expected_exception', [
+  ('invaliduser', 'invalidpass', ConfigValidationException),
+  ('cool.user', 'invalidpass', ConfigValidationException),
+  ('invaliduser', 'somepass', ConfigValidationException),
+  ('cool.user', 'password', None),
+])
+def test_validated_jwt(username, password, expected_exception, app):
+  with fake_jwt() as jwt_auth:
+    config = {}
+    config['AUTHENTICATION_TYPE'] = 'JWT'
+    config['JWT_AUTH_ISSUER'] = jwt_auth.issuer
+    config['JWT_VERIFY_ENDPOINT'] = jwt_auth.verify_url
+    config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
+    config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
+
+    unvalidated_config = ValidatorContext(config)
+    unvalidated_config.user = AttrDict(dict(username=username))
+    unvalidated_config.user_password = password
+    unvalidated_config.config_provider = config_provider
+
+    unvalidated_config.http_client = build_requests_session()
+
+    if expected_exception is not None:
+      with pytest.raises(ConfigValidationException):
+        JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
+    else:
+        JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)

util/config/validators/test/test_validate_keystone.py

@@ -0,0 +1,54 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_keystone import KeystoneValidator
+
+from test.test_keystone_auth import fake_keystone
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'AUTHENTICATION_TYPE': 'Database'}),
+])
+def test_validate_noop(unvalidated_config, app):
+  KeystoneValidator.validate(ValidatorContext(unvalidated_config))
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({'AUTHENTICATION_TYPE': 'Keystone'}),
+  ({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo'}),
+  ({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo',
+    'KEYSTONE_ADMIN_USERNAME': 'bar'}),
+  ({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo',
+    'KEYSTONE_ADMIN_USERNAME': 'bar', 'KEYSTONE_ADMIN_PASSWORD': 'baz'}),
+])
+def test_invalid_config(unvalidated_config, app):
+  with pytest.raises(ConfigValidationException):
+    KeystoneValidator.validate(ValidatorContext(unvalidated_config))
+
+
+@pytest.mark.parametrize('admin_tenant_id, expected_exception', [
+  ('somegroupid', None),
+  ('groupwithnousers', ConfigValidationException),
+  ('somegroupid', None),
+  ('groupwithnousers', ConfigValidationException),
+])
+def test_validated_keystone(admin_tenant_id, expected_exception, app):
+  with fake_keystone(2) as keystone_auth:
+    auth_url = keystone_auth.auth_url
+
+    config = {}
+    config['AUTHENTICATION_TYPE'] = 'Keystone'
+    config['KEYSTONE_AUTH_URL'] = auth_url
+    config['KEYSTONE_ADMIN_USERNAME'] = 'adminuser'
+    config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
+    config['KEYSTONE_ADMIN_TENANT'] = admin_tenant_id
+
+    unvalidated_config = ValidatorContext(config)
+
+    if expected_exception is not None:
+      with pytest.raises(ConfigValidationException):
+        KeystoneValidator.validate(unvalidated_config)
+    else:
+      KeystoneValidator.validate(unvalidated_config)

util/config/validators/test/test_validate_ldap.py

@@ -0,0 +1,72 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_ldap import LDAPValidator
+from util.morecollections import AttrDict
+
+from test.test_ldap import mock_ldap
+
+from test.fixtures import *
+from app import config_provider
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'AUTHENTICATION_TYPE': 'Database'}),
+])
+def test_validate_noop(unvalidated_config, app):
+  config = ValidatorContext(unvalidated_config, config_provider=config_provider)
+  LDAPValidator.validate(config)
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({'AUTHENTICATION_TYPE': 'LDAP'}),
+  ({'AUTHENTICATION_TYPE': 'LDAP', 'LDAP_ADMIN_DN': 'foo'}),
+])
+def test_invalid_config(unvalidated_config, app):
+  with pytest.raises(ConfigValidationException):
+    config = ValidatorContext(unvalidated_config, config_provider=config_provider)
+    LDAPValidator.validate(config)
+
+
+@pytest.mark.parametrize('uri', [
+  'foo',
+  'http://foo',
+  'ldap:foo',
+])
+def test_invalid_uri(uri, app):
+  config = {}
+  config['AUTHENTICATION_TYPE'] = 'LDAP'
+  config['LDAP_BASE_DN'] = ['dc=quay', 'dc=io']
+  config['LDAP_ADMIN_DN'] = 'uid=testy,ou=employees,dc=quay,dc=io'
+  config['LDAP_ADMIN_PASSWD'] = 'password'
+  config['LDAP_USER_RDN'] = ['ou=employees']
+  config['LDAP_URI'] = uri
+
+  with pytest.raises(ConfigValidationException):
+    config = ValidatorContext(config, config_provider=config_provider)
+    LDAPValidator.validate(config)
+
+
+@pytest.mark.parametrize('admin_dn, admin_passwd, user_rdn, expected_exception', [
+  ('uid=testy,ou=employees,dc=quay,dc=io', 'password', ['ou=employees'], None),
+  ('uid=invalidadmindn', 'password', ['ou=employees'], ConfigValidationException),
+  ('uid=testy,ou=employees,dc=quay,dc=io', 'invalid_password', ['ou=employees'], ConfigValidationException),
+  ('uid=testy,ou=employees,dc=quay,dc=io', 'password', ['ou=invalidgroup'], ConfigValidationException),
+])
+def test_validated_ldap(admin_dn, admin_passwd, user_rdn, expected_exception, app):
+  config = {}
+  config['AUTHENTICATION_TYPE'] = 'LDAP'
+  config['LDAP_BASE_DN'] = ['dc=quay', 'dc=io']
+  config['LDAP_ADMIN_DN'] = admin_dn
+  config['LDAP_ADMIN_PASSWD'] = admin_passwd
+  config['LDAP_USER_RDN'] = user_rdn
+
+  unvalidated_config = ValidatorContext(config, config_provider=config_provider)
+
+  if expected_exception is not None:
+    with pytest.raises(ConfigValidationException):
+      with mock_ldap():
+        LDAPValidator.validate(unvalidated_config)
+  else:
+    with mock_ldap():
+      LDAPValidator.validate(unvalidated_config)

util/config/validators/test/test_validate_oidc.py

@@ -0,0 +1,50 @@
+import json
+import pytest
+
+from httmock import urlmatch, HTTMock
+
+from config import build_requests_session
+from oauth.oidc import OIDC_WELLKNOWN
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_oidc import OIDCLoginValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({'SOMETHING_LOGIN_CONFIG': {}}),
+  ({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo'}}),
+  ({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo', 'CLIENT_ID': 'foobar'}}),
+  ({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo', 'CLIENT_SECRET': 'foobar'}}),
+])
+def test_validate_invalid_oidc_login_config(unvalidated_config, app):
+  validator = OIDCLoginValidator()
+
+  with pytest.raises(ConfigValidationException):
+    validator.validate(ValidatorContext(unvalidated_config))
+
+def test_validate_oidc_login(app):
+  url_hit = [False]
+  @urlmatch(netloc=r'someserver', path=r'/\.well-known/openid-configuration')
+  def handler(_, __):
+    url_hit[0] = True
+    data = {
+      'token_endpoint': 'foobar',
+    }
+    return {'status_code': 200, 'content': json.dumps(data)}
+
+  with HTTMock(handler):
+    validator = OIDCLoginValidator()
+    unvalidated_config = ValidatorContext({
+      'SOMETHING_LOGIN_CONFIG': {
+        'CLIENT_ID': 'foo',
+        'CLIENT_SECRET': 'bar',
+        'OIDC_SERVER': 'http://someserver',
+        'DEBUGGING': True, # Allows for HTTP.
+      },
+    })
+    unvalidated_config.http_client = build_requests_session()
+
+    validator.validate(unvalidated_config)
+
+  assert url_hit[0]

util/config/validators/test/test_validate_redis.py

@@ -0,0 +1,34 @@
+import pytest
+import redis
+
+from mock import patch
+
+from mockredis import mock_strict_redis_client
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_redis import RedisValidator
+
+from test.fixtures import *
+from util.morecollections import AttrDict
+
+
+@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
+  ({}, None, None, False, ConfigValidationException),
+  ({'BUILDLOGS_REDIS': {}}, None, None, False, ConfigValidationException),
+  ({'BUILDLOGS_REDIS': {'host': 'somehost'}}, None, None, False, redis.ConnectionError),
+  ({'BUILDLOGS_REDIS': {'host': 'localhost'}}, None, None, True, None),
+])
+def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
+  with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
+    validator = RedisValidator()
+    unvalidated_config = ValidatorContext(unvalidated_config)
+
+    unvalidated_config.user = AttrDict(dict(username=user))
+    unvalidated_config.user_password = user_password
+
+    if expected is not None:
+      with pytest.raises(expected):
+        validator.validate(unvalidated_config)
+    else:
+      validator.validate(unvalidated_config)

util/config/validators/test/test_validate_secscan.py

@@ -0,0 +1,48 @@
+import pytest
+
+from config import build_requests_session
+from util.config import URLSchemeAndHostname
+from util.config.validator import ValidatorContext
+from util.config.validators.validate_secscan import SecurityScannerValidator
+from util.secscan.fake import fake_security_scanner
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
+])
+def test_validate_noop(unvalidated_config, app):
+
+  unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True,
+                                        http_client=build_requests_session(),
+                                        url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
+
+  SecurityScannerValidator.validate(unvalidated_config)
+
+
+@pytest.mark.parametrize('unvalidated_config, expected_error', [
+  ({
+    'TESTING': True,
+    'DISTRIBUTED_STORAGE_PREFERENCE': [],
+    'FEATURE_SECURITY_SCANNER': True,
+    'SECURITY_SCANNER_ENDPOINT': 'http://invalidhost',
+  }, Exception),
+
+  ({
+    'TESTING': True,
+    'DISTRIBUTED_STORAGE_PREFERENCE': [],
+    'FEATURE_SECURITY_SCANNER': True,
+    'SECURITY_SCANNER_ENDPOINT': 'http://fakesecurityscanner',
+  }, None),
+])
+def test_validate(unvalidated_config, expected_error, app):
+  unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True,
+                                        http_client=build_requests_session(),
+                                        url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
+
+  with fake_security_scanner(hostname='fakesecurityscanner'):
+    if expected_error is not None:
+      with pytest.raises(expected_error):
+        SecurityScannerValidator.validate(unvalidated_config)
+    else:
+      SecurityScannerValidator.validate(unvalidated_config)

util/config/validators/test/test_validate_signer.py

@@ -0,0 +1,20 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_signer import SignerValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config,expected', [
+  ({}, None),
+  ({'SIGNING_ENGINE': 'foobar'}, ConfigValidationException),
+  ({'SIGNING_ENGINE': 'gpg2'}, Exception),
+])
+def test_validate_signer(unvalidated_config, expected, app):
+  validator = SignerValidator()
+  if expected is not None:
+    with pytest.raises(expected):
+      validator.validate(ValidatorContext(unvalidated_config))
+  else:
+    validator.validate(ValidatorContext(unvalidated_config))

util/config/validators/test/test_validate_ssl.py

@@ -0,0 +1,75 @@
+import pytest
+
+from mock import patch
+from tempfile import NamedTemporaryFile
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
+from util.security.test.test_ssl_util import generate_test_cert
+
+from test.fixtures import *
+from app import config_provider
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'PREFERRED_URL_SCHEME': 'http'}),
+  ({'PREFERRED_URL_SCHEME': 'https', 'EXTERNAL_TLS_TERMINATION': True}),
+])
+def test_skip_validate_ssl(unvalidated_config, app):
+  validator = SSLValidator()
+  validator.validate(ValidatorContext(unvalidated_config))
+
+
+@pytest.mark.parametrize('cert, server_hostname, expected_error, error_message', [
+  ('invalidcert', 'someserver', ConfigValidationException, 'Could not load SSL certificate: no start line'),
+  (generate_test_cert(hostname='someserver'), 'someserver', None, None),
+  (generate_test_cert(hostname='invalidserver'), 'someserver', ConfigValidationException,
+   'Supported names "invalidserver" in SSL cert do not match server hostname "someserver"'),
+  (generate_test_cert(hostname='someserver'), 'someserver:1234', None, None),
+  (generate_test_cert(hostname='invalidserver'), 'someserver:1234', ConfigValidationException,
+   'Supported names "invalidserver" in SSL cert do not match server hostname "someserver"'),
+  (generate_test_cert(hostname='someserver:1234'), 'someserver:1234', ConfigValidationException,
+   'Supported names "someserver:1234" in SSL cert do not match server hostname "someserver"'),
+  (generate_test_cert(hostname='someserver:more'), 'someserver:more', None, None),
+  (generate_test_cert(hostname='someserver:more'), 'someserver:more:1234', None, None),
+])
+def test_validate_ssl(cert, server_hostname, expected_error, error_message, app):
+  with NamedTemporaryFile(delete=False) as cert_file:
+    cert_file.write(cert[0])
+    cert_file.seek(0)
+
+    with NamedTemporaryFile(delete=False) as key_file:
+      key_file.write(cert[1])
+      key_file.seek(0)
+
+    def return_true(filename):
+      return True
+
+    def get_volume_file(filename):
+      if filename == SSL_FILENAMES[0]:
+        return open(cert_file.name)
+
+      if filename == SSL_FILENAMES[1]:
+        return open(key_file.name)
+
+      return None
+
+    config = {
+      'PREFERRED_URL_SCHEME': 'https',
+      'SERVER_HOSTNAME': server_hostname,
+    }
+
+    with patch('app.config_provider.volume_file_exists', return_true):
+      with patch('app.config_provider.get_volume_file', get_volume_file):
+        validator = SSLValidator()
+        config = ValidatorContext(config)
+        config.config_provider = config_provider
+
+        if expected_error is not None:
+          with pytest.raises(expected_error) as ipe:
+            validator.validate(config)
+
+          assert str(ipe.value) == error_message
+        else:
+          validator.validate(config)

util/config/validators/test/test_validate_storage.py

@@ -0,0 +1,40 @@
+import pytest
+
+from moto import mock_s3_deprecated as mock_s3
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_storage import StorageValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config, expected', [
+  ({}, ConfigValidationException),
+  ({'DISTRIBUTED_STORAGE_CONFIG': {}}, ConfigValidationException),
+  ({'DISTRIBUTED_STORAGE_CONFIG': {'local': None}}, ConfigValidationException),
+  ({'DISTRIBUTED_STORAGE_CONFIG': {'local': ['FakeStorage', {}]}}, None),
+])
+def test_validate_storage(unvalidated_config, expected, app):
+  validator = StorageValidator()
+  if expected is not None:
+    with pytest.raises(expected):
+      validator.validate(ValidatorContext(unvalidated_config))
+  else:
+    validator.validate(ValidatorContext(unvalidated_config))
+
+def test_validate_s3_storage(app):
+  validator = StorageValidator()
+  with mock_s3():
+    with pytest.raises(ConfigValidationException) as ipe:
+      validator.validate(ValidatorContext({
+        'DISTRIBUTED_STORAGE_CONFIG': {
+          'default': ('S3Storage', {
+            's3_access_key': 'invalid',
+            's3_secret_key': 'invalid',
+            's3_bucket': 'somebucket',
+            'storage_path': ''
+          }),
+        }
+      }))
+
+    assert str(ipe.value) == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

util/config/validators/test/test_validate_timemachine.py

@@ -0,0 +1,32 @@
+import pytest
+
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_timemachine import TimeMachineValidator
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+])
+def test_validate_noop(unvalidated_config):
+  TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
+
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('default_exp,options,expected_exception', [
+  ('2d', ['1w', '2d'], None),
+
+  ('2d', ['1w'], 'Default expiration must be in expiration options set'),
+  ('2d', ['2d', '1M'], 'Invalid tag expiration option: 1M'),
+])
+def test_validate(default_exp, options, expected_exception, app):
+  config = {}
+  config['DEFAULT_TAG_EXPIRATION'] = default_exp
+  config['TAG_EXPIRATION_OPTIONS'] = options
+
+  if expected_exception is not None:
+    with pytest.raises(ConfigValidationException) as cve:
+      TimeMachineValidator.validate(ValidatorContext(config))
+    assert str(cve.value) == str(expected_exception)
+  else:
+    TimeMachineValidator.validate(ValidatorContext(config))

util/config/validators/test/test_validate_torrent.py

@@ -0,0 +1,39 @@
+import pytest
+
+from config import build_requests_session
+from httmock import urlmatch, HTTMock
+
+from app import instance_keys
+from util.config.validator import ValidatorContext
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_torrent import BittorrentValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config,expected', [
+  ({}, ConfigValidationException),
+  ({'BITTORRENT_ANNOUNCE_URL': 'http://faketorrent/announce'}, None),
+])
+def test_validate_torrent(unvalidated_config, expected, app):
+  announcer_hit = [False]
+
+  @urlmatch(netloc=r'faketorrent', path='/announce')
+  def handler(url, request):
+    announcer_hit[0] = True
+    return {'status_code': 200, 'content': ''}
+
+  with HTTMock(handler):
+    validator = BittorrentValidator()
+    if expected is not None:
+      with pytest.raises(expected):
+        config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
+        config.http_client = build_requests_session()
+
+        validator.validate(config)
+      assert not announcer_hit[0]
+    else:
+      config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
+      config.http_client = build_requests_session()
+
+      validator.validate(config)
+      assert announcer_hit[0]

util/config/validators/validate_access.py

@@ -0,0 +1,28 @@
+from util.config.validators import BaseValidator, ConfigValidationException
+from oauth.loginmanager import OAuthLoginManager
+from oauth.oidc import OIDCLoginService
+
+class AccessSettingsValidator(BaseValidator):
+  name = "access"
+
+  @classmethod
+  def validate(cls, validator_context):
+    config = validator_context.config
+    client = validator_context.http_client
+
+    if not config.get('FEATURE_DIRECT_LOGIN', True):
+      # Make sure we have at least one OIDC enabled.
+      github_login = config.get('FEATURE_GITHUB_LOGIN', False)
+      google_login = config.get('FEATURE_GOOGLE_LOGIN', False)
+
+      login_manager = OAuthLoginManager(config, client=client)
+      custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)]
+
+      if not github_login and not google_login and not custom_oidc:
+        msg = 'Cannot disable credentials login to UI without configured OIDC service'
+        raise ConfigValidationException(msg)
+
+    if (not config.get('FEATURE_USER_CREATION', True) and
+        config.get('FEATURE_INVITE_ONLY_USER_CREATION', False)):
+      msg = "Invite only user creation requires user creation to be enabled"
+      raise ConfigValidationException(msg)

util/config/validators/validate_actionlog_archiving.py

@@ -0,0 +1,24 @@
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class ActionLogArchivingValidator(BaseValidator):
+  name = "actionlogarchiving"
+
+  @classmethod
+  def validate(cls, validator_context):
+    config = validator_context.config
+
+    """ Validates the action log archiving configuration. """
+    if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
+      return
+
+    if not config.get('ACTION_LOG_ARCHIVE_PATH'):
+      raise ConfigValidationException('Missing action log archive path')
+
+    if not config.get('ACTION_LOG_ARCHIVE_LOCATION'):
+      raise ConfigValidationException('Missing action log archive storage location')
+
+    location = config['ACTION_LOG_ARCHIVE_LOCATION']
+    storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG') or {}
+    if location not in storage_config:
+      msg = 'Action log archive storage location `%s` not found in storage config' % location
+      raise ConfigValidationException(msg)

util/config/validators/validate_apptokenauth.py

@@ -0,0 +1,21 @@
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class AppTokenAuthValidator(BaseValidator):
+  name = "apptoken-auth"
+
+  @classmethod
+  def validate(cls, validator_context):
+    config = validator_context.config
+
+    if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken':
+      return
+
+    # Ensure that app tokens are enabled, as they are required.
+    if not config.get('FEATURE_APP_SPECIFIC_TOKENS', False):
+      msg = 'Application token support must be enabled to use External Application Token auth'
+      raise ConfigValidationException(msg)
+
+    # Ensure that direct login is disabled.
+    if config.get('FEATURE_DIRECT_LOGIN', True):
+      msg = 'Direct login must be disabled to use External Application Token auth'
+      raise ConfigValidationException(msg)

util/config/validators/validate_bitbucket_trigger.py

@@ -0,0 +1,30 @@
+from bitbucket import BitBucket
+
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class BitbucketTriggerValidator(BaseValidator):
+  name = "bitbucket-trigger"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the config for BitBucket. """
+    config = validator_context.config
+
+    trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
+    if not trigger_config:
+      raise ConfigValidationException('Missing client ID and client secret')
+
+    if not trigger_config.get('CONSUMER_KEY'):
+      raise ConfigValidationException('Missing Consumer Key')
+
+    if not trigger_config.get('CONSUMER_SECRET'):
+      raise ConfigValidationException('Missing Consumer Secret')
+
+    key = trigger_config['CONSUMER_KEY']
+    secret = trigger_config['CONSUMER_SECRET']
+    callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url())
+
+    bitbucket_client = BitBucket(key, secret, callback_url)
+    (result, _, _) = bitbucket_client.get_authorization_url()
+    if not result:
+      raise ConfigValidationException('Invalid consumer key or secret')

util/config/validators/validate_database.py

@@ -0,0 +1,20 @@
+from peewee import OperationalError
+
+from data.database import validate_database_precondition
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class DatabaseValidator(BaseValidator):
+  name = "database"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates connecting to the database. """
+    config = validator_context.config
+
+    try:
+      validate_database_precondition(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
+    except OperationalError as ex:
+      if ex.args and len(ex.args) > 1:
+        raise ConfigValidationException(ex.args[1])
+      else:
+        raise ex

util/config/validators/validate_github.py

@@ -0,0 +1,53 @@
+from oauth.services.github import GithubOAuthService
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class BaseGitHubValidator(BaseValidator):
+  name = None
+  config_key = None
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the OAuth credentials and API endpoint for a Github service. """
+    config = validator_context.config
+    client = validator_context.http_client
+    url_scheme_and_hostname = validator_context.url_scheme_and_hostname
+
+    github_config = config.get(cls.config_key)
+    if not github_config:
+      raise ConfigValidationException('Missing GitHub client id and client secret')
+
+    endpoint = github_config.get('GITHUB_ENDPOINT')
+    if not endpoint:
+      raise ConfigValidationException('Missing GitHub Endpoint')
+
+    if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
+      raise ConfigValidationException('Github Endpoint must start with http:// or https://')
+
+    if not github_config.get('CLIENT_ID'):
+      raise ConfigValidationException('Missing Client ID')
+
+    if not github_config.get('CLIENT_SECRET'):
+      raise ConfigValidationException('Missing Client Secret')
+
+    if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
+      raise ConfigValidationException('Organization restriction must have at least one allowed ' +
+                                      'organization')
+
+    oauth = GithubOAuthService(config, cls.config_key)
+    result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
+    if not result:
+      raise ConfigValidationException('Invalid client id or client secret')
+
+    if github_config.get('ALLOWED_ORGANIZATIONS'):
+      for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
+        if not oauth.validate_organization(org_id, client):
+          raise ConfigValidationException('Invalid organization: %s' % org_id)
+
+
+class GitHubLoginValidator(BaseGitHubValidator):
+  name = "github-login"
+  config_key = "GITHUB_LOGIN_CONFIG"
+
+class GitHubTriggerValidator(BaseGitHubValidator):
+  name = "github-trigger"
+  config_key = "GITHUB_TRIGGER_CONFIG"

util/config/validators/validate_gitlab_trigger.py

@@ -0,0 +1,32 @@
+from oauth.services.gitlab import GitLabOAuthService
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class GitLabTriggerValidator(BaseValidator):
+  name = "gitlab-trigger"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the OAuth credentials and API endpoint for a GitLab service. """
+    config = validator_context.config
+    url_scheme_and_hostname = validator_context.url_scheme_and_hostname
+    client = validator_context.http_client
+
+    github_config = config.get('GITLAB_TRIGGER_CONFIG')
+    if not github_config:
+      raise ConfigValidationException('Missing GitLab client id and client secret')
+
+    endpoint = github_config.get('GITLAB_ENDPOINT')
+    if endpoint:
+      if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
+        raise ConfigValidationException('GitLab Endpoint must start with http:// or https://')
+
+    if not github_config.get('CLIENT_ID'):
+      raise ConfigValidationException('Missing Client ID')
+
+    if not github_config.get('CLIENT_SECRET'):
+      raise ConfigValidationException('Missing Client Secret')
+
+    oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
+    result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
+    if not result:
+      raise ConfigValidationException('Invalid client id or client secret')

util/config/validators/validate_google_login.py

@@ -0,0 +1,27 @@
+from oauth.services.google import GoogleOAuthService
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class GoogleLoginValidator(BaseValidator):
+  name = "google-login"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the Google Login client ID and secret. """
+    config = validator_context.config
+    client = validator_context.http_client
+    url_scheme_and_hostname = validator_context.url_scheme_and_hostname
+
+    google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
+    if not google_login_config:
+      raise ConfigValidationException('Missing client ID and client secret')
+
+    if not google_login_config.get('CLIENT_ID'):
+      raise ConfigValidationException('Missing Client ID')
+
+    if not google_login_config.get('CLIENT_SECRET'):
+      raise ConfigValidationException('Missing Client Secret')
+
+    oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
+    result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
+    if not result:
+      raise ConfigValidationException('Invalid client id or client secret')

util/config/validators/validate_jwt.py

@@ -0,0 +1,48 @@
+import os
+from data.users.externaljwt import ExternalJWTAuthN
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class JWTAuthValidator(BaseValidator):
+  name = "jwt"
+
+  @classmethod
+  def validate(cls, validator_context, public_key_path=None):
+    """ Validates the JWT authentication system. """
+    config = validator_context.config
+    http_client = validator_context.http_client
+    jwt_auth_max = validator_context.jwt_auth_max
+    config_provider = validator_context.config_provider
+
+    if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
+      return
+
+    verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
+    query_endpoint = config.get('JWT_QUERY_ENDPOINT', None)
+    getuser_endpoint = config.get('JWT_GETUSER_ENDPOINT', None)
+
+    issuer = config.get('JWT_AUTH_ISSUER')
+
+    if not verify_endpoint:
+      raise ConfigValidationException('Missing JWT Verification endpoint')
+
+    if not issuer:
+      raise ConfigValidationException('Missing JWT Issuer ID')
+
+
+    override_config_directory = config_provider.get_config_dir_path()
+
+    # Try to instatiate the JWT authentication mechanism. This will raise an exception if
+    # the key cannot be found.
+    users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
+                             override_config_directory,
+                             http_client,
+                             jwt_auth_max,
+                             public_key_path=public_key_path,
+                             requires_email=config.get('FEATURE_MAILING', True))
+
+    # Verify that we can reach the jwt server
+    (result, err_msg) = users.ping()
+    if not result:
+      msg = ('Verification of JWT failed: %s. \n\nWe cannot reach the JWT server' +
+             'OR JWT auth is misconfigured') % err_msg
+      raise ConfigValidationException(msg)

util/config/validators/validate_keystone.py

@@ -0,0 +1,44 @@
+from util.config.validators import BaseValidator, ConfigValidationException
+from data.users.keystone import get_keystone_users
+
+class KeystoneValidator(BaseValidator):
+  name = "keystone"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the Keystone authentication system. """
+    config = validator_context.config
+
+    if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
+      return
+
+    auth_url = config.get('KEYSTONE_AUTH_URL')
+    auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
+    admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
+    admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
+    admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')
+
+    if not auth_url:
+      raise ConfigValidationException('Missing authentication URL')
+
+    if not admin_username:
+      raise ConfigValidationException('Missing admin username')
+
+    if not admin_password:
+      raise ConfigValidationException('Missing admin password')
+
+    if not admin_tenant:
+      raise ConfigValidationException('Missing admin tenant')
+
+    requires_email = config.get('FEATURE_MAILING', True)
+    users = get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
+                               requires_email)
+
+    # Verify that the superuser exists. If not, raise an exception.
+    (result, err_msg) = users.at_least_one_user_exists()
+    if not result:
+      msg = ('Verification that users exist failed: %s. \n\nNo users exist ' +
+             'in the admin tenant/project ' +
+             'in the remote authentication system ' +
+             'OR Keystone auth is misconfigured.') % err_msg
+      raise ConfigValidationException(msg)

util/config/validators/validate_ldap.py

@@ -0,0 +1,68 @@
+import os
+import ldap
+import subprocess
+
+from data.users import LDAP_CERT_FILENAME
+from data.users.externalldap import LDAPConnection, LDAPUsers
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class LDAPValidator(BaseValidator):
+  name = "ldap"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the LDAP connection. """
+    config = validator_context.config
+    config_provider = validator_context.config_provider
+    init_scripts_location = validator_context.init_scripts_location
+
+    if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
+      return
+
+    # If there is a custom LDAP certificate, then reinstall the certificates for the container.
+    if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
+      subprocess.check_call([os.path.join(init_scripts_location, 'certs_install.sh')],
+                            env={ 'QUAYCONFIG': config_provider.get_config_dir_path() })
+
+    # Note: raises ldap.INVALID_CREDENTIALS on failure
+    admin_dn = config.get('LDAP_ADMIN_DN')
+    admin_passwd = config.get('LDAP_ADMIN_PASSWD')
+
+    if not admin_dn:
+      raise ConfigValidationException('Missing Admin DN for LDAP configuration')
+
+    if not admin_passwd:
+      raise ConfigValidationException('Missing Admin Password for LDAP configuration')
+
+    ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
+    if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
+      raise ConfigValidationException('LDAP URI must start with ldap:// or ldaps://')
+
+    allow_tls_fallback = config.get('LDAP_ALLOW_INSECURE_FALLBACK', False)
+
+    try:
+      with LDAPConnection(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback):
+        pass
+    except ldap.LDAPError as ex:
+      values = ex.args[0] if ex.args else {}
+      if not isinstance(values, dict):
+        raise ConfigValidationException(str(ex.args))
+
+      raise ConfigValidationException(values.get('desc', 'Unknown error'))
+
+    base_dn = config.get('LDAP_BASE_DN')
+    user_rdn = config.get('LDAP_USER_RDN', [])
+    uid_attr = config.get('LDAP_UID_ATTR', 'uid')
+    email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')
+    requires_email = config.get('FEATURE_MAILING', True)
+
+    users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
+                      allow_tls_fallback, requires_email=requires_email)
+
+    # Ensure at least one user exists to verify the connection is setup properly.
+    (result, err_msg) = users.at_least_one_user_exists()
+    if not result:
+      msg = ('Verification that users exist failed: %s. \n\nNo users exist ' +
+             'in the remote authentication system ' +
+             'OR LDAP auth is misconfigured.') % err_msg
+      raise ConfigValidationException(msg)

util/config/validators/validate_oidc.py

@@ -0,0 +1,36 @@
+from oauth.loginmanager import OAuthLoginManager
+from oauth.oidc import OIDCLoginService, DiscoveryFailureException
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class OIDCLoginValidator(BaseValidator):
+  name = "oidc-login"
+
+  @classmethod
+  def validate(cls, validator_context):
+    config = validator_context.config
+    client = validator_context.http_client
+
+    login_manager = OAuthLoginManager(config, client=client)
+    for service in login_manager.services:
+      if not isinstance(service, OIDCLoginService):
+        continue
+
+      if service.config.get('OIDC_SERVER') is None:
+        msg = 'Missing OIDC_SERVER on OIDC service %s' % service.service_id()
+        raise ConfigValidationException(msg)
+
+      if service.config.get('CLIENT_ID') is None:
+        msg = 'Missing CLIENT_ID on OIDC service %s' % service.service_id()
+        raise ConfigValidationException(msg)
+
+      if service.config.get('CLIENT_SECRET') is None:
+        msg = 'Missing CLIENT_SECRET on OIDC service %s' % service.service_id()
+        raise ConfigValidationException(msg)
+
+      try:
+        if not service.validate():
+          msg = 'Could not validate OIDC service %s' % service.service_id()
+          raise ConfigValidationException(msg)
+      except DiscoveryFailureException as dfe:
+        msg = 'Could not validate OIDC service %s: %s' % (service.service_id(), dfe.message)
+        raise ConfigValidationException(msg)

util/config/validators/validate_redis.py

@@ -0,0 +1,18 @@
+import redis
+
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class RedisValidator(BaseValidator):
+  name = "redis"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates connecting to redis. """
+    config = validator_context.config
+
+    redis_config = config.get('BUILDLOGS_REDIS', {})
+    if not 'host' in  redis_config:
+      raise ConfigValidationException('Missing redis hostname')
+
+    client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
+    client.ping()

util/config/validators/validate_secscan.py

@@ -0,0 +1,52 @@
+import time
+
+# from boot import setup_jwt_proxy
+from util.secscan.api import SecurityScannerAPI
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class SecurityScannerValidator(BaseValidator):
+  name = "security-scanner"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the configuration for talking to a Quay Security Scanner. """
+    config = validator_context.config
+    client = validator_context.http_client
+    feature_sec_scanner = validator_context.feature_sec_scanner
+    is_testing = validator_context.is_testing
+
+    server_hostname = validator_context.url_scheme_and_hostname.hostname
+    uri_creator = validator_context.uri_creator
+
+    if not feature_sec_scanner:
+      return
+
+    api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
+
+    # if not is_testing:
+      # Generate a temporary Quay key to use for signing the outgoing requests.
+      # setup_jwt_proxy()
+
+    # We have to wait for JWT proxy to restart with the newly generated key.
+    max_tries = 5
+    response = None
+    last_exception = None
+
+    while max_tries > 0:
+      try:
+        response = api.ping()
+        last_exception = None
+        if response.status_code == 200:
+          return
+      except Exception as ex:
+        last_exception = ex
+
+      time.sleep(1)
+      max_tries = max_tries - 1
+
+    if last_exception is not None:
+      message = str(last_exception)
+      raise ConfigValidationException('Could not ping security scanner: %s' % message)
+    else:
+      message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
+      raise ConfigValidationException('Could not ping security scanner: %s' % message)

util/config/validators/validate_signer.py

@@ -0,0 +1,22 @@
+from StringIO import StringIO
+
+from util.config.validators import BaseValidator, ConfigValidationException
+from util.security.signing import SIGNING_ENGINES
+
+class SignerValidator(BaseValidator):
+  name = "signer"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the GPG public+private key pair used for signing converted ACIs. """
+    config = validator_context.config
+    config_provider = validator_context.config_provider
+
+    if config.get('SIGNING_ENGINE') is None:
+      return
+
+    if config['SIGNING_ENGINE'] not in SIGNING_ENGINES:
+      raise ConfigValidationException('Unknown signing engine: %s' % config['SIGNING_ENGINE'])
+
+    engine = SIGNING_ENGINES[config['SIGNING_ENGINE']](config, config_provider)
+    engine.detached_sign(StringIO('test string'))

util/config/validators/validate_ssl.py

@@ -0,0 +1,72 @@
+from util.config.validators import BaseValidator, ConfigValidationException
+from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
+
+SSL_FILENAMES = ['ssl.cert', 'ssl.key']
+
+class SSLValidator(BaseValidator):
+  name = "ssl"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the SSL configuration (if enabled). """
+    config = validator_context.config
+    config_provider = validator_context.config_provider
+
+    # Skip if non-SSL.
+    if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
+      return
+
+    # Skip if externally terminated.
+    if config.get('EXTERNAL_TLS_TERMINATION', False) is True:
+      return
+
+    # Verify that we have all the required SSL files.
+    for filename in SSL_FILENAMES:
+      if not config_provider.volume_file_exists(filename):
+        raise ConfigValidationException('Missing required SSL file: %s' % filename)
+
+    # Read the contents of the SSL certificate.
+    with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
+      cert_contents = f.read()
+
+    # Validate the certificate.
+    try:
+      certificate = load_certificate(cert_contents)
+    except CertInvalidException as cie:
+      raise ConfigValidationException('Could not load SSL certificate: %s' % cie)
+
+    # Verify the certificate has not expired.
+    if certificate.expired:
+      raise ConfigValidationException('The specified SSL certificate has expired.')
+
+    # Verify the hostname matches the name in the certificate.
+    if not certificate.matches_name(_ssl_cn(config['SERVER_HOSTNAME'])):
+      msg = ('Supported names "%s" in SSL cert do not match server hostname "%s"' %
+             (', '.join(list(certificate.names)), _ssl_cn(config['SERVER_HOSTNAME'])))
+      raise ConfigValidationException(msg)
+
+    # Verify the private key against the certificate.
+    private_key_path = None
+    with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
+      private_key_path = f.name
+
+    if not private_key_path:
+      # Only in testing.
+      return
+
+    try:
+      certificate.validate_private_key(private_key_path)
+    except KeyInvalidException as kie:
+      raise ConfigValidationException('SSL private key failed to validate: %s' % kie)
+
+
+def _ssl_cn(server_hostname):
+  """ Return the common name (fully qualified host name) from the SERVER_HOSTNAME. """
+  host_port = server_hostname.rsplit(':', 1)
+
+  # SERVER_HOSTNAME includes the port
+  if len(host_port) == 2:
+    if host_port[-1].isdigit():
+      return host_port[-2]
+
+  return server_hostname

util/config/validators/validate_storage.py

@@ -0,0 +1,54 @@
+from storage import get_storage_driver, TYPE_LOCAL_STORAGE
+from util.config.validators import BaseValidator, ConfigValidationException
+
+
+class StorageValidator(BaseValidator):
+  name = "registry-storage"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates registry storage. """
+    config = validator_context.config
+    client = validator_context.http_client
+    ip_resolver = validator_context.ip_resolver
+    config_provider = validator_context.config_provider
+
+    replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
+
+    providers = _get_storage_providers(config, ip_resolver, config_provider).items()
+    if not providers:
+      raise ConfigValidationException('Storage configuration required')
+
+    for name, (storage_type, driver) in providers:
+      # We can skip localstorage validation, since we can't guarantee that
+      # this will be the same machine Q.E. will run under
+      if storage_type == TYPE_LOCAL_STORAGE:
+          continue
+
+      try:
+        if replication_enabled and storage_type == 'LocalStorage':
+          raise ConfigValidationException('Locally mounted directory not supported ' +
+                                          'with storage replication')
+
+        # Run validation on the driver.
+        driver.validate(client)
+
+        # Run setup on the driver if the read/write succeeded.
+        driver.setup()
+      except Exception as ex:
+        msg = str(ex).strip().split("\n")[0]
+        raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
+
+
+def _get_storage_providers(config, ip_resolver, config_provider):
+  storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
+  drivers = {}
+
+  try:
+    for name, parameters in storage_config.items():
+      driver = get_storage_driver(None, None, None, config_provider, ip_resolver, parameters)
+      drivers[name] = (parameters[0], driver)
+  except TypeError:
+    raise ConfigValidationException('Missing required parameter(s) for storage %s' % name)
+
+  return drivers

util/config/validators/validate_timemachine.py

@@ -0,0 +1,31 @@
+import logging
+
+from util.config.validators import BaseValidator, ConfigValidationException
+from util.timedeltastring import convert_to_timedelta
+
+logger = logging.getLogger(__name__)
+
+class TimeMachineValidator(BaseValidator):
+  name = "time-machine"
+
+  @classmethod
+  def validate(cls, validator_context):
+    config = validator_context.config
+
+    if not 'DEFAULT_TAG_EXPIRATION' in config:
+      # Old style config
+      return
+
+    try:
+      convert_to_timedelta(config['DEFAULT_TAG_EXPIRATION']).total_seconds()
+    except ValueError as ve:
+      raise ConfigValidationException('Invalid default expiration: %s' % ve.message)
+
+    if not config['DEFAULT_TAG_EXPIRATION'] in config.get('TAG_EXPIRATION_OPTIONS', []):
+      raise ConfigValidationException('Default expiration must be in expiration options set')
+
+    for ts in config.get('TAG_EXPIRATION_OPTIONS', []):
+      try:
+        convert_to_timedelta(ts)
+      except ValueError as ve:
+        raise ConfigValidationException('Invalid tag expiration option: %s' % ts)

util/config/validators/validate_torrent.py

@@ -0,0 +1,59 @@
+import logging
+
+from hashlib import sha1
+
+from util.config.validators import BaseValidator, ConfigValidationException
+from util.registry.torrent import jwt_from_infohash, TorrentConfiguration
+
+logger = logging.getLogger(__name__)
+
+class BittorrentValidator(BaseValidator):
+  name = "bittorrent"
+
+  @classmethod
+  def validate(cls, validator_context):
+    """ Validates the configuration for using BitTorrent for downloads. """
+    config = validator_context.config
+    client = validator_context.http_client
+
+    announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
+    if not announce_url:
+      raise ConfigValidationException('Missing announce URL')
+
+    # Ensure that the tracker is reachable and accepts requests signed with a registry key.
+    params = {
+      'info_hash': sha1('test').digest(),
+      'peer_id': '-QUAY00-6wfG2wk6wWLc',
+      'uploaded': 0,
+      'downloaded': 0,
+      'left': 0,
+      'numwant': 0,
+      'port': 80,
+    }
+
+    torrent_config = TorrentConfiguration.for_testing(validator_context.instance_keys, announce_url,
+                                                      validator_context.registry_title)
+    encoded_jwt = jwt_from_infohash(torrent_config, params['info_hash'])
+    params['jwt'] = encoded_jwt
+
+    resp = client.get(announce_url, timeout=5, params=params)
+    logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)
+
+    if resp.status_code == 404:
+      raise ConfigValidationException('Announce path not found; did you forget `/announce`?')
+
+    if resp.status_code == 500:
+      raise ConfigValidationException('Did not get expected response from Tracker; ' +
+                                      'please check your settings')
+
+    if resp.status_code == 200:
+      if 'invalid jwt' in resp.text:
+        raise ConfigValidationException('Could not authorize to Tracker; is your Tracker ' +
+                                        'properly configured?')
+
+      if 'failure reason' in resp.text:
+        raise ConfigValidationException('Could not validate signed announce request: ' + resp.text)
+
+      if 'go_goroutines' in resp.text:
+        raise ConfigValidationException('Could not validate signed announce request: ' +
+                                        'provided port is used for Prometheus')