diff --git a/_init.py b/_init.py index 804323555..3acbc3742 100644 --- a/_init.py +++ b/_init.py @@ -2,6 +2,8 @@ import os import re import subprocess +from util.config.provider import get_config_provider + ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) CONF_DIR = os.getenv("QUAYCONF", os.path.join(ROOT_DIR, "conf/")) @@ -10,6 +12,14 @@ STATIC_LDN_DIR = os.path.join(STATIC_DIR, 'ldn/') STATIC_FONTS_DIR = os.path.join(STATIC_DIR, 'fonts/') TEMPLATE_DIR = os.path.join(ROOT_DIR, 'templates/') +IS_TESTING = 'TEST' in os.environ +IS_KUBERNETES = 'KUBERNETES_SERVICE_HOST' in os.environ +OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/') + + +config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', + testing=IS_TESTING, kubernetes=IS_KUBERNETES) + def _get_version_number_changelog(): try: diff --git a/app.py b/app.py index 90f5771b6..d38e9549c 100644 --- a/app.py +++ b/app.py @@ -13,7 +13,9 @@ from flask_principal import Principal from jwkest.jwk import RSAKey import features -from _init import CONF_DIR + +from _init import config_provider, CONF_DIR, IS_KUBERNETES, IS_TESTING, OVERRIDE_CONFIG_DIRECTORY + from auth.auth_context import get_authenticated_user from avatars.avatars import Avatar from buildman.manager.buildcanceller import BuildCanceller @@ -41,7 +43,6 @@ from util.saas.useranalytics import UserAnalytics from util.saas.exceptionlog import Sentry from util.names import urn_generator from util.config.configutil import generate_secret_key -from util.config.provider import get_config_provider from util.config.superusermanager import SuperUserManager from util.label_validator import LabelValidator from util.metrics.metricqueue import MetricQueue @@ -53,7 +54,6 @@ from util.security.instancekeys import InstanceKeys from util.security.signing import Signer -OVERRIDE_CONFIG_DIRECTORY = os.path.join(CONF_DIR, 'stack/') OVERRIDE_CONFIG_YAML_FILENAME = os.path.join(CONF_DIR, 'stack/config.yaml') OVERRIDE_CONFIG_PY_FILENAME = os.path.join(CONF_DIR, 'stack/config.py') @@ -65,10 +65,8 @@ app = Flask(__name__) logger = logging.getLogger(__name__) # Instantiate the configuration. -is_testing = 'TEST' in os.environ -is_kubernetes = 'KUBERNETES_SERVICE_HOST' in os.environ -config_provider = get_config_provider(OVERRIDE_CONFIG_DIRECTORY, 'config.yaml', 'config.py', - testing=is_testing, kubernetes=is_kubernetes) +is_testing = IS_TESTING +is_kubernetes = IS_KUBERNETES if is_testing: from test.testconfig import TestConfig diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 836f09f80..a852150fd 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -405,6 +405,6 @@ class SuperUserConfigValidate(ApiResource): # this is also safe since this method does not access any information not given in the request. if not config_provider.config_exists() or SuperUserPermission().can(): config = request.get_json()['config'] - return validate_service_for_config(service, config, request.get_json().get('password', '')) + return validate_service_for_config(service, config, request.get_json().get('password', ''), app) abort(403) diff --git a/util/config/validator.py b/util/config/validator.py index c45a89e03..febd88d48 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -64,7 +64,7 @@ VALIDATORS = { AppTokenAuthValidator.name: AppTokenAuthValidator.validate, } -def validate_service_for_config(service, config, password=None): +def validate_service_for_config(service, config, password=None, app=None): """ Attempts to validate the configuration for the given service. """ if not service in VALIDATORS: return { @@ -72,7 +72,7 @@ def validate_service_for_config(service, config, password=None): } try: - VALIDATORS[service](config, get_authenticated_user(), password) + VALIDATORS[service](config, get_authenticated_user(), password, app) return { 'status': True } diff --git a/util/config/validators/__init__.py b/util/config/validators/__init__.py index a3edeeb12..dfa18e1b0 100644 --- a/util/config/validators/__init__.py +++ b/util/config/validators/__init__.py @@ -15,6 +15,6 @@ class BaseValidator(object): @classmethod @abstractmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Raises Exception if failure to validate. """ pass diff --git a/util/config/validators/test/test_validate_jwt.py b/util/config/validators/test/test_validate_jwt.py index 0a29b1953..e535f7f9e 100644 --- a/util/config/validators/test/test_validate_jwt.py +++ b/util/config/validators/test/test_validate_jwt.py @@ -14,7 +14,7 @@ from test.fixtures import * ({'AUTHENTICATION_TYPE': 'Database'}), ]) def test_validate_noop(unvalidated_config, app): - JWTAuthValidator.validate(unvalidated_config, None, None) + JWTAuthValidator.validate(unvalidated_config, None, None, app) @pytest.mark.parametrize('unvalidated_config', [ @@ -24,7 +24,7 @@ def test_validate_noop(unvalidated_config, app): ]) def test_invalid_config(unvalidated_config, app): with pytest.raises(ConfigValidationException): - JWTAuthValidator.validate(unvalidated_config, None, None) + JWTAuthValidator.validate(unvalidated_config, None, None, app) @pytest.mark.parametrize('username, password, expected_exception', [ @@ -44,8 +44,8 @@ def test_validated_jwt(username, password, expected_exception, app): if expected_exception is not None: with pytest.raises(ConfigValidationException): - JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, + JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app, public_key_path=jwt_auth.public_key_path) else: - JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, + JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app, public_key_path=jwt_auth.public_key_path) diff --git a/util/config/validators/validate_access.py b/util/config/validators/validate_access.py index eb80090d8..ed4d0ef8a 100644 --- a/util/config/validators/validate_access.py +++ b/util/config/validators/validate_access.py @@ -1,4 +1,3 @@ -from app import app from util.config.validators import BaseValidator, ConfigValidationException from oauth.loginmanager import OAuthLoginManager from oauth.oidc import OIDCLoginService @@ -7,7 +6,7 @@ class AccessSettingsValidator(BaseValidator): name = "access" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): if not config.get('FEATURE_DIRECT_LOGIN', True): # Make sure we have at least one OIDC enabled. github_login = config.get('FEATURE_GITHUB_LOGIN', False) diff --git a/util/config/validators/validate_actionlog_archiving.py b/util/config/validators/validate_actionlog_archiving.py index e8fb79a50..256e2c6b8 100644 --- a/util/config/validators/validate_actionlog_archiving.py +++ b/util/config/validators/validate_actionlog_archiving.py @@ -4,7 +4,7 @@ class ActionLogArchivingValidator(BaseValidator): name = "actionlogarchiving" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the action log archiving configuration. """ if not config.get('FEATURE_ACTION_LOG_ROTATION', False): return diff --git a/util/config/validators/validate_apptokenauth.py b/util/config/validators/validate_apptokenauth.py index 6d7be1f1b..e77e50b4d 100644 --- a/util/config/validators/validate_apptokenauth.py +++ b/util/config/validators/validate_apptokenauth.py @@ -4,7 +4,7 @@ class AppTokenAuthValidator(BaseValidator): name = "apptoken-auth" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken': return diff --git a/util/config/validators/validate_bitbucket_trigger.py b/util/config/validators/validate_bitbucket_trigger.py index 15378c1b4..db25e43a9 100644 --- a/util/config/validators/validate_bitbucket_trigger.py +++ b/util/config/validators/validate_bitbucket_trigger.py @@ -1,13 +1,13 @@ from bitbucket import BitBucket -from app import get_app_url +from util import get_app_url from util.config.validators import BaseValidator, ConfigValidationException class BitbucketTriggerValidator(BaseValidator): name = "bitbucket-trigger" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the config for BitBucket. """ trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG') if not trigger_config: @@ -21,7 +21,7 @@ class BitbucketTriggerValidator(BaseValidator): key = trigger_config['CONSUMER_KEY'] secret = trigger_config['CONSUMER_SECRET'] - callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url()) + callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url(app.config)) bitbucket_client = BitBucket(key, secret, callback_url) (result, _, _) = bitbucket_client.get_authorization_url() diff --git a/util/config/validators/validate_database.py b/util/config/validators/validate_database.py index 5fb27fa80..4484edfbe 100644 --- a/util/config/validators/validate_database.py +++ b/util/config/validators/validate_database.py @@ -7,7 +7,7 @@ class DatabaseValidator(BaseValidator): name = "database" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates connecting to the database. """ try: validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {})) diff --git a/util/config/validators/validate_email.py b/util/config/validators/validate_email.py index b394fdad4..4a7756b27 100644 --- a/util/config/validators/validate_email.py +++ b/util/config/validators/validate_email.py @@ -1,14 +1,13 @@ from flask import Flask from flask_mail import Mail, Message -from app import app from util.config.validators import BaseValidator class EmailValidator(BaseValidator): name = "mail" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates sending email. """ with app.app_context(): test_app = Flask("mail-test-app") diff --git a/util/config/validators/validate_github.py b/util/config/validators/validate_github.py index 39293a11d..a0d2aaaf8 100644 --- a/util/config/validators/validate_github.py +++ b/util/config/validators/validate_github.py @@ -1,4 +1,3 @@ -from app import app from oauth.services.github import GithubOAuthService from util.config.validators import BaseValidator, ConfigValidationException @@ -7,7 +6,7 @@ class BaseGitHubValidator(BaseValidator): config_key = None @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the OAuth credentials and API endpoint for a Github service. """ github_config = config.get(cls.config_key) if not github_config: diff --git a/util/config/validators/validate_gitlab_trigger.py b/util/config/validators/validate_gitlab_trigger.py index 7f9e8c28e..7871a3fb1 100644 --- a/util/config/validators/validate_gitlab_trigger.py +++ b/util/config/validators/validate_gitlab_trigger.py @@ -1,4 +1,3 @@ -from app import app from oauth.services.gitlab import GitLabOAuthService from util.config.validators import BaseValidator, ConfigValidationException @@ -6,7 +5,7 @@ class GitLabTriggerValidator(BaseValidator): name = "gitlab-trigger" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the OAuth credentials and API endpoint for a GitLab service. """ github_config = config.get('GITLAB_TRIGGER_CONFIG') if not github_config: diff --git a/util/config/validators/validate_google_login.py b/util/config/validators/validate_google_login.py index 80e1537f0..4867045f9 100644 --- a/util/config/validators/validate_google_login.py +++ b/util/config/validators/validate_google_login.py @@ -1,4 +1,3 @@ -from app import app from oauth.services.google import GoogleOAuthService from util.config.validators import BaseValidator, ConfigValidationException @@ -6,7 +5,7 @@ class GoogleLoginValidator(BaseValidator): name = "google-login" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the Google Login client ID and secret. """ google_login_config = config.get('GOOGLE_LOGIN_CONFIG') if not google_login_config: diff --git a/util/config/validators/validate_jwt.py b/util/config/validators/validate_jwt.py index 808e74152..438b93ca8 100644 --- a/util/config/validators/validate_jwt.py +++ b/util/config/validators/validate_jwt.py @@ -1,4 +1,4 @@ -from app import app, OVERRIDE_CONFIG_DIRECTORY +from _init import OVERRIDE_CONFIG_DIRECTORY from data.users.externaljwt import ExternalJWTAuthN from util.config.validators import BaseValidator, ConfigValidationException @@ -6,7 +6,7 @@ class JWTAuthValidator(BaseValidator): name = "jwt" @classmethod - def validate(cls, config, user, user_password, public_key_path=None): + def validate(cls, config, user, user_password, app, public_key_path=None): """ Validates the JWT authentication system. """ if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT': return diff --git a/util/config/validators/validate_keystone.py b/util/config/validators/validate_keystone.py index 415f7958b..553c0487b 100644 --- a/util/config/validators/validate_keystone.py +++ b/util/config/validators/validate_keystone.py @@ -5,7 +5,7 @@ class KeystoneValidator(BaseValidator): name = "keystone" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the Keystone authentication system. """ if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone': return diff --git a/util/config/validators/validate_ldap.py b/util/config/validators/validate_ldap.py index 1c4fb9df7..67221e1a2 100644 --- a/util/config/validators/validate_ldap.py +++ b/util/config/validators/validate_ldap.py @@ -2,17 +2,16 @@ import os import ldap import subprocess -from app import app, config_provider from data.users import LDAP_CERT_FILENAME from data.users.externalldap import LDAPConnection, LDAPUsers from util.config.validators import BaseValidator, ConfigValidationException -from _init import CONF_DIR +from _init import CONF_DIR, config_provider class LDAPValidator(BaseValidator): name = "ldap" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the LDAP connection. """ if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP': return diff --git a/util/config/validators/validate_oidc.py b/util/config/validators/validate_oidc.py index ba94b0362..54baeaefd 100644 --- a/util/config/validators/validate_oidc.py +++ b/util/config/validators/validate_oidc.py @@ -1,4 +1,3 @@ -from app import app from oauth.loginmanager import OAuthLoginManager from oauth.oidc import OIDCLoginService, DiscoveryFailureException from util.config.validators import BaseValidator, ConfigValidationException @@ -7,7 +6,7 @@ class OIDCLoginValidator(BaseValidator): name = "oidc-login" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): client = app.config['HTTPCLIENT'] login_manager = OAuthLoginManager(config, client=client) for service in login_manager.services: diff --git a/util/config/validators/validate_redis.py b/util/config/validators/validate_redis.py index 92909dbf4..9b1254d04 100644 --- a/util/config/validators/validate_redis.py +++ b/util/config/validators/validate_redis.py @@ -6,7 +6,7 @@ class RedisValidator(BaseValidator): name = "redis" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates connecting to redis. """ redis_config = config.get('BUILDLOGS_REDIS', {}) if not 'host' in redis_config: diff --git a/util/config/validators/validate_secscan.py b/util/config/validators/validate_secscan.py index c5efd34b0..845934bf9 100644 --- a/util/config/validators/validate_secscan.py +++ b/util/config/validators/validate_secscan.py @@ -1,6 +1,5 @@ import time -from app import app from boot import setup_jwt_proxy from util.secscan.api import SecurityScannerAPI from util.config.validators import BaseValidator, ConfigValidationException @@ -9,7 +8,7 @@ class SecurityScannerValidator(BaseValidator): name = "security-scanner" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the configuration for talking to a Quay Security Scanner. """ if not config.get('FEATURE_SECURITY_SCANNER', False): return diff --git a/util/config/validators/validate_signer.py b/util/config/validators/validate_signer.py index b44cb3c3d..2f9801f63 100644 --- a/util/config/validators/validate_signer.py +++ b/util/config/validators/validate_signer.py @@ -1,6 +1,6 @@ from StringIO import StringIO -from app import config_provider +from _init import config_provider from util.config.validators import BaseValidator, ConfigValidationException from util.security.signing import SIGNING_ENGINES @@ -8,7 +8,7 @@ class SignerValidator(BaseValidator): name = "signer" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the GPG public+private key pair used for signing converted ACIs. """ if config.get('SIGNING_ENGINE') is None: return diff --git a/util/config/validators/validate_ssl.py b/util/config/validators/validate_ssl.py index ea1ae3188..bd3e2c3e1 100644 --- a/util/config/validators/validate_ssl.py +++ b/util/config/validators/validate_ssl.py @@ -1,4 +1,4 @@ -from app import config_provider +from _init import config_provider from util.config.validators import BaseValidator, ConfigValidationException from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException @@ -8,7 +8,7 @@ class SSLValidator(BaseValidator): name = "ssl" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the SSL configuration (if enabled). """ # Skip if non-SSL. diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index faded5c36..01e9d8cca 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,12 +1,16 @@ -from app import app, ip_resolver, config_provider +from _init import config_provider from storage import get_storage_driver from util.config.validators import BaseValidator, ConfigValidationException +from util.ipresolver import NoopIPResolver + +ip_resolver = NoopIPResolver() + class StorageValidator(BaseValidator): name = "registry-storage" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates registry storage. """ replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) diff --git a/util/config/validators/validate_timemachine.py b/util/config/validators/validate_timemachine.py index 750e1695e..f2671bc13 100644 --- a/util/config/validators/validate_timemachine.py +++ b/util/config/validators/validate_timemachine.py @@ -9,7 +9,7 @@ class TimeMachineValidator(BaseValidator): name = "time-machine" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): if not 'DEFAULT_TAG_EXPIRATION' in config: # Old style config return diff --git a/util/config/validators/validate_torrent.py b/util/config/validators/validate_torrent.py index f63029c46..80fe8916e 100644 --- a/util/config/validators/validate_torrent.py +++ b/util/config/validators/validate_torrent.py @@ -2,7 +2,6 @@ import logging from hashlib import sha1 -from app import app from util.config.validators import BaseValidator, ConfigValidationException from util.registry.torrent import jwt_from_infohash @@ -12,7 +11,7 @@ class BittorrentValidator(BaseValidator): name = "bittorrent" @classmethod - def validate(cls, config, user, user_password): + def validate(cls, config, user, user_password, app): """ Validates the configuration for using BitTorrent for downloads. """ announce_url = config.get('BITTORRENT_ANNOUNCE_URL') if not announce_url: