From 554d4f47a8ad72ec956041ce218718b11c331209 Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Fri, 25 May 2018 15:42:27 -0400 Subject: [PATCH] Change validators to use the validator_context Change InstanceKeys to take a namedtuple for context --- app.py | 6 ++-- endpoints/api/suconfig.py | 7 +++-- oauth/services/gitlab.py | 2 ++ test/test_storageproxy.py | 4 +-- util/__init__.py | 3 ++ util/config/validator.py | 30 +++++++++++++++++-- util/config/validators/__init__.py | 2 +- util/config/validators/validate_access.py | 6 ++-- .../validate_actionlog_archiving.py | 4 ++- .../validators/validate_apptokenauth.py | 4 ++- .../validators/validate_bitbucket_trigger.py | 8 +++-- util/config/validators/validate_database.py | 4 ++- util/config/validators/validate_email.py | 11 +++++-- util/config/validators/validate_github.py | 8 +++-- .../validators/validate_gitlab_trigger.py | 9 +++--- .../validators/validate_google_login.py | 9 ++++-- util/config/validators/validate_jwt.py | 12 ++++++-- util/config/validators/validate_keystone.py | 6 +++- util/config/validators/validate_ldap.py | 5 +++- util/config/validators/validate_oidc.py | 6 ++-- util/config/validators/validate_redis.py | 4 ++- util/config/validators/validate_secscan.py | 9 ++++-- util/config/validators/validate_signer.py | 4 ++- util/config/validators/validate_ssl.py | 3 +- util/config/validators/validate_storage.py | 15 +++++----- .../config/validators/validate_timemachine.py | 4 ++- util/config/validators/validate_torrent.py | 7 +++-- util/secscan/__init__.py | 9 ++++++ util/secscan/api.py | 10 +++---- util/security/instancekeys.py | 26 ++++++++++++---- util/tufmetadata/api.py | 4 +-- 31 files changed, 172 insertions(+), 69 deletions(-) diff --git a/app.py b/app.py index d38e9549c..24071c5be 100644 --- a/app.py +++ b/app.py @@ -50,7 +50,7 @@ from util.metrics.prometheus import PrometheusPlugin from util.saas.cloudwatch import start_cloudwatch_sender from util.secscan.api import SecurityScannerAPI from util.tufmetadata.api import TUFMetadataAPI -from util.security.instancekeys import InstanceKeys +from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config from util.security.signing import Signer @@ -182,7 +182,7 @@ mail = Mail(app) prometheus = PrometheusPlugin(app) metric_queue = MetricQueue(prometheus) chunk_cleanup_queue = WorkQueue(app.config['CHUNK_CLEANUP_QUEUE_NAME'], tf, metric_queue=metric_queue) -instance_keys = InstanceKeys(app) +instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config)) ip_resolver = IPResolver(app) storage = Storage(app, metric_queue, chunk_cleanup_queue, instance_keys, config_provider, ip_resolver) userfiles = Userfiles(app, storage) @@ -196,7 +196,7 @@ authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECT userevents = UserEventsBuilderModule(app) superusers = SuperUserManager(app) signer = Signer(app, config_provider) -instance_keys = InstanceKeys(app) +instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config)) label_validator = LabelValidator(app) build_canceller = BuildCanceller(app) diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index a852150fd..12b4918bb 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -7,7 +7,7 @@ import subprocess from flask import abort -from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY +from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY, ip_resolver from auth.permissions import SuperUserPermission from auth.auth_context import get_authenticated_user from data.database import configure @@ -20,7 +20,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_ from endpoints.common import common_login from util.config.configutil import add_enterprise_config_defaults from util.config.database import sync_database_with_config -from util.config.validator import validate_service_for_config, is_valid_config_upload_filename +from util.config.validator import validate_service_for_config, is_valid_config_upload_filename, ValidatorContext import features @@ -405,6 +405,7 @@ class SuperUserConfigValidate(ApiResource): # this is also safe since this method does not access any information not given in the request. if not config_provider.config_exists() or SuperUserPermission().can(): config = request.get_json()['config'] - return validate_service_for_config(service, config, request.get_json().get('password', ''), app) + validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app, ip_resolver) + return validate_service_for_config(service, validator_context) abort(403) diff --git a/oauth/services/gitlab.py b/oauth/services/gitlab.py index 9b0dcc2ec..344709563 100644 --- a/oauth/services/gitlab.py +++ b/oauth/services/gitlab.py @@ -29,6 +29,8 @@ class GitLabOAuthService(OAuthService): def token_endpoint(self): return OAuthEndpoint(slash_join(self._endpoint(), '/oauth/token')) + # TODO(sam): this signature does not match its parent class. refactor the base method to take the namedtuple URLSchemeAndHostname + # TODO cont: reason I did this was to decouple the app, but it requires more refactoring def validate_client_id_and_secret(self, http_client, url_scheme_and_hostname): # We validate the client ID and secret by hitting the OAuth token exchange endpoint with # the real client ID and secret, but a fake auth code to exchange. Gitlab's implementation will diff --git a/test/test_storageproxy.py b/test/test_storageproxy.py index 174c3c854..3078018dd 100644 --- a/test/test_storageproxy.py +++ b/test/test_storageproxy.py @@ -7,7 +7,7 @@ from flask_testing import LiveServerTestCase from initdb import setup_database_for_testing, finished_database_for_testing from storage import Storage -from util.security.instancekeys import InstanceKeys +from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config _PORT_NUMBER = 5001 @@ -42,7 +42,7 @@ class TestStorageProxy(LiveServerTestCase): 'test': ['FakeStorage', {}], } - instance_keys = InstanceKeys(self.test_app) + instance_keys = InstanceKeys(instance_keys_context_from_app_config(self.test_app.config)) self.storage = Storage(self.test_app, instance_keys=instance_keys) self.test_app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = ['test'] return self.test_app diff --git a/util/__init__.py b/util/__init__.py index f4744897a..53a25fb85 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -2,6 +2,9 @@ def get_app_url(config): """ Returns the application's URL, based on the given config. """ return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME']) +def get_app_url_from_scheme_hostname(url_scheme_and_hostname): + """ Returns the application's URL, based on the given url scheme and hostname. """ + return '%s://%s' % (url_scheme_and_hostname.url_scheme, url_scheme_and_hostname.hostname) def slash_join(*args): """ diff --git a/util/config/validator.py b/util/config/validator.py index febd88d48..3e6aaf787 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -2,6 +2,7 @@ import logging from auth.auth_context import get_authenticated_user from data.users import LDAP_CERT_FILENAME +from util.config import URLSchemeAndHostname from util.config.validators.validate_database import DatabaseValidator from util.config.validators.validate_redis import RedisValidator @@ -64,7 +65,7 @@ VALIDATORS = { AppTokenAuthValidator.name: AppTokenAuthValidator.validate, } -def validate_service_for_config(service, config, password=None, app=None): +def validate_service_for_config(service, validator_context): """ Attempts to validate the configuration for the given service. """ if not service in VALIDATORS: return { @@ -72,7 +73,7 @@ def validate_service_for_config(service, config, password=None, app=None): } try: - VALIDATORS[service](config, get_authenticated_user(), password, app) + VALIDATORS[service](validator_context) return { 'status': True } @@ -92,3 +93,28 @@ def is_valid_config_upload_filename(filename): return True return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES]) + + +class ValidatorContext(object): + """ Context to run validators in, with any additional runtime configuration they need + """ + def __init__(self, config, user_password=None, http_client=None, context=None, + url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None): + self.config = config + self.user = get_authenticated_user() + self.user_password = user_password + self.http_client = http_client + self.context = context + self.scheme_and_hostname = url_scheme_and_hostname + self.jwt_auth_max = jwt_auth_max + self.registry_title = registry_title + self.ip_resolver = ip_resolver + + @classmethod + def from_app(cls, config, user_password, app, ip_resolver): + cls(config, user_password, app.config['HTTP_CLIENT'], app.app_context, + URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']), + app.config.get('JWT_AUTH_MAX_FRESH_S', 300), app.config['REGISTRY_TITLE'], ip_resolver) + + + diff --git a/util/config/validators/__init__.py b/util/config/validators/__init__.py index dfa18e1b0..31268c45f 100644 --- a/util/config/validators/__init__.py +++ b/util/config/validators/__init__.py @@ -15,6 +15,6 @@ class BaseValidator(object): @classmethod @abstractmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Raises Exception if failure to validate. """ pass diff --git a/util/config/validators/validate_access.py b/util/config/validators/validate_access.py index ed4d0ef8a..552e3b176 100644 --- a/util/config/validators/validate_access.py +++ b/util/config/validators/validate_access.py @@ -6,13 +6,15 @@ class AccessSettingsValidator(BaseValidator): name = "access" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): + config = validator_context.config + client = validator_context.http_client + if not config.get('FEATURE_DIRECT_LOGIN', True): # Make sure we have at least one OIDC enabled. github_login = config.get('FEATURE_GITHUB_LOGIN', False) google_login = config.get('FEATURE_GOOGLE_LOGIN', False) - client = app.config['HTTPCLIENT'] login_manager = OAuthLoginManager(config, client=client) custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)] diff --git a/util/config/validators/validate_actionlog_archiving.py b/util/config/validators/validate_actionlog_archiving.py index 256e2c6b8..03c63fe95 100644 --- a/util/config/validators/validate_actionlog_archiving.py +++ b/util/config/validators/validate_actionlog_archiving.py @@ -4,7 +4,9 @@ class ActionLogArchivingValidator(BaseValidator): name = "actionlogarchiving" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): + config = validator_context.config + """ Validates the action log archiving configuration. """ if not config.get('FEATURE_ACTION_LOG_ROTATION', False): return diff --git a/util/config/validators/validate_apptokenauth.py b/util/config/validators/validate_apptokenauth.py index e77e50b4d..be2096117 100644 --- a/util/config/validators/validate_apptokenauth.py +++ b/util/config/validators/validate_apptokenauth.py @@ -4,7 +4,9 @@ class AppTokenAuthValidator(BaseValidator): name = "apptoken-auth" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): + config = validator_context.config + if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken': return diff --git a/util/config/validators/validate_bitbucket_trigger.py b/util/config/validators/validate_bitbucket_trigger.py index db25e43a9..04ae1de04 100644 --- a/util/config/validators/validate_bitbucket_trigger.py +++ b/util/config/validators/validate_bitbucket_trigger.py @@ -1,14 +1,16 @@ from bitbucket import BitBucket -from util import get_app_url +from util import get_app_url_from_scheme_hostname from util.config.validators import BaseValidator, ConfigValidationException class BitbucketTriggerValidator(BaseValidator): name = "bitbucket-trigger" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the config for BitBucket. """ + config = validator_context.config + trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG') if not trigger_config: raise ConfigValidationException('Missing client ID and client secret') @@ -21,7 +23,7 @@ class BitbucketTriggerValidator(BaseValidator): key = trigger_config['CONSUMER_KEY'] secret = trigger_config['CONSUMER_SECRET'] - callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url(app.config)) + callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url(validator_context.scheme_and_hostname)) bitbucket_client = BitBucket(key, secret, callback_url) (result, _, _) = bitbucket_client.get_authorization_url() diff --git a/util/config/validators/validate_database.py b/util/config/validators/validate_database.py index 4484edfbe..30fcc6d0e 100644 --- a/util/config/validators/validate_database.py +++ b/util/config/validators/validate_database.py @@ -7,8 +7,10 @@ class DatabaseValidator(BaseValidator): name = "database" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates connecting to the database. """ + config = validator_context.config + try: validate_database_url(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {})) except OperationalError as ex: diff --git a/util/config/validators/validate_email.py b/util/config/validators/validate_email.py index 4a7756b27..7adb7f6fa 100644 --- a/util/config/validators/validate_email.py +++ b/util/config/validators/validate_email.py @@ -7,9 +7,14 @@ class EmailValidator(BaseValidator): name = "mail" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates sending email. """ - with app.app_context(): + config = validator_context.config + user = validator_context.user + app_context = validator_context.context + registry_title = validator_context.registry_title + + with app_context(): test_app = Flask("mail-test-app") test_app.config.update(config) test_app.config.update({ @@ -18,7 +23,7 @@ class EmailValidator(BaseValidator): }) test_mail = Mail(test_app) - test_msg = Message("Test e-mail from %s" % app.config['REGISTRY_TITLE'], + test_msg = Message("Test e-mail from %s" % registry_title, sender=config.get('MAIL_DEFAULT_SENDER')) test_msg.add_recipient(user.email) test_mail.send(test_msg) diff --git a/util/config/validators/validate_github.py b/util/config/validators/validate_github.py index a0d2aaaf8..9156b8a15 100644 --- a/util/config/validators/validate_github.py +++ b/util/config/validators/validate_github.py @@ -6,8 +6,11 @@ class BaseGitHubValidator(BaseValidator): config_key = None @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the OAuth credentials and API endpoint for a Github service. """ + config = validator_context.config + client = validator_context.http_client + github_config = config.get(cls.config_key) if not github_config: raise ConfigValidationException('Missing GitHub client id and client secret') @@ -29,9 +32,8 @@ class BaseGitHubValidator(BaseValidator): raise ConfigValidationException('Organization restriction must have at least one allowed ' + 'organization') - client = app.config['HTTPCLIENT'] oauth = GithubOAuthService(config, cls.config_key) - result = oauth.validate_client_id_and_secret(client, app.config) + result = oauth.validate_client_id_and_secret(client) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_gitlab_trigger.py b/util/config/validators/validate_gitlab_trigger.py index 90590128c..50c381881 100644 --- a/util/config/validators/validate_gitlab_trigger.py +++ b/util/config/validators/validate_gitlab_trigger.py @@ -1,13 +1,16 @@ from oauth.services.gitlab import GitLabOAuthService -from util.config import URLSchemeAndHostname from util.config.validators import BaseValidator, ConfigValidationException class GitLabTriggerValidator(BaseValidator): name = "gitlab-trigger" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the OAuth credentials and API endpoint for a GitLab service. """ + config = validator_context.config + url_scheme_and_hostname = validator_context.url_scheme_and_hostname + client = validator_context.http_client + github_config = config.get('GITLAB_TRIGGER_CONFIG') if not github_config: raise ConfigValidationException('Missing GitLab client id and client secret') @@ -23,9 +26,7 @@ class GitLabTriggerValidator(BaseValidator): if not github_config.get('CLIENT_SECRET'): raise ConfigValidationException('Missing Client Secret') - client = app.config['HTTPCLIENT'] oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG') - url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']) result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_google_login.py b/util/config/validators/validate_google_login.py index 4867045f9..7af89ff0c 100644 --- a/util/config/validators/validate_google_login.py +++ b/util/config/validators/validate_google_login.py @@ -5,8 +5,11 @@ class GoogleLoginValidator(BaseValidator): name = "google-login" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the Google Login client ID and secret. """ + config = validator_context.config + client = validator_context.http_client + google_login_config = config.get('GOOGLE_LOGIN_CONFIG') if not google_login_config: raise ConfigValidationException('Missing client ID and client secret') @@ -17,8 +20,8 @@ class GoogleLoginValidator(BaseValidator): if not google_login_config.get('CLIENT_SECRET'): raise ConfigValidationException('Missing Client Secret') - client = app.config['HTTPCLIENT'] oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG') - result = oauth.validate_client_id_and_secret(client, app.config) + # TODO(sam): the google oauth doesn't need the app config, but when refactoring pass in the URLSchemeandHostname + result = oauth.validate_client_id_and_secret(client) if not result: raise ConfigValidationException('Invalid client id or client secret') diff --git a/util/config/validators/validate_jwt.py b/util/config/validators/validate_jwt.py index 438b93ca8..83b3bfe19 100644 --- a/util/config/validators/validate_jwt.py +++ b/util/config/validators/validate_jwt.py @@ -6,8 +6,14 @@ class JWTAuthValidator(BaseValidator): name = "jwt" @classmethod - def validate(cls, config, user, user_password, app, public_key_path=None): + def validate(cls, validator_context, public_key_path=None): """ Validates the JWT authentication system. """ + config = validator_context.config + user = validator_context.user + user_password = validator_context.user_password + http_client = validator_context.http_client + jwt_auth_max = validator_context.jwt_auth_max + if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT': return @@ -27,8 +33,8 @@ class JWTAuthValidator(BaseValidator): # the key cannot be found. users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer, OVERRIDE_CONFIG_DIRECTORY, - app.config['HTTPCLIENT'], - app.config.get('JWT_AUTH_MAX_FRESH_S', 300), + http_client, + jwt_auth_max, public_key_path=public_key_path, requires_email=config.get('FEATURE_MAILING', True)) diff --git a/util/config/validators/validate_keystone.py b/util/config/validators/validate_keystone.py index 553c0487b..2d4f7bbe0 100644 --- a/util/config/validators/validate_keystone.py +++ b/util/config/validators/validate_keystone.py @@ -5,8 +5,12 @@ class KeystoneValidator(BaseValidator): name = "keystone" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the Keystone authentication system. """ + config = validator_context.config + user = validator_context.user + user_password = validator_context.user_password + if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone': return diff --git a/util/config/validators/validate_ldap.py b/util/config/validators/validate_ldap.py index 67221e1a2..febb5aec1 100644 --- a/util/config/validators/validate_ldap.py +++ b/util/config/validators/validate_ldap.py @@ -11,8 +11,11 @@ class LDAPValidator(BaseValidator): name = "ldap" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the LDAP connection. """ + config = validator_context.config + user = validator_context.user + user_password = validator_context.user_password if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP': return diff --git a/util/config/validators/validate_oidc.py b/util/config/validators/validate_oidc.py index 54baeaefd..8126d4db7 100644 --- a/util/config/validators/validate_oidc.py +++ b/util/config/validators/validate_oidc.py @@ -6,8 +6,10 @@ class OIDCLoginValidator(BaseValidator): name = "oidc-login" @classmethod - def validate(cls, config, user, user_password, app): - client = app.config['HTTPCLIENT'] + def validate(cls, validator_context): + config = validator_context.config + client = validator_context.http_client + login_manager = OAuthLoginManager(config, client=client) for service in login_manager.services: if not isinstance(service, OIDCLoginService): diff --git a/util/config/validators/validate_redis.py b/util/config/validators/validate_redis.py index 9b1254d04..55beffb84 100644 --- a/util/config/validators/validate_redis.py +++ b/util/config/validators/validate_redis.py @@ -6,8 +6,10 @@ class RedisValidator(BaseValidator): name = "redis" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates connecting to redis. """ + config = validator_context.config + redis_config = config.get('BUILDLOGS_REDIS', {}) if not 'host' in redis_config: raise ConfigValidationException('Missing redis hostname') diff --git a/util/config/validators/validate_secscan.py b/util/config/validators/validate_secscan.py index 845934bf9..0e1eac870 100644 --- a/util/config/validators/validate_secscan.py +++ b/util/config/validators/validate_secscan.py @@ -8,13 +8,16 @@ class SecurityScannerValidator(BaseValidator): name = "security-scanner" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the configuration for talking to a Quay Security Scanner. """ + config = validator_context.config + client = validator_context.http_client + app = None #TODO(sam) validate with joey's pr about security scanner api + if not config.get('FEATURE_SECURITY_SCANNER', False): return - client = app.config['HTTPCLIENT'] - api = SecurityScannerAPI(app, config, None, client=client, skip_validation=True) + api = SecurityScannerAPI(app.config, config, None, client=client, skip_validation=True) if not config.get('TESTING', False): # Generate a temporary Quay key to use for signing the outgoing requests. diff --git a/util/config/validators/validate_signer.py b/util/config/validators/validate_signer.py index 2f9801f63..241ecf227 100644 --- a/util/config/validators/validate_signer.py +++ b/util/config/validators/validate_signer.py @@ -8,8 +8,10 @@ class SignerValidator(BaseValidator): name = "signer" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the GPG public+private key pair used for signing converted ACIs. """ + config = validator_context.config + if config.get('SIGNING_ENGINE') is None: return diff --git a/util/config/validators/validate_ssl.py b/util/config/validators/validate_ssl.py index bd3e2c3e1..b76645e82 100644 --- a/util/config/validators/validate_ssl.py +++ b/util/config/validators/validate_ssl.py @@ -8,8 +8,9 @@ class SSLValidator(BaseValidator): name = "ssl" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the SSL configuration (if enabled). """ + config = validator_context.config # Skip if non-SSL. if config.get('PREFERRED_URL_SCHEME', 'http') != 'https': diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index 01e9d8cca..bb3c0e685 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,20 +1,21 @@ from _init import config_provider from storage import get_storage_driver from util.config.validators import BaseValidator, ConfigValidationException -from util.ipresolver import NoopIPResolver - -ip_resolver = NoopIPResolver() class StorageValidator(BaseValidator): name = "registry-storage" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates registry storage. """ + config = validator_context.config + client = validator_context.http_client + ip_resolver = validator_context.ip_resolver + replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) - providers = _get_storage_providers(config).items() + providers = _get_storage_providers(config, ip_resolver).items() if not providers: raise ConfigValidationException('Storage configuration required') @@ -25,7 +26,7 @@ class StorageValidator(BaseValidator): 'with storage replication') # Run validation on the driver. - driver.validate(app.config['HTTPCLIENT']) + driver.validate(client) # Run setup on the driver if the read/write succeeded. driver.setup() @@ -34,7 +35,7 @@ class StorageValidator(BaseValidator): raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg)) -def _get_storage_providers(config): +def _get_storage_providers(config, ip_resolver): storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {}) drivers = {} diff --git a/util/config/validators/validate_timemachine.py b/util/config/validators/validate_timemachine.py index f2671bc13..c246a2686 100644 --- a/util/config/validators/validate_timemachine.py +++ b/util/config/validators/validate_timemachine.py @@ -9,7 +9,9 @@ class TimeMachineValidator(BaseValidator): name = "time-machine" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): + config = validator_context.config + if not 'DEFAULT_TAG_EXPIRATION' in config: # Old style config return diff --git a/util/config/validators/validate_torrent.py b/util/config/validators/validate_torrent.py index 80fe8916e..d8137e12c 100644 --- a/util/config/validators/validate_torrent.py +++ b/util/config/validators/validate_torrent.py @@ -11,15 +11,16 @@ class BittorrentValidator(BaseValidator): name = "bittorrent" @classmethod - def validate(cls, config, user, user_password, app): + def validate(cls, validator_context): """ Validates the configuration for using BitTorrent for downloads. """ + config = validator_context.config + client = validator_context.http_client + announce_url = config.get('BITTORRENT_ANNOUNCE_URL') if not announce_url: raise ConfigValidationException('Missing announce URL') # Ensure that the tracker is reachable and accepts requests signed with a registry key. - client = app.config['HTTPCLIENT'] - params = { 'info_hash': sha1('test').digest(), 'peer_id': '-QUAY00-6wfG2wk6wWLc', diff --git a/util/secscan/__init__.py b/util/secscan/__init__.py index 1e3ac04aa..7871f28b4 100644 --- a/util/secscan/__init__.py +++ b/util/secscan/__init__.py @@ -107,3 +107,12 @@ def get_priority_for_index(index): return priority return 'Unknown' + + +def create_url_from_app(app): + """ + Higher order function that returns a function that when called, will generate a url for that given app + :param app: Flask app + :return: + type: Flask -> (str -> url) + """ diff --git a/util/secscan/api.py b/util/secscan/api.py index fee1b97aa..4bae2034d 100644 --- a/util/secscan/api.py +++ b/util/secscan/api.py @@ -16,7 +16,7 @@ from util import get_app_url, slash_join from util.abchelpers import nooper from util.failover import failover, FailoverException from util.secscan.validator import SecurityConfigValidator -from util.security.instancekeys import InstanceKeys +from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config from util.security.registry_jwt import generate_bearer_token, build_context_and_subject from _init import CONF_DIR @@ -150,10 +150,10 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface): class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): """ Helper class for talking to the Security Scan service (Clair). """ - def __init__(self, app, config, storage, client=None): - self._app = app + def __init__(self, app_config, config, storage, client=None): + self._app_config = app_config self._config = config - self._instance_keys = InstanceKeys(app) + self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app_config)) self._client = client or config['HTTPCLIENT'] self._storage = storage self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE'] @@ -183,7 +183,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface): repository_and_namespace = '/'.join([namespace_name, repo_name]) # Generate the JWT which will authorize this - audience = self._app.config['SERVER_HOSTNAME'] + audience = self._app_config['SERVER_HOSTNAME'] context, subject = build_context_and_subject() access = [{ 'type': 'repository', diff --git a/util/security/instancekeys.py b/util/security/instancekeys.py index 75269552c..c5b12c3b0 100644 --- a/util/security/instancekeys.py +++ b/util/security/instancekeys.py @@ -1,3 +1,4 @@ +from collections import namedtuple from cachetools import lru_cache from data import model from util.expiresdict import ExpiresDict, ExpiresEntry @@ -25,9 +26,10 @@ class InstanceKeys(object): """ InstanceKeys defines a helper class for interacting with the Quay instance service keys used for JWT signing of registry tokens as well as requests from Quay to other services such as Clair. Each container will have a single registered instance key. + :param keys_context: InstanceKeysContext """ - def __init__(self, app): - self.app = app + def __init__(self, keys_context): + self.keys_context = keys_context self.instance_keys = ExpiresDict(self._load_instance_keys) def clear_cache(self): @@ -45,24 +47,24 @@ class InstanceKeys(object): @property def service_name(self): """ Returns the name of the instance key's service (i.e. 'quay'). """ - return self.app.config['INSTANCE_SERVICE_KEY_SERVICE'] + return self.keys_context.instance_key_service @property def service_key_expiration(self): """ Returns the defined expiration for instance service keys, in minutes. """ - return self.app.config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120) + return self.keys_context.service_key_expiration @property @lru_cache(maxsize=1) def local_key_id(self): """ Returns the ID of the local instance service key. """ - return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_KID_LOCATION']) + return _load_file_contents(self.keys_context.service_key_kid_location) @property @lru_cache(maxsize=1) def local_private_key(self): """ Returns the private key of the local instance service key. """ - return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_LOCATION']) + return _load_file_contents(self.keys_context.service_key_location) def get_service_key_public_key(self, kid): """ Returns the public key associated with the given instance service key or None if none. """ @@ -77,3 +79,15 @@ def _load_file_contents(path): """ Returns the contents of the specified file path. """ with open(path) as f: return f.read() + + +InstanceKeysContext = namedtuple('InstanceKeysContext', ['instance_key_service', + 'service_key_expiration', + 'service_key_kid_location', + 'service_key_location']) + +def instance_keys_context_from_app_config(app_config): + return InstanceKeysContext(app_config['INSTANCE_SERVICE_KEY_SERVICE'], + app_config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120), + app_config['INSTANCE_SERVICE_KEY_KID_LOCATION'], + app_config['INSTANCE_SERVICE_KEY_LOCATION']) diff --git a/util/tufmetadata/api.py b/util/tufmetadata/api.py index 9c039477a..0f9f123b7 100644 --- a/util/tufmetadata/api.py +++ b/util/tufmetadata/api.py @@ -11,7 +11,7 @@ import requests from data.database import CloseForLongOperation from util.abchelpers import nooper from util.failover import failover, FailoverException -from util.security.instancekeys import InstanceKeys +from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config from util.security.registry_jwt import (build_context_and_subject, generate_bearer_token, SIGNER_TUF_ROOT) @@ -108,7 +108,7 @@ class NoopTUFMetadataAPI(TUFMetadataAPIInterface): class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface): def __init__(self, app, config, client=None): self._app = app - self._instance_keys = InstanceKeys(app) + self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config)) self._config = config self._client = client or config['HTTPCLIENT'] self._gun_prefix = config['TUF_GUN_PREFIX'] or config['SERVER_HOSTNAME']