vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add a security scanner api config object for params

Browse source 
Change SecScanAPI to use a uri creation func instead of test context

Pass config provider through validator context

Remove app config dependency for validators
This commit is contained in:
Sam Chow 2018-05-29 13:50:51 -04:00
parent 554d4f47a8
commit 7df8ed4a60
47 changed files with 305 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
app.py
View file

@ -35,8 +35,9 @@ from oauth.services.github import GithubOAuthService
from oauth.services.gitlab import GitLabOAuthService
from oauth.loginmanager import OAuthLoginManager
from storage import Storage
from util.config import URLSchemeAndHostname
from util.log import filter_logs
from util import get_app_url
from util import get_app_url, create_uri_func_from_context
from util.ipresolver import IPResolver
from util.saas.analytics import Analytics
from util.saas.useranalytics import UserAnalytics
@ -50,7 +51,7 @@ from util.metrics.prometheus import PrometheusPlugin
from util.saas.cloudwatch import start_cloudwatch_sender
from util.secscan.api import SecurityScannerAPI
from util.tufmetadata.api import TUFMetadataAPI
from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
from util.security.instancekeys import InstanceKeys
from util.security.signing import Signer
@ -182,7 +183,7 @@ mail = Mail(app)
prometheus = PrometheusPlugin(app)
metric_queue = MetricQueue(prometheus)
chunk_cleanup_queue = WorkQueue(app.config['CHUNK_CLEANUP_QUEUE_NAME'], tf, metric_queue=metric_queue)
instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config))
instance_keys = InstanceKeys(app)
ip_resolver = IPResolver(app)
storage = Storage(app, metric_queue, chunk_cleanup_queue, instance_keys, config_provider, ip_resolver)
userfiles = Userfiles(app, storage)
@ -196,7 +197,7 @@ authentication = UserAuthentication(app, config_provider, OVERRIDE_CONFIG_DIRECT
userevents = UserEventsBuilderModule(app)
superusers = SuperUserManager(app)
signer = Signer(app, config_provider)
instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config))
instance_keys = InstanceKeys(app)
label_validator = LabelValidator(app)
build_canceller = BuildCanceller(app)
@ -228,7 +229,11 @@ namespace_gc_queue = WorkQueue(app.config['NAMESPACE_GC_QUEUE_NAME'], tf, has_na
all_queues = [image_replication_queue, dockerfile_build_queue, notification_queue,
secscan_notification_queue, chunk_cleanup_queue, namespace_gc_queue]
secscan_api = SecurityScannerAPI(app, app.config, storage)
url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
secscan_api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
uri_creator=create_uri_func_from_context(app.test_request_context('/'), url_scheme_and_hostname),
instance_keys=instance_keys)
tuf_metadata_api = TUFMetadataAPI(app, app.config)
# Check for a key in config. If none found, generate a new signing key for Docker V2 manifests.

5
endpoints/api/suconfig.py
View file

@ -405,7 +405,10 @@ class SuperUserConfigValidate(ApiResource):
# this is also safe since this method does not access any information not given in the request.
if not config_provider.config_exists() or SuperUserPermission().can():
config = request.get_json()['config']
validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app, ip_resolver)
validator_context = ValidatorContext.from_app(config, request.get_json().get('password', ''), app,
ip_resolver=ip_resolver,
config_provider=config_provider)
return validate_service_for_config(service, validator_context)
abort(403)

2
endpoints/githubtrigger.py
View file

@ -10,7 +10,6 @@ from auth.decorators import require_session_login
from auth.permissions import AdministerRepositoryPermission
from data import model
from endpoints.decorators import route_show_if, parse_repository_name
from util.config import URLSchemeAndHostname
from util.http import abort
@ -27,7 +26,6 @@ def attach_github_build_trigger(namespace_name, repo_name):
permission = AdministerRepositoryPermission(namespace_name, repo_name)
if permission.can():
code = request.args.get('code')
# url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
token = github_trigger.exchange_code_for_token(app.config, client, code)
repo = model.repository.get_repository(namespace_name, repo_name)
if not repo:

2
endpoints/gitlabtrigger.py
View file

@ -10,7 +10,6 @@ from auth.decorators import require_session_login
from auth.permissions import AdministerRepositoryPermission
from data import model
from endpoints.decorators import route_show_if
from util.config import URLSchemeAndHostname
from util.http import abort
@ -35,7 +34,6 @@ def attach_gitlab_build_trigger():
permission = AdministerRepositoryPermission(namespace, repository)
if permission.can():
code = request.args.get('code')
# url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
token = gitlab_trigger.exchange_code_for_token(app.config, client, code,
redirect_suffix='/trigger')
if not token:

2
oauth/login.py
View file

@ -6,7 +6,6 @@ from six import add_metaclass
import features
from oauth.base import OAuthService, OAuthExchangeCodeException, OAuthGetUserInfoException
from util.config import URLSchemeAndHostname
logger = logging.getLogger(__name__)
@ -65,7 +64,6 @@ class OAuthLoginService(OAuthService):
# Retrieve the token for the OAuth code.
try:
# url_scheme_and_hostname = URLSchemeAndHostname(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
token = self.exchange_code_for_token(app_config, http_client, code,
redirect_suffix=redirect_suffix,
form_encode=self.requires_form_encoding())

3
oauth/services/github.py
View file

@ -75,7 +75,8 @@ class GithubOAuthService(OAuthLoginService):
def orgs_endpoint(self):
return slash_join(self._api_endpoint(), 'user/orgs')
def validate_client_id_and_secret(self, http_client, app_config):
# TODO(sam): refactor the base method to not take app config
def validate_client_id_and_secret(self, http_client):
# First: Verify that the github endpoint is actually Github by checking for the
# X-GitHub-Request-Id here.
api_endpoint = self._api_endpoint()

3
oauth/services/google.py
View file

@ -41,7 +41,8 @@ class GoogleOAuthService(OAuthLoginService):
def requires_form_encoding(self):
return True
def validate_client_id_and_secret(self, http_client, app_config):
# TODO(sam): this signature does not match its parent class. refactor the base method to take the namedtuple URLSchemeAndHostname
def validate_client_id_and_secret(self, http_client):
# To verify the Google client ID and secret, we hit the
# https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client
# ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned

11
test/test_secscan.py
View file

@ -8,11 +8,14 @@ from data.database import Image, IMAGE_NOT_SCANNED_ENGINE_VERSION
from endpoints.v2 import v2_bp
from initdb import setup_database_for_testing, finished_database_for_testing
from notifications.notificationevent import VulnerabilityFoundEvent
from util import create_uri_func_from_context
from util.config import URLSchemeAndHostname
from util.morecollections import AttrDict
from util.secscan.api import SecurityScannerAPI, APIRequestFailure
from util.secscan.analyzer import LayerAnalyzer
from util.secscan.fake import fake_security_scanner
from util.secscan.notifier import SecurityNotificationHandler, ProcessNotificationPageResult
from util.security.instancekeys import InstanceKeys
from workers.security_notification_worker import SecurityNotificationWorker
@ -42,7 +45,13 @@ class TestSecurityScanner(unittest.TestCase):
self.ctx = app.test_request_context()
self.ctx.__enter__()
self.api = SecurityScannerAPI(app, app.config, storage)
url_scheme_and_hostname = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
instance_keys = InstanceKeys(app)
self.api = SecurityScannerAPI(app.config, storage, app.config['SERVER_HOSTNAME'], app.config['HTTPCLIENT'],
uri_creator=create_uri_func_from_context(app.test_request_context('/'),
url_scheme_and_hostname),
instance_keys=instance_keys)
def tearDown(self):
storage.remove(['local_us'], 'supports_direct_download')

4
test/test_storageproxy.py
View file

@ -7,7 +7,7 @@ from flask_testing import LiveServerTestCase
from initdb import setup_database_for_testing, finished_database_for_testing
from storage import Storage
from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
from util.security.instancekeys import InstanceKeys
_PORT_NUMBER = 5001
@ -42,7 +42,7 @@ class TestStorageProxy(LiveServerTestCase):
'test': ['FakeStorage', {}],
}
instance_keys = InstanceKeys(instance_keys_context_from_app_config(self.test_app.config))
instance_keys = InstanceKeys(self.test_app)
self.storage = Storage(self.test_app, instance_keys=instance_keys)
self.test_app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = ['test']
return self.test_app

13
util/__init__.py
View file

@ -1,3 +1,6 @@
from flask import url_for
from urlparse import urljoin
def get_app_url(config):
""" Returns the application's URL, based on the given config. """
return '%s://%s' % (config['PREFERRED_URL_SCHEME'], config['SERVER_HOSTNAME'])
@ -19,3 +22,13 @@ def slash_join(*args):
args = [rmslash(path) for path in args]
return '/'.join(args)
def create_uri_func_from_context(context, url_scheme_and_hostname):
def create_uri(repository_and_namespace, checksum):
with context:
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=checksum)
return urljoin(get_app_url_from_scheme_hostname(url_scheme_and_hostname), relative_layer_url)
return create_uri

4
util/config/provider/basefileprovider.py
View file

@ -68,3 +68,7 @@ class BaseFileProvider(BaseProvider):
return True
return False
def get_config_root(self):
return self.config_volume

5
util/config/provider/baseprovider.py
View file

@ -123,3 +123,8 @@ class BaseProvider(object):
def get_volume_path(self, directory, filename):
""" Helper for constructing relative file paths, which may differ between providers.
For example, kubernetes can't have subfolders in configmaps """
@abstractmethod
def get_config_root(self):
""" Returns the config root directory. """

5
util/config/provider/testprovider.py
View file

@ -10,6 +10,11 @@ REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', '
class TestConfigProvider(BaseProvider):
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
the real file system. """
def get_config_root(self):
raise Exception('Test Config does not have a config root')
# return ''
def __init__(self):
self.clear()

31
util/config/validator.py
View file

@ -2,6 +2,7 @@ import logging
from auth.auth_context import get_authenticated_user
from data.users import LDAP_CERT_FILENAME
from util import create_uri_func_from_context
from util.config import URLSchemeAndHostname
from util.config.validators.validate_database import DatabaseValidator
@ -99,22 +100,40 @@ class ValidatorContext(object):
""" Context to run validators in, with any additional runtime configuration they need
"""
def __init__(self, config, user_password=None, http_client=None, context=None,
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None):
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
uri_creator=None, config_provider=None):
self.config = config
self.user = get_authenticated_user()
self.user_password = user_password
self.http_client = http_client
self.context = context
self.scheme_and_hostname = url_scheme_and_hostname
self.url_scheme_and_hostname = url_scheme_and_hostname
self.jwt_auth_max = jwt_auth_max
self.registry_title = registry_title
self.ip_resolver = ip_resolver
self.feature_sec_scanner = feature_sec_scanner
self.is_testing = is_testing
self.uri_creator = uri_creator
self.config_provider = config_provider
@classmethod
def from_app(cls, config, user_password, app, ip_resolver):
cls(config, user_password, app.config['HTTP_CLIENT'], app.app_context,
URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME']),
app.config.get('JWT_AUTH_MAX_FRESH_S', 300), app.config['REGISTRY_TITLE'], ip_resolver)
def from_app(cls, config, user_password, app, ip_resolver, client=None, config_provider=None):
url_scheme = URLSchemeAndHostname(app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'])
cls(config,
user_password,
client or app.config['HTTPCLIENT'],
app.app_context,
url_scheme,
app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
app.config['REGISTRY_TITLE'],
ip_resolver,
app.config.get('FEATURE_SECURITY_SCANNER', False),
app.config.get('TESTING', False),
create_uri_func_from_context(app.test_request_context('/'), url_scheme),
config_provider)

5
util/config/validators/test/test_validate_access.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_access import AccessSettingsValidator
@ -17,6 +18,6 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, expected_excepti
if expected_exception is not None:
with pytest.raises(expected_exception):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

11
util/config/validators/test/test_validate_actionlog_archiving.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
@ -12,7 +13,7 @@ from test.fixtures import *
])
def test_skip_validate_actionlog(unvalidated_config, app):
validator = ActionLogArchivingValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('config, expected_error', [
@ -33,19 +34,19 @@ def test_invalid_config(config, expected_error, app):
validator = ActionLogArchivingValidator()
with pytest.raises(ConfigValidationException) as ipe:
validator.validate(config, None, None)
validator.validate(ValidatorContext(config))
assert ipe.value.message == expected_error
def test_valid_config(app):
config = {
config = ValidatorContext({
'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'somepath',
'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
'DISTRIBUTED_STORAGE_CONFIG': {
'somelocation': {},
},
}
})
validator = ActionLogArchivingValidator()
validator.validate(config, None, None)
validator.validate(config)

9
util/config/validators/test/test_validate_apptokenauth.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
@ -15,15 +16,15 @@ def test_validate_invalid_auth_config(unvalidated_config, app):
validator = AppTokenAuthValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_auth(app):
config = {
config = ValidatorContext({
'AUTHENTICATION_TYPE': 'AppToken',
'FEATURE_APP_SPECIFIC_TOKENS': True,
'FEATURE_DIRECT_LOGIN': False,
}
})
validator = AppTokenAuthValidator()
validator.validate(config, None, None)
validator.validate(config)

21
util/config/validators/test/test_validate_bitbucket_trigger.py
View file

@ -2,22 +2,24 @@ import pytest
from httmock import urlmatch, HTTMock
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'BITBUCKET_TRIGGER_CONFIG': {}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}}),
({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}}),
(ValidatorContext({})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
])
def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
validator = BitbucketTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(unvalidated_config)
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@ -32,11 +34,16 @@ def test_validate_bitbucket_trigger(app):
with HTTMock(handler):
validator = BitbucketTriggerValidator()
validator.validate({
unvalidated_config = ValidatorContext({
'BITBUCKET_TRIGGER_CONFIG': {
'CONSUMER_KEY': 'foo',
'CONSUMER_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
validator.validate(unvalidated_config)
assert url_hit[0]

15
util/config/validators/test/test_validate_database.py
View file

@ -1,22 +1,23 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_database import DatabaseValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
(None, None, None, TypeError),
({}, None, None, KeyError),
({'DB_URI': 'sqlite:///:memory:'}, None, None, None),
({'DB_URI': 'invalid:///:memory:'}, None, None, KeyError),
({'DB_NOTURI': 'sqlite:///:memory:'}, None, None, KeyError),
(ValidatorContext(None), None, None, TypeError),
(ValidatorContext({}), None, None, KeyError),
(ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
(ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
(ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
])
def test_validate_database(unvalidated_config, user, user_password, expected, app):
validator = DatabaseValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

11
util/config/validators/test/test_validate_github.py
View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
@ -36,7 +38,7 @@ def test_validate_invalid_github_config(github_config, github_validator, app):
with pytest.raises(ConfigValidationException):
unvalidated_config = {}
unvalidated_config[github_validator.config_key] = github_config
github_validator.validate(unvalidated_config, None, None)
github_validator.validate(ValidatorContext(unvalidated_config))
def test_validate_github(github_validator, app):
url_hit = [False, False]
@ -52,13 +54,16 @@ def test_validate_github(github_validator, app):
return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
with HTTMock(app_handler, handler):
github_validator.validate({
unvalidated_config = ValidatorContext({
github_validator.config_key: {
'GITHUB_ENDPOINT': 'http://somehost',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
github_validator.validate(unvalidated_config)
assert url_hit[0]
assert url_hit[1]

13
util/config/validators/test/test_validate_gitlab_trigger.py
View file

@ -3,6 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
@ -18,7 +21,7 @@ def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
validator = GitLabTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@ -30,12 +33,16 @@ def test_validate_gitlab_enterprise_trigger(app):
with HTTMock(handler):
validator = GitLabTriggerValidator()
validator.validate({
unvalidated_config = ValidatorContext({
'GITLAB_TRIGGER_CONFIG': {
'GITLAB_ENDPOINT': 'http://somegitlab',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
validator.validate(unvalidated_config)
assert url_hit[0]

12
util/config/validators/test/test_validate_google_login.py
View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_google_login import GoogleLoginValidator
@ -17,7 +19,7 @@ def test_validate_invalid_google_login_config(unvalidated_config, app):
validator = GoogleLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_google_login(app):
url_hit = [False]
@ -29,11 +31,15 @@ def test_validate_google_login(app):
validator = GoogleLoginValidator()
with HTTMock(handler):
validator.validate({
unvalidated_config = ValidatorContext({
'GOOGLE_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

24
util/config/validators/test/test_validate_jwt.py
View file

@ -1,5 +1,7 @@
import pytest
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_jwt import JWTAuthValidator
from util.morecollections import AttrDict
@ -7,6 +9,7 @@ from util.morecollections import AttrDict
from test.test_external_jwt_authn import fake_jwt
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
@ -14,7 +17,9 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
JWTAuthValidator.validate(unvalidated_config, None, None, app)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
@ -24,7 +29,9 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(unvalidated_config, None, None, app)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
@pytest.mark.parametrize('username, password, expected_exception', [
@ -42,10 +49,15 @@ def test_validated_jwt(username, password, expected_exception, app):
config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
unvalidated_config.config_provider = config_provider
unvalidated_config.http_client = build_requests_session()
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
else:
JWTAuthValidator.validate(config, AttrDict(dict(username=username)), password, app,
public_key_path=jwt_auth.public_key_path)
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)

13
util/config/validators/test/test_validate_keystone.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_keystone import KeystoneValidator
from util.morecollections import AttrDict
@ -13,7 +14,7 @@ from test.fixtures import *
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'Keystone'}),
@ -25,7 +26,7 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(unvalidated_config, None, None)
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('username, password, expected_exception', [
@ -45,8 +46,12 @@ def test_validated_keystone(username, password, expected_exception, app):
config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
config['KEYSTONE_ADMIN_TENANT'] = 'admintenant'
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)
else:
KeystoneValidator.validate(config, AttrDict(dict(username=username)), password)
KeystoneValidator.validate(unvalidated_config)

23
util/config/validators/test/test_validate_ldap.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ldap import LDAPValidator
from util.morecollections import AttrDict
@ -7,13 +8,16 @@ from util.morecollections import AttrDict
from test.test_ldap import mock_ldap
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
LDAPValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'LDAP'}),
@ -21,7 +25,9 @@ def test_validate_noop(unvalidated_config, app):
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
LDAPValidator.validate(config)
@pytest.mark.parametrize('uri', [
@ -39,7 +45,9 @@ def test_invalid_uri(uri, app):
config['LDAP_URI'] = uri
with pytest.raises(ConfigValidationException):
LDAPValidator.validate(config, None, None)
config = ValidatorContext(config)
config.config_provider = config_provider
LDAPValidator.validate(config)
@pytest.mark.parametrize('username, password, expected_exception', [
@ -56,10 +64,15 @@ def test_validated_ldap(username, password, expected_exception, app):
config['LDAP_ADMIN_PASSWD'] = 'password'
config['LDAP_USER_RDN'] = ['ou=employees']
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
unvalidated_config.config_provider = config_provider
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)
else:
with mock_ldap():
LDAPValidator.validate(config, AttrDict(dict(username=username)), password)
LDAPValidator.validate(unvalidated_config)

11
util/config/validators/test/test_validate_oidc.py
View file

@ -3,7 +3,9 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from oauth.oidc import OIDC_WELLKNOWN
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_oidc import OIDCLoginValidator
@ -19,7 +21,7 @@ def test_validate_invalid_oidc_login_config(unvalidated_config, app):
validator = OIDCLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_oidc_login(app):
url_hit = [False]
@ -33,13 +35,16 @@ def test_validate_oidc_login(app):
with HTTMock(handler):
validator = OIDCLoginValidator()
validator.validate({
unvalidated_config = ValidatorContext({
'SOMETHING_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
'OIDC_SERVER': 'http://someserver',
'DEBUGGING': True, # Allows for HTTP.
},
}, None, None)
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

12
util/config/validators/test/test_validate_redis.py
View file

@ -5,10 +5,13 @@ from mock import patch
from mockredis import mock_strict_redis_client
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_redis import RedisValidator
from test.fixtures import *
from util.morecollections import AttrDict
@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
({}, None, None, False, ConfigValidationException),
@ -19,8 +22,13 @@ from test.fixtures import *
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
validator = RedisValidator()
unvalidated_config = ValidatorContext(unvalidated_config)
unvalidated_config.user = AttrDict(dict(username=user))
unvalidated_config.user_password = user_password
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config, user, user_password)
validator.validate(unvalidated_config)

18
util/config/validators/test/test_validate_secscan.py
View file

@ -1,6 +1,8 @@
import pytest
from util.config.validators import ConfigValidationException
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.secscan.fake import fake_security_scanner
@ -10,7 +12,11 @@ from test.fixtures import *
({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
])
def test_validate_noop(unvalidated_config, app):
SecurityScannerValidator.validate(unvalidated_config, None, None)
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True)
unvalidated_config.http_client = build_requests_session()
unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
SecurityScannerValidator.validate(unvalidated_config)
@pytest.mark.parametrize('unvalidated_config, expected_error', [
@ -29,9 +35,13 @@ def test_validate_noop(unvalidated_config, app):
}, None),
])
def test_validate(unvalidated_config, expected_error, app):
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True)
unvalidated_config.http_client = build_requests_session()
unvalidated_config.url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
with fake_security_scanner(hostname='fakesecurityscanner'):
if expected_error is not None:
with pytest.raises(expected_error):
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)
else:
SecurityScannerValidator.validate(unvalidated_config, None, None)
SecurityScannerValidator.validate(unvalidated_config)

5
util/config/validators/test/test_validate_signer.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_signer import SignerValidator
@ -14,6 +15,6 @@ def test_validate_signer(unvalidated_config, expected, app):
validator = SignerValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))

10
util/config/validators/test/test_validate_ssl.py
View file

@ -3,11 +3,13 @@ import pytest
from mock import patch
from tempfile import NamedTemporaryFile
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from test.test_ssl_util import generate_test_cert
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
@ -16,7 +18,7 @@ from test.fixtures import *
])
def test_skip_validate_ssl(unvalidated_config, app):
validator = SSLValidator()
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('cert, expected_error, error_message', [
@ -54,11 +56,13 @@ def test_validate_ssl(cert, expected_error, error_message, app):
with patch('app.config_provider.volume_file_exists', return_true):
with patch('app.config_provider.get_volume_file', get_volume_file):
validator = SSLValidator()
config = ValidatorContext(config)
config.config_provider = config_provider
if expected_error is not None:
with pytest.raises(expected_error) as ipe:
validator.validate(config, None, None)
validator.validate(config)
assert ipe.value.message == error_message
else:
validator.validate(config, None, None)
validator.validate(config)

9
util/config/validators/test/test_validate_storage.py
View file

@ -1,6 +1,7 @@
import moto
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_storage import StorageValidator
@ -16,15 +17,15 @@ def test_validate_storage(unvalidated_config, expected, app):
validator = StorageValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(unvalidated_config, None, None)
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_s3_storage(app):
validator = StorageValidator()
with moto.mock_s3():
with pytest.raises(ConfigValidationException) as ipe:
validator.validate({
validator.validate(ValidatorContext({
'DISTRIBUTED_STORAGE_CONFIG': {
'default': ('S3Storage', {
's3_access_key': 'invalid',
@ -33,6 +34,6 @@ def test_validate_s3_storage(app):
'storage_path': ''
}),
}
}, None, None)
}))
assert ipe.value.message == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

7
util/config/validators/test/test_validate_timemachine.py
View file

@ -1,5 +1,6 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_timemachine import TimeMachineValidator
@ -7,7 +8,7 @@ from util.config.validators.validate_timemachine import TimeMachineValidator
({}),
])
def test_validate_noop(unvalidated_config):
TimeMachineValidator.validate(unvalidated_config, None, None)
TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
from test.fixtures import *
@ -25,7 +26,7 @@ def test_validate(default_exp, options, expected_exception, app):
if expected_exception is not None:
with pytest.raises(ConfigValidationException) as cve:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))
assert str(cve.value) == str(expected_exception)
else:
TimeMachineValidator.validate(config, None, None)
TimeMachineValidator.validate(ValidatorContext(config))

12
util/config/validators/test/test_validate_torrent.py
View file

@ -2,6 +2,8 @@ import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_torrent import BittorrentValidator
@ -23,8 +25,14 @@ def test_validate_torrent(unvalidated_config, expected, app):
validator = BittorrentValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert not announcer_hit[0]
else:
validator.validate(unvalidated_config, None, None)
config = ValidatorContext(unvalidated_config)
config.http_client = build_requests_session()
validator.validate(config)
assert announcer_hit[0]

2
util/config/validators/validate_bitbucket_trigger.py
View file

@ -23,7 +23,7 @@ class BitbucketTriggerValidator(BaseValidator):
key = trigger_config['CONSUMER_KEY']
secret = trigger_config['CONSUMER_SECRET']
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url(validator_context.scheme_and_hostname))
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (get_app_url_from_scheme_hostname(validator_context.url_scheme_and_hostname))
bitbucket_client = BitBucket(key, secret, callback_url)
(result, _, _) = bitbucket_client.get_authorization_url()

1
util/config/validators/validate_google_login.py
View file

@ -21,7 +21,6 @@ class GoogleLoginValidator(BaseValidator):
raise ConfigValidationException('Missing Client Secret')
oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
# TODO(sam): the google oauth doesn't need the app config, but when refactoring pass in the URLSchemeandHostname
result = oauth.validate_client_id_and_secret(client)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

8
util/config/validators/validate_jwt.py
View file

@ -1,4 +1,4 @@
from _init import OVERRIDE_CONFIG_DIRECTORY
import os
from data.users.externaljwt import ExternalJWTAuthN
from util.config.validators import BaseValidator, ConfigValidationException
@ -13,6 +13,7 @@ class JWTAuthValidator(BaseValidator):
user_password = validator_context.user_password
http_client = validator_context.http_client
jwt_auth_max = validator_context.jwt_auth_max
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
return
@ -29,10 +30,13 @@ class JWTAuthValidator(BaseValidator):
if not issuer:
raise ConfigValidationException('Missing JWT Issuer ID')
override_config_directory = os.path.join(config_provider.get_config_root(), 'stack/')
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
# the key cannot be found.
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
OVERRIDE_CONFIG_DIRECTORY,
override_config_directory,
http_client,
jwt_auth_max,
public_key_path=public_key_path,

5
util/config/validators/validate_ldap.py
View file

@ -5,7 +5,6 @@ import subprocess
from data.users import LDAP_CERT_FILENAME
from data.users.externalldap import LDAPConnection, LDAPUsers
from util.config.validators import BaseValidator, ConfigValidationException
from _init import CONF_DIR, config_provider
class LDAPValidator(BaseValidator):
name = "ldap"
@ -16,12 +15,14 @@ class LDAPValidator(BaseValidator):
config = validator_context.config
user = validator_context.user
user_password = validator_context.user_password
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
return
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
subprocess.check_call([os.path.join(CONF_DIR, 'init/certs_install.sh')])
subprocess.check_call([os.path.join(config_provider.get_config_root(), '../init/certs_install.sh')])
# Note: raises ldap.INVALID_CREDENTIALS on failure
admin_dn = config.get('LDAP_ADMIN_DN')

12
util/config/validators/validate_secscan.py
View file

@ -12,14 +12,18 @@ class SecurityScannerValidator(BaseValidator):
""" Validates the configuration for talking to a Quay Security Scanner. """
config = validator_context.config
client = validator_context.http_client
app = None #TODO(sam) validate with joey's pr about security scanner api
feature_sec_scanner = validator_context.feature_sec_scanner
is_testing = validator_context.is_testing
if not config.get('FEATURE_SECURITY_SCANNER', False):
server_hostname = validator_context.url_scheme_and_hostname.hostname
uri_creator = validator_context.uri_creator
if not feature_sec_scanner:
return
api = SecurityScannerAPI(app.config, config, None, client=client, skip_validation=True)
api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
if not config.get('TESTING', False):
if not is_testing:
# Generate a temporary Quay key to use for signing the outgoing requests.
setup_jwt_proxy()

2
util/config/validators/validate_signer.py
View file

@ -1,6 +1,5 @@
from StringIO import StringIO
from _init import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.signing import SIGNING_ENGINES
@ -11,6 +10,7 @@ class SignerValidator(BaseValidator):
def validate(cls, validator_context):
""" Validates the GPG public+private key pair used for signing converted ACIs. """
config = validator_context.config
config_provider = validator_context.config_provider
if config.get('SIGNING_ENGINE') is None:
return

2
util/config/validators/validate_ssl.py
View file

@ -1,4 +1,3 @@
from _init import config_provider
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
@ -11,6 +10,7 @@ class SSLValidator(BaseValidator):
def validate(cls, validator_context):
""" Validates the SSL configuration (if enabled). """
config = validator_context.config
config_provider = validator_context.config_provider
# Skip if non-SSL.
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':

7
util/config/validators/validate_storage.py
View file

@ -1,4 +1,3 @@
from _init import config_provider
from storage import get_storage_driver
from util.config.validators import BaseValidator, ConfigValidationException
@ -12,10 +11,12 @@ class StorageValidator(BaseValidator):
config = validator_context.config
client = validator_context.http_client
ip_resolver = validator_context.ip_resolver
config_provider = validator_context.config_provider
# replication_enabled = app.config.get('FEATURE_STORAGE_REPLICATION', False)
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
providers = _get_storage_providers(config, ip_resolver).items()
providers = _get_storage_providers(config, ip_resolver, config_provider).items()
if not providers:
raise ConfigValidationException('Storage configuration required')
@ -35,7 +36,7 @@ class StorageValidator(BaseValidator):
raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
def _get_storage_providers(config, ip_resolver):
def _get_storage_providers(config, ip_resolver, config_provider):
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
drivers = {}

9
util/secscan/__init__.py
View file

@ -107,12 +107,3 @@ def get_priority_for_index(index):
return priority
return 'Unknown'
def create_url_from_app(app):
"""
Higher order function that returns a function that when called, will generate a url for that given app
:param app: Flask app
:return:
type: Flask -> (str -> url)
"""

30
util/secscan/api.py
View file

@ -7,16 +7,12 @@ from urlparse import urljoin
import requests
from flask import url_for
from data.database import CloseForLongOperation
from data import model
from data.model.storage import get_storage_locations
from util import get_app_url, slash_join
from util.abchelpers import nooper
from util.failover import failover, FailoverException
from util.secscan.validator import SecurityConfigValidator
from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
from util.security.registry_jwt import generate_bearer_token, build_context_and_subject
from _init import CONF_DIR
@ -70,16 +66,16 @@ def compute_layer_id(layer):
class SecurityScannerAPI(object):
""" Helper class for talking to the Security Scan service (usually Clair). """
def __init__(self, app, config, storage, client=None, skip_validation=False):
def __init__(self, config, storage, server_hostname=None, client=None, skip_validation=False, uri_creator=None, instance_keys=None):
feature_enabled = config.get('FEATURE_SECURITY_SCANNER', False)
has_valid_config = skip_validation
if not skip_validation and feature_enabled:
config_validator = SecurityConfigValidator(config)
config_validator = SecurityConfigValidator(feature_enabled, config.get('SECURITY_SCANNER_ENDPOINT'))
has_valid_config = config_validator.valid()
if feature_enabled and has_valid_config:
self.state = ImplementedSecurityScannerAPI(app, config, storage, client=client)
self.state = ImplementedSecurityScannerAPI(config, storage, server_hostname, client=client, uri_creator=uri_creator, instance_keys=instance_keys)
else:
self.state = NoopSecurityScannerAPI()
@ -150,20 +146,25 @@ class NoopSecurityScannerAPI(SecurityScannerAPIInterface):
class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
""" Helper class for talking to the Security Scan service (Clair). """
def __init__(self, app_config, config, storage, client=None):
self._app_config = app_config
# TODO(sam) refactor this to not take an app config, and instead just the things it needs as a config object
def __init__(self, config, storage, server_hostname, client=None, uri_creator=None, instance_keys=None):
self._config = config
self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app_config))
self._client = client or config['HTTPCLIENT']
self._instance_keys = instance_keys
self._client = client
self._storage = storage
self._server_hostname = server_hostname
self._default_storage_locations = config['DISTRIBUTED_STORAGE_PREFERENCE']
self._target_version = config.get('SECURITY_SCANNER_ENGINE_VERSION_TARGET', 2)
self._uri_creator = uri_creator
def _get_image_url_and_auth(self, image):
""" Returns a tuple of the url and the auth header value that must be used
to fetch the layer data itself. If the image can't be addressed, we return
None.
"""
if self._instance_keys is None:
raise Exception('No Instance keys provided to Security Scanner API')
path = model.storage.get_layer_path(image.storage)
locations = self._default_storage_locations
@ -183,7 +184,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
repository_and_namespace = '/'.join([namespace_name, repo_name])
# Generate the JWT which will authorize this
audience = self._app_config['SERVER_HOSTNAME']
audience = self._server_hostname
context, subject = build_context_and_subject()
access = [{
'type': 'repository',
@ -195,10 +196,7 @@ class ImplementedSecurityScannerAPI(SecurityScannerAPIInterface):
TOKEN_VALIDITY_LIFETIME_S, self._instance_keys)
auth_header = 'Bearer ' + auth_token
with self._app.test_request_context('/'):
relative_layer_url = url_for('v2.download_blob', repository=repository_and_namespace,
digest=image.storage.content_checksum)
uri = urljoin(get_app_url(self._config), relative_layer_url)
uri = self._uri_creator(repository_and_namespace, image.storage.content_checksum)
return uri, auth_header

15
util/secscan/validator.py
View file

@ -1,28 +1,27 @@
import logging
import features
logger = logging.getLogger(__name__)
class SecurityConfigValidator(object):
""" Helper class for validating the security scanner configuration. """
def __init__(self, config):
if not features.SECURITY_SCANNER:
def __init__(self, feature_sec_scan, sec_scan_endpoint):
if not feature_sec_scan:
return
self._config = config
self._feature_sec_scan = feature_sec_scan
self._sec_scan_endpoint = sec_scan_endpoint
def valid(self):
if not features.SECURITY_SCANNER:
if not self._feature_sec_scan:
return False
if self._config.get('SECURITY_SCANNER_ENDPOINT') is None:
if self._sec_scan_endpoint is None:
logger.debug('Missing SECURITY_SCANNER_ENDPOINT configuration')
return False
endpoint = self._config.get('SECURITY_SCANNER_ENDPOINT')
endpoint = self._sec_scan_endpoint
if not endpoint.startswith('http://') and not endpoint.startswith('https://'):
logger.debug('SECURITY_SCANNER_ENDPOINT configuration must start with http or https')
return False

26
util/security/instancekeys.py
View file

@ -1,4 +1,3 @@
from collections import namedtuple
from cachetools import lru_cache
from data import model
from util.expiresdict import ExpiresDict, ExpiresEntry
@ -26,10 +25,9 @@ class InstanceKeys(object):
""" InstanceKeys defines a helper class for interacting with the Quay instance service keys
used for JWT signing of registry tokens as well as requests from Quay to other services
such as Clair. Each container will have a single registered instance key.
:param keys_context: InstanceKeysContext
"""
def __init__(self, keys_context):
self.keys_context = keys_context
def __init__(self, app):
self.app = app
self.instance_keys = ExpiresDict(self._load_instance_keys)
def clear_cache(self):
@ -47,24 +45,24 @@ class InstanceKeys(object):
@property
def service_name(self):
""" Returns the name of the instance key's service (i.e. 'quay'). """
return self.keys_context.instance_key_service
return self.app.config['INSTANCE_SERVICE_KEY_SERVICE']
@property
def service_key_expiration(self):
""" Returns the defined expiration for instance service keys, in minutes. """
return self.keys_context.service_key_expiration
return self.app.config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120)
@property
@lru_cache(maxsize=1)
def local_key_id(self):
""" Returns the ID of the local instance service key. """
return _load_file_contents(self.keys_context.service_key_kid_location)
return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_KID_LOCATION'])
@property
@lru_cache(maxsize=1)
def local_private_key(self):
""" Returns the private key of the local instance service key. """
return _load_file_contents(self.keys_context.service_key_location)
return _load_file_contents(self.app.config['INSTANCE_SERVICE_KEY_LOCATION'])
def get_service_key_public_key(self, kid):
""" Returns the public key associated with the given instance service key or None if none. """
@ -79,15 +77,3 @@ def _load_file_contents(path):
""" Returns the contents of the specified file path. """
with open(path) as f:
return f.read()
InstanceKeysContext = namedtuple('InstanceKeysContext', ['instance_key_service',
'service_key_expiration',
'service_key_kid_location',
'service_key_location'])
def instance_keys_context_from_app_config(app_config):
return InstanceKeysContext(app_config['INSTANCE_SERVICE_KEY_SERVICE'],
app_config.get('INSTANCE_SERVICE_KEY_EXPIRATION', 120),
app_config['INSTANCE_SERVICE_KEY_KID_LOCATION'],
app_config['INSTANCE_SERVICE_KEY_LOCATION'])

4
util/tufmetadata/api.py
View file

@ -11,7 +11,7 @@ import requests
from data.database import CloseForLongOperation
from util.abchelpers import nooper
from util.failover import failover, FailoverException
from util.security.instancekeys import InstanceKeys, instance_keys_context_from_app_config
from util.security.instancekeys import InstanceKeys
from util.security.registry_jwt import (build_context_and_subject, generate_bearer_token,
SIGNER_TUF_ROOT)
@ -108,7 +108,7 @@ class NoopTUFMetadataAPI(TUFMetadataAPIInterface):
class ImplementedTUFMetadataAPI(TUFMetadataAPIInterface):
def __init__(self, app, config, client=None):
self._app = app
self._instance_keys = InstanceKeys(instance_keys_context_from_app_config(app.config))
self._instance_keys = InstanceKeys(app)
self._config = config
self._client = client or config['HTTPCLIENT']
self._gun_prefix = config['TUF_GUN_PREFIX'] or config['SERVER_HOSTNAME']

2
workers/securityworker/securityworker.py
View file

@ -19,7 +19,7 @@ DEFAULT_INDEXING_INTERVAL = 30
class SecurityWorker(Worker):
def __init__(self):
super(SecurityWorker, self).__init__()
validator = SecurityConfigValidator(app.config)
validator = SecurityConfigValidator(app.config.get('FEATURE_SECURITY_SCANNER', False), app.config.get('SECURITY_SCANNER_ENDPOINT'))
if not validator.valid():
logger.warning('Failed to validate security scan configuration')
return