initial import for Open Source 🎉
This commit is contained in:
parent
1898c361f3
commit
9c0dd3b722
2048 changed files with 218743 additions and 0 deletions
29
util/config/__init__.py
Normal file
29
util/config/__init__.py
Normal file
|
@ -0,0 +1,29 @@
|
|||
class URLSchemeAndHostname:
|
||||
"""
|
||||
Immutable configuration for a given preferred url scheme (e.g. http or https), and a hostname (e.g. localhost:5000)
|
||||
"""
|
||||
def __init__(self, url_scheme, hostname):
|
||||
self._url_scheme = url_scheme
|
||||
self._hostname = hostname
|
||||
|
||||
@classmethod
|
||||
def from_app_config(cls, app_config):
|
||||
"""
|
||||
Helper method to instantiate class from app config, a frequent pattern
|
||||
:param app_config:
|
||||
:return:
|
||||
"""
|
||||
return cls(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
|
||||
|
||||
@property
|
||||
def url_scheme(self):
|
||||
return self._url_scheme
|
||||
|
||||
@property
|
||||
def hostname(self):
|
||||
return self._hostname
|
||||
|
||||
def get_url(self):
|
||||
""" Returns the application's URL, based on the given url scheme and hostname. """
|
||||
return '%s://%s' % (self._url_scheme, self._hostname)
|
||||
|
43
util/config/configdocs/configdoc.py
Normal file
43
util/config/configdocs/configdoc.py
Normal file
|
@ -0,0 +1,43 @@
|
|||
""" Generates html documentation from JSON Schema """
|
||||
|
||||
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
|
||||
import docsmodel
|
||||
import html_output
|
||||
|
||||
from util.config.schema import CONFIG_SCHEMA
|
||||
|
||||
|
||||
def make_custom_sort(orders):
|
||||
""" Sort in a specified order any dictionary nested in a complex structure """
|
||||
|
||||
orders = [{k: -i for (i, k) in enumerate(reversed(order), 1)} for order in orders]
|
||||
def process(stuff):
|
||||
if isinstance(stuff, dict):
|
||||
l = [(k, process(v)) for (k, v) in stuff.iteritems()]
|
||||
keys = set(stuff)
|
||||
for order in orders:
|
||||
if keys.issubset(order) or keys.issuperset(order):
|
||||
return OrderedDict(sorted(l, key=lambda x: order.get(x[0], 0)))
|
||||
return OrderedDict(sorted(l))
|
||||
if isinstance(stuff, list):
|
||||
return [process(x) for x in stuff]
|
||||
return stuff
|
||||
return process
|
||||
|
||||
SCHEMA_HTML_FILE = "schema.html"
|
||||
|
||||
schema = json.dumps(CONFIG_SCHEMA, sort_keys = True)
|
||||
schema = json.loads(schema, object_pairs_hook = OrderedDict)
|
||||
|
||||
req = sorted(schema["required"])
|
||||
custom_sort = make_custom_sort([req])
|
||||
schema = custom_sort(schema)
|
||||
|
||||
parsed_items = docsmodel.DocsModel().parse(schema)[1:]
|
||||
output = html_output.HtmlOutput().generate_output(parsed_items)
|
||||
|
||||
with open(SCHEMA_HTML_FILE, 'wt') as f:
|
||||
f.write(output)
|
93
util/config/configdocs/docsmodel.py
Normal file
93
util/config/configdocs/docsmodel.py
Normal file
|
@ -0,0 +1,93 @@
|
|||
import json
|
||||
import collections
|
||||
|
||||
class ParsedItem(dict):
|
||||
""" Parsed Schema item """
|
||||
|
||||
def __init__(self, json_object, name, required, level):
|
||||
"""Fills dict with basic item information"""
|
||||
super(ParsedItem, self).__init__()
|
||||
self['name'] = name
|
||||
self['title'] = json_object.get('title', '')
|
||||
self['type'] = json_object.get('type')
|
||||
self['description'] = json_object.get('description', '')
|
||||
self['level'] = level
|
||||
self['required'] = required
|
||||
self['x-reference'] = json_object.get('x-reference', '')
|
||||
self['x-example'] = json_object.get('x-example', '')
|
||||
self['pattern'] = json_object.get('pattern', '')
|
||||
self['enum'] = json_object.get('enum', '')
|
||||
|
||||
class DocsModel:
|
||||
""" Documentation model and Schema Parser """
|
||||
|
||||
def __init__(self):
|
||||
self.__parsed_items = None
|
||||
|
||||
def parse(self, json_object):
|
||||
""" Returns multi-level list of recursively parsed items """
|
||||
|
||||
self.__parsed_items = list()
|
||||
self.__parse_schema(json_object, 'root', True, 0)
|
||||
return self.__parsed_items
|
||||
|
||||
def __parse_schema(self, schema, name, required, level):
|
||||
""" Parses schema, which type is object, array or leaf.
|
||||
Appends new ParsedItem to self.__parsed_items lis """
|
||||
parsed_item = ParsedItem(schema, name, required, level)
|
||||
self.__parsed_items.append(parsed_item)
|
||||
required = schema.get('required', [])
|
||||
|
||||
if 'enum' in schema:
|
||||
parsed_item['item'] = schema.get('enum')
|
||||
item_type = schema.get('type')
|
||||
if item_type == 'object' and name != 'DISTRIBUTED_STORAGE_CONFIG':
|
||||
self.__parse_object(parsed_item, schema, required, level)
|
||||
elif item_type == 'array':
|
||||
self.__parse_array(parsed_item, schema, required, level)
|
||||
else:
|
||||
parse_leaf(parsed_item, schema)
|
||||
|
||||
def __parse_object(self, parsed_item, schema, required, level):
|
||||
""" Parses schema of type object """
|
||||
for key, value in schema.get('properties', {}).items():
|
||||
self.__parse_schema(value, key, key in required, level + 1)
|
||||
|
||||
def __parse_array(self, parsed_item, schema, required, level):
|
||||
""" Parses schema of type array """
|
||||
items = schema.get('items')
|
||||
parsed_item['minItems'] = schema.get('minItems', None)
|
||||
parsed_item['maxItems'] = schema.get('maxItems', None)
|
||||
parsed_item['uniqueItems'] = schema.get('uniqueItems', False)
|
||||
if isinstance(items, dict):
|
||||
# item is single schema describing all elements in an array
|
||||
self.__parse_schema(
|
||||
items,
|
||||
'array item',
|
||||
required,
|
||||
level + 1)
|
||||
|
||||
elif isinstance(items, list):
|
||||
# item is a list of schemas
|
||||
for index, list_item in enumerate(items):
|
||||
self.__parse_schema(
|
||||
list_item,
|
||||
'array item {}'.format(index),
|
||||
index in required,
|
||||
level + 1)
|
||||
|
||||
def parse_leaf(parsed_item, schema):
|
||||
""" Parses schema of a number and a string """
|
||||
if parsed_item['name'] != 'root':
|
||||
parsed_item['description'] = schema.get('description','')
|
||||
parsed_item['x-reference'] = schema.get('x-reference','')
|
||||
parsed_item['pattern'] = schema.get('pattern','')
|
||||
parsed_item['enum'] = ", ".join(schema.get('enum','')).encode()
|
||||
|
||||
ex = schema.get('x-example', '')
|
||||
if isinstance(ex, list):
|
||||
parsed_item['x-example'] = ", ".join(ex).encode()
|
||||
elif isinstance(ex, collections.OrderedDict):
|
||||
parsed_item['x-example'] = json.dumps(ex)
|
||||
else:
|
||||
parsed_item['x-example'] = ex
|
63
util/config/configdocs/html_output.py
Normal file
63
util/config/configdocs/html_output.py
Normal file
|
@ -0,0 +1,63 @@
|
|||
class HtmlOutput:
|
||||
""" Generates HTML from documentation model """
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def generate_output(self, parsed_items):
|
||||
"""Returns generated HTML strin"""
|
||||
return self.__get_html_begin() + \
|
||||
self.__get_html_middle(parsed_items) + \
|
||||
self.__get_html_end()
|
||||
|
||||
def __get_html_begin(self):
|
||||
return '<!DOCTYPE html>\n<html>\n<head>\n<link rel="stylesheet" type="text/css" href="style.css" />\n</head>\n<body>\n'
|
||||
|
||||
def __get_html_end(self):
|
||||
return '</body>\n</html>'
|
||||
|
||||
def __get_html_middle(self, parsed_items):
|
||||
output = ''
|
||||
root_item = parsed_items[0]
|
||||
|
||||
#output += '<h1 class="root_title">{}</h1>\n'.format(root_item['title'])
|
||||
#output += '<h1 class="root_title">{}</h1>\n'.format(root_item['title'])
|
||||
output += "Schema for Red Hat Quay"
|
||||
|
||||
output += '<ul class="level0">\n'
|
||||
last_level = 0
|
||||
is_root = True
|
||||
for item in parsed_items:
|
||||
level = item['level'] - 1
|
||||
if last_level < level:
|
||||
output += '<ul class="level{}">\n'.format(level)
|
||||
for i in range(last_level - level):
|
||||
output += '</ul>\n'
|
||||
last_level = level
|
||||
output += self.__get_html_item(item, is_root)
|
||||
is_root = False
|
||||
output += '</ul>\n'
|
||||
return output
|
||||
|
||||
def __get_required_field(self, parsed_item):
|
||||
return 'required' if parsed_item['required'] else ''
|
||||
|
||||
def __get_html_item(self, parsed_item, is_root):
|
||||
item = '<li class="schema item"> \n'
|
||||
item += '<div class="name">{}</div> \n'.format(parsed_item['name'])
|
||||
item += '<div class="type">[{}]</div> \n'.format(parsed_item['type'])
|
||||
item += '<div class="required">{}</div> \n'.format(self.__get_required_field(parsed_item))
|
||||
item += '<div class="docs">\n' if not is_root else '<div class="root_docs">\n'
|
||||
item += '<div class="title">{}</div>\n'.format(parsed_item['title'])
|
||||
item += ': ' if parsed_item['title'] != '' and parsed_item['description'] != '' else ''
|
||||
item += '<div class="description">{}</div>\n'.format(parsed_item['description'])
|
||||
item += '<div class="enum">enum: {}</div>\n'.format(parsed_item['enum']) if parsed_item['enum'] != '' else ''
|
||||
item += '<div class="minItems">Min Items: {}</div>\n'.format(parsed_item['minItems']) if parsed_item['type'] == "array" and parsed_item['minItems'] != "None" else ''
|
||||
item += '<div class="uniqueItems">Unique Items: {}</div>\n'.format(parsed_item['uniqueItems']) if parsed_item['type'] == "array" and parsed_item['uniqueItems'] else ''
|
||||
item += '<div class="pattern">Pattern: {}</div>\n'.format(parsed_item['pattern']) if parsed_item['pattern'] != 'None' and parsed_item['pattern'] != '' else ''
|
||||
item += '<div class="x-reference"><a href="{}">Reference: {}</a></div>\n'.format(parsed_item['x-reference'],parsed_item['x-reference']) if parsed_item['x-reference'] != '' else ''
|
||||
item += '<div class="x-example">Example: <code>{}</code></div>\n'.format(parsed_item['x-example']) if parsed_item['x-example'] != '' else ''
|
||||
item += '</div>\n'
|
||||
item += '</li>\n'
|
||||
return item
|
||||
|
1062
util/config/configdocs/schema.html
Normal file
1062
util/config/configdocs/schema.html
Normal file
File diff suppressed because it is too large
Load diff
78
util/config/configdocs/style.css
Normal file
78
util/config/configdocs/style.css
Normal file
|
@ -0,0 +1,78 @@
|
|||
body {
|
||||
font-family: sans-serif;
|
||||
}
|
||||
|
||||
pre, code{
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
div.root docs{
|
||||
display: none;
|
||||
}
|
||||
|
||||
div.name {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
div.type {
|
||||
display: inline;
|
||||
font-weight: bold;
|
||||
color: blue;
|
||||
}
|
||||
|
||||
div.required {
|
||||
display: inline;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
div.docs {
|
||||
display: block;
|
||||
}
|
||||
|
||||
div.title {
|
||||
display: block;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
div.description {
|
||||
display: block;
|
||||
font-family: serif;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
div.enum {
|
||||
display: block;
|
||||
font-family: serif;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
div.x-example {
|
||||
display: block;
|
||||
font-family: serif;
|
||||
font-style: italic;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
div.pattern {
|
||||
display: block;
|
||||
font-family: serif;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
div.x-reference {
|
||||
display: block;
|
||||
font-family: serif;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
div.uniqueItems {
|
||||
display: block;
|
||||
}
|
||||
|
||||
div.minItems {
|
||||
display: block;
|
||||
}
|
||||
|
||||
div.maxItems {
|
||||
display: block;
|
||||
}
|
108
util/config/configutil.py
Normal file
108
util/config/configutil.py
Normal file
|
@ -0,0 +1,108 @@
|
|||
from random import SystemRandom
|
||||
from uuid import uuid4
|
||||
|
||||
def generate_secret_key():
|
||||
cryptogen = SystemRandom()
|
||||
return str(cryptogen.getrandbits(256))
|
||||
|
||||
|
||||
def add_enterprise_config_defaults(config_obj, current_secret_key):
|
||||
""" Adds/Sets the config defaults for enterprise registry config. """
|
||||
# These have to be false.
|
||||
config_obj['TESTING'] = False
|
||||
config_obj['USE_CDN'] = False
|
||||
|
||||
# Default for V3 upgrade.
|
||||
config_obj['V3_UPGRADE_MODE'] = config_obj.get('V3_UPGRADE_MODE', 'complete')
|
||||
|
||||
# Defaults for Red Hat Quay.
|
||||
config_obj['REGISTRY_TITLE'] = config_obj.get('REGISTRY_TITLE', 'Red Hat Quay')
|
||||
config_obj['REGISTRY_TITLE_SHORT'] = config_obj.get('REGISTRY_TITLE_SHORT', 'Red Hat Quay')
|
||||
|
||||
# Default features that are on.
|
||||
config_obj['FEATURE_USER_LOG_ACCESS'] = config_obj.get('FEATURE_USER_LOG_ACCESS', True)
|
||||
config_obj['FEATURE_USER_CREATION'] = config_obj.get('FEATURE_USER_CREATION', True)
|
||||
config_obj['FEATURE_ANONYMOUS_ACCESS'] = config_obj.get('FEATURE_ANONYMOUS_ACCESS', True)
|
||||
config_obj['FEATURE_REQUIRE_TEAM_INVITE'] = config_obj.get('FEATURE_REQUIRE_TEAM_INVITE', True)
|
||||
config_obj['FEATURE_CHANGE_TAG_EXPIRATION'] = config_obj.get('FEATURE_CHANGE_TAG_EXPIRATION',
|
||||
True)
|
||||
config_obj['FEATURE_DIRECT_LOGIN'] = config_obj.get('FEATURE_DIRECT_LOGIN', True)
|
||||
config_obj['FEATURE_APP_SPECIFIC_TOKENS'] = config_obj.get('FEATURE_APP_SPECIFIC_TOKENS', True)
|
||||
config_obj['FEATURE_PARTIAL_USER_AUTOCOMPLETE'] = config_obj.get('FEATURE_PARTIAL_USER_AUTOCOMPLETE', True)
|
||||
config_obj['FEATURE_USERNAME_CONFIRMATION'] = config_obj.get('FEATURE_USERNAME_CONFIRMATION', True)
|
||||
config_obj['FEATURE_RESTRICTED_V1_PUSH'] = config_obj.get('FEATURE_RESTRICTED_V1_PUSH', True)
|
||||
|
||||
# Default features that are off.
|
||||
config_obj['FEATURE_MAILING'] = config_obj.get('FEATURE_MAILING', False)
|
||||
config_obj['FEATURE_BUILD_SUPPORT'] = config_obj.get('FEATURE_BUILD_SUPPORT', False)
|
||||
config_obj['FEATURE_ACI_CONVERSION'] = config_obj.get('FEATURE_ACI_CONVERSION', False)
|
||||
config_obj['FEATURE_APP_REGISTRY'] = config_obj.get('FEATURE_APP_REGISTRY', False)
|
||||
config_obj['FEATURE_REPO_MIRROR'] = config_obj.get('FEATURE_REPO_MIRROR', False)
|
||||
|
||||
# Default repo mirror config.
|
||||
config_obj['REPO_MIRROR_TLS_VERIFY'] = config_obj.get('REPO_MIRROR_TLS_VERIFY', True)
|
||||
config_obj['REPO_MIRROR_SERVER_HOSTNAME'] = config_obj.get('REPO_MIRROR_SERVER_HOSTNAME', None)
|
||||
|
||||
# Default the signer config.
|
||||
config_obj['GPG2_PRIVATE_KEY_FILENAME'] = config_obj.get('GPG2_PRIVATE_KEY_FILENAME',
|
||||
'signing-private.gpg')
|
||||
config_obj['GPG2_PUBLIC_KEY_FILENAME'] = config_obj.get('GPG2_PUBLIC_KEY_FILENAME',
|
||||
'signing-public.gpg')
|
||||
config_obj['SIGNING_ENGINE'] = config_obj.get('SIGNING_ENGINE', 'gpg2')
|
||||
|
||||
# Default security scanner config.
|
||||
config_obj['FEATURE_SECURITY_NOTIFICATIONS'] = config_obj.get(
|
||||
'FEATURE_SECURITY_NOTIFICATIONS', True)
|
||||
|
||||
config_obj['FEATURE_SECURITY_SCANNER'] = config_obj.get(
|
||||
'FEATURE_SECURITY_SCANNER', False)
|
||||
|
||||
config_obj['SECURITY_SCANNER_ISSUER_NAME'] = config_obj.get(
|
||||
'SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
|
||||
|
||||
# Default time machine config.
|
||||
config_obj['TAG_EXPIRATION_OPTIONS'] = config_obj.get('TAG_EXPIRATION_OPTIONS',
|
||||
['0s', '1d', '1w', '2w', '4w'])
|
||||
config_obj['DEFAULT_TAG_EXPIRATION'] = config_obj.get('DEFAULT_TAG_EXPIRATION', '2w')
|
||||
|
||||
# Default mail setings.
|
||||
config_obj['MAIL_USE_TLS'] = config_obj.get('MAIL_USE_TLS', True)
|
||||
config_obj['MAIL_PORT'] = config_obj.get('MAIL_PORT', 587)
|
||||
config_obj['MAIL_DEFAULT_SENDER'] = config_obj.get('MAIL_DEFAULT_SENDER', 'support@quay.io')
|
||||
|
||||
# Default auth type.
|
||||
if not 'AUTHENTICATION_TYPE' in config_obj:
|
||||
config_obj['AUTHENTICATION_TYPE'] = 'Database'
|
||||
|
||||
# Default secret key.
|
||||
if not 'SECRET_KEY' in config_obj:
|
||||
if current_secret_key:
|
||||
config_obj['SECRET_KEY'] = current_secret_key
|
||||
else:
|
||||
config_obj['SECRET_KEY'] = generate_secret_key()
|
||||
|
||||
# Default database secret key.
|
||||
if not 'DATABASE_SECRET_KEY' in config_obj:
|
||||
config_obj['DATABASE_SECRET_KEY'] = generate_secret_key()
|
||||
|
||||
# Default torrent pepper.
|
||||
if not 'BITTORRENT_FILENAME_PEPPER' in config_obj:
|
||||
config_obj['BITTORRENT_FILENAME_PEPPER'] = str(uuid4())
|
||||
|
||||
# Default storage configuration.
|
||||
if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj:
|
||||
config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['default']
|
||||
config_obj['DISTRIBUTED_STORAGE_CONFIG'] = {
|
||||
'default': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
|
||||
}
|
||||
|
||||
config_obj['USERFILES_LOCATION'] = 'default'
|
||||
config_obj['USERFILES_PATH'] = 'userfiles/'
|
||||
|
||||
config_obj['LOG_ARCHIVE_LOCATION'] = 'default'
|
||||
|
||||
# Misc configuration.
|
||||
config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
|
||||
config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get(
|
||||
'ENTERPRISE_LOGO_URL', '/static/img/quay-horizontal-color.svg')
|
||||
config_obj['TEAM_RESYNC_STALE_TIME'] = config_obj.get('TEAM_RESYNC_STALE_TIME', '60m')
|
12
util/config/database.py
Normal file
12
util/config/database.py
Normal file
|
@ -0,0 +1,12 @@
|
|||
from data import model
|
||||
from data.appr_model import blob
|
||||
from data.appr_model.models import NEW_MODELS
|
||||
|
||||
|
||||
def sync_database_with_config(config):
|
||||
""" This ensures all implicitly required reference table entries exist in the database. """
|
||||
|
||||
location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
|
||||
if location_names:
|
||||
model.image.ensure_image_locations(*location_names)
|
||||
blob.ensure_blob_locations(NEW_MODELS, *location_names)
|
13
util/config/provider/__init__.py
Normal file
13
util/config/provider/__init__.py
Normal file
|
@ -0,0 +1,13 @@
|
|||
from util.config.provider.fileprovider import FileConfigProvider
|
||||
from util.config.provider.testprovider import TestConfigProvider
|
||||
from util.config.provider.k8sprovider import KubernetesConfigProvider
|
||||
|
||||
def get_config_provider(config_volume, yaml_filename, py_filename, testing=False, kubernetes=False):
|
||||
""" Loads and returns the config provider for the current environment. """
|
||||
if testing:
|
||||
return TestConfigProvider()
|
||||
|
||||
if kubernetes:
|
||||
return KubernetesConfigProvider(config_volume, yaml_filename, py_filename)
|
||||
|
||||
return FileConfigProvider(config_volume, yaml_filename, py_filename)
|
62
util/config/provider/basefileprovider.py
Normal file
62
util/config/provider/basefileprovider.py
Normal file
|
@ -0,0 +1,62 @@
|
|||
import os
|
||||
import logging
|
||||
|
||||
from util.config.provider.baseprovider import (BaseProvider, import_yaml, export_yaml,
|
||||
CannotWriteConfigException)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class BaseFileProvider(BaseProvider):
|
||||
""" Base implementation of the config provider that reads the data from the file system. """
|
||||
def __init__(self, config_volume, yaml_filename, py_filename):
|
||||
self.config_volume = config_volume
|
||||
self.yaml_filename = yaml_filename
|
||||
self.py_filename = py_filename
|
||||
|
||||
self.yaml_path = os.path.join(config_volume, yaml_filename)
|
||||
self.py_path = os.path.join(config_volume, py_filename)
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
if os.path.exists(self.py_path):
|
||||
logger.debug('Applying config file: %s', self.py_path)
|
||||
app_config.from_pyfile(self.py_path)
|
||||
|
||||
if os.path.exists(self.yaml_path):
|
||||
logger.debug('Applying config file: %s', self.yaml_path)
|
||||
import_yaml(app_config, self.yaml_path)
|
||||
|
||||
def get_config(self):
|
||||
if not self.config_exists():
|
||||
return None
|
||||
|
||||
config_obj = {}
|
||||
import_yaml(config_obj, self.yaml_path)
|
||||
return config_obj
|
||||
|
||||
def config_exists(self):
|
||||
return self.volume_file_exists(self.yaml_filename)
|
||||
|
||||
def volume_exists(self):
|
||||
return os.path.exists(self.config_volume)
|
||||
|
||||
def volume_file_exists(self, relative_file_path):
|
||||
return os.path.exists(os.path.join(self.config_volume, relative_file_path))
|
||||
|
||||
def get_volume_file(self, relative_file_path, mode='r'):
|
||||
return open(os.path.join(self.config_volume, relative_file_path), mode=mode)
|
||||
|
||||
def get_volume_path(self, directory, relative_file_path):
|
||||
return os.path.join(directory, relative_file_path)
|
||||
|
||||
def list_volume_directory(self, path):
|
||||
dirpath = os.path.join(self.config_volume, path)
|
||||
if not os.path.exists(dirpath):
|
||||
return None
|
||||
|
||||
if not os.path.isdir(dirpath):
|
||||
return None
|
||||
|
||||
return os.listdir(dirpath)
|
||||
|
||||
def get_config_root(self):
|
||||
return self.config_volume
|
123
util/config/provider/baseprovider.py
Normal file
123
util/config/provider/baseprovider.py
Normal file
|
@ -0,0 +1,123 @@
|
|||
import logging
|
||||
import yaml
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from six import add_metaclass
|
||||
|
||||
from jsonschema import validate, ValidationError
|
||||
|
||||
from util.config.schema import CONFIG_SCHEMA
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CannotWriteConfigException(Exception):
|
||||
""" Exception raised when the config cannot be written. """
|
||||
pass
|
||||
|
||||
|
||||
class SetupIncompleteException(Exception):
|
||||
""" Exception raised when attempting to verify config that has not yet been setup. """
|
||||
pass
|
||||
|
||||
|
||||
def import_yaml(config_obj, config_file):
|
||||
with open(config_file) as f:
|
||||
c = yaml.safe_load(f)
|
||||
if not c:
|
||||
logger.debug('Empty YAML config file')
|
||||
return
|
||||
|
||||
if isinstance(c, str):
|
||||
raise Exception('Invalid YAML config file: ' + str(c))
|
||||
|
||||
for key in c.iterkeys():
|
||||
if key.isupper():
|
||||
config_obj[key] = c[key]
|
||||
|
||||
if config_obj.get('SETUP_COMPLETE', True):
|
||||
try:
|
||||
validate(config_obj, CONFIG_SCHEMA)
|
||||
except ValidationError:
|
||||
# TODO: Change this into a real error
|
||||
logger.exception('Could not validate config schema')
|
||||
else:
|
||||
logger.debug('Skipping config schema validation because setup is not complete')
|
||||
|
||||
return config_obj
|
||||
|
||||
|
||||
def get_yaml(config_obj):
|
||||
return yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)
|
||||
|
||||
|
||||
def export_yaml(config_obj, config_file):
|
||||
try:
|
||||
with open(config_file, 'w') as f:
|
||||
f.write(get_yaml(config_obj))
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
|
||||
@add_metaclass(ABCMeta)
|
||||
class BaseProvider(object):
|
||||
""" A configuration provider helps to load, save, and handle config override in the application.
|
||||
"""
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def update_app_config(self, app_config):
|
||||
""" Updates the given application config object with the loaded override config. """
|
||||
|
||||
@abstractmethod
|
||||
def get_config(self):
|
||||
""" Returns the contents of the config override file, or None if none. """
|
||||
|
||||
@abstractmethod
|
||||
def save_config(self, config_object):
|
||||
""" Updates the contents of the config override file to those given. """
|
||||
|
||||
@abstractmethod
|
||||
def config_exists(self):
|
||||
""" Returns true if a config override file exists in the config volume. """
|
||||
|
||||
@abstractmethod
|
||||
def volume_exists(self):
|
||||
""" Returns whether the config override volume exists. """
|
||||
|
||||
@abstractmethod
|
||||
def volume_file_exists(self, relative_file_path):
|
||||
""" Returns whether the file with the given relative path exists under the config override
|
||||
volume. """
|
||||
|
||||
@abstractmethod
|
||||
def get_volume_file(self, relative_file_path, mode='r'):
|
||||
""" Returns a Python file referring to the given path under the config override volume. """
|
||||
|
||||
@abstractmethod
|
||||
def remove_volume_file(self, relative_file_path):
|
||||
""" Removes the config override volume file with the given path. """
|
||||
|
||||
@abstractmethod
|
||||
def list_volume_directory(self, path):
|
||||
""" Returns a list of strings representing the names of the files found in the config override
|
||||
directory under the given path. If the path doesn't exist, returns None.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def save_volume_file(self, flask_file, relative_file_path):
|
||||
""" Saves the given flask file to the config override volume, with the given
|
||||
relative path.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_volume_path(self, directory, filename):
|
||||
""" Helper for constructing relative file paths, which may differ between providers.
|
||||
For example, kubernetes can't have subfolders in configmaps """
|
||||
|
||||
@abstractmethod
|
||||
def get_config_root(self):
|
||||
""" Returns the config root directory. """
|
47
util/config/provider/fileprovider.py
Normal file
47
util/config/provider/fileprovider.py
Normal file
|
@ -0,0 +1,47 @@
|
|||
import os
|
||||
import logging
|
||||
|
||||
from util.config.provider.baseprovider import export_yaml, CannotWriteConfigException
|
||||
from util.config.provider.basefileprovider import BaseFileProvider
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _ensure_parent_dir(filepath):
|
||||
""" Ensures that the parent directory of the given file path exists. """
|
||||
try:
|
||||
parentpath = os.path.abspath(os.path.join(filepath, os.pardir))
|
||||
if not os.path.isdir(parentpath):
|
||||
os.makedirs(parentpath)
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
|
||||
class FileConfigProvider(BaseFileProvider):
|
||||
""" Implementation of the config provider that reads and writes the data
|
||||
from/to the file system. """
|
||||
def __init__(self, config_volume, yaml_filename, py_filename):
|
||||
super(FileConfigProvider, self).__init__(config_volume, yaml_filename, py_filename)
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return 'file'
|
||||
|
||||
def save_config(self, config_obj):
|
||||
export_yaml(config_obj, self.yaml_path)
|
||||
|
||||
def remove_volume_file(self, relative_file_path):
|
||||
filepath = os.path.join(self.config_volume, relative_file_path)
|
||||
os.remove(filepath)
|
||||
|
||||
def save_volume_file(self, flask_file, relative_file_path):
|
||||
filepath = os.path.join(self.config_volume, relative_file_path)
|
||||
_ensure_parent_dir(filepath)
|
||||
|
||||
# Write the file.
|
||||
try:
|
||||
flask_file.save(filepath)
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
return filepath
|
188
util/config/provider/k8sprovider.py
Normal file
188
util/config/provider/k8sprovider.py
Normal file
|
@ -0,0 +1,188 @@
|
|||
import os
|
||||
import logging
|
||||
import json
|
||||
import base64
|
||||
import time
|
||||
|
||||
from cStringIO import StringIO
|
||||
from requests import Request, Session
|
||||
|
||||
from util.config.provider.baseprovider import CannotWriteConfigException, get_yaml
|
||||
from util.config.provider.basefileprovider import BaseFileProvider
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KUBERNETES_API_HOST = os.environ.get('KUBERNETES_SERVICE_HOST', '')
|
||||
port = os.environ.get('KUBERNETES_SERVICE_PORT')
|
||||
if port:
|
||||
KUBERNETES_API_HOST += ':' + port
|
||||
|
||||
SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
|
||||
|
||||
QE_NAMESPACE = os.environ.get('QE_K8S_NAMESPACE', 'quay-enterprise')
|
||||
QE_CONFIG_SECRET = os.environ.get('QE_K8S_CONFIG_SECRET', 'quay-enterprise-config-secret')
|
||||
|
||||
class KubernetesConfigProvider(BaseFileProvider):
|
||||
""" Implementation of the config provider that reads and writes configuration
|
||||
data from a Kubernetes Secret. """
|
||||
def __init__(self, config_volume, yaml_filename, py_filename, api_host=None,
|
||||
service_account_token_path=None):
|
||||
super(KubernetesConfigProvider, self).__init__(config_volume, yaml_filename, py_filename)
|
||||
service_account_token_path = service_account_token_path or SERVICE_ACCOUNT_TOKEN_PATH
|
||||
api_host = api_host or KUBERNETES_API_HOST
|
||||
|
||||
# Load the service account token from the local store.
|
||||
if not os.path.exists(service_account_token_path):
|
||||
raise Exception('Cannot load Kubernetes service account token')
|
||||
|
||||
with open(service_account_token_path, 'r') as f:
|
||||
self._service_token = f.read()
|
||||
|
||||
self._api_host = api_host
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return 'k8s'
|
||||
|
||||
def get_volume_path(self, directory, filename):
|
||||
# NOTE: Overridden to ensure we don't have subdirectories, which aren't supported
|
||||
# in Kubernetes secrets.
|
||||
return "_".join([directory.rstrip('/'), filename])
|
||||
|
||||
def volume_exists(self):
|
||||
secret = self._lookup_secret()
|
||||
return secret is not None
|
||||
|
||||
def volume_file_exists(self, relative_file_path):
|
||||
if '/' in relative_file_path:
|
||||
raise Exception('Expected path from get_volume_path, but found slashes')
|
||||
|
||||
# NOTE: Overridden because we don't have subdirectories, which aren't supported
|
||||
# in Kubernetes secrets.
|
||||
secret = self._lookup_secret()
|
||||
if not secret or not secret.get('data'):
|
||||
return False
|
||||
return relative_file_path in secret['data']
|
||||
|
||||
def list_volume_directory(self, path):
|
||||
# NOTE: Overridden because we don't have subdirectories, which aren't supported
|
||||
# in Kubernetes secrets.
|
||||
secret = self._lookup_secret()
|
||||
|
||||
if not secret:
|
||||
return []
|
||||
|
||||
paths = []
|
||||
for filename in secret.get('data', {}):
|
||||
if filename.startswith(path):
|
||||
paths.append(filename[len(path) + 1:])
|
||||
return paths
|
||||
|
||||
def save_config(self, config_obj):
|
||||
self._update_secret_file(self.yaml_filename, get_yaml(config_obj))
|
||||
|
||||
def remove_volume_file(self, relative_file_path):
|
||||
try:
|
||||
self._update_secret_file(relative_file_path, None)
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
def save_volume_file(self, flask_file, relative_file_path):
|
||||
# Write the file to a temp location.
|
||||
buf = StringIO()
|
||||
try:
|
||||
try:
|
||||
flask_file.save(buf)
|
||||
except IOError as ioe:
|
||||
raise CannotWriteConfigException(str(ioe))
|
||||
|
||||
self._update_secret_file(relative_file_path, buf.getvalue())
|
||||
finally:
|
||||
buf.close()
|
||||
|
||||
def _assert_success(self, response):
|
||||
if response.status_code != 200:
|
||||
logger.error('Kubernetes API call failed with response: %s => %s', response.status_code,
|
||||
response.text)
|
||||
raise CannotWriteConfigException('Kubernetes API call failed: %s' % response.text)
|
||||
|
||||
def _update_secret_file(self, relative_file_path, value=None):
|
||||
if '/' in relative_file_path:
|
||||
raise Exception('Expected path from get_volume_path, but found slashes')
|
||||
|
||||
# Check first that the namespace for Red Hat Quay exists. If it does not, report that
|
||||
# as an error, as it seems to be a common issue.
|
||||
namespace_url = 'namespaces/%s' % (QE_NAMESPACE)
|
||||
response = self._execute_k8s_api('GET', namespace_url)
|
||||
if response.status_code // 100 != 2:
|
||||
msg = 'A Kubernetes namespace with name `%s` must be created to save config' % QE_NAMESPACE
|
||||
raise CannotWriteConfigException(msg)
|
||||
|
||||
# Check if the secret exists. If not, then we create an empty secret and then update the file
|
||||
# inside.
|
||||
secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET)
|
||||
secret = self._lookup_secret()
|
||||
if secret is None:
|
||||
self._assert_success(self._execute_k8s_api('POST', secret_url, {
|
||||
"kind": "Secret",
|
||||
"apiVersion": "v1",
|
||||
"metadata": {
|
||||
"name": QE_CONFIG_SECRET
|
||||
},
|
||||
"data": {}
|
||||
}))
|
||||
|
||||
# Update the secret to reflect the file change.
|
||||
secret['data'] = secret.get('data', {})
|
||||
|
||||
if value is not None:
|
||||
secret['data'][relative_file_path] = base64.b64encode(value)
|
||||
else:
|
||||
secret['data'].pop(relative_file_path)
|
||||
|
||||
self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
|
||||
|
||||
# Wait until the local mounted copy of the secret has been updated, as
|
||||
# this is an eventual consistency operation, but the caller expects immediate
|
||||
# consistency.
|
||||
while True:
|
||||
matching_files = set()
|
||||
for secret_filename, encoded_value in secret['data'].iteritems():
|
||||
expected_value = base64.b64decode(encoded_value)
|
||||
try:
|
||||
with self.get_volume_file(secret_filename) as f:
|
||||
contents = f.read()
|
||||
|
||||
if contents == expected_value:
|
||||
matching_files.add(secret_filename)
|
||||
except IOError:
|
||||
continue
|
||||
|
||||
if matching_files == set(secret['data'].keys()):
|
||||
break
|
||||
|
||||
# Sleep for a second and then try again.
|
||||
time.sleep(1)
|
||||
|
||||
def _lookup_secret(self):
|
||||
secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET)
|
||||
response = self._execute_k8s_api('GET', secret_url)
|
||||
if response.status_code != 200:
|
||||
return None
|
||||
return json.loads(response.text)
|
||||
|
||||
def _execute_k8s_api(self, method, relative_url, data=None):
|
||||
headers = {
|
||||
'Authorization': 'Bearer ' + self._service_token
|
||||
}
|
||||
|
||||
if data:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
|
||||
data = json.dumps(data) if data else None
|
||||
session = Session()
|
||||
url = 'https://%s/api/v1/%s' % (self._api_host, relative_url)
|
||||
|
||||
request = Request(method, url, data=data, headers=headers)
|
||||
return session.send(request.prepare(), verify=False, timeout=2)
|
29
util/config/provider/test/test_fileprovider.py
Normal file
29
util/config/provider/test/test_fileprovider.py
Normal file
|
@ -0,0 +1,29 @@
|
|||
import pytest
|
||||
|
||||
from util.config.provider import FileConfigProvider
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
|
||||
class TestFileConfigProvider(FileConfigProvider):
|
||||
def __init__(self):
|
||||
self.yaml_filename = 'yaml_filename'
|
||||
self._service_token = 'service_token'
|
||||
self.config_volume = 'config_volume'
|
||||
self.py_filename = 'py_filename'
|
||||
self.yaml_path = os.path.join(self.config_volume, self.yaml_filename)
|
||||
self.py_path = os.path.join(self.config_volume, self.py_filename)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('directory,filename,expected', [
|
||||
("directory", "file", "directory/file"),
|
||||
("directory/dir", "file", "directory/dir/file"),
|
||||
("directory/dir/", "file", "directory/dir/file"),
|
||||
("directory", "file/test", "directory/file/test"),
|
||||
])
|
||||
def test_get_volume_path(directory, filename, expected):
|
||||
provider = TestFileConfigProvider()
|
||||
|
||||
assert expected == provider.get_volume_path(directory, filename)
|
||||
|
||||
|
138
util/config/provider/test/test_k8sprovider.py
Normal file
138
util/config/provider/test/test_k8sprovider.py
Normal file
|
@ -0,0 +1,138 @@
|
|||
import base64
|
||||
import os
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from contextlib import contextmanager
|
||||
from collections import namedtuple
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from util.config.provider import KubernetesConfigProvider
|
||||
|
||||
def normalize_path(path):
|
||||
return path.replace('/', '_')
|
||||
|
||||
@contextmanager
|
||||
def fake_kubernetes_api(tmpdir_factory, files=None):
|
||||
hostname = 'kubapi'
|
||||
service_account_token_path = str(tmpdir_factory.mktemp("k8s").join("serviceaccount"))
|
||||
auth_header = str(uuid.uuid4())
|
||||
|
||||
with open(service_account_token_path, 'w') as f:
|
||||
f.write(auth_header)
|
||||
|
||||
global secret
|
||||
secret = {
|
||||
'data': {}
|
||||
}
|
||||
|
||||
def write_file(config_dir, filepath, value):
|
||||
normalized_path = normalize_path(filepath)
|
||||
absolute_path = str(config_dir.join(normalized_path))
|
||||
try:
|
||||
os.makedirs(os.path.dirname(absolute_path))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with open(absolute_path, 'w') as f:
|
||||
f.write(value)
|
||||
|
||||
config_dir = tmpdir_factory.mktemp("config")
|
||||
if files:
|
||||
for filepath, value in files.iteritems():
|
||||
normalized_path = normalize_path(filepath)
|
||||
write_file(config_dir, filepath, value)
|
||||
secret['data'][normalized_path] = base64.b64encode(value)
|
||||
|
||||
@urlmatch(netloc=hostname,
|
||||
path='/api/v1/namespaces/quay-enterprise/secrets/quay-enterprise-config-secret$',
|
||||
method='get')
|
||||
def get_secret(_, __):
|
||||
return {'status_code': 200, 'content': json.dumps(secret)}
|
||||
|
||||
@urlmatch(netloc=hostname,
|
||||
path='/api/v1/namespaces/quay-enterprise/secrets/quay-enterprise-config-secret$',
|
||||
method='put')
|
||||
def put_secret(_, request):
|
||||
updated_secret = json.loads(request.body)
|
||||
for filepath, value in updated_secret['data'].iteritems():
|
||||
if filepath not in secret['data']:
|
||||
# Add
|
||||
write_file(config_dir, filepath, base64.b64decode(value))
|
||||
|
||||
for filepath in secret['data']:
|
||||
if filepath not in updated_secret['data']:
|
||||
# Remove.
|
||||
normalized_path = normalize_path(filepath)
|
||||
os.remove(str(config_dir.join(normalized_path)))
|
||||
|
||||
secret['data'] = updated_secret['data']
|
||||
return {'status_code': 200, 'content': json.dumps(secret)}
|
||||
|
||||
@urlmatch(netloc=hostname, path='/api/v1/namespaces/quay-enterprise$')
|
||||
def get_namespace(_, __):
|
||||
return {'status_code': 200, 'content': json.dumps({})}
|
||||
|
||||
@urlmatch(netloc=hostname)
|
||||
def catch_all(url, _):
|
||||
print url
|
||||
return {'status_code': 404, 'content': '{}'}
|
||||
|
||||
with HTTMock(get_secret, put_secret, get_namespace, catch_all):
|
||||
provider = KubernetesConfigProvider(str(config_dir), 'config.yaml', 'config.py',
|
||||
api_host=hostname,
|
||||
service_account_token_path=service_account_token_path)
|
||||
|
||||
# Validate all the files.
|
||||
for filepath, value in files.iteritems():
|
||||
normalized_path = normalize_path(filepath)
|
||||
assert provider.volume_file_exists(normalized_path)
|
||||
with provider.get_volume_file(normalized_path) as f:
|
||||
assert f.read() == value
|
||||
|
||||
yield provider
|
||||
|
||||
|
||||
def test_basic_config(tmpdir_factory):
|
||||
basic_files = {
|
||||
'config.yaml': 'FOO: bar',
|
||||
}
|
||||
|
||||
with fake_kubernetes_api(tmpdir_factory, files=basic_files) as provider:
|
||||
assert provider.config_exists()
|
||||
assert provider.get_config() is not None
|
||||
assert provider.get_config()['FOO'] == 'bar'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('filepath', [
|
||||
'foo',
|
||||
'foo/meh',
|
||||
'foo/bar/baz',
|
||||
])
|
||||
def test_remove_file(filepath, tmpdir_factory):
|
||||
basic_files = {
|
||||
filepath: 'foo',
|
||||
}
|
||||
|
||||
with fake_kubernetes_api(tmpdir_factory, files=basic_files) as provider:
|
||||
normalized_path = normalize_path(filepath)
|
||||
assert provider.volume_file_exists(normalized_path)
|
||||
provider.remove_volume_file(normalized_path)
|
||||
assert not provider.volume_file_exists(normalized_path)
|
||||
|
||||
|
||||
class TestFlaskFile(object):
|
||||
def save(self, buf):
|
||||
buf.write('hello world!')
|
||||
|
||||
|
||||
def test_save_file(tmpdir_factory):
|
||||
basic_files = {}
|
||||
|
||||
with fake_kubernetes_api(tmpdir_factory, files=basic_files) as provider:
|
||||
assert not provider.volume_file_exists('testfile')
|
||||
flask_file = TestFlaskFile()
|
||||
provider.save_volume_file(flask_file, 'testfile')
|
||||
assert provider.volume_file_exists('testfile')
|
77
util/config/provider/testprovider.py
Normal file
77
util/config/provider/testprovider.py
Normal file
|
@ -0,0 +1,77 @@
|
|||
import json
|
||||
import io
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from util.config.provider.baseprovider import BaseProvider
|
||||
|
||||
REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', 'test/data/test.pem']
|
||||
|
||||
class TestConfigProvider(BaseProvider):
|
||||
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
|
||||
the real file system. """
|
||||
|
||||
def get_config_root(self):
|
||||
raise Exception('Test Config does not have a config root')
|
||||
|
||||
def __init__(self):
|
||||
self.clear()
|
||||
|
||||
def clear(self):
|
||||
self.files = {}
|
||||
self._config = {}
|
||||
|
||||
@property
|
||||
def provider_id(self):
|
||||
return 'test'
|
||||
|
||||
def update_app_config(self, app_config):
|
||||
self._config = app_config
|
||||
|
||||
def get_config(self):
|
||||
if not 'config.yaml' in self.files:
|
||||
return None
|
||||
|
||||
return json.loads(self.files.get('config.yaml', '{}'))
|
||||
|
||||
def save_config(self, config_obj):
|
||||
self.files['config.yaml'] = json.dumps(config_obj)
|
||||
|
||||
def config_exists(self):
|
||||
return 'config.yaml' in self.files
|
||||
|
||||
def volume_exists(self):
|
||||
return True
|
||||
|
||||
def volume_file_exists(self, filename):
|
||||
if filename in REAL_FILES:
|
||||
return True
|
||||
|
||||
return filename in self.files
|
||||
|
||||
def save_volume_file(self, flask_file, filename):
|
||||
self.files[filename] = flask_file.read()
|
||||
|
||||
def get_volume_file(self, filename, mode='r'):
|
||||
if filename in REAL_FILES:
|
||||
return open(filename, mode=mode)
|
||||
|
||||
return io.BytesIO(self.files[filename])
|
||||
|
||||
def remove_volume_file(self, filename):
|
||||
self.files.pop(filename, None)
|
||||
|
||||
def list_volume_directory(self, path):
|
||||
paths = []
|
||||
for filename in self.files:
|
||||
if filename.startswith(path):
|
||||
paths.append(filename[len(path)+1:])
|
||||
|
||||
return paths
|
||||
|
||||
def reset_for_test(self):
|
||||
self._config['SUPER_USERS'] = ['devtable']
|
||||
self.files = {}
|
||||
|
||||
def get_volume_path(self, directory, filename):
|
||||
return os.path.join(directory, filename)
|
1232
util/config/schema.py
Normal file
1232
util/config/schema.py
Normal file
File diff suppressed because it is too large
Load diff
38
util/config/superusermanager.py
Normal file
38
util/config/superusermanager.py
Normal file
|
@ -0,0 +1,38 @@
|
|||
from multiprocessing.sharedctypes import Array
|
||||
from util.validation import MAX_USERNAME_LENGTH
|
||||
|
||||
class SuperUserManager(object):
|
||||
""" In-memory helper class for quickly accessing (and updating) the valid
|
||||
set of super users. This class communicates across processes to ensure
|
||||
that the shared set is always the same.
|
||||
"""
|
||||
|
||||
def __init__(self, app):
|
||||
usernames = app.config.get('SUPER_USERS', [])
|
||||
usernames_str = ','.join(usernames)
|
||||
|
||||
self._max_length = len(usernames_str) + MAX_USERNAME_LENGTH + 1
|
||||
self._array = Array('c', self._max_length, lock=True)
|
||||
self._array.value = usernames_str
|
||||
|
||||
def is_superuser(self, username):
|
||||
""" Returns if the given username represents a super user. """
|
||||
usernames = self._array.value.split(',')
|
||||
return username in usernames
|
||||
|
||||
def register_superuser(self, username):
|
||||
""" Registers a new username as a super user for the duration of the container.
|
||||
Note that this does *not* change any underlying config files.
|
||||
"""
|
||||
usernames = self._array.value.split(',')
|
||||
usernames.append(username)
|
||||
new_string = ','.join(usernames)
|
||||
|
||||
if len(new_string) <= self._max_length:
|
||||
self._array.value = new_string
|
||||
else:
|
||||
raise Exception('Maximum superuser count reached. Please report this to support.')
|
||||
|
||||
def has_superusers(self):
|
||||
""" Returns whether there are any superusers defined. """
|
||||
return bool(self._array.value)
|
7
util/config/test/test_schema.py
Normal file
7
util/config/test/test_schema.py
Normal file
|
@ -0,0 +1,7 @@
|
|||
from config import DefaultConfig
|
||||
from util.config.schema import CONFIG_SCHEMA, INTERNAL_ONLY_PROPERTIES
|
||||
|
||||
def test_ensure_schema_defines_all_fields():
|
||||
for key in vars(DefaultConfig):
|
||||
has_key = key in CONFIG_SCHEMA['properties'] or key in INTERNAL_ONLY_PROPERTIES
|
||||
assert has_key, "Property `%s` is missing from config schema" % key
|
32
util/config/test/test_validator.py
Normal file
32
util/config/test/test_validator.py
Normal file
|
@ -0,0 +1,32 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import is_valid_config_upload_filename
|
||||
from util.config.validator import CONFIG_FILENAMES, CONFIG_FILE_SUFFIXES
|
||||
|
||||
def test_valid_config_upload_filenames():
|
||||
for filename in CONFIG_FILENAMES:
|
||||
assert is_valid_config_upload_filename(filename)
|
||||
|
||||
for suffix in CONFIG_FILE_SUFFIXES:
|
||||
assert is_valid_config_upload_filename('foo' + suffix)
|
||||
assert not is_valid_config_upload_filename(suffix + 'foo')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('filename, expect_valid', [
|
||||
('', False),
|
||||
('foo', False),
|
||||
('config.yaml', False),
|
||||
|
||||
('ssl.cert', True),
|
||||
('ssl.key', True),
|
||||
|
||||
('ssl.crt', False),
|
||||
|
||||
('foobar-cloudfront-signing-key.pem', True),
|
||||
('foobaz-cloudfront-signing-key.pem', True),
|
||||
('barbaz-cloudfront-signing-key.pem', True),
|
||||
|
||||
('barbaz-cloudfront-signing-key.pem.bak', False),
|
||||
])
|
||||
def test_is_valid_config_upload_filename(filename, expect_valid):
|
||||
assert is_valid_config_upload_filename(filename) == expect_valid
|
152
util/config/validator.py
Normal file
152
util/config/validator.py
Normal file
|
@ -0,0 +1,152 @@
|
|||
import logging
|
||||
|
||||
from auth.auth_context import get_authenticated_user
|
||||
from data.users import LDAP_CERT_FILENAME
|
||||
from util.secscan.secscan_util import get_blob_download_uri_getter
|
||||
from util.config import URLSchemeAndHostname
|
||||
|
||||
from util.config.validators.validate_database import DatabaseValidator
|
||||
from util.config.validators.validate_redis import RedisValidator
|
||||
from util.config.validators.validate_storage import StorageValidator
|
||||
from util.config.validators.validate_ldap import LDAPValidator
|
||||
from util.config.validators.validate_keystone import KeystoneValidator
|
||||
from util.config.validators.validate_jwt import JWTAuthValidator
|
||||
from util.config.validators.validate_secscan import SecurityScannerValidator
|
||||
from util.config.validators.validate_signer import SignerValidator
|
||||
from util.config.validators.validate_torrent import BittorrentValidator
|
||||
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
|
||||
from util.config.validators.validate_google_login import GoogleLoginValidator
|
||||
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
|
||||
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
|
||||
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
|
||||
from util.config.validators.validate_oidc import OIDCLoginValidator
|
||||
from util.config.validators.validate_timemachine import TimeMachineValidator
|
||||
from util.config.validators.validate_access import AccessSettingsValidator
|
||||
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
|
||||
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class ConfigValidationException(Exception):
|
||||
""" Exception raised when the configuration fails to validate for a known reason. """
|
||||
pass
|
||||
|
||||
# Note: Only add files required for HTTPS to the SSL_FILESNAMES list.
|
||||
DB_SSL_FILENAMES = ['database.pem']
|
||||
JWT_FILENAMES = ['jwt-authn.cert']
|
||||
ACI_CERT_FILENAMES = ['signing-public.gpg', 'signing-private.gpg']
|
||||
LDAP_FILENAMES = [LDAP_CERT_FILENAME]
|
||||
CONFIG_FILENAMES = (SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES + ACI_CERT_FILENAMES +
|
||||
LDAP_FILENAMES)
|
||||
CONFIG_FILE_SUFFIXES = ['-cloudfront-signing-key.pem']
|
||||
EXTRA_CA_DIRECTORY = 'extra_ca_certs'
|
||||
EXTRA_CA_DIRECTORY_PREFIX = 'extra_ca_certs_'
|
||||
|
||||
VALIDATORS = {
|
||||
DatabaseValidator.name: DatabaseValidator.validate,
|
||||
RedisValidator.name: RedisValidator.validate,
|
||||
StorageValidator.name: StorageValidator.validate,
|
||||
GitHubLoginValidator.name: GitHubLoginValidator.validate,
|
||||
GitHubTriggerValidator.name: GitHubTriggerValidator.validate,
|
||||
GitLabTriggerValidator.name: GitLabTriggerValidator.validate,
|
||||
BitbucketTriggerValidator.name: BitbucketTriggerValidator.validate,
|
||||
GoogleLoginValidator.name: GoogleLoginValidator.validate,
|
||||
SSLValidator.name: SSLValidator.validate,
|
||||
LDAPValidator.name: LDAPValidator.validate,
|
||||
JWTAuthValidator.name: JWTAuthValidator.validate,
|
||||
KeystoneValidator.name: KeystoneValidator.validate,
|
||||
SignerValidator.name: SignerValidator.validate,
|
||||
SecurityScannerValidator.name: SecurityScannerValidator.validate,
|
||||
BittorrentValidator.name: BittorrentValidator.validate,
|
||||
OIDCLoginValidator.name: OIDCLoginValidator.validate,
|
||||
TimeMachineValidator.name: TimeMachineValidator.validate,
|
||||
AccessSettingsValidator.name: AccessSettingsValidator.validate,
|
||||
ActionLogArchivingValidator.name: ActionLogArchivingValidator.validate,
|
||||
AppTokenAuthValidator.name: AppTokenAuthValidator.validate,
|
||||
}
|
||||
|
||||
def validate_service_for_config(service, validator_context):
|
||||
""" Attempts to validate the configuration for the given service. """
|
||||
if not service in VALIDATORS:
|
||||
return {
|
||||
'status': False
|
||||
}
|
||||
|
||||
try:
|
||||
VALIDATORS[service](validator_context)
|
||||
return {
|
||||
'status': True
|
||||
}
|
||||
except Exception as ex:
|
||||
logger.exception('Validation exception')
|
||||
return {
|
||||
'status': False,
|
||||
'reason': str(ex)
|
||||
}
|
||||
|
||||
|
||||
def is_valid_config_upload_filename(filename):
|
||||
""" Returns true if and only if the given filename is one which is supported for upload
|
||||
from the configuration UI tool.
|
||||
"""
|
||||
if filename in CONFIG_FILENAMES:
|
||||
return True
|
||||
|
||||
return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES])
|
||||
|
||||
|
||||
class ValidatorContext(object):
|
||||
""" Context to run validators in, with any additional runtime configuration they need
|
||||
"""
|
||||
def __init__(self, config, user_password=None, http_client=None, context=None,
|
||||
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
|
||||
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
|
||||
uri_creator=None, config_provider=None, instance_keys=None,
|
||||
init_scripts_location=None):
|
||||
self.config = config
|
||||
self.user = get_authenticated_user()
|
||||
self.user_password = user_password
|
||||
self.http_client = http_client
|
||||
self.context = context
|
||||
self.url_scheme_and_hostname = url_scheme_and_hostname
|
||||
self.jwt_auth_max = jwt_auth_max
|
||||
self.registry_title = registry_title
|
||||
self.ip_resolver = ip_resolver
|
||||
self.feature_sec_scanner = feature_sec_scanner
|
||||
self.is_testing = is_testing
|
||||
self.uri_creator = uri_creator
|
||||
self.config_provider = config_provider
|
||||
self.instance_keys = instance_keys
|
||||
self.init_scripts_location = init_scripts_location
|
||||
|
||||
@classmethod
|
||||
def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None,
|
||||
config_provider=None, init_scripts_location=None):
|
||||
"""
|
||||
Creates a ValidatorContext from an app config, with a given config to validate
|
||||
:param app: the Flask app to pull configuration information from
|
||||
:param config: the config to validate
|
||||
:param user_password: request password
|
||||
:param instance_keys: The instance keys handler
|
||||
:param ip_resolver: an App
|
||||
:param client: http client used to connect to services
|
||||
:param config_provider: config provider used to access config volume(s)
|
||||
:param init_scripts_location: location where initial load scripts are stored
|
||||
:return: ValidatorContext
|
||||
"""
|
||||
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config)
|
||||
|
||||
return cls(config,
|
||||
user_password=user_password,
|
||||
http_client=client or app.config['HTTPCLIENT'],
|
||||
context=app.app_context,
|
||||
url_scheme_and_hostname=url_scheme_and_hostname,
|
||||
jwt_auth_max=app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
|
||||
registry_title=app.config['REGISTRY_TITLE'],
|
||||
ip_resolver=ip_resolver,
|
||||
feature_sec_scanner=app.config.get('FEATURE_SECURITY_SCANNER', False),
|
||||
is_testing=app.config.get('TESTING', False),
|
||||
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
|
||||
config_provider=config_provider,
|
||||
instance_keys=instance_keys,
|
||||
init_scripts_location=init_scripts_location)
|
20
util/config/validators/__init__.py
Normal file
20
util/config/validators/__init__.py
Normal file
|
@ -0,0 +1,20 @@
|
|||
from abc import ABCMeta, abstractmethod, abstractproperty
|
||||
from six import add_metaclass
|
||||
|
||||
class ConfigValidationException(Exception):
|
||||
""" Exception raised when the configuration fails to validate for a known reason. """
|
||||
pass
|
||||
|
||||
|
||||
@add_metaclass(ABCMeta)
|
||||
class BaseValidator(object):
|
||||
@abstractproperty
|
||||
def name(self):
|
||||
""" The key for the validation API. """
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Raises Exception if failure to validate. """
|
||||
pass
|
29
util/config/validators/test/test_validate_access.py
Normal file
29
util/config/validators/test/test_validate_access.py
Normal file
|
@ -0,0 +1,29 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_access import AccessSettingsValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config, expected_exception', [
|
||||
({}, None),
|
||||
({'FEATURE_DIRECT_LOGIN': False}, ConfigValidationException),
|
||||
({'FEATURE_DIRECT_LOGIN': False, 'SOMETHING_LOGIN_CONFIG': {}}, None),
|
||||
({'FEATURE_DIRECT_LOGIN': False, 'FEATURE_GITHUB_LOGIN': True}, None),
|
||||
({'FEATURE_DIRECT_LOGIN': False, 'FEATURE_GOOGLE_LOGIN': True}, None),
|
||||
({'FEATURE_USER_CREATION': True, 'FEATURE_INVITE_ONLY_USER_CREATION': False}, None),
|
||||
({'FEATURE_USER_CREATION': True, 'FEATURE_INVITE_ONLY_USER_CREATION': True}, None),
|
||||
({'FEATURE_INVITE_ONLY_USER_CREATION': True}, None),
|
||||
({'FEATURE_USER_CREATION': False, 'FEATURE_INVITE_ONLY_USER_CREATION': True},
|
||||
ConfigValidationException),
|
||||
({'FEATURE_USER_CREATION': False, 'FEATURE_INVITE_ONLY_USER_CREATION': False}, None),
|
||||
])
|
||||
def test_validate_invalid_oidc_login_config(unvalidated_config, expected_exception, app):
|
||||
validator = AccessSettingsValidator()
|
||||
|
||||
if expected_exception is not None:
|
||||
with pytest.raises(expected_exception):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
else:
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
|
@ -0,0 +1,52 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'ACTION_LOG_ARCHIVE_PATH': 'foo'}),
|
||||
({'ACTION_LOG_ARCHIVE_LOCATION': ''}),
|
||||
])
|
||||
def test_skip_validate_actionlog(unvalidated_config, app):
|
||||
validator = ActionLogArchivingValidator()
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('config, expected_error', [
|
||||
({'FEATURE_ACTION_LOG_ROTATION': True}, 'Missing action log archive path'),
|
||||
({'FEATURE_ACTION_LOG_ROTATION': True,
|
||||
'ACTION_LOG_ARCHIVE_PATH': ''}, 'Missing action log archive path'),
|
||||
({'FEATURE_ACTION_LOG_ROTATION': True,
|
||||
'ACTION_LOG_ARCHIVE_PATH': 'foo'}, 'Missing action log archive storage location'),
|
||||
({'FEATURE_ACTION_LOG_ROTATION': True,
|
||||
'ACTION_LOG_ARCHIVE_PATH': 'foo',
|
||||
'ACTION_LOG_ARCHIVE_LOCATION': ''}, 'Missing action log archive storage location'),
|
||||
({'FEATURE_ACTION_LOG_ROTATION': True,
|
||||
'ACTION_LOG_ARCHIVE_PATH': 'foo',
|
||||
'ACTION_LOG_ARCHIVE_LOCATION': 'invalid'},
|
||||
'Action log archive storage location `invalid` not found in storage config'),
|
||||
])
|
||||
def test_invalid_config(config, expected_error, app):
|
||||
validator = ActionLogArchivingValidator()
|
||||
|
||||
with pytest.raises(ConfigValidationException) as ipe:
|
||||
validator.validate(ValidatorContext(config))
|
||||
|
||||
assert str(ipe.value) == expected_error
|
||||
|
||||
def test_valid_config(app):
|
||||
config = ValidatorContext({
|
||||
'FEATURE_ACTION_LOG_ROTATION': True,
|
||||
'ACTION_LOG_ARCHIVE_PATH': 'somepath',
|
||||
'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
|
||||
'DISTRIBUTED_STORAGE_CONFIG': {
|
||||
'somelocation': {},
|
||||
},
|
||||
})
|
||||
|
||||
validator = ActionLogArchivingValidator()
|
||||
validator.validate(config)
|
30
util/config/validators/test/test_validate_apptokenauth.py
Normal file
30
util/config/validators/test/test_validate_apptokenauth.py
Normal file
|
@ -0,0 +1,30 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({'AUTHENTICATION_TYPE': 'AppToken'}),
|
||||
({'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': False}),
|
||||
({'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': True,
|
||||
'FEATURE_DIRECT_LOGIN': True}),
|
||||
])
|
||||
def test_validate_invalid_auth_config(unvalidated_config, app):
|
||||
validator = AppTokenAuthValidator()
|
||||
|
||||
with pytest.raises(ConfigValidationException):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
|
||||
def test_validate_auth(app):
|
||||
config = ValidatorContext({
|
||||
'AUTHENTICATION_TYPE': 'AppToken',
|
||||
'FEATURE_APP_SPECIFIC_TOKENS': True,
|
||||
'FEATURE_DIRECT_LOGIN': False,
|
||||
})
|
||||
|
||||
validator = AppTokenAuthValidator()
|
||||
validator.validate(config)
|
|
@ -0,0 +1,48 @@
|
|||
import pytest
|
||||
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from util.config import URLSchemeAndHostname
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
(ValidatorContext({})),
|
||||
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
|
||||
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
|
||||
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
|
||||
])
|
||||
def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
|
||||
validator = BitbucketTriggerValidator()
|
||||
|
||||
with pytest.raises(ConfigValidationException):
|
||||
validator.validate(unvalidated_config)
|
||||
|
||||
def test_validate_bitbucket_trigger(app):
|
||||
url_hit = [False]
|
||||
|
||||
@urlmatch(netloc=r'bitbucket.org')
|
||||
def handler(url, request):
|
||||
url_hit[0] = True
|
||||
return {
|
||||
'status_code': 200,
|
||||
'content': 'oauth_token=foo&oauth_token_secret=bar',
|
||||
}
|
||||
|
||||
with HTTMock(handler):
|
||||
validator = BitbucketTriggerValidator()
|
||||
|
||||
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
|
||||
unvalidated_config = ValidatorContext({
|
||||
'BITBUCKET_TRIGGER_CONFIG': {
|
||||
'CONSUMER_KEY': 'foo',
|
||||
'CONSUMER_SECRET': 'bar',
|
||||
},
|
||||
}, url_scheme_and_hostname=url_scheme_and_hostname)
|
||||
|
||||
validator.validate(unvalidated_config)
|
||||
|
||||
assert url_hit[0]
|
23
util/config/validators/test/test_validate_database.py
Normal file
23
util/config/validators/test/test_validate_database.py
Normal file
|
@ -0,0 +1,23 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_database import DatabaseValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
|
||||
(ValidatorContext(None), None, None, TypeError),
|
||||
(ValidatorContext({}), None, None, KeyError),
|
||||
(ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
|
||||
(ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
|
||||
(ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
|
||||
])
|
||||
def test_validate_database(unvalidated_config, user, user_password, expected, app):
|
||||
validator = DatabaseValidator()
|
||||
|
||||
if expected is not None:
|
||||
with pytest.raises(expected):
|
||||
validator.validate(unvalidated_config)
|
||||
else:
|
||||
validator.validate(unvalidated_config)
|
69
util/config/validators/test/test_validate_github.py
Normal file
69
util/config/validators/test/test_validate_github.py
Normal file
|
@ -0,0 +1,69 @@
|
|||
import pytest
|
||||
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from config import build_requests_session
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.fixture(params=[GitHubLoginValidator, GitHubTriggerValidator])
|
||||
def github_validator(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.mark.parametrize('github_config', [
|
||||
({}),
|
||||
({'GITHUB_ENDPOINT': 'foo'}),
|
||||
({'GITHUB_ENDPOINT': 'http://github.com'}),
|
||||
({'GITHUB_ENDPOINT': 'http://github.com', 'CLIENT_ID': 'foo'}),
|
||||
({'GITHUB_ENDPOINT': 'http://github.com', 'CLIENT_SECRET': 'foo'}),
|
||||
({
|
||||
'GITHUB_ENDPOINT': 'http://github.com',
|
||||
'CLIENT_ID': 'foo',
|
||||
'CLIENT_SECRET': 'foo',
|
||||
'ORG_RESTRICT': True
|
||||
}),
|
||||
({
|
||||
'GITHUB_ENDPOINT': 'http://github.com',
|
||||
'CLIENT_ID': 'foo',
|
||||
'CLIENT_SECRET': 'foo',
|
||||
'ORG_RESTRICT': True,
|
||||
'ALLOWED_ORGANIZATIONS': [],
|
||||
}),
|
||||
])
|
||||
def test_validate_invalid_github_config(github_config, github_validator, app):
|
||||
with pytest.raises(ConfigValidationException):
|
||||
unvalidated_config = {}
|
||||
unvalidated_config[github_validator.config_key] = github_config
|
||||
github_validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
def test_validate_github(github_validator, app):
|
||||
url_hit = [False, False]
|
||||
|
||||
@urlmatch(netloc=r'somehost')
|
||||
def handler(url, request):
|
||||
url_hit[0] = True
|
||||
return {'status_code': 200, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
|
||||
|
||||
@urlmatch(netloc=r'somehost', path=r'/api/v3/applications/foo/tokens/foo')
|
||||
def app_handler(url, request):
|
||||
url_hit[1] = True
|
||||
return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
|
||||
|
||||
with HTTMock(app_handler, handler):
|
||||
unvalidated_config = ValidatorContext({
|
||||
github_validator.config_key: {
|
||||
'GITHUB_ENDPOINT': 'http://somehost',
|
||||
'CLIENT_ID': 'foo',
|
||||
'CLIENT_SECRET': 'bar',
|
||||
},
|
||||
})
|
||||
|
||||
unvalidated_config.http_client = build_requests_session()
|
||||
github_validator.validate(unvalidated_config)
|
||||
|
||||
assert url_hit[0]
|
||||
assert url_hit[1]
|
49
util/config/validators/test/test_validate_gitlab_trigger.py
Normal file
49
util/config/validators/test/test_validate_gitlab_trigger.py
Normal file
|
@ -0,0 +1,49 @@
|
|||
import json
|
||||
import pytest
|
||||
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from config import build_requests_session
|
||||
from util.config import URLSchemeAndHostname
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'foo'}}),
|
||||
({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'http://someendpoint', 'CLIENT_ID': 'foo'}}),
|
||||
({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'http://someendpoint', 'CLIENT_SECRET': 'foo'}}),
|
||||
])
|
||||
def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
|
||||
validator = GitLabTriggerValidator()
|
||||
|
||||
with pytest.raises(ConfigValidationException):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
def test_validate_gitlab_enterprise_trigger(app):
|
||||
url_hit = [False]
|
||||
|
||||
@urlmatch(netloc=r'somegitlab', path='/oauth/token')
|
||||
def handler(_, __):
|
||||
url_hit[0] = True
|
||||
return {'status_code': 400, 'content': json.dumps({'error': 'invalid code'})}
|
||||
|
||||
with HTTMock(handler):
|
||||
validator = GitLabTriggerValidator()
|
||||
|
||||
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
|
||||
|
||||
unvalidated_config = ValidatorContext({
|
||||
'GITLAB_TRIGGER_CONFIG': {
|
||||
'GITLAB_ENDPOINT': 'http://somegitlab',
|
||||
'CLIENT_ID': 'foo',
|
||||
'CLIENT_SECRET': 'bar',
|
||||
},
|
||||
}, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname)
|
||||
|
||||
validator.validate(unvalidated_config)
|
||||
|
||||
assert url_hit[0]
|
45
util/config/validators/test/test_validate_google_login.py
Normal file
45
util/config/validators/test/test_validate_google_login.py
Normal file
|
@ -0,0 +1,45 @@
|
|||
import pytest
|
||||
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from config import build_requests_session
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_google_login import GoogleLoginValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'GOOGLE_LOGIN_CONFIG': {}}),
|
||||
({'GOOGLE_LOGIN_CONFIG': {'CLIENT_ID': 'foo'}}),
|
||||
({'GOOGLE_LOGIN_CONFIG': {'CLIENT_SECRET': 'foo'}}),
|
||||
])
|
||||
def test_validate_invalid_google_login_config(unvalidated_config, app):
|
||||
validator = GoogleLoginValidator()
|
||||
|
||||
with pytest.raises(ConfigValidationException):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
def test_validate_google_login(app):
|
||||
url_hit = [False]
|
||||
@urlmatch(netloc=r'www.googleapis.com', path='/oauth2/v3/token')
|
||||
def handler(_, __):
|
||||
url_hit[0] = True
|
||||
return {'status_code': 200, 'content': ''}
|
||||
|
||||
validator = GoogleLoginValidator()
|
||||
|
||||
with HTTMock(handler):
|
||||
unvalidated_config = ValidatorContext({
|
||||
'GOOGLE_LOGIN_CONFIG': {
|
||||
'CLIENT_ID': 'foo',
|
||||
'CLIENT_SECRET': 'bar',
|
||||
},
|
||||
})
|
||||
|
||||
unvalidated_config.http_client = build_requests_session()
|
||||
|
||||
validator.validate(unvalidated_config)
|
||||
|
||||
assert url_hit[0]
|
65
util/config/validators/test/test_validate_jwt.py
Normal file
65
util/config/validators/test/test_validate_jwt.py
Normal file
|
@ -0,0 +1,65 @@
|
|||
import pytest
|
||||
|
||||
from config import build_requests_session
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_jwt import JWTAuthValidator
|
||||
from util.morecollections import AttrDict
|
||||
|
||||
from test.test_external_jwt_authn import fake_jwt
|
||||
|
||||
from test.fixtures import *
|
||||
from app import config_provider
|
||||
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'AUTHENTICATION_TYPE': 'Database'}),
|
||||
])
|
||||
def test_validate_noop(unvalidated_config, app):
|
||||
config = ValidatorContext(unvalidated_config)
|
||||
config.config_provider = config_provider
|
||||
JWTAuthValidator.validate(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({'AUTHENTICATION_TYPE': 'JWT'}),
|
||||
({'AUTHENTICATION_TYPE': 'JWT', 'JWT_AUTH_ISSUER': 'foo'}),
|
||||
({'AUTHENTICATION_TYPE': 'JWT', 'JWT_VERIFY_ENDPOINT': 'foo'}),
|
||||
])
|
||||
def test_invalid_config(unvalidated_config, app):
|
||||
with pytest.raises(ConfigValidationException):
|
||||
config = ValidatorContext(unvalidated_config)
|
||||
config.config_provider = config_provider
|
||||
JWTAuthValidator.validate(config)
|
||||
|
||||
|
||||
# TODO: fix these when re-adding jwt auth mechanism to jwt validators
|
||||
@pytest.mark.skip(reason='No way of currently testing this')
|
||||
@pytest.mark.parametrize('username, password, expected_exception', [
|
||||
('invaliduser', 'invalidpass', ConfigValidationException),
|
||||
('cool.user', 'invalidpass', ConfigValidationException),
|
||||
('invaliduser', 'somepass', ConfigValidationException),
|
||||
('cool.user', 'password', None),
|
||||
])
|
||||
def test_validated_jwt(username, password, expected_exception, app):
|
||||
with fake_jwt() as jwt_auth:
|
||||
config = {}
|
||||
config['AUTHENTICATION_TYPE'] = 'JWT'
|
||||
config['JWT_AUTH_ISSUER'] = jwt_auth.issuer
|
||||
config['JWT_VERIFY_ENDPOINT'] = jwt_auth.verify_url
|
||||
config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
|
||||
config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
|
||||
|
||||
unvalidated_config = ValidatorContext(config)
|
||||
unvalidated_config.user = AttrDict(dict(username=username))
|
||||
unvalidated_config.user_password = password
|
||||
unvalidated_config.config_provider = config_provider
|
||||
|
||||
unvalidated_config.http_client = build_requests_session()
|
||||
|
||||
if expected_exception is not None:
|
||||
with pytest.raises(ConfigValidationException):
|
||||
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
|
||||
else:
|
||||
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
|
54
util/config/validators/test/test_validate_keystone.py
Normal file
54
util/config/validators/test/test_validate_keystone.py
Normal file
|
@ -0,0 +1,54 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_keystone import KeystoneValidator
|
||||
|
||||
from test.test_keystone_auth import fake_keystone
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'AUTHENTICATION_TYPE': 'Database'}),
|
||||
])
|
||||
def test_validate_noop(unvalidated_config, app):
|
||||
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({'AUTHENTICATION_TYPE': 'Keystone'}),
|
||||
({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo'}),
|
||||
({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo',
|
||||
'KEYSTONE_ADMIN_USERNAME': 'bar'}),
|
||||
({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo',
|
||||
'KEYSTONE_ADMIN_USERNAME': 'bar', 'KEYSTONE_ADMIN_PASSWORD': 'baz'}),
|
||||
])
|
||||
def test_invalid_config(unvalidated_config, app):
|
||||
with pytest.raises(ConfigValidationException):
|
||||
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('admin_tenant_id, expected_exception', [
|
||||
('somegroupid', None),
|
||||
('groupwithnousers', ConfigValidationException),
|
||||
('somegroupid', None),
|
||||
('groupwithnousers', ConfigValidationException),
|
||||
])
|
||||
def test_validated_keystone(admin_tenant_id, expected_exception, app):
|
||||
with fake_keystone(2) as keystone_auth:
|
||||
auth_url = keystone_auth.auth_url
|
||||
|
||||
config = {}
|
||||
config['AUTHENTICATION_TYPE'] = 'Keystone'
|
||||
config['KEYSTONE_AUTH_URL'] = auth_url
|
||||
config['KEYSTONE_ADMIN_USERNAME'] = 'adminuser'
|
||||
config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
|
||||
config['KEYSTONE_ADMIN_TENANT'] = admin_tenant_id
|
||||
|
||||
unvalidated_config = ValidatorContext(config)
|
||||
|
||||
if expected_exception is not None:
|
||||
with pytest.raises(ConfigValidationException):
|
||||
KeystoneValidator.validate(unvalidated_config)
|
||||
else:
|
||||
KeystoneValidator.validate(unvalidated_config)
|
72
util/config/validators/test/test_validate_ldap.py
Normal file
72
util/config/validators/test/test_validate_ldap.py
Normal file
|
@ -0,0 +1,72 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_ldap import LDAPValidator
|
||||
from util.morecollections import AttrDict
|
||||
|
||||
from test.test_ldap import mock_ldap
|
||||
|
||||
from test.fixtures import *
|
||||
from app import config_provider
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'AUTHENTICATION_TYPE': 'Database'}),
|
||||
])
|
||||
def test_validate_noop(unvalidated_config, app):
|
||||
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
|
||||
LDAPValidator.validate(config)
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({'AUTHENTICATION_TYPE': 'LDAP'}),
|
||||
({'AUTHENTICATION_TYPE': 'LDAP', 'LDAP_ADMIN_DN': 'foo'}),
|
||||
])
|
||||
def test_invalid_config(unvalidated_config, app):
|
||||
with pytest.raises(ConfigValidationException):
|
||||
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
|
||||
LDAPValidator.validate(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('uri', [
|
||||
'foo',
|
||||
'http://foo',
|
||||
'ldap:foo',
|
||||
])
|
||||
def test_invalid_uri(uri, app):
|
||||
config = {}
|
||||
config['AUTHENTICATION_TYPE'] = 'LDAP'
|
||||
config['LDAP_BASE_DN'] = ['dc=quay', 'dc=io']
|
||||
config['LDAP_ADMIN_DN'] = 'uid=testy,ou=employees,dc=quay,dc=io'
|
||||
config['LDAP_ADMIN_PASSWD'] = 'password'
|
||||
config['LDAP_USER_RDN'] = ['ou=employees']
|
||||
config['LDAP_URI'] = uri
|
||||
|
||||
with pytest.raises(ConfigValidationException):
|
||||
config = ValidatorContext(config, config_provider=config_provider)
|
||||
LDAPValidator.validate(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('admin_dn, admin_passwd, user_rdn, expected_exception', [
|
||||
('uid=testy,ou=employees,dc=quay,dc=io', 'password', ['ou=employees'], None),
|
||||
('uid=invalidadmindn', 'password', ['ou=employees'], ConfigValidationException),
|
||||
('uid=testy,ou=employees,dc=quay,dc=io', 'invalid_password', ['ou=employees'], ConfigValidationException),
|
||||
('uid=testy,ou=employees,dc=quay,dc=io', 'password', ['ou=invalidgroup'], ConfigValidationException),
|
||||
])
|
||||
def test_validated_ldap(admin_dn, admin_passwd, user_rdn, expected_exception, app):
|
||||
config = {}
|
||||
config['AUTHENTICATION_TYPE'] = 'LDAP'
|
||||
config['LDAP_BASE_DN'] = ['dc=quay', 'dc=io']
|
||||
config['LDAP_ADMIN_DN'] = admin_dn
|
||||
config['LDAP_ADMIN_PASSWD'] = admin_passwd
|
||||
config['LDAP_USER_RDN'] = user_rdn
|
||||
|
||||
unvalidated_config = ValidatorContext(config, config_provider=config_provider)
|
||||
|
||||
if expected_exception is not None:
|
||||
with pytest.raises(ConfigValidationException):
|
||||
with mock_ldap():
|
||||
LDAPValidator.validate(unvalidated_config)
|
||||
else:
|
||||
with mock_ldap():
|
||||
LDAPValidator.validate(unvalidated_config)
|
50
util/config/validators/test/test_validate_oidc.py
Normal file
50
util/config/validators/test/test_validate_oidc.py
Normal file
|
@ -0,0 +1,50 @@
|
|||
import json
|
||||
import pytest
|
||||
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from config import build_requests_session
|
||||
from oauth.oidc import OIDC_WELLKNOWN
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_oidc import OIDCLoginValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({'SOMETHING_LOGIN_CONFIG': {}}),
|
||||
({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo'}}),
|
||||
({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo', 'CLIENT_ID': 'foobar'}}),
|
||||
({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo', 'CLIENT_SECRET': 'foobar'}}),
|
||||
])
|
||||
def test_validate_invalid_oidc_login_config(unvalidated_config, app):
|
||||
validator = OIDCLoginValidator()
|
||||
|
||||
with pytest.raises(ConfigValidationException):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
def test_validate_oidc_login(app):
|
||||
url_hit = [False]
|
||||
@urlmatch(netloc=r'someserver', path=r'/\.well-known/openid-configuration')
|
||||
def handler(_, __):
|
||||
url_hit[0] = True
|
||||
data = {
|
||||
'token_endpoint': 'foobar',
|
||||
}
|
||||
return {'status_code': 200, 'content': json.dumps(data)}
|
||||
|
||||
with HTTMock(handler):
|
||||
validator = OIDCLoginValidator()
|
||||
unvalidated_config = ValidatorContext({
|
||||
'SOMETHING_LOGIN_CONFIG': {
|
||||
'CLIENT_ID': 'foo',
|
||||
'CLIENT_SECRET': 'bar',
|
||||
'OIDC_SERVER': 'http://someserver',
|
||||
'DEBUGGING': True, # Allows for HTTP.
|
||||
},
|
||||
})
|
||||
unvalidated_config.http_client = build_requests_session()
|
||||
|
||||
validator.validate(unvalidated_config)
|
||||
|
||||
assert url_hit[0]
|
34
util/config/validators/test/test_validate_redis.py
Normal file
34
util/config/validators/test/test_validate_redis.py
Normal file
|
@ -0,0 +1,34 @@
|
|||
import pytest
|
||||
import redis
|
||||
|
||||
from mock import patch
|
||||
|
||||
from mockredis import mock_strict_redis_client
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_redis import RedisValidator
|
||||
|
||||
from test.fixtures import *
|
||||
from util.morecollections import AttrDict
|
||||
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
|
||||
({}, None, None, False, ConfigValidationException),
|
||||
({'BUILDLOGS_REDIS': {}}, None, None, False, ConfigValidationException),
|
||||
({'BUILDLOGS_REDIS': {'host': 'somehost'}}, None, None, False, redis.ConnectionError),
|
||||
({'BUILDLOGS_REDIS': {'host': 'localhost'}}, None, None, True, None),
|
||||
])
|
||||
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
|
||||
with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
|
||||
validator = RedisValidator()
|
||||
unvalidated_config = ValidatorContext(unvalidated_config)
|
||||
|
||||
unvalidated_config.user = AttrDict(dict(username=user))
|
||||
unvalidated_config.user_password = user_password
|
||||
|
||||
if expected is not None:
|
||||
with pytest.raises(expected):
|
||||
validator.validate(unvalidated_config)
|
||||
else:
|
||||
validator.validate(unvalidated_config)
|
48
util/config/validators/test/test_validate_secscan.py
Normal file
48
util/config/validators/test/test_validate_secscan.py
Normal file
|
@ -0,0 +1,48 @@
|
|||
import pytest
|
||||
|
||||
from config import build_requests_session
|
||||
from util.config import URLSchemeAndHostname
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators.validate_secscan import SecurityScannerValidator
|
||||
from util.secscan.fake import fake_security_scanner
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
|
||||
])
|
||||
def test_validate_noop(unvalidated_config, app):
|
||||
|
||||
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True,
|
||||
http_client=build_requests_session(),
|
||||
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
|
||||
|
||||
SecurityScannerValidator.validate(unvalidated_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config, expected_error', [
|
||||
({
|
||||
'TESTING': True,
|
||||
'DISTRIBUTED_STORAGE_PREFERENCE': [],
|
||||
'FEATURE_SECURITY_SCANNER': True,
|
||||
'SECURITY_SCANNER_ENDPOINT': 'http://invalidhost',
|
||||
}, Exception),
|
||||
|
||||
({
|
||||
'TESTING': True,
|
||||
'DISTRIBUTED_STORAGE_PREFERENCE': [],
|
||||
'FEATURE_SECURITY_SCANNER': True,
|
||||
'SECURITY_SCANNER_ENDPOINT': 'http://fakesecurityscanner',
|
||||
}, None),
|
||||
])
|
||||
def test_validate(unvalidated_config, expected_error, app):
|
||||
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True,
|
||||
http_client=build_requests_session(),
|
||||
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
|
||||
|
||||
with fake_security_scanner(hostname='fakesecurityscanner'):
|
||||
if expected_error is not None:
|
||||
with pytest.raises(expected_error):
|
||||
SecurityScannerValidator.validate(unvalidated_config)
|
||||
else:
|
||||
SecurityScannerValidator.validate(unvalidated_config)
|
20
util/config/validators/test/test_validate_signer.py
Normal file
20
util/config/validators/test/test_validate_signer.py
Normal file
|
@ -0,0 +1,20 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_signer import SignerValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config,expected', [
|
||||
({}, None),
|
||||
({'SIGNING_ENGINE': 'foobar'}, ConfigValidationException),
|
||||
({'SIGNING_ENGINE': 'gpg2'}, Exception),
|
||||
])
|
||||
def test_validate_signer(unvalidated_config, expected, app):
|
||||
validator = SignerValidator()
|
||||
if expected is not None:
|
||||
with pytest.raises(expected):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
else:
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
75
util/config/validators/test/test_validate_ssl.py
Normal file
75
util/config/validators/test/test_validate_ssl.py
Normal file
|
@ -0,0 +1,75 @@
|
|||
import pytest
|
||||
|
||||
from mock import patch
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
|
||||
from util.security.test.test_ssl_util import generate_test_cert
|
||||
|
||||
from test.fixtures import *
|
||||
from app import config_provider
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
({'PREFERRED_URL_SCHEME': 'http'}),
|
||||
({'PREFERRED_URL_SCHEME': 'https', 'EXTERNAL_TLS_TERMINATION': True}),
|
||||
])
|
||||
def test_skip_validate_ssl(unvalidated_config, app):
|
||||
validator = SSLValidator()
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('cert, server_hostname, expected_error, error_message', [
|
||||
('invalidcert', 'someserver', ConfigValidationException, 'Could not load SSL certificate: no start line'),
|
||||
(generate_test_cert(hostname='someserver'), 'someserver', None, None),
|
||||
(generate_test_cert(hostname='invalidserver'), 'someserver', ConfigValidationException,
|
||||
'Supported names "invalidserver" in SSL cert do not match server hostname "someserver"'),
|
||||
(generate_test_cert(hostname='someserver'), 'someserver:1234', None, None),
|
||||
(generate_test_cert(hostname='invalidserver'), 'someserver:1234', ConfigValidationException,
|
||||
'Supported names "invalidserver" in SSL cert do not match server hostname "someserver"'),
|
||||
(generate_test_cert(hostname='someserver:1234'), 'someserver:1234', ConfigValidationException,
|
||||
'Supported names "someserver:1234" in SSL cert do not match server hostname "someserver"'),
|
||||
(generate_test_cert(hostname='someserver:more'), 'someserver:more', None, None),
|
||||
(generate_test_cert(hostname='someserver:more'), 'someserver:more:1234', None, None),
|
||||
])
|
||||
def test_validate_ssl(cert, server_hostname, expected_error, error_message, app):
|
||||
with NamedTemporaryFile(delete=False) as cert_file:
|
||||
cert_file.write(cert[0])
|
||||
cert_file.seek(0)
|
||||
|
||||
with NamedTemporaryFile(delete=False) as key_file:
|
||||
key_file.write(cert[1])
|
||||
key_file.seek(0)
|
||||
|
||||
def return_true(filename):
|
||||
return True
|
||||
|
||||
def get_volume_file(filename):
|
||||
if filename == SSL_FILENAMES[0]:
|
||||
return open(cert_file.name)
|
||||
|
||||
if filename == SSL_FILENAMES[1]:
|
||||
return open(key_file.name)
|
||||
|
||||
return None
|
||||
|
||||
config = {
|
||||
'PREFERRED_URL_SCHEME': 'https',
|
||||
'SERVER_HOSTNAME': server_hostname,
|
||||
}
|
||||
|
||||
with patch('app.config_provider.volume_file_exists', return_true):
|
||||
with patch('app.config_provider.get_volume_file', get_volume_file):
|
||||
validator = SSLValidator()
|
||||
config = ValidatorContext(config)
|
||||
config.config_provider = config_provider
|
||||
|
||||
if expected_error is not None:
|
||||
with pytest.raises(expected_error) as ipe:
|
||||
validator.validate(config)
|
||||
|
||||
assert str(ipe.value) == error_message
|
||||
else:
|
||||
validator.validate(config)
|
40
util/config/validators/test/test_validate_storage.py
Normal file
40
util/config/validators/test/test_validate_storage.py
Normal file
|
@ -0,0 +1,40 @@
|
|||
import pytest
|
||||
|
||||
from moto import mock_s3_deprecated as mock_s3
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_storage import StorageValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config, expected', [
|
||||
({}, ConfigValidationException),
|
||||
({'DISTRIBUTED_STORAGE_CONFIG': {}}, ConfigValidationException),
|
||||
({'DISTRIBUTED_STORAGE_CONFIG': {'local': None}}, ConfigValidationException),
|
||||
({'DISTRIBUTED_STORAGE_CONFIG': {'local': ['FakeStorage', {}]}}, None),
|
||||
])
|
||||
def test_validate_storage(unvalidated_config, expected, app):
|
||||
validator = StorageValidator()
|
||||
if expected is not None:
|
||||
with pytest.raises(expected):
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
else:
|
||||
validator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
def test_validate_s3_storage(app):
|
||||
validator = StorageValidator()
|
||||
with mock_s3():
|
||||
with pytest.raises(ConfigValidationException) as ipe:
|
||||
validator.validate(ValidatorContext({
|
||||
'DISTRIBUTED_STORAGE_CONFIG': {
|
||||
'default': ('S3Storage', {
|
||||
's3_access_key': 'invalid',
|
||||
's3_secret_key': 'invalid',
|
||||
's3_bucket': 'somebucket',
|
||||
'storage_path': ''
|
||||
}),
|
||||
}
|
||||
}))
|
||||
|
||||
assert str(ipe.value) == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'
|
32
util/config/validators/test/test_validate_timemachine.py
Normal file
32
util/config/validators/test/test_validate_timemachine.py
Normal file
|
@ -0,0 +1,32 @@
|
|||
import pytest
|
||||
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_timemachine import TimeMachineValidator
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config', [
|
||||
({}),
|
||||
])
|
||||
def test_validate_noop(unvalidated_config):
|
||||
TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
|
||||
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('default_exp,options,expected_exception', [
|
||||
('2d', ['1w', '2d'], None),
|
||||
|
||||
('2d', ['1w'], 'Default expiration must be in expiration options set'),
|
||||
('2d', ['2d', '1M'], 'Invalid tag expiration option: 1M'),
|
||||
])
|
||||
def test_validate(default_exp, options, expected_exception, app):
|
||||
config = {}
|
||||
config['DEFAULT_TAG_EXPIRATION'] = default_exp
|
||||
config['TAG_EXPIRATION_OPTIONS'] = options
|
||||
|
||||
if expected_exception is not None:
|
||||
with pytest.raises(ConfigValidationException) as cve:
|
||||
TimeMachineValidator.validate(ValidatorContext(config))
|
||||
assert str(cve.value) == str(expected_exception)
|
||||
else:
|
||||
TimeMachineValidator.validate(ValidatorContext(config))
|
39
util/config/validators/test/test_validate_torrent.py
Normal file
39
util/config/validators/test/test_validate_torrent.py
Normal file
|
@ -0,0 +1,39 @@
|
|||
import pytest
|
||||
|
||||
from config import build_requests_session
|
||||
from httmock import urlmatch, HTTMock
|
||||
|
||||
from app import instance_keys
|
||||
from util.config.validator import ValidatorContext
|
||||
from util.config.validators import ConfigValidationException
|
||||
from util.config.validators.validate_torrent import BittorrentValidator
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
@pytest.mark.parametrize('unvalidated_config,expected', [
|
||||
({}, ConfigValidationException),
|
||||
({'BITTORRENT_ANNOUNCE_URL': 'http://faketorrent/announce'}, None),
|
||||
])
|
||||
def test_validate_torrent(unvalidated_config, expected, app):
|
||||
announcer_hit = [False]
|
||||
|
||||
@urlmatch(netloc=r'faketorrent', path='/announce')
|
||||
def handler(url, request):
|
||||
announcer_hit[0] = True
|
||||
return {'status_code': 200, 'content': ''}
|
||||
|
||||
with HTTMock(handler):
|
||||
validator = BittorrentValidator()
|
||||
if expected is not None:
|
||||
with pytest.raises(expected):
|
||||
config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
|
||||
config.http_client = build_requests_session()
|
||||
|
||||
validator.validate(config)
|
||||
assert not announcer_hit[0]
|
||||
else:
|
||||
config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
|
||||
config.http_client = build_requests_session()
|
||||
|
||||
validator.validate(config)
|
||||
assert announcer_hit[0]
|
28
util/config/validators/validate_access.py
Normal file
28
util/config/validators/validate_access.py
Normal file
|
@ -0,0 +1,28 @@
|
|||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
from oauth.loginmanager import OAuthLoginManager
|
||||
from oauth.oidc import OIDCLoginService
|
||||
|
||||
class AccessSettingsValidator(BaseValidator):
|
||||
name = "access"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
|
||||
if not config.get('FEATURE_DIRECT_LOGIN', True):
|
||||
# Make sure we have at least one OIDC enabled.
|
||||
github_login = config.get('FEATURE_GITHUB_LOGIN', False)
|
||||
google_login = config.get('FEATURE_GOOGLE_LOGIN', False)
|
||||
|
||||
login_manager = OAuthLoginManager(config, client=client)
|
||||
custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)]
|
||||
|
||||
if not github_login and not google_login and not custom_oidc:
|
||||
msg = 'Cannot disable credentials login to UI without configured OIDC service'
|
||||
raise ConfigValidationException(msg)
|
||||
|
||||
if (not config.get('FEATURE_USER_CREATION', True) and
|
||||
config.get('FEATURE_INVITE_ONLY_USER_CREATION', False)):
|
||||
msg = "Invite only user creation requires user creation to be enabled"
|
||||
raise ConfigValidationException(msg)
|
24
util/config/validators/validate_actionlog_archiving.py
Normal file
24
util/config/validators/validate_actionlog_archiving.py
Normal file
|
@ -0,0 +1,24 @@
|
|||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class ActionLogArchivingValidator(BaseValidator):
|
||||
name = "actionlogarchiving"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
config = validator_context.config
|
||||
|
||||
""" Validates the action log archiving configuration. """
|
||||
if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
|
||||
return
|
||||
|
||||
if not config.get('ACTION_LOG_ARCHIVE_PATH'):
|
||||
raise ConfigValidationException('Missing action log archive path')
|
||||
|
||||
if not config.get('ACTION_LOG_ARCHIVE_LOCATION'):
|
||||
raise ConfigValidationException('Missing action log archive storage location')
|
||||
|
||||
location = config['ACTION_LOG_ARCHIVE_LOCATION']
|
||||
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG') or {}
|
||||
if location not in storage_config:
|
||||
msg = 'Action log archive storage location `%s` not found in storage config' % location
|
||||
raise ConfigValidationException(msg)
|
21
util/config/validators/validate_apptokenauth.py
Normal file
21
util/config/validators/validate_apptokenauth.py
Normal file
|
@ -0,0 +1,21 @@
|
|||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class AppTokenAuthValidator(BaseValidator):
|
||||
name = "apptoken-auth"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
config = validator_context.config
|
||||
|
||||
if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken':
|
||||
return
|
||||
|
||||
# Ensure that app tokens are enabled, as they are required.
|
||||
if not config.get('FEATURE_APP_SPECIFIC_TOKENS', False):
|
||||
msg = 'Application token support must be enabled to use External Application Token auth'
|
||||
raise ConfigValidationException(msg)
|
||||
|
||||
# Ensure that direct login is disabled.
|
||||
if config.get('FEATURE_DIRECT_LOGIN', True):
|
||||
msg = 'Direct login must be disabled to use External Application Token auth'
|
||||
raise ConfigValidationException(msg)
|
30
util/config/validators/validate_bitbucket_trigger.py
Normal file
30
util/config/validators/validate_bitbucket_trigger.py
Normal file
|
@ -0,0 +1,30 @@
|
|||
from bitbucket import BitBucket
|
||||
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class BitbucketTriggerValidator(BaseValidator):
|
||||
name = "bitbucket-trigger"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the config for BitBucket. """
|
||||
config = validator_context.config
|
||||
|
||||
trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
|
||||
if not trigger_config:
|
||||
raise ConfigValidationException('Missing client ID and client secret')
|
||||
|
||||
if not trigger_config.get('CONSUMER_KEY'):
|
||||
raise ConfigValidationException('Missing Consumer Key')
|
||||
|
||||
if not trigger_config.get('CONSUMER_SECRET'):
|
||||
raise ConfigValidationException('Missing Consumer Secret')
|
||||
|
||||
key = trigger_config['CONSUMER_KEY']
|
||||
secret = trigger_config['CONSUMER_SECRET']
|
||||
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url())
|
||||
|
||||
bitbucket_client = BitBucket(key, secret, callback_url)
|
||||
(result, _, _) = bitbucket_client.get_authorization_url()
|
||||
if not result:
|
||||
raise ConfigValidationException('Invalid consumer key or secret')
|
20
util/config/validators/validate_database.py
Normal file
20
util/config/validators/validate_database.py
Normal file
|
@ -0,0 +1,20 @@
|
|||
from peewee import OperationalError
|
||||
|
||||
from data.database import validate_database_precondition
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class DatabaseValidator(BaseValidator):
|
||||
name = "database"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates connecting to the database. """
|
||||
config = validator_context.config
|
||||
|
||||
try:
|
||||
validate_database_precondition(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
|
||||
except OperationalError as ex:
|
||||
if ex.args and len(ex.args) > 1:
|
||||
raise ConfigValidationException(ex.args[1])
|
||||
else:
|
||||
raise ex
|
53
util/config/validators/validate_github.py
Normal file
53
util/config/validators/validate_github.py
Normal file
|
@ -0,0 +1,53 @@
|
|||
from oauth.services.github import GithubOAuthService
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class BaseGitHubValidator(BaseValidator):
|
||||
name = None
|
||||
config_key = None
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the OAuth credentials and API endpoint for a Github service. """
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
|
||||
|
||||
github_config = config.get(cls.config_key)
|
||||
if not github_config:
|
||||
raise ConfigValidationException('Missing GitHub client id and client secret')
|
||||
|
||||
endpoint = github_config.get('GITHUB_ENDPOINT')
|
||||
if not endpoint:
|
||||
raise ConfigValidationException('Missing GitHub Endpoint')
|
||||
|
||||
if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
|
||||
raise ConfigValidationException('Github Endpoint must start with http:// or https://')
|
||||
|
||||
if not github_config.get('CLIENT_ID'):
|
||||
raise ConfigValidationException('Missing Client ID')
|
||||
|
||||
if not github_config.get('CLIENT_SECRET'):
|
||||
raise ConfigValidationException('Missing Client Secret')
|
||||
|
||||
if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
|
||||
raise ConfigValidationException('Organization restriction must have at least one allowed ' +
|
||||
'organization')
|
||||
|
||||
oauth = GithubOAuthService(config, cls.config_key)
|
||||
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
|
||||
if not result:
|
||||
raise ConfigValidationException('Invalid client id or client secret')
|
||||
|
||||
if github_config.get('ALLOWED_ORGANIZATIONS'):
|
||||
for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
|
||||
if not oauth.validate_organization(org_id, client):
|
||||
raise ConfigValidationException('Invalid organization: %s' % org_id)
|
||||
|
||||
|
||||
class GitHubLoginValidator(BaseGitHubValidator):
|
||||
name = "github-login"
|
||||
config_key = "GITHUB_LOGIN_CONFIG"
|
||||
|
||||
class GitHubTriggerValidator(BaseGitHubValidator):
|
||||
name = "github-trigger"
|
||||
config_key = "GITHUB_TRIGGER_CONFIG"
|
32
util/config/validators/validate_gitlab_trigger.py
Normal file
32
util/config/validators/validate_gitlab_trigger.py
Normal file
|
@ -0,0 +1,32 @@
|
|||
from oauth.services.gitlab import GitLabOAuthService
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class GitLabTriggerValidator(BaseValidator):
|
||||
name = "gitlab-trigger"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the OAuth credentials and API endpoint for a GitLab service. """
|
||||
config = validator_context.config
|
||||
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
|
||||
client = validator_context.http_client
|
||||
|
||||
github_config = config.get('GITLAB_TRIGGER_CONFIG')
|
||||
if not github_config:
|
||||
raise ConfigValidationException('Missing GitLab client id and client secret')
|
||||
|
||||
endpoint = github_config.get('GITLAB_ENDPOINT')
|
||||
if endpoint:
|
||||
if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
|
||||
raise ConfigValidationException('GitLab Endpoint must start with http:// or https://')
|
||||
|
||||
if not github_config.get('CLIENT_ID'):
|
||||
raise ConfigValidationException('Missing Client ID')
|
||||
|
||||
if not github_config.get('CLIENT_SECRET'):
|
||||
raise ConfigValidationException('Missing Client Secret')
|
||||
|
||||
oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
|
||||
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
|
||||
if not result:
|
||||
raise ConfigValidationException('Invalid client id or client secret')
|
27
util/config/validators/validate_google_login.py
Normal file
27
util/config/validators/validate_google_login.py
Normal file
|
@ -0,0 +1,27 @@
|
|||
from oauth.services.google import GoogleOAuthService
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class GoogleLoginValidator(BaseValidator):
|
||||
name = "google-login"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the Google Login client ID and secret. """
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
|
||||
|
||||
google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
|
||||
if not google_login_config:
|
||||
raise ConfigValidationException('Missing client ID and client secret')
|
||||
|
||||
if not google_login_config.get('CLIENT_ID'):
|
||||
raise ConfigValidationException('Missing Client ID')
|
||||
|
||||
if not google_login_config.get('CLIENT_SECRET'):
|
||||
raise ConfigValidationException('Missing Client Secret')
|
||||
|
||||
oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
|
||||
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
|
||||
if not result:
|
||||
raise ConfigValidationException('Invalid client id or client secret')
|
48
util/config/validators/validate_jwt.py
Normal file
48
util/config/validators/validate_jwt.py
Normal file
|
@ -0,0 +1,48 @@
|
|||
import os
|
||||
from data.users.externaljwt import ExternalJWTAuthN
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class JWTAuthValidator(BaseValidator):
|
||||
name = "jwt"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context, public_key_path=None):
|
||||
""" Validates the JWT authentication system. """
|
||||
config = validator_context.config
|
||||
http_client = validator_context.http_client
|
||||
jwt_auth_max = validator_context.jwt_auth_max
|
||||
config_provider = validator_context.config_provider
|
||||
|
||||
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
|
||||
return
|
||||
|
||||
verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
|
||||
query_endpoint = config.get('JWT_QUERY_ENDPOINT', None)
|
||||
getuser_endpoint = config.get('JWT_GETUSER_ENDPOINT', None)
|
||||
|
||||
issuer = config.get('JWT_AUTH_ISSUER')
|
||||
|
||||
if not verify_endpoint:
|
||||
raise ConfigValidationException('Missing JWT Verification endpoint')
|
||||
|
||||
if not issuer:
|
||||
raise ConfigValidationException('Missing JWT Issuer ID')
|
||||
|
||||
|
||||
override_config_directory = config_provider.get_config_dir_path()
|
||||
|
||||
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
|
||||
# the key cannot be found.
|
||||
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
|
||||
override_config_directory,
|
||||
http_client,
|
||||
jwt_auth_max,
|
||||
public_key_path=public_key_path,
|
||||
requires_email=config.get('FEATURE_MAILING', True))
|
||||
|
||||
# Verify that we can reach the jwt server
|
||||
(result, err_msg) = users.ping()
|
||||
if not result:
|
||||
msg = ('Verification of JWT failed: %s. \n\nWe cannot reach the JWT server' +
|
||||
'OR JWT auth is misconfigured') % err_msg
|
||||
raise ConfigValidationException(msg)
|
44
util/config/validators/validate_keystone.py
Normal file
44
util/config/validators/validate_keystone.py
Normal file
|
@ -0,0 +1,44 @@
|
|||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
from data.users.keystone import get_keystone_users
|
||||
|
||||
class KeystoneValidator(BaseValidator):
|
||||
name = "keystone"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the Keystone authentication system. """
|
||||
config = validator_context.config
|
||||
|
||||
if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
|
||||
return
|
||||
|
||||
auth_url = config.get('KEYSTONE_AUTH_URL')
|
||||
auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
|
||||
admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
|
||||
admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
|
||||
admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')
|
||||
|
||||
if not auth_url:
|
||||
raise ConfigValidationException('Missing authentication URL')
|
||||
|
||||
if not admin_username:
|
||||
raise ConfigValidationException('Missing admin username')
|
||||
|
||||
if not admin_password:
|
||||
raise ConfigValidationException('Missing admin password')
|
||||
|
||||
if not admin_tenant:
|
||||
raise ConfigValidationException('Missing admin tenant')
|
||||
|
||||
requires_email = config.get('FEATURE_MAILING', True)
|
||||
users = get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
|
||||
requires_email)
|
||||
|
||||
# Verify that the superuser exists. If not, raise an exception.
|
||||
(result, err_msg) = users.at_least_one_user_exists()
|
||||
if not result:
|
||||
msg = ('Verification that users exist failed: %s. \n\nNo users exist ' +
|
||||
'in the admin tenant/project ' +
|
||||
'in the remote authentication system ' +
|
||||
'OR Keystone auth is misconfigured.') % err_msg
|
||||
raise ConfigValidationException(msg)
|
68
util/config/validators/validate_ldap.py
Normal file
68
util/config/validators/validate_ldap.py
Normal file
|
@ -0,0 +1,68 @@
|
|||
import os
|
||||
import ldap
|
||||
import subprocess
|
||||
|
||||
from data.users import LDAP_CERT_FILENAME
|
||||
from data.users.externalldap import LDAPConnection, LDAPUsers
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class LDAPValidator(BaseValidator):
|
||||
name = "ldap"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the LDAP connection. """
|
||||
config = validator_context.config
|
||||
config_provider = validator_context.config_provider
|
||||
init_scripts_location = validator_context.init_scripts_location
|
||||
|
||||
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
|
||||
return
|
||||
|
||||
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
|
||||
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
|
||||
subprocess.check_call([os.path.join(init_scripts_location, 'certs_install.sh')],
|
||||
env={ 'QUAYCONFIG': config_provider.get_config_dir_path() })
|
||||
|
||||
# Note: raises ldap.INVALID_CREDENTIALS on failure
|
||||
admin_dn = config.get('LDAP_ADMIN_DN')
|
||||
admin_passwd = config.get('LDAP_ADMIN_PASSWD')
|
||||
|
||||
if not admin_dn:
|
||||
raise ConfigValidationException('Missing Admin DN for LDAP configuration')
|
||||
|
||||
if not admin_passwd:
|
||||
raise ConfigValidationException('Missing Admin Password for LDAP configuration')
|
||||
|
||||
ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
|
||||
if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
|
||||
raise ConfigValidationException('LDAP URI must start with ldap:// or ldaps://')
|
||||
|
||||
allow_tls_fallback = config.get('LDAP_ALLOW_INSECURE_FALLBACK', False)
|
||||
|
||||
try:
|
||||
with LDAPConnection(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback):
|
||||
pass
|
||||
except ldap.LDAPError as ex:
|
||||
values = ex.args[0] if ex.args else {}
|
||||
if not isinstance(values, dict):
|
||||
raise ConfigValidationException(str(ex.args))
|
||||
|
||||
raise ConfigValidationException(values.get('desc', 'Unknown error'))
|
||||
|
||||
base_dn = config.get('LDAP_BASE_DN')
|
||||
user_rdn = config.get('LDAP_USER_RDN', [])
|
||||
uid_attr = config.get('LDAP_UID_ATTR', 'uid')
|
||||
email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')
|
||||
requires_email = config.get('FEATURE_MAILING', True)
|
||||
|
||||
users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
|
||||
allow_tls_fallback, requires_email=requires_email)
|
||||
|
||||
# Ensure at least one user exists to verify the connection is setup properly.
|
||||
(result, err_msg) = users.at_least_one_user_exists()
|
||||
if not result:
|
||||
msg = ('Verification that users exist failed: %s. \n\nNo users exist ' +
|
||||
'in the remote authentication system ' +
|
||||
'OR LDAP auth is misconfigured.') % err_msg
|
||||
raise ConfigValidationException(msg)
|
36
util/config/validators/validate_oidc.py
Normal file
36
util/config/validators/validate_oidc.py
Normal file
|
@ -0,0 +1,36 @@
|
|||
from oauth.loginmanager import OAuthLoginManager
|
||||
from oauth.oidc import OIDCLoginService, DiscoveryFailureException
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class OIDCLoginValidator(BaseValidator):
|
||||
name = "oidc-login"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
|
||||
login_manager = OAuthLoginManager(config, client=client)
|
||||
for service in login_manager.services:
|
||||
if not isinstance(service, OIDCLoginService):
|
||||
continue
|
||||
|
||||
if service.config.get('OIDC_SERVER') is None:
|
||||
msg = 'Missing OIDC_SERVER on OIDC service %s' % service.service_id()
|
||||
raise ConfigValidationException(msg)
|
||||
|
||||
if service.config.get('CLIENT_ID') is None:
|
||||
msg = 'Missing CLIENT_ID on OIDC service %s' % service.service_id()
|
||||
raise ConfigValidationException(msg)
|
||||
|
||||
if service.config.get('CLIENT_SECRET') is None:
|
||||
msg = 'Missing CLIENT_SECRET on OIDC service %s' % service.service_id()
|
||||
raise ConfigValidationException(msg)
|
||||
|
||||
try:
|
||||
if not service.validate():
|
||||
msg = 'Could not validate OIDC service %s' % service.service_id()
|
||||
raise ConfigValidationException(msg)
|
||||
except DiscoveryFailureException as dfe:
|
||||
msg = 'Could not validate OIDC service %s: %s' % (service.service_id(), dfe.message)
|
||||
raise ConfigValidationException(msg)
|
18
util/config/validators/validate_redis.py
Normal file
18
util/config/validators/validate_redis.py
Normal file
|
@ -0,0 +1,18 @@
|
|||
import redis
|
||||
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class RedisValidator(BaseValidator):
|
||||
name = "redis"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates connecting to redis. """
|
||||
config = validator_context.config
|
||||
|
||||
redis_config = config.get('BUILDLOGS_REDIS', {})
|
||||
if not 'host' in redis_config:
|
||||
raise ConfigValidationException('Missing redis hostname')
|
||||
|
||||
client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
|
||||
client.ping()
|
52
util/config/validators/validate_secscan.py
Normal file
52
util/config/validators/validate_secscan.py
Normal file
|
@ -0,0 +1,52 @@
|
|||
import time
|
||||
|
||||
# from boot import setup_jwt_proxy
|
||||
from util.secscan.api import SecurityScannerAPI
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
class SecurityScannerValidator(BaseValidator):
|
||||
name = "security-scanner"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the configuration for talking to a Quay Security Scanner. """
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
feature_sec_scanner = validator_context.feature_sec_scanner
|
||||
is_testing = validator_context.is_testing
|
||||
|
||||
server_hostname = validator_context.url_scheme_and_hostname.hostname
|
||||
uri_creator = validator_context.uri_creator
|
||||
|
||||
if not feature_sec_scanner:
|
||||
return
|
||||
|
||||
api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
|
||||
|
||||
# if not is_testing:
|
||||
# Generate a temporary Quay key to use for signing the outgoing requests.
|
||||
# setup_jwt_proxy()
|
||||
|
||||
# We have to wait for JWT proxy to restart with the newly generated key.
|
||||
max_tries = 5
|
||||
response = None
|
||||
last_exception = None
|
||||
|
||||
while max_tries > 0:
|
||||
try:
|
||||
response = api.ping()
|
||||
last_exception = None
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception as ex:
|
||||
last_exception = ex
|
||||
|
||||
time.sleep(1)
|
||||
max_tries = max_tries - 1
|
||||
|
||||
if last_exception is not None:
|
||||
message = str(last_exception)
|
||||
raise ConfigValidationException('Could not ping security scanner: %s' % message)
|
||||
else:
|
||||
message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
|
||||
raise ConfigValidationException('Could not ping security scanner: %s' % message)
|
22
util/config/validators/validate_signer.py
Normal file
22
util/config/validators/validate_signer.py
Normal file
|
@ -0,0 +1,22 @@
|
|||
from StringIO import StringIO
|
||||
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
from util.security.signing import SIGNING_ENGINES
|
||||
|
||||
class SignerValidator(BaseValidator):
|
||||
name = "signer"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the GPG public+private key pair used for signing converted ACIs. """
|
||||
config = validator_context.config
|
||||
config_provider = validator_context.config_provider
|
||||
|
||||
if config.get('SIGNING_ENGINE') is None:
|
||||
return
|
||||
|
||||
if config['SIGNING_ENGINE'] not in SIGNING_ENGINES:
|
||||
raise ConfigValidationException('Unknown signing engine: %s' % config['SIGNING_ENGINE'])
|
||||
|
||||
engine = SIGNING_ENGINES[config['SIGNING_ENGINE']](config, config_provider)
|
||||
engine.detached_sign(StringIO('test string'))
|
72
util/config/validators/validate_ssl.py
Normal file
72
util/config/validators/validate_ssl.py
Normal file
|
@ -0,0 +1,72 @@
|
|||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
|
||||
|
||||
SSL_FILENAMES = ['ssl.cert', 'ssl.key']
|
||||
|
||||
class SSLValidator(BaseValidator):
|
||||
name = "ssl"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the SSL configuration (if enabled). """
|
||||
config = validator_context.config
|
||||
config_provider = validator_context.config_provider
|
||||
|
||||
# Skip if non-SSL.
|
||||
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
|
||||
return
|
||||
|
||||
# Skip if externally terminated.
|
||||
if config.get('EXTERNAL_TLS_TERMINATION', False) is True:
|
||||
return
|
||||
|
||||
# Verify that we have all the required SSL files.
|
||||
for filename in SSL_FILENAMES:
|
||||
if not config_provider.volume_file_exists(filename):
|
||||
raise ConfigValidationException('Missing required SSL file: %s' % filename)
|
||||
|
||||
# Read the contents of the SSL certificate.
|
||||
with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
|
||||
cert_contents = f.read()
|
||||
|
||||
# Validate the certificate.
|
||||
try:
|
||||
certificate = load_certificate(cert_contents)
|
||||
except CertInvalidException as cie:
|
||||
raise ConfigValidationException('Could not load SSL certificate: %s' % cie)
|
||||
|
||||
# Verify the certificate has not expired.
|
||||
if certificate.expired:
|
||||
raise ConfigValidationException('The specified SSL certificate has expired.')
|
||||
|
||||
# Verify the hostname matches the name in the certificate.
|
||||
if not certificate.matches_name(_ssl_cn(config['SERVER_HOSTNAME'])):
|
||||
msg = ('Supported names "%s" in SSL cert do not match server hostname "%s"' %
|
||||
(', '.join(list(certificate.names)), _ssl_cn(config['SERVER_HOSTNAME'])))
|
||||
raise ConfigValidationException(msg)
|
||||
|
||||
# Verify the private key against the certificate.
|
||||
private_key_path = None
|
||||
with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
|
||||
private_key_path = f.name
|
||||
|
||||
if not private_key_path:
|
||||
# Only in testing.
|
||||
return
|
||||
|
||||
try:
|
||||
certificate.validate_private_key(private_key_path)
|
||||
except KeyInvalidException as kie:
|
||||
raise ConfigValidationException('SSL private key failed to validate: %s' % kie)
|
||||
|
||||
|
||||
def _ssl_cn(server_hostname):
|
||||
""" Return the common name (fully qualified host name) from the SERVER_HOSTNAME. """
|
||||
host_port = server_hostname.rsplit(':', 1)
|
||||
|
||||
# SERVER_HOSTNAME includes the port
|
||||
if len(host_port) == 2:
|
||||
if host_port[-1].isdigit():
|
||||
return host_port[-2]
|
||||
|
||||
return server_hostname
|
54
util/config/validators/validate_storage.py
Normal file
54
util/config/validators/validate_storage.py
Normal file
|
@ -0,0 +1,54 @@
|
|||
from storage import get_storage_driver, TYPE_LOCAL_STORAGE
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
|
||||
|
||||
class StorageValidator(BaseValidator):
|
||||
name = "registry-storage"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates registry storage. """
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
ip_resolver = validator_context.ip_resolver
|
||||
config_provider = validator_context.config_provider
|
||||
|
||||
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
|
||||
|
||||
providers = _get_storage_providers(config, ip_resolver, config_provider).items()
|
||||
if not providers:
|
||||
raise ConfigValidationException('Storage configuration required')
|
||||
|
||||
for name, (storage_type, driver) in providers:
|
||||
# We can skip localstorage validation, since we can't guarantee that
|
||||
# this will be the same machine Q.E. will run under
|
||||
if storage_type == TYPE_LOCAL_STORAGE:
|
||||
continue
|
||||
|
||||
try:
|
||||
if replication_enabled and storage_type == 'LocalStorage':
|
||||
raise ConfigValidationException('Locally mounted directory not supported ' +
|
||||
'with storage replication')
|
||||
|
||||
# Run validation on the driver.
|
||||
driver.validate(client)
|
||||
|
||||
# Run setup on the driver if the read/write succeeded.
|
||||
driver.setup()
|
||||
except Exception as ex:
|
||||
msg = str(ex).strip().split("\n")[0]
|
||||
raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
|
||||
|
||||
|
||||
def _get_storage_providers(config, ip_resolver, config_provider):
|
||||
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
|
||||
drivers = {}
|
||||
|
||||
try:
|
||||
for name, parameters in storage_config.items():
|
||||
driver = get_storage_driver(None, None, None, config_provider, ip_resolver, parameters)
|
||||
drivers[name] = (parameters[0], driver)
|
||||
except TypeError:
|
||||
raise ConfigValidationException('Missing required parameter(s) for storage %s' % name)
|
||||
|
||||
return drivers
|
31
util/config/validators/validate_timemachine.py
Normal file
31
util/config/validators/validate_timemachine.py
Normal file
|
@ -0,0 +1,31 @@
|
|||
import logging
|
||||
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
from util.timedeltastring import convert_to_timedelta
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class TimeMachineValidator(BaseValidator):
|
||||
name = "time-machine"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
config = validator_context.config
|
||||
|
||||
if not 'DEFAULT_TAG_EXPIRATION' in config:
|
||||
# Old style config
|
||||
return
|
||||
|
||||
try:
|
||||
convert_to_timedelta(config['DEFAULT_TAG_EXPIRATION']).total_seconds()
|
||||
except ValueError as ve:
|
||||
raise ConfigValidationException('Invalid default expiration: %s' % ve.message)
|
||||
|
||||
if not config['DEFAULT_TAG_EXPIRATION'] in config.get('TAG_EXPIRATION_OPTIONS', []):
|
||||
raise ConfigValidationException('Default expiration must be in expiration options set')
|
||||
|
||||
for ts in config.get('TAG_EXPIRATION_OPTIONS', []):
|
||||
try:
|
||||
convert_to_timedelta(ts)
|
||||
except ValueError as ve:
|
||||
raise ConfigValidationException('Invalid tag expiration option: %s' % ts)
|
59
util/config/validators/validate_torrent.py
Normal file
59
util/config/validators/validate_torrent.py
Normal file
|
@ -0,0 +1,59 @@
|
|||
import logging
|
||||
|
||||
from hashlib import sha1
|
||||
|
||||
from util.config.validators import BaseValidator, ConfigValidationException
|
||||
from util.registry.torrent import jwt_from_infohash, TorrentConfiguration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class BittorrentValidator(BaseValidator):
|
||||
name = "bittorrent"
|
||||
|
||||
@classmethod
|
||||
def validate(cls, validator_context):
|
||||
""" Validates the configuration for using BitTorrent for downloads. """
|
||||
config = validator_context.config
|
||||
client = validator_context.http_client
|
||||
|
||||
announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
|
||||
if not announce_url:
|
||||
raise ConfigValidationException('Missing announce URL')
|
||||
|
||||
# Ensure that the tracker is reachable and accepts requests signed with a registry key.
|
||||
params = {
|
||||
'info_hash': sha1('test').digest(),
|
||||
'peer_id': '-QUAY00-6wfG2wk6wWLc',
|
||||
'uploaded': 0,
|
||||
'downloaded': 0,
|
||||
'left': 0,
|
||||
'numwant': 0,
|
||||
'port': 80,
|
||||
}
|
||||
|
||||
torrent_config = TorrentConfiguration.for_testing(validator_context.instance_keys, announce_url,
|
||||
validator_context.registry_title)
|
||||
encoded_jwt = jwt_from_infohash(torrent_config, params['info_hash'])
|
||||
params['jwt'] = encoded_jwt
|
||||
|
||||
resp = client.get(announce_url, timeout=5, params=params)
|
||||
logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)
|
||||
|
||||
if resp.status_code == 404:
|
||||
raise ConfigValidationException('Announce path not found; did you forget `/announce`?')
|
||||
|
||||
if resp.status_code == 500:
|
||||
raise ConfigValidationException('Did not get expected response from Tracker; ' +
|
||||
'please check your settings')
|
||||
|
||||
if resp.status_code == 200:
|
||||
if 'invalid jwt' in resp.text:
|
||||
raise ConfigValidationException('Could not authorize to Tracker; is your Tracker ' +
|
||||
'properly configured?')
|
||||
|
||||
if 'failure reason' in resp.text:
|
||||
raise ConfigValidationException('Could not validate signed announce request: ' + resp.text)
|
||||
|
||||
if 'go_goroutines' in resp.text:
|
||||
raise ConfigValidationException('Could not validate signed announce request: ' +
|
||||
'provided port is used for Prometheus')
|
Reference in a new issue