initial import for Open Source 🎉

This commit is contained in:
Jimmy Zelinskie 2019-11-12 11:09:47 -05:00
parent 1898c361f3
commit 9c0dd3b722
2048 changed files with 218743 additions and 0 deletions

29
util/config/__init__.py Normal file
View file

@ -0,0 +1,29 @@
class URLSchemeAndHostname:
"""
Immutable configuration for a given preferred url scheme (e.g. http or https), and a hostname (e.g. localhost:5000)
"""
def __init__(self, url_scheme, hostname):
self._url_scheme = url_scheme
self._hostname = hostname
@classmethod
def from_app_config(cls, app_config):
"""
Helper method to instantiate class from app config, a frequent pattern
:param app_config:
:return:
"""
return cls(app_config['PREFERRED_URL_SCHEME'], app_config['SERVER_HOSTNAME'])
@property
def url_scheme(self):
return self._url_scheme
@property
def hostname(self):
return self._hostname
def get_url(self):
""" Returns the application's URL, based on the given url scheme and hostname. """
return '%s://%s' % (self._url_scheme, self._hostname)

View file

@ -0,0 +1,43 @@
""" Generates html documentation from JSON Schema """
import json
from collections import OrderedDict
import docsmodel
import html_output
from util.config.schema import CONFIG_SCHEMA
def make_custom_sort(orders):
""" Sort in a specified order any dictionary nested in a complex structure """
orders = [{k: -i for (i, k) in enumerate(reversed(order), 1)} for order in orders]
def process(stuff):
if isinstance(stuff, dict):
l = [(k, process(v)) for (k, v) in stuff.iteritems()]
keys = set(stuff)
for order in orders:
if keys.issubset(order) or keys.issuperset(order):
return OrderedDict(sorted(l, key=lambda x: order.get(x[0], 0)))
return OrderedDict(sorted(l))
if isinstance(stuff, list):
return [process(x) for x in stuff]
return stuff
return process
SCHEMA_HTML_FILE = "schema.html"
schema = json.dumps(CONFIG_SCHEMA, sort_keys = True)
schema = json.loads(schema, object_pairs_hook = OrderedDict)
req = sorted(schema["required"])
custom_sort = make_custom_sort([req])
schema = custom_sort(schema)
parsed_items = docsmodel.DocsModel().parse(schema)[1:]
output = html_output.HtmlOutput().generate_output(parsed_items)
with open(SCHEMA_HTML_FILE, 'wt') as f:
f.write(output)

View file

@ -0,0 +1,93 @@
import json
import collections
class ParsedItem(dict):
""" Parsed Schema item """
def __init__(self, json_object, name, required, level):
"""Fills dict with basic item information"""
super(ParsedItem, self).__init__()
self['name'] = name
self['title'] = json_object.get('title', '')
self['type'] = json_object.get('type')
self['description'] = json_object.get('description', '')
self['level'] = level
self['required'] = required
self['x-reference'] = json_object.get('x-reference', '')
self['x-example'] = json_object.get('x-example', '')
self['pattern'] = json_object.get('pattern', '')
self['enum'] = json_object.get('enum', '')
class DocsModel:
""" Documentation model and Schema Parser """
def __init__(self):
self.__parsed_items = None
def parse(self, json_object):
""" Returns multi-level list of recursively parsed items """
self.__parsed_items = list()
self.__parse_schema(json_object, 'root', True, 0)
return self.__parsed_items
def __parse_schema(self, schema, name, required, level):
""" Parses schema, which type is object, array or leaf.
Appends new ParsedItem to self.__parsed_items lis """
parsed_item = ParsedItem(schema, name, required, level)
self.__parsed_items.append(parsed_item)
required = schema.get('required', [])
if 'enum' in schema:
parsed_item['item'] = schema.get('enum')
item_type = schema.get('type')
if item_type == 'object' and name != 'DISTRIBUTED_STORAGE_CONFIG':
self.__parse_object(parsed_item, schema, required, level)
elif item_type == 'array':
self.__parse_array(parsed_item, schema, required, level)
else:
parse_leaf(parsed_item, schema)
def __parse_object(self, parsed_item, schema, required, level):
""" Parses schema of type object """
for key, value in schema.get('properties', {}).items():
self.__parse_schema(value, key, key in required, level + 1)
def __parse_array(self, parsed_item, schema, required, level):
""" Parses schema of type array """
items = schema.get('items')
parsed_item['minItems'] = schema.get('minItems', None)
parsed_item['maxItems'] = schema.get('maxItems', None)
parsed_item['uniqueItems'] = schema.get('uniqueItems', False)
if isinstance(items, dict):
# item is single schema describing all elements in an array
self.__parse_schema(
items,
'array item',
required,
level + 1)
elif isinstance(items, list):
# item is a list of schemas
for index, list_item in enumerate(items):
self.__parse_schema(
list_item,
'array item {}'.format(index),
index in required,
level + 1)
def parse_leaf(parsed_item, schema):
""" Parses schema of a number and a string """
if parsed_item['name'] != 'root':
parsed_item['description'] = schema.get('description','')
parsed_item['x-reference'] = schema.get('x-reference','')
parsed_item['pattern'] = schema.get('pattern','')
parsed_item['enum'] = ", ".join(schema.get('enum','')).encode()
ex = schema.get('x-example', '')
if isinstance(ex, list):
parsed_item['x-example'] = ", ".join(ex).encode()
elif isinstance(ex, collections.OrderedDict):
parsed_item['x-example'] = json.dumps(ex)
else:
parsed_item['x-example'] = ex

View file

@ -0,0 +1,63 @@
class HtmlOutput:
""" Generates HTML from documentation model """
def __init__(self):
pass
def generate_output(self, parsed_items):
"""Returns generated HTML strin"""
return self.__get_html_begin() + \
self.__get_html_middle(parsed_items) + \
self.__get_html_end()
def __get_html_begin(self):
return '<!DOCTYPE html>\n<html>\n<head>\n<link rel="stylesheet" type="text/css" href="style.css" />\n</head>\n<body>\n'
def __get_html_end(self):
return '</body>\n</html>'
def __get_html_middle(self, parsed_items):
output = ''
root_item = parsed_items[0]
#output += '<h1 class="root_title">{}</h1>\n'.format(root_item['title'])
#output += '<h1 class="root_title">{}</h1>\n'.format(root_item['title'])
output += "Schema for Red Hat Quay"
output += '<ul class="level0">\n'
last_level = 0
is_root = True
for item in parsed_items:
level = item['level'] - 1
if last_level < level:
output += '<ul class="level{}">\n'.format(level)
for i in range(last_level - level):
output += '</ul>\n'
last_level = level
output += self.__get_html_item(item, is_root)
is_root = False
output += '</ul>\n'
return output
def __get_required_field(self, parsed_item):
return 'required' if parsed_item['required'] else ''
def __get_html_item(self, parsed_item, is_root):
item = '<li class="schema item"> \n'
item += '<div class="name">{}</div> \n'.format(parsed_item['name'])
item += '<div class="type">[{}]</div> \n'.format(parsed_item['type'])
item += '<div class="required">{}</div> \n'.format(self.__get_required_field(parsed_item))
item += '<div class="docs">\n' if not is_root else '<div class="root_docs">\n'
item += '<div class="title">{}</div>\n'.format(parsed_item['title'])
item += ': ' if parsed_item['title'] != '' and parsed_item['description'] != '' else ''
item += '<div class="description">{}</div>\n'.format(parsed_item['description'])
item += '<div class="enum">enum: {}</div>\n'.format(parsed_item['enum']) if parsed_item['enum'] != '' else ''
item += '<div class="minItems">Min Items: {}</div>\n'.format(parsed_item['minItems']) if parsed_item['type'] == "array" and parsed_item['minItems'] != "None" else ''
item += '<div class="uniqueItems">Unique Items: {}</div>\n'.format(parsed_item['uniqueItems']) if parsed_item['type'] == "array" and parsed_item['uniqueItems'] else ''
item += '<div class="pattern">Pattern: {}</div>\n'.format(parsed_item['pattern']) if parsed_item['pattern'] != 'None' and parsed_item['pattern'] != '' else ''
item += '<div class="x-reference"><a href="{}">Reference: {}</a></div>\n'.format(parsed_item['x-reference'],parsed_item['x-reference']) if parsed_item['x-reference'] != '' else ''
item += '<div class="x-example">Example: <code>{}</code></div>\n'.format(parsed_item['x-example']) if parsed_item['x-example'] != '' else ''
item += '</div>\n'
item += '</li>\n'
return item

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,78 @@
body {
font-family: sans-serif;
}
pre, code{
white-space: normal;
}
div.root docs{
display: none;
}
div.name {
display: inline;
}
div.type {
display: inline;
font-weight: bold;
color: blue;
}
div.required {
display: inline;
font-weight: bold;
}
div.docs {
display: block;
}
div.title {
display: block;
font-weight: bold;
}
div.description {
display: block;
font-family: serif;
font-style: italic;
}
div.enum {
display: block;
font-family: serif;
font-style: italic;
}
div.x-example {
display: block;
font-family: serif;
font-style: italic;
margin-bottom: 10px;
}
div.pattern {
display: block;
font-family: serif;
font-style: italic;
}
div.x-reference {
display: block;
font-family: serif;
font-style: italic;
}
div.uniqueItems {
display: block;
}
div.minItems {
display: block;
}
div.maxItems {
display: block;
}

108
util/config/configutil.py Normal file
View file

@ -0,0 +1,108 @@
from random import SystemRandom
from uuid import uuid4
def generate_secret_key():
cryptogen = SystemRandom()
return str(cryptogen.getrandbits(256))
def add_enterprise_config_defaults(config_obj, current_secret_key):
""" Adds/Sets the config defaults for enterprise registry config. """
# These have to be false.
config_obj['TESTING'] = False
config_obj['USE_CDN'] = False
# Default for V3 upgrade.
config_obj['V3_UPGRADE_MODE'] = config_obj.get('V3_UPGRADE_MODE', 'complete')
# Defaults for Red Hat Quay.
config_obj['REGISTRY_TITLE'] = config_obj.get('REGISTRY_TITLE', 'Red Hat Quay')
config_obj['REGISTRY_TITLE_SHORT'] = config_obj.get('REGISTRY_TITLE_SHORT', 'Red Hat Quay')
# Default features that are on.
config_obj['FEATURE_USER_LOG_ACCESS'] = config_obj.get('FEATURE_USER_LOG_ACCESS', True)
config_obj['FEATURE_USER_CREATION'] = config_obj.get('FEATURE_USER_CREATION', True)
config_obj['FEATURE_ANONYMOUS_ACCESS'] = config_obj.get('FEATURE_ANONYMOUS_ACCESS', True)
config_obj['FEATURE_REQUIRE_TEAM_INVITE'] = config_obj.get('FEATURE_REQUIRE_TEAM_INVITE', True)
config_obj['FEATURE_CHANGE_TAG_EXPIRATION'] = config_obj.get('FEATURE_CHANGE_TAG_EXPIRATION',
True)
config_obj['FEATURE_DIRECT_LOGIN'] = config_obj.get('FEATURE_DIRECT_LOGIN', True)
config_obj['FEATURE_APP_SPECIFIC_TOKENS'] = config_obj.get('FEATURE_APP_SPECIFIC_TOKENS', True)
config_obj['FEATURE_PARTIAL_USER_AUTOCOMPLETE'] = config_obj.get('FEATURE_PARTIAL_USER_AUTOCOMPLETE', True)
config_obj['FEATURE_USERNAME_CONFIRMATION'] = config_obj.get('FEATURE_USERNAME_CONFIRMATION', True)
config_obj['FEATURE_RESTRICTED_V1_PUSH'] = config_obj.get('FEATURE_RESTRICTED_V1_PUSH', True)
# Default features that are off.
config_obj['FEATURE_MAILING'] = config_obj.get('FEATURE_MAILING', False)
config_obj['FEATURE_BUILD_SUPPORT'] = config_obj.get('FEATURE_BUILD_SUPPORT', False)
config_obj['FEATURE_ACI_CONVERSION'] = config_obj.get('FEATURE_ACI_CONVERSION', False)
config_obj['FEATURE_APP_REGISTRY'] = config_obj.get('FEATURE_APP_REGISTRY', False)
config_obj['FEATURE_REPO_MIRROR'] = config_obj.get('FEATURE_REPO_MIRROR', False)
# Default repo mirror config.
config_obj['REPO_MIRROR_TLS_VERIFY'] = config_obj.get('REPO_MIRROR_TLS_VERIFY', True)
config_obj['REPO_MIRROR_SERVER_HOSTNAME'] = config_obj.get('REPO_MIRROR_SERVER_HOSTNAME', None)
# Default the signer config.
config_obj['GPG2_PRIVATE_KEY_FILENAME'] = config_obj.get('GPG2_PRIVATE_KEY_FILENAME',
'signing-private.gpg')
config_obj['GPG2_PUBLIC_KEY_FILENAME'] = config_obj.get('GPG2_PUBLIC_KEY_FILENAME',
'signing-public.gpg')
config_obj['SIGNING_ENGINE'] = config_obj.get('SIGNING_ENGINE', 'gpg2')
# Default security scanner config.
config_obj['FEATURE_SECURITY_NOTIFICATIONS'] = config_obj.get(
'FEATURE_SECURITY_NOTIFICATIONS', True)
config_obj['FEATURE_SECURITY_SCANNER'] = config_obj.get(
'FEATURE_SECURITY_SCANNER', False)
config_obj['SECURITY_SCANNER_ISSUER_NAME'] = config_obj.get(
'SECURITY_SCANNER_ISSUER_NAME', 'security_scanner')
# Default time machine config.
config_obj['TAG_EXPIRATION_OPTIONS'] = config_obj.get('TAG_EXPIRATION_OPTIONS',
['0s', '1d', '1w', '2w', '4w'])
config_obj['DEFAULT_TAG_EXPIRATION'] = config_obj.get('DEFAULT_TAG_EXPIRATION', '2w')
# Default mail setings.
config_obj['MAIL_USE_TLS'] = config_obj.get('MAIL_USE_TLS', True)
config_obj['MAIL_PORT'] = config_obj.get('MAIL_PORT', 587)
config_obj['MAIL_DEFAULT_SENDER'] = config_obj.get('MAIL_DEFAULT_SENDER', 'support@quay.io')
# Default auth type.
if not 'AUTHENTICATION_TYPE' in config_obj:
config_obj['AUTHENTICATION_TYPE'] = 'Database'
# Default secret key.
if not 'SECRET_KEY' in config_obj:
if current_secret_key:
config_obj['SECRET_KEY'] = current_secret_key
else:
config_obj['SECRET_KEY'] = generate_secret_key()
# Default database secret key.
if not 'DATABASE_SECRET_KEY' in config_obj:
config_obj['DATABASE_SECRET_KEY'] = generate_secret_key()
# Default torrent pepper.
if not 'BITTORRENT_FILENAME_PEPPER' in config_obj:
config_obj['BITTORRENT_FILENAME_PEPPER'] = str(uuid4())
# Default storage configuration.
if not 'DISTRIBUTED_STORAGE_CONFIG' in config_obj:
config_obj['DISTRIBUTED_STORAGE_PREFERENCE'] = ['default']
config_obj['DISTRIBUTED_STORAGE_CONFIG'] = {
'default': ['LocalStorage', {'storage_path': '/datastorage/registry'}]
}
config_obj['USERFILES_LOCATION'] = 'default'
config_obj['USERFILES_PATH'] = 'userfiles/'
config_obj['LOG_ARCHIVE_LOCATION'] = 'default'
# Misc configuration.
config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get(
'ENTERPRISE_LOGO_URL', '/static/img/quay-horizontal-color.svg')
config_obj['TEAM_RESYNC_STALE_TIME'] = config_obj.get('TEAM_RESYNC_STALE_TIME', '60m')

12
util/config/database.py Normal file
View file

@ -0,0 +1,12 @@
from data import model
from data.appr_model import blob
from data.appr_model.models import NEW_MODELS
def sync_database_with_config(config):
""" This ensures all implicitly required reference table entries exist in the database. """
location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
if location_names:
model.image.ensure_image_locations(*location_names)
blob.ensure_blob_locations(NEW_MODELS, *location_names)

View file

@ -0,0 +1,13 @@
from util.config.provider.fileprovider import FileConfigProvider
from util.config.provider.testprovider import TestConfigProvider
from util.config.provider.k8sprovider import KubernetesConfigProvider
def get_config_provider(config_volume, yaml_filename, py_filename, testing=False, kubernetes=False):
""" Loads and returns the config provider for the current environment. """
if testing:
return TestConfigProvider()
if kubernetes:
return KubernetesConfigProvider(config_volume, yaml_filename, py_filename)
return FileConfigProvider(config_volume, yaml_filename, py_filename)

View file

@ -0,0 +1,62 @@
import os
import logging
from util.config.provider.baseprovider import (BaseProvider, import_yaml, export_yaml,
CannotWriteConfigException)
logger = logging.getLogger(__name__)
class BaseFileProvider(BaseProvider):
""" Base implementation of the config provider that reads the data from the file system. """
def __init__(self, config_volume, yaml_filename, py_filename):
self.config_volume = config_volume
self.yaml_filename = yaml_filename
self.py_filename = py_filename
self.yaml_path = os.path.join(config_volume, yaml_filename)
self.py_path = os.path.join(config_volume, py_filename)
def update_app_config(self, app_config):
if os.path.exists(self.py_path):
logger.debug('Applying config file: %s', self.py_path)
app_config.from_pyfile(self.py_path)
if os.path.exists(self.yaml_path):
logger.debug('Applying config file: %s', self.yaml_path)
import_yaml(app_config, self.yaml_path)
def get_config(self):
if not self.config_exists():
return None
config_obj = {}
import_yaml(config_obj, self.yaml_path)
return config_obj
def config_exists(self):
return self.volume_file_exists(self.yaml_filename)
def volume_exists(self):
return os.path.exists(self.config_volume)
def volume_file_exists(self, relative_file_path):
return os.path.exists(os.path.join(self.config_volume, relative_file_path))
def get_volume_file(self, relative_file_path, mode='r'):
return open(os.path.join(self.config_volume, relative_file_path), mode=mode)
def get_volume_path(self, directory, relative_file_path):
return os.path.join(directory, relative_file_path)
def list_volume_directory(self, path):
dirpath = os.path.join(self.config_volume, path)
if not os.path.exists(dirpath):
return None
if not os.path.isdir(dirpath):
return None
return os.listdir(dirpath)
def get_config_root(self):
return self.config_volume

View file

@ -0,0 +1,123 @@
import logging
import yaml
from abc import ABCMeta, abstractmethod
from six import add_metaclass
from jsonschema import validate, ValidationError
from util.config.schema import CONFIG_SCHEMA
logger = logging.getLogger(__name__)
class CannotWriteConfigException(Exception):
""" Exception raised when the config cannot be written. """
pass
class SetupIncompleteException(Exception):
""" Exception raised when attempting to verify config that has not yet been setup. """
pass
def import_yaml(config_obj, config_file):
with open(config_file) as f:
c = yaml.safe_load(f)
if not c:
logger.debug('Empty YAML config file')
return
if isinstance(c, str):
raise Exception('Invalid YAML config file: ' + str(c))
for key in c.iterkeys():
if key.isupper():
config_obj[key] = c[key]
if config_obj.get('SETUP_COMPLETE', True):
try:
validate(config_obj, CONFIG_SCHEMA)
except ValidationError:
# TODO: Change this into a real error
logger.exception('Could not validate config schema')
else:
logger.debug('Skipping config schema validation because setup is not complete')
return config_obj
def get_yaml(config_obj):
return yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True)
def export_yaml(config_obj, config_file):
try:
with open(config_file, 'w') as f:
f.write(get_yaml(config_obj))
except IOError as ioe:
raise CannotWriteConfigException(str(ioe))
@add_metaclass(ABCMeta)
class BaseProvider(object):
""" A configuration provider helps to load, save, and handle config override in the application.
"""
@property
def provider_id(self):
raise NotImplementedError
@abstractmethod
def update_app_config(self, app_config):
""" Updates the given application config object with the loaded override config. """
@abstractmethod
def get_config(self):
""" Returns the contents of the config override file, or None if none. """
@abstractmethod
def save_config(self, config_object):
""" Updates the contents of the config override file to those given. """
@abstractmethod
def config_exists(self):
""" Returns true if a config override file exists in the config volume. """
@abstractmethod
def volume_exists(self):
""" Returns whether the config override volume exists. """
@abstractmethod
def volume_file_exists(self, relative_file_path):
""" Returns whether the file with the given relative path exists under the config override
volume. """
@abstractmethod
def get_volume_file(self, relative_file_path, mode='r'):
""" Returns a Python file referring to the given path under the config override volume. """
@abstractmethod
def remove_volume_file(self, relative_file_path):
""" Removes the config override volume file with the given path. """
@abstractmethod
def list_volume_directory(self, path):
""" Returns a list of strings representing the names of the files found in the config override
directory under the given path. If the path doesn't exist, returns None.
"""
@abstractmethod
def save_volume_file(self, flask_file, relative_file_path):
""" Saves the given flask file to the config override volume, with the given
relative path.
"""
@abstractmethod
def get_volume_path(self, directory, filename):
""" Helper for constructing relative file paths, which may differ between providers.
For example, kubernetes can't have subfolders in configmaps """
@abstractmethod
def get_config_root(self):
""" Returns the config root directory. """

View file

@ -0,0 +1,47 @@
import os
import logging
from util.config.provider.baseprovider import export_yaml, CannotWriteConfigException
from util.config.provider.basefileprovider import BaseFileProvider
logger = logging.getLogger(__name__)
def _ensure_parent_dir(filepath):
""" Ensures that the parent directory of the given file path exists. """
try:
parentpath = os.path.abspath(os.path.join(filepath, os.pardir))
if not os.path.isdir(parentpath):
os.makedirs(parentpath)
except IOError as ioe:
raise CannotWriteConfigException(str(ioe))
class FileConfigProvider(BaseFileProvider):
""" Implementation of the config provider that reads and writes the data
from/to the file system. """
def __init__(self, config_volume, yaml_filename, py_filename):
super(FileConfigProvider, self).__init__(config_volume, yaml_filename, py_filename)
@property
def provider_id(self):
return 'file'
def save_config(self, config_obj):
export_yaml(config_obj, self.yaml_path)
def remove_volume_file(self, relative_file_path):
filepath = os.path.join(self.config_volume, relative_file_path)
os.remove(filepath)
def save_volume_file(self, flask_file, relative_file_path):
filepath = os.path.join(self.config_volume, relative_file_path)
_ensure_parent_dir(filepath)
# Write the file.
try:
flask_file.save(filepath)
except IOError as ioe:
raise CannotWriteConfigException(str(ioe))
return filepath

View file

@ -0,0 +1,188 @@
import os
import logging
import json
import base64
import time
from cStringIO import StringIO
from requests import Request, Session
from util.config.provider.baseprovider import CannotWriteConfigException, get_yaml
from util.config.provider.basefileprovider import BaseFileProvider
logger = logging.getLogger(__name__)
KUBERNETES_API_HOST = os.environ.get('KUBERNETES_SERVICE_HOST', '')
port = os.environ.get('KUBERNETES_SERVICE_PORT')
if port:
KUBERNETES_API_HOST += ':' + port
SERVICE_ACCOUNT_TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
QE_NAMESPACE = os.environ.get('QE_K8S_NAMESPACE', 'quay-enterprise')
QE_CONFIG_SECRET = os.environ.get('QE_K8S_CONFIG_SECRET', 'quay-enterprise-config-secret')
class KubernetesConfigProvider(BaseFileProvider):
""" Implementation of the config provider that reads and writes configuration
data from a Kubernetes Secret. """
def __init__(self, config_volume, yaml_filename, py_filename, api_host=None,
service_account_token_path=None):
super(KubernetesConfigProvider, self).__init__(config_volume, yaml_filename, py_filename)
service_account_token_path = service_account_token_path or SERVICE_ACCOUNT_TOKEN_PATH
api_host = api_host or KUBERNETES_API_HOST
# Load the service account token from the local store.
if not os.path.exists(service_account_token_path):
raise Exception('Cannot load Kubernetes service account token')
with open(service_account_token_path, 'r') as f:
self._service_token = f.read()
self._api_host = api_host
@property
def provider_id(self):
return 'k8s'
def get_volume_path(self, directory, filename):
# NOTE: Overridden to ensure we don't have subdirectories, which aren't supported
# in Kubernetes secrets.
return "_".join([directory.rstrip('/'), filename])
def volume_exists(self):
secret = self._lookup_secret()
return secret is not None
def volume_file_exists(self, relative_file_path):
if '/' in relative_file_path:
raise Exception('Expected path from get_volume_path, but found slashes')
# NOTE: Overridden because we don't have subdirectories, which aren't supported
# in Kubernetes secrets.
secret = self._lookup_secret()
if not secret or not secret.get('data'):
return False
return relative_file_path in secret['data']
def list_volume_directory(self, path):
# NOTE: Overridden because we don't have subdirectories, which aren't supported
# in Kubernetes secrets.
secret = self._lookup_secret()
if not secret:
return []
paths = []
for filename in secret.get('data', {}):
if filename.startswith(path):
paths.append(filename[len(path) + 1:])
return paths
def save_config(self, config_obj):
self._update_secret_file(self.yaml_filename, get_yaml(config_obj))
def remove_volume_file(self, relative_file_path):
try:
self._update_secret_file(relative_file_path, None)
except IOError as ioe:
raise CannotWriteConfigException(str(ioe))
def save_volume_file(self, flask_file, relative_file_path):
# Write the file to a temp location.
buf = StringIO()
try:
try:
flask_file.save(buf)
except IOError as ioe:
raise CannotWriteConfigException(str(ioe))
self._update_secret_file(relative_file_path, buf.getvalue())
finally:
buf.close()
def _assert_success(self, response):
if response.status_code != 200:
logger.error('Kubernetes API call failed with response: %s => %s', response.status_code,
response.text)
raise CannotWriteConfigException('Kubernetes API call failed: %s' % response.text)
def _update_secret_file(self, relative_file_path, value=None):
if '/' in relative_file_path:
raise Exception('Expected path from get_volume_path, but found slashes')
# Check first that the namespace for Red Hat Quay exists. If it does not, report that
# as an error, as it seems to be a common issue.
namespace_url = 'namespaces/%s' % (QE_NAMESPACE)
response = self._execute_k8s_api('GET', namespace_url)
if response.status_code // 100 != 2:
msg = 'A Kubernetes namespace with name `%s` must be created to save config' % QE_NAMESPACE
raise CannotWriteConfigException(msg)
# Check if the secret exists. If not, then we create an empty secret and then update the file
# inside.
secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET)
secret = self._lookup_secret()
if secret is None:
self._assert_success(self._execute_k8s_api('POST', secret_url, {
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": QE_CONFIG_SECRET
},
"data": {}
}))
# Update the secret to reflect the file change.
secret['data'] = secret.get('data', {})
if value is not None:
secret['data'][relative_file_path] = base64.b64encode(value)
else:
secret['data'].pop(relative_file_path)
self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
# Wait until the local mounted copy of the secret has been updated, as
# this is an eventual consistency operation, but the caller expects immediate
# consistency.
while True:
matching_files = set()
for secret_filename, encoded_value in secret['data'].iteritems():
expected_value = base64.b64decode(encoded_value)
try:
with self.get_volume_file(secret_filename) as f:
contents = f.read()
if contents == expected_value:
matching_files.add(secret_filename)
except IOError:
continue
if matching_files == set(secret['data'].keys()):
break
# Sleep for a second and then try again.
time.sleep(1)
def _lookup_secret(self):
secret_url = 'namespaces/%s/secrets/%s' % (QE_NAMESPACE, QE_CONFIG_SECRET)
response = self._execute_k8s_api('GET', secret_url)
if response.status_code != 200:
return None
return json.loads(response.text)
def _execute_k8s_api(self, method, relative_url, data=None):
headers = {
'Authorization': 'Bearer ' + self._service_token
}
if data:
headers['Content-Type'] = 'application/json'
data = json.dumps(data) if data else None
session = Session()
url = 'https://%s/api/v1/%s' % (self._api_host, relative_url)
request = Request(method, url, data=data, headers=headers)
return session.send(request.prepare(), verify=False, timeout=2)

View file

@ -0,0 +1,29 @@
import pytest
from util.config.provider import FileConfigProvider
from test.fixtures import *
class TestFileConfigProvider(FileConfigProvider):
def __init__(self):
self.yaml_filename = 'yaml_filename'
self._service_token = 'service_token'
self.config_volume = 'config_volume'
self.py_filename = 'py_filename'
self.yaml_path = os.path.join(self.config_volume, self.yaml_filename)
self.py_path = os.path.join(self.config_volume, self.py_filename)
@pytest.mark.parametrize('directory,filename,expected', [
("directory", "file", "directory/file"),
("directory/dir", "file", "directory/dir/file"),
("directory/dir/", "file", "directory/dir/file"),
("directory", "file/test", "directory/file/test"),
])
def test_get_volume_path(directory, filename, expected):
provider = TestFileConfigProvider()
assert expected == provider.get_volume_path(directory, filename)

View file

@ -0,0 +1,138 @@
import base64
import os
import json
import uuid
import pytest
from contextlib import contextmanager
from collections import namedtuple
from httmock import urlmatch, HTTMock
from util.config.provider import KubernetesConfigProvider
def normalize_path(path):
return path.replace('/', '_')
@contextmanager
def fake_kubernetes_api(tmpdir_factory, files=None):
hostname = 'kubapi'
service_account_token_path = str(tmpdir_factory.mktemp("k8s").join("serviceaccount"))
auth_header = str(uuid.uuid4())
with open(service_account_token_path, 'w') as f:
f.write(auth_header)
global secret
secret = {
'data': {}
}
def write_file(config_dir, filepath, value):
normalized_path = normalize_path(filepath)
absolute_path = str(config_dir.join(normalized_path))
try:
os.makedirs(os.path.dirname(absolute_path))
except OSError:
pass
with open(absolute_path, 'w') as f:
f.write(value)
config_dir = tmpdir_factory.mktemp("config")
if files:
for filepath, value in files.iteritems():
normalized_path = normalize_path(filepath)
write_file(config_dir, filepath, value)
secret['data'][normalized_path] = base64.b64encode(value)
@urlmatch(netloc=hostname,
path='/api/v1/namespaces/quay-enterprise/secrets/quay-enterprise-config-secret$',
method='get')
def get_secret(_, __):
return {'status_code': 200, 'content': json.dumps(secret)}
@urlmatch(netloc=hostname,
path='/api/v1/namespaces/quay-enterprise/secrets/quay-enterprise-config-secret$',
method='put')
def put_secret(_, request):
updated_secret = json.loads(request.body)
for filepath, value in updated_secret['data'].iteritems():
if filepath not in secret['data']:
# Add
write_file(config_dir, filepath, base64.b64decode(value))
for filepath in secret['data']:
if filepath not in updated_secret['data']:
# Remove.
normalized_path = normalize_path(filepath)
os.remove(str(config_dir.join(normalized_path)))
secret['data'] = updated_secret['data']
return {'status_code': 200, 'content': json.dumps(secret)}
@urlmatch(netloc=hostname, path='/api/v1/namespaces/quay-enterprise$')
def get_namespace(_, __):
return {'status_code': 200, 'content': json.dumps({})}
@urlmatch(netloc=hostname)
def catch_all(url, _):
print url
return {'status_code': 404, 'content': '{}'}
with HTTMock(get_secret, put_secret, get_namespace, catch_all):
provider = KubernetesConfigProvider(str(config_dir), 'config.yaml', 'config.py',
api_host=hostname,
service_account_token_path=service_account_token_path)
# Validate all the files.
for filepath, value in files.iteritems():
normalized_path = normalize_path(filepath)
assert provider.volume_file_exists(normalized_path)
with provider.get_volume_file(normalized_path) as f:
assert f.read() == value
yield provider
def test_basic_config(tmpdir_factory):
basic_files = {
'config.yaml': 'FOO: bar',
}
with fake_kubernetes_api(tmpdir_factory, files=basic_files) as provider:
assert provider.config_exists()
assert provider.get_config() is not None
assert provider.get_config()['FOO'] == 'bar'
@pytest.mark.parametrize('filepath', [
'foo',
'foo/meh',
'foo/bar/baz',
])
def test_remove_file(filepath, tmpdir_factory):
basic_files = {
filepath: 'foo',
}
with fake_kubernetes_api(tmpdir_factory, files=basic_files) as provider:
normalized_path = normalize_path(filepath)
assert provider.volume_file_exists(normalized_path)
provider.remove_volume_file(normalized_path)
assert not provider.volume_file_exists(normalized_path)
class TestFlaskFile(object):
def save(self, buf):
buf.write('hello world!')
def test_save_file(tmpdir_factory):
basic_files = {}
with fake_kubernetes_api(tmpdir_factory, files=basic_files) as provider:
assert not provider.volume_file_exists('testfile')
flask_file = TestFlaskFile()
provider.save_volume_file(flask_file, 'testfile')
assert provider.volume_file_exists('testfile')

View file

@ -0,0 +1,77 @@
import json
import io
import os
from datetime import datetime, timedelta
from util.config.provider.baseprovider import BaseProvider
REAL_FILES = ['test/data/signing-private.gpg', 'test/data/signing-public.gpg', 'test/data/test.pem']
class TestConfigProvider(BaseProvider):
""" Implementation of the config provider for testing. Everything is kept in-memory instead on
the real file system. """
def get_config_root(self):
raise Exception('Test Config does not have a config root')
def __init__(self):
self.clear()
def clear(self):
self.files = {}
self._config = {}
@property
def provider_id(self):
return 'test'
def update_app_config(self, app_config):
self._config = app_config
def get_config(self):
if not 'config.yaml' in self.files:
return None
return json.loads(self.files.get('config.yaml', '{}'))
def save_config(self, config_obj):
self.files['config.yaml'] = json.dumps(config_obj)
def config_exists(self):
return 'config.yaml' in self.files
def volume_exists(self):
return True
def volume_file_exists(self, filename):
if filename in REAL_FILES:
return True
return filename in self.files
def save_volume_file(self, flask_file, filename):
self.files[filename] = flask_file.read()
def get_volume_file(self, filename, mode='r'):
if filename in REAL_FILES:
return open(filename, mode=mode)
return io.BytesIO(self.files[filename])
def remove_volume_file(self, filename):
self.files.pop(filename, None)
def list_volume_directory(self, path):
paths = []
for filename in self.files:
if filename.startswith(path):
paths.append(filename[len(path)+1:])
return paths
def reset_for_test(self):
self._config['SUPER_USERS'] = ['devtable']
self.files = {}
def get_volume_path(self, directory, filename):
return os.path.join(directory, filename)

1232
util/config/schema.py Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,38 @@
from multiprocessing.sharedctypes import Array
from util.validation import MAX_USERNAME_LENGTH
class SuperUserManager(object):
""" In-memory helper class for quickly accessing (and updating) the valid
set of super users. This class communicates across processes to ensure
that the shared set is always the same.
"""
def __init__(self, app):
usernames = app.config.get('SUPER_USERS', [])
usernames_str = ','.join(usernames)
self._max_length = len(usernames_str) + MAX_USERNAME_LENGTH + 1
self._array = Array('c', self._max_length, lock=True)
self._array.value = usernames_str
def is_superuser(self, username):
""" Returns if the given username represents a super user. """
usernames = self._array.value.split(',')
return username in usernames
def register_superuser(self, username):
""" Registers a new username as a super user for the duration of the container.
Note that this does *not* change any underlying config files.
"""
usernames = self._array.value.split(',')
usernames.append(username)
new_string = ','.join(usernames)
if len(new_string) <= self._max_length:
self._array.value = new_string
else:
raise Exception('Maximum superuser count reached. Please report this to support.')
def has_superusers(self):
""" Returns whether there are any superusers defined. """
return bool(self._array.value)

View file

@ -0,0 +1,7 @@
from config import DefaultConfig
from util.config.schema import CONFIG_SCHEMA, INTERNAL_ONLY_PROPERTIES
def test_ensure_schema_defines_all_fields():
for key in vars(DefaultConfig):
has_key = key in CONFIG_SCHEMA['properties'] or key in INTERNAL_ONLY_PROPERTIES
assert has_key, "Property `%s` is missing from config schema" % key

View file

@ -0,0 +1,32 @@
import pytest
from util.config.validator import is_valid_config_upload_filename
from util.config.validator import CONFIG_FILENAMES, CONFIG_FILE_SUFFIXES
def test_valid_config_upload_filenames():
for filename in CONFIG_FILENAMES:
assert is_valid_config_upload_filename(filename)
for suffix in CONFIG_FILE_SUFFIXES:
assert is_valid_config_upload_filename('foo' + suffix)
assert not is_valid_config_upload_filename(suffix + 'foo')
@pytest.mark.parametrize('filename, expect_valid', [
('', False),
('foo', False),
('config.yaml', False),
('ssl.cert', True),
('ssl.key', True),
('ssl.crt', False),
('foobar-cloudfront-signing-key.pem', True),
('foobaz-cloudfront-signing-key.pem', True),
('barbaz-cloudfront-signing-key.pem', True),
('barbaz-cloudfront-signing-key.pem.bak', False),
])
def test_is_valid_config_upload_filename(filename, expect_valid):
assert is_valid_config_upload_filename(filename) == expect_valid

152
util/config/validator.py Normal file
View file

@ -0,0 +1,152 @@
import logging
from auth.auth_context import get_authenticated_user
from data.users import LDAP_CERT_FILENAME
from util.secscan.secscan_util import get_blob_download_uri_getter
from util.config import URLSchemeAndHostname
from util.config.validators.validate_database import DatabaseValidator
from util.config.validators.validate_redis import RedisValidator
from util.config.validators.validate_storage import StorageValidator
from util.config.validators.validate_ldap import LDAPValidator
from util.config.validators.validate_keystone import KeystoneValidator
from util.config.validators.validate_jwt import JWTAuthValidator
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.config.validators.validate_signer import SignerValidator
from util.config.validators.validate_torrent import BittorrentValidator
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from util.config.validators.validate_google_login import GoogleLoginValidator
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
from util.config.validators.validate_oidc import OIDCLoginValidator
from util.config.validators.validate_timemachine import TimeMachineValidator
from util.config.validators.validate_access import AccessSettingsValidator
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
logger = logging.getLogger(__name__)
class ConfigValidationException(Exception):
""" Exception raised when the configuration fails to validate for a known reason. """
pass
# Note: Only add files required for HTTPS to the SSL_FILESNAMES list.
DB_SSL_FILENAMES = ['database.pem']
JWT_FILENAMES = ['jwt-authn.cert']
ACI_CERT_FILENAMES = ['signing-public.gpg', 'signing-private.gpg']
LDAP_FILENAMES = [LDAP_CERT_FILENAME]
CONFIG_FILENAMES = (SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES + ACI_CERT_FILENAMES +
LDAP_FILENAMES)
CONFIG_FILE_SUFFIXES = ['-cloudfront-signing-key.pem']
EXTRA_CA_DIRECTORY = 'extra_ca_certs'
EXTRA_CA_DIRECTORY_PREFIX = 'extra_ca_certs_'
VALIDATORS = {
DatabaseValidator.name: DatabaseValidator.validate,
RedisValidator.name: RedisValidator.validate,
StorageValidator.name: StorageValidator.validate,
GitHubLoginValidator.name: GitHubLoginValidator.validate,
GitHubTriggerValidator.name: GitHubTriggerValidator.validate,
GitLabTriggerValidator.name: GitLabTriggerValidator.validate,
BitbucketTriggerValidator.name: BitbucketTriggerValidator.validate,
GoogleLoginValidator.name: GoogleLoginValidator.validate,
SSLValidator.name: SSLValidator.validate,
LDAPValidator.name: LDAPValidator.validate,
JWTAuthValidator.name: JWTAuthValidator.validate,
KeystoneValidator.name: KeystoneValidator.validate,
SignerValidator.name: SignerValidator.validate,
SecurityScannerValidator.name: SecurityScannerValidator.validate,
BittorrentValidator.name: BittorrentValidator.validate,
OIDCLoginValidator.name: OIDCLoginValidator.validate,
TimeMachineValidator.name: TimeMachineValidator.validate,
AccessSettingsValidator.name: AccessSettingsValidator.validate,
ActionLogArchivingValidator.name: ActionLogArchivingValidator.validate,
AppTokenAuthValidator.name: AppTokenAuthValidator.validate,
}
def validate_service_for_config(service, validator_context):
""" Attempts to validate the configuration for the given service. """
if not service in VALIDATORS:
return {
'status': False
}
try:
VALIDATORS[service](validator_context)
return {
'status': True
}
except Exception as ex:
logger.exception('Validation exception')
return {
'status': False,
'reason': str(ex)
}
def is_valid_config_upload_filename(filename):
""" Returns true if and only if the given filename is one which is supported for upload
from the configuration UI tool.
"""
if filename in CONFIG_FILENAMES:
return True
return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES])
class ValidatorContext(object):
""" Context to run validators in, with any additional runtime configuration they need
"""
def __init__(self, config, user_password=None, http_client=None, context=None,
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
uri_creator=None, config_provider=None, instance_keys=None,
init_scripts_location=None):
self.config = config
self.user = get_authenticated_user()
self.user_password = user_password
self.http_client = http_client
self.context = context
self.url_scheme_and_hostname = url_scheme_and_hostname
self.jwt_auth_max = jwt_auth_max
self.registry_title = registry_title
self.ip_resolver = ip_resolver
self.feature_sec_scanner = feature_sec_scanner
self.is_testing = is_testing
self.uri_creator = uri_creator
self.config_provider = config_provider
self.instance_keys = instance_keys
self.init_scripts_location = init_scripts_location
@classmethod
def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None,
config_provider=None, init_scripts_location=None):
"""
Creates a ValidatorContext from an app config, with a given config to validate
:param app: the Flask app to pull configuration information from
:param config: the config to validate
:param user_password: request password
:param instance_keys: The instance keys handler
:param ip_resolver: an App
:param client: http client used to connect to services
:param config_provider: config provider used to access config volume(s)
:param init_scripts_location: location where initial load scripts are stored
:return: ValidatorContext
"""
url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config)
return cls(config,
user_password=user_password,
http_client=client or app.config['HTTPCLIENT'],
context=app.app_context,
url_scheme_and_hostname=url_scheme_and_hostname,
jwt_auth_max=app.config.get('JWT_AUTH_MAX_FRESH_S', 300),
registry_title=app.config['REGISTRY_TITLE'],
ip_resolver=ip_resolver,
feature_sec_scanner=app.config.get('FEATURE_SECURITY_SCANNER', False),
is_testing=app.config.get('TESTING', False),
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
config_provider=config_provider,
instance_keys=instance_keys,
init_scripts_location=init_scripts_location)

View file

@ -0,0 +1,20 @@
from abc import ABCMeta, abstractmethod, abstractproperty
from six import add_metaclass
class ConfigValidationException(Exception):
""" Exception raised when the configuration fails to validate for a known reason. """
pass
@add_metaclass(ABCMeta)
class BaseValidator(object):
@abstractproperty
def name(self):
""" The key for the validation API. """
pass
@classmethod
@abstractmethod
def validate(cls, validator_context):
""" Raises Exception if failure to validate. """
pass

View file

@ -0,0 +1,29 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_access import AccessSettingsValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config, expected_exception', [
({}, None),
({'FEATURE_DIRECT_LOGIN': False}, ConfigValidationException),
({'FEATURE_DIRECT_LOGIN': False, 'SOMETHING_LOGIN_CONFIG': {}}, None),
({'FEATURE_DIRECT_LOGIN': False, 'FEATURE_GITHUB_LOGIN': True}, None),
({'FEATURE_DIRECT_LOGIN': False, 'FEATURE_GOOGLE_LOGIN': True}, None),
({'FEATURE_USER_CREATION': True, 'FEATURE_INVITE_ONLY_USER_CREATION': False}, None),
({'FEATURE_USER_CREATION': True, 'FEATURE_INVITE_ONLY_USER_CREATION': True}, None),
({'FEATURE_INVITE_ONLY_USER_CREATION': True}, None),
({'FEATURE_USER_CREATION': False, 'FEATURE_INVITE_ONLY_USER_CREATION': True},
ConfigValidationException),
({'FEATURE_USER_CREATION': False, 'FEATURE_INVITE_ONLY_USER_CREATION': False}, None),
])
def test_validate_invalid_oidc_login_config(unvalidated_config, expected_exception, app):
validator = AccessSettingsValidator()
if expected_exception is not None:
with pytest.raises(expected_exception):
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(ValidatorContext(unvalidated_config))

View file

@ -0,0 +1,52 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'ACTION_LOG_ARCHIVE_PATH': 'foo'}),
({'ACTION_LOG_ARCHIVE_LOCATION': ''}),
])
def test_skip_validate_actionlog(unvalidated_config, app):
validator = ActionLogArchivingValidator()
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('config, expected_error', [
({'FEATURE_ACTION_LOG_ROTATION': True}, 'Missing action log archive path'),
({'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': ''}, 'Missing action log archive path'),
({'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'foo'}, 'Missing action log archive storage location'),
({'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'foo',
'ACTION_LOG_ARCHIVE_LOCATION': ''}, 'Missing action log archive storage location'),
({'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'foo',
'ACTION_LOG_ARCHIVE_LOCATION': 'invalid'},
'Action log archive storage location `invalid` not found in storage config'),
])
def test_invalid_config(config, expected_error, app):
validator = ActionLogArchivingValidator()
with pytest.raises(ConfigValidationException) as ipe:
validator.validate(ValidatorContext(config))
assert str(ipe.value) == expected_error
def test_valid_config(app):
config = ValidatorContext({
'FEATURE_ACTION_LOG_ROTATION': True,
'ACTION_LOG_ARCHIVE_PATH': 'somepath',
'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
'DISTRIBUTED_STORAGE_CONFIG': {
'somelocation': {},
},
})
validator = ActionLogArchivingValidator()
validator.validate(config)

View file

@ -0,0 +1,30 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_apptokenauth import AppTokenAuthValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'AppToken'}),
({'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': False}),
({'AUTHENTICATION_TYPE': 'AppToken', 'FEATURE_APP_SPECIFIC_TOKENS': True,
'FEATURE_DIRECT_LOGIN': True}),
])
def test_validate_invalid_auth_config(unvalidated_config, app):
validator = AppTokenAuthValidator()
with pytest.raises(ConfigValidationException):
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_auth(app):
config = ValidatorContext({
'AUTHENTICATION_TYPE': 'AppToken',
'FEATURE_APP_SPECIFIC_TOKENS': True,
'FEATURE_DIRECT_LOGIN': False,
})
validator = AppTokenAuthValidator()
validator.validate(config)

View file

@ -0,0 +1,48 @@
import pytest
from httmock import urlmatch, HTTMock
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_bitbucket_trigger import BitbucketTriggerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
(ValidatorContext({})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_KEY': 'foo'}})),
(ValidatorContext({'BITBUCKET_TRIGGER_CONFIG': {'CONSUMER_SECRET': 'foo'}})),
])
def test_validate_invalid_bitbucket_trigger_config(unvalidated_config, app):
validator = BitbucketTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(unvalidated_config)
def test_validate_bitbucket_trigger(app):
url_hit = [False]
@urlmatch(netloc=r'bitbucket.org')
def handler(url, request):
url_hit[0] = True
return {
'status_code': 200,
'content': 'oauth_token=foo&oauth_token_secret=bar',
}
with HTTMock(handler):
validator = BitbucketTriggerValidator()
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'BITBUCKET_TRIGGER_CONFIG': {
'CONSUMER_KEY': 'foo',
'CONSUMER_SECRET': 'bar',
},
}, url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -0,0 +1,23 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_database import DatabaseValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,user,user_password,expected', [
(ValidatorContext(None), None, None, TypeError),
(ValidatorContext({}), None, None, KeyError),
(ValidatorContext({'DB_URI': 'sqlite:///:memory:'}), None, None, None),
(ValidatorContext({'DB_URI': 'invalid:///:memory:'}), None, None, KeyError),
(ValidatorContext({'DB_NOTURI': 'sqlite:///:memory:'}), None, None, KeyError),
])
def test_validate_database(unvalidated_config, user, user_password, expected, app):
validator = DatabaseValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config)

View file

@ -0,0 +1,69 @@
import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_github import GitHubLoginValidator, GitHubTriggerValidator
from test.fixtures import *
@pytest.fixture(params=[GitHubLoginValidator, GitHubTriggerValidator])
def github_validator(request):
return request.param
@pytest.mark.parametrize('github_config', [
({}),
({'GITHUB_ENDPOINT': 'foo'}),
({'GITHUB_ENDPOINT': 'http://github.com'}),
({'GITHUB_ENDPOINT': 'http://github.com', 'CLIENT_ID': 'foo'}),
({'GITHUB_ENDPOINT': 'http://github.com', 'CLIENT_SECRET': 'foo'}),
({
'GITHUB_ENDPOINT': 'http://github.com',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'foo',
'ORG_RESTRICT': True
}),
({
'GITHUB_ENDPOINT': 'http://github.com',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'foo',
'ORG_RESTRICT': True,
'ALLOWED_ORGANIZATIONS': [],
}),
])
def test_validate_invalid_github_config(github_config, github_validator, app):
with pytest.raises(ConfigValidationException):
unvalidated_config = {}
unvalidated_config[github_validator.config_key] = github_config
github_validator.validate(ValidatorContext(unvalidated_config))
def test_validate_github(github_validator, app):
url_hit = [False, False]
@urlmatch(netloc=r'somehost')
def handler(url, request):
url_hit[0] = True
return {'status_code': 200, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
@urlmatch(netloc=r'somehost', path=r'/api/v3/applications/foo/tokens/foo')
def app_handler(url, request):
url_hit[1] = True
return {'status_code': 404, 'content': '', 'headers': {'X-GitHub-Request-Id': 'foo'}}
with HTTMock(app_handler, handler):
unvalidated_config = ValidatorContext({
github_validator.config_key: {
'GITHUB_ENDPOINT': 'http://somehost',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
})
unvalidated_config.http_client = build_requests_session()
github_validator.validate(unvalidated_config)
assert url_hit[0]
assert url_hit[1]

View file

@ -0,0 +1,49 @@
import json
import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_gitlab_trigger import GitLabTriggerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'foo'}}),
({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'http://someendpoint', 'CLIENT_ID': 'foo'}}),
({'GITLAB_TRIGGER_CONFIG': {'GITLAB_ENDPOINT': 'http://someendpoint', 'CLIENT_SECRET': 'foo'}}),
])
def test_validate_invalid_gitlab_trigger_config(unvalidated_config, app):
validator = GitLabTriggerValidator()
with pytest.raises(ConfigValidationException):
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_gitlab_enterprise_trigger(app):
url_hit = [False]
@urlmatch(netloc=r'somegitlab', path='/oauth/token')
def handler(_, __):
url_hit[0] = True
return {'status_code': 400, 'content': json.dumps({'error': 'invalid code'})}
with HTTMock(handler):
validator = GitLabTriggerValidator()
url_scheme_and_hostname = URLSchemeAndHostname('http', 'localhost:5000')
unvalidated_config = ValidatorContext({
'GITLAB_TRIGGER_CONFIG': {
'GITLAB_ENDPOINT': 'http://somegitlab',
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
}, http_client=build_requests_session(), url_scheme_and_hostname=url_scheme_and_hostname)
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -0,0 +1,45 @@
import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_google_login import GoogleLoginValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'GOOGLE_LOGIN_CONFIG': {}}),
({'GOOGLE_LOGIN_CONFIG': {'CLIENT_ID': 'foo'}}),
({'GOOGLE_LOGIN_CONFIG': {'CLIENT_SECRET': 'foo'}}),
])
def test_validate_invalid_google_login_config(unvalidated_config, app):
validator = GoogleLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_google_login(app):
url_hit = [False]
@urlmatch(netloc=r'www.googleapis.com', path='/oauth2/v3/token')
def handler(_, __):
url_hit[0] = True
return {'status_code': 200, 'content': ''}
validator = GoogleLoginValidator()
with HTTMock(handler):
unvalidated_config = ValidatorContext({
'GOOGLE_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
},
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -0,0 +1,65 @@
import pytest
from config import build_requests_session
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_jwt import JWTAuthValidator
from util.morecollections import AttrDict
from test.test_external_jwt_authn import fake_jwt
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'JWT'}),
({'AUTHENTICATION_TYPE': 'JWT', 'JWT_AUTH_ISSUER': 'foo'}),
({'AUTHENTICATION_TYPE': 'JWT', 'JWT_VERIFY_ENDPOINT': 'foo'}),
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
config = ValidatorContext(unvalidated_config)
config.config_provider = config_provider
JWTAuthValidator.validate(config)
# TODO: fix these when re-adding jwt auth mechanism to jwt validators
@pytest.mark.skip(reason='No way of currently testing this')
@pytest.mark.parametrize('username, password, expected_exception', [
('invaliduser', 'invalidpass', ConfigValidationException),
('cool.user', 'invalidpass', ConfigValidationException),
('invaliduser', 'somepass', ConfigValidationException),
('cool.user', 'password', None),
])
def test_validated_jwt(username, password, expected_exception, app):
with fake_jwt() as jwt_auth:
config = {}
config['AUTHENTICATION_TYPE'] = 'JWT'
config['JWT_AUTH_ISSUER'] = jwt_auth.issuer
config['JWT_VERIFY_ENDPOINT'] = jwt_auth.verify_url
config['JWT_QUERY_ENDPOINT'] = jwt_auth.query_url
config['JWT_GETUSER_ENDPOINT'] = jwt_auth.getuser_url
unvalidated_config = ValidatorContext(config)
unvalidated_config.user = AttrDict(dict(username=username))
unvalidated_config.user_password = password
unvalidated_config.config_provider = config_provider
unvalidated_config.http_client = build_requests_session()
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)
else:
JWTAuthValidator.validate(unvalidated_config, public_key_path=jwt_auth.public_key_path)

View file

@ -0,0 +1,54 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_keystone import KeystoneValidator
from test.test_keystone_auth import fake_keystone
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'Keystone'}),
({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo'}),
({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo',
'KEYSTONE_ADMIN_USERNAME': 'bar'}),
({'AUTHENTICATION_TYPE': 'Keystone', 'KEYSTONE_AUTH_URL': 'foo',
'KEYSTONE_ADMIN_USERNAME': 'bar', 'KEYSTONE_ADMIN_PASSWORD': 'baz'}),
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('admin_tenant_id, expected_exception', [
('somegroupid', None),
('groupwithnousers', ConfigValidationException),
('somegroupid', None),
('groupwithnousers', ConfigValidationException),
])
def test_validated_keystone(admin_tenant_id, expected_exception, app):
with fake_keystone(2) as keystone_auth:
auth_url = keystone_auth.auth_url
config = {}
config['AUTHENTICATION_TYPE'] = 'Keystone'
config['KEYSTONE_AUTH_URL'] = auth_url
config['KEYSTONE_ADMIN_USERNAME'] = 'adminuser'
config['KEYSTONE_ADMIN_PASSWORD'] = 'adminpass'
config['KEYSTONE_ADMIN_TENANT'] = admin_tenant_id
unvalidated_config = ValidatorContext(config)
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
KeystoneValidator.validate(unvalidated_config)
else:
KeystoneValidator.validate(unvalidated_config)

View file

@ -0,0 +1,72 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ldap import LDAPValidator
from util.morecollections import AttrDict
from test.test_ldap import mock_ldap
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'AUTHENTICATION_TYPE': 'Database'}),
])
def test_validate_noop(unvalidated_config, app):
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('unvalidated_config', [
({'AUTHENTICATION_TYPE': 'LDAP'}),
({'AUTHENTICATION_TYPE': 'LDAP', 'LDAP_ADMIN_DN': 'foo'}),
])
def test_invalid_config(unvalidated_config, app):
with pytest.raises(ConfigValidationException):
config = ValidatorContext(unvalidated_config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('uri', [
'foo',
'http://foo',
'ldap:foo',
])
def test_invalid_uri(uri, app):
config = {}
config['AUTHENTICATION_TYPE'] = 'LDAP'
config['LDAP_BASE_DN'] = ['dc=quay', 'dc=io']
config['LDAP_ADMIN_DN'] = 'uid=testy,ou=employees,dc=quay,dc=io'
config['LDAP_ADMIN_PASSWD'] = 'password'
config['LDAP_USER_RDN'] = ['ou=employees']
config['LDAP_URI'] = uri
with pytest.raises(ConfigValidationException):
config = ValidatorContext(config, config_provider=config_provider)
LDAPValidator.validate(config)
@pytest.mark.parametrize('admin_dn, admin_passwd, user_rdn, expected_exception', [
('uid=testy,ou=employees,dc=quay,dc=io', 'password', ['ou=employees'], None),
('uid=invalidadmindn', 'password', ['ou=employees'], ConfigValidationException),
('uid=testy,ou=employees,dc=quay,dc=io', 'invalid_password', ['ou=employees'], ConfigValidationException),
('uid=testy,ou=employees,dc=quay,dc=io', 'password', ['ou=invalidgroup'], ConfigValidationException),
])
def test_validated_ldap(admin_dn, admin_passwd, user_rdn, expected_exception, app):
config = {}
config['AUTHENTICATION_TYPE'] = 'LDAP'
config['LDAP_BASE_DN'] = ['dc=quay', 'dc=io']
config['LDAP_ADMIN_DN'] = admin_dn
config['LDAP_ADMIN_PASSWD'] = admin_passwd
config['LDAP_USER_RDN'] = user_rdn
unvalidated_config = ValidatorContext(config, config_provider=config_provider)
if expected_exception is not None:
with pytest.raises(ConfigValidationException):
with mock_ldap():
LDAPValidator.validate(unvalidated_config)
else:
with mock_ldap():
LDAPValidator.validate(unvalidated_config)

View file

@ -0,0 +1,50 @@
import json
import pytest
from httmock import urlmatch, HTTMock
from config import build_requests_session
from oauth.oidc import OIDC_WELLKNOWN
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_oidc import OIDCLoginValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({'SOMETHING_LOGIN_CONFIG': {}}),
({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo'}}),
({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo', 'CLIENT_ID': 'foobar'}}),
({'SOMETHING_LOGIN_CONFIG': {'OIDC_SERVER': 'foo', 'CLIENT_SECRET': 'foobar'}}),
])
def test_validate_invalid_oidc_login_config(unvalidated_config, app):
validator = OIDCLoginValidator()
with pytest.raises(ConfigValidationException):
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_oidc_login(app):
url_hit = [False]
@urlmatch(netloc=r'someserver', path=r'/\.well-known/openid-configuration')
def handler(_, __):
url_hit[0] = True
data = {
'token_endpoint': 'foobar',
}
return {'status_code': 200, 'content': json.dumps(data)}
with HTTMock(handler):
validator = OIDCLoginValidator()
unvalidated_config = ValidatorContext({
'SOMETHING_LOGIN_CONFIG': {
'CLIENT_ID': 'foo',
'CLIENT_SECRET': 'bar',
'OIDC_SERVER': 'http://someserver',
'DEBUGGING': True, # Allows for HTTP.
},
})
unvalidated_config.http_client = build_requests_session()
validator.validate(unvalidated_config)
assert url_hit[0]

View file

@ -0,0 +1,34 @@
import pytest
import redis
from mock import patch
from mockredis import mock_strict_redis_client
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_redis import RedisValidator
from test.fixtures import *
from util.morecollections import AttrDict
@pytest.mark.parametrize('unvalidated_config,user,user_password,use_mock,expected', [
({}, None, None, False, ConfigValidationException),
({'BUILDLOGS_REDIS': {}}, None, None, False, ConfigValidationException),
({'BUILDLOGS_REDIS': {'host': 'somehost'}}, None, None, False, redis.ConnectionError),
({'BUILDLOGS_REDIS': {'host': 'localhost'}}, None, None, True, None),
])
def test_validate_redis(unvalidated_config, user, user_password, use_mock, expected, app):
with patch('redis.StrictRedis' if use_mock else 'redis.None', mock_strict_redis_client):
validator = RedisValidator()
unvalidated_config = ValidatorContext(unvalidated_config)
unvalidated_config.user = AttrDict(dict(username=user))
unvalidated_config.user_password = user_password
if expected is not None:
with pytest.raises(expected):
validator.validate(unvalidated_config)
else:
validator.validate(unvalidated_config)

View file

@ -0,0 +1,48 @@
import pytest
from config import build_requests_session
from util.config import URLSchemeAndHostname
from util.config.validator import ValidatorContext
from util.config.validators.validate_secscan import SecurityScannerValidator
from util.secscan.fake import fake_security_scanner
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config', [
({'DISTRIBUTED_STORAGE_PREFERENCE': []}),
])
def test_validate_noop(unvalidated_config, app):
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=False, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
SecurityScannerValidator.validate(unvalidated_config)
@pytest.mark.parametrize('unvalidated_config, expected_error', [
({
'TESTING': True,
'DISTRIBUTED_STORAGE_PREFERENCE': [],
'FEATURE_SECURITY_SCANNER': True,
'SECURITY_SCANNER_ENDPOINT': 'http://invalidhost',
}, Exception),
({
'TESTING': True,
'DISTRIBUTED_STORAGE_PREFERENCE': [],
'FEATURE_SECURITY_SCANNER': True,
'SECURITY_SCANNER_ENDPOINT': 'http://fakesecurityscanner',
}, None),
])
def test_validate(unvalidated_config, expected_error, app):
unvalidated_config = ValidatorContext(unvalidated_config, feature_sec_scanner=True, is_testing=True,
http_client=build_requests_session(),
url_scheme_and_hostname=URLSchemeAndHostname('http', 'localhost:5000'))
with fake_security_scanner(hostname='fakesecurityscanner'):
if expected_error is not None:
with pytest.raises(expected_error):
SecurityScannerValidator.validate(unvalidated_config)
else:
SecurityScannerValidator.validate(unvalidated_config)

View file

@ -0,0 +1,20 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_signer import SignerValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,expected', [
({}, None),
({'SIGNING_ENGINE': 'foobar'}, ConfigValidationException),
({'SIGNING_ENGINE': 'gpg2'}, Exception),
])
def test_validate_signer(unvalidated_config, expected, app):
validator = SignerValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(ValidatorContext(unvalidated_config))

View file

@ -0,0 +1,75 @@
import pytest
from mock import patch
from tempfile import NamedTemporaryFile
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_ssl import SSLValidator, SSL_FILENAMES
from util.security.test.test_ssl_util import generate_test_cert
from test.fixtures import *
from app import config_provider
@pytest.mark.parametrize('unvalidated_config', [
({}),
({'PREFERRED_URL_SCHEME': 'http'}),
({'PREFERRED_URL_SCHEME': 'https', 'EXTERNAL_TLS_TERMINATION': True}),
])
def test_skip_validate_ssl(unvalidated_config, app):
validator = SSLValidator()
validator.validate(ValidatorContext(unvalidated_config))
@pytest.mark.parametrize('cert, server_hostname, expected_error, error_message', [
('invalidcert', 'someserver', ConfigValidationException, 'Could not load SSL certificate: no start line'),
(generate_test_cert(hostname='someserver'), 'someserver', None, None),
(generate_test_cert(hostname='invalidserver'), 'someserver', ConfigValidationException,
'Supported names "invalidserver" in SSL cert do not match server hostname "someserver"'),
(generate_test_cert(hostname='someserver'), 'someserver:1234', None, None),
(generate_test_cert(hostname='invalidserver'), 'someserver:1234', ConfigValidationException,
'Supported names "invalidserver" in SSL cert do not match server hostname "someserver"'),
(generate_test_cert(hostname='someserver:1234'), 'someserver:1234', ConfigValidationException,
'Supported names "someserver:1234" in SSL cert do not match server hostname "someserver"'),
(generate_test_cert(hostname='someserver:more'), 'someserver:more', None, None),
(generate_test_cert(hostname='someserver:more'), 'someserver:more:1234', None, None),
])
def test_validate_ssl(cert, server_hostname, expected_error, error_message, app):
with NamedTemporaryFile(delete=False) as cert_file:
cert_file.write(cert[0])
cert_file.seek(0)
with NamedTemporaryFile(delete=False) as key_file:
key_file.write(cert[1])
key_file.seek(0)
def return_true(filename):
return True
def get_volume_file(filename):
if filename == SSL_FILENAMES[0]:
return open(cert_file.name)
if filename == SSL_FILENAMES[1]:
return open(key_file.name)
return None
config = {
'PREFERRED_URL_SCHEME': 'https',
'SERVER_HOSTNAME': server_hostname,
}
with patch('app.config_provider.volume_file_exists', return_true):
with patch('app.config_provider.get_volume_file', get_volume_file):
validator = SSLValidator()
config = ValidatorContext(config)
config.config_provider = config_provider
if expected_error is not None:
with pytest.raises(expected_error) as ipe:
validator.validate(config)
assert str(ipe.value) == error_message
else:
validator.validate(config)

View file

@ -0,0 +1,40 @@
import pytest
from moto import mock_s3_deprecated as mock_s3
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_storage import StorageValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config, expected', [
({}, ConfigValidationException),
({'DISTRIBUTED_STORAGE_CONFIG': {}}, ConfigValidationException),
({'DISTRIBUTED_STORAGE_CONFIG': {'local': None}}, ConfigValidationException),
({'DISTRIBUTED_STORAGE_CONFIG': {'local': ['FakeStorage', {}]}}, None),
])
def test_validate_storage(unvalidated_config, expected, app):
validator = StorageValidator()
if expected is not None:
with pytest.raises(expected):
validator.validate(ValidatorContext(unvalidated_config))
else:
validator.validate(ValidatorContext(unvalidated_config))
def test_validate_s3_storage(app):
validator = StorageValidator()
with mock_s3():
with pytest.raises(ConfigValidationException) as ipe:
validator.validate(ValidatorContext({
'DISTRIBUTED_STORAGE_CONFIG': {
'default': ('S3Storage', {
's3_access_key': 'invalid',
's3_secret_key': 'invalid',
's3_bucket': 'somebucket',
'storage_path': ''
}),
}
}))
assert str(ipe.value) == 'Invalid storage configuration: default: S3ResponseError: 404 Not Found'

View file

@ -0,0 +1,32 @@
import pytest
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_timemachine import TimeMachineValidator
@pytest.mark.parametrize('unvalidated_config', [
({}),
])
def test_validate_noop(unvalidated_config):
TimeMachineValidator.validate(ValidatorContext(unvalidated_config))
from test.fixtures import *
@pytest.mark.parametrize('default_exp,options,expected_exception', [
('2d', ['1w', '2d'], None),
('2d', ['1w'], 'Default expiration must be in expiration options set'),
('2d', ['2d', '1M'], 'Invalid tag expiration option: 1M'),
])
def test_validate(default_exp, options, expected_exception, app):
config = {}
config['DEFAULT_TAG_EXPIRATION'] = default_exp
config['TAG_EXPIRATION_OPTIONS'] = options
if expected_exception is not None:
with pytest.raises(ConfigValidationException) as cve:
TimeMachineValidator.validate(ValidatorContext(config))
assert str(cve.value) == str(expected_exception)
else:
TimeMachineValidator.validate(ValidatorContext(config))

View file

@ -0,0 +1,39 @@
import pytest
from config import build_requests_session
from httmock import urlmatch, HTTMock
from app import instance_keys
from util.config.validator import ValidatorContext
from util.config.validators import ConfigValidationException
from util.config.validators.validate_torrent import BittorrentValidator
from test.fixtures import *
@pytest.mark.parametrize('unvalidated_config,expected', [
({}, ConfigValidationException),
({'BITTORRENT_ANNOUNCE_URL': 'http://faketorrent/announce'}, None),
])
def test_validate_torrent(unvalidated_config, expected, app):
announcer_hit = [False]
@urlmatch(netloc=r'faketorrent', path='/announce')
def handler(url, request):
announcer_hit[0] = True
return {'status_code': 200, 'content': ''}
with HTTMock(handler):
validator = BittorrentValidator()
if expected is not None:
with pytest.raises(expected):
config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
config.http_client = build_requests_session()
validator.validate(config)
assert not announcer_hit[0]
else:
config = ValidatorContext(unvalidated_config, instance_keys=instance_keys)
config.http_client = build_requests_session()
validator.validate(config)
assert announcer_hit[0]

View file

@ -0,0 +1,28 @@
from util.config.validators import BaseValidator, ConfigValidationException
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService
class AccessSettingsValidator(BaseValidator):
name = "access"
@classmethod
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
if not config.get('FEATURE_DIRECT_LOGIN', True):
# Make sure we have at least one OIDC enabled.
github_login = config.get('FEATURE_GITHUB_LOGIN', False)
google_login = config.get('FEATURE_GOOGLE_LOGIN', False)
login_manager = OAuthLoginManager(config, client=client)
custom_oidc = [s for s in login_manager.services if isinstance(s, OIDCLoginService)]
if not github_login and not google_login and not custom_oidc:
msg = 'Cannot disable credentials login to UI without configured OIDC service'
raise ConfigValidationException(msg)
if (not config.get('FEATURE_USER_CREATION', True) and
config.get('FEATURE_INVITE_ONLY_USER_CREATION', False)):
msg = "Invite only user creation requires user creation to be enabled"
raise ConfigValidationException(msg)

View file

@ -0,0 +1,24 @@
from util.config.validators import BaseValidator, ConfigValidationException
class ActionLogArchivingValidator(BaseValidator):
name = "actionlogarchiving"
@classmethod
def validate(cls, validator_context):
config = validator_context.config
""" Validates the action log archiving configuration. """
if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
return
if not config.get('ACTION_LOG_ARCHIVE_PATH'):
raise ConfigValidationException('Missing action log archive path')
if not config.get('ACTION_LOG_ARCHIVE_LOCATION'):
raise ConfigValidationException('Missing action log archive storage location')
location = config['ACTION_LOG_ARCHIVE_LOCATION']
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG') or {}
if location not in storage_config:
msg = 'Action log archive storage location `%s` not found in storage config' % location
raise ConfigValidationException(msg)

View file

@ -0,0 +1,21 @@
from util.config.validators import BaseValidator, ConfigValidationException
class AppTokenAuthValidator(BaseValidator):
name = "apptoken-auth"
@classmethod
def validate(cls, validator_context):
config = validator_context.config
if config.get('AUTHENTICATION_TYPE', 'Database') != 'AppToken':
return
# Ensure that app tokens are enabled, as they are required.
if not config.get('FEATURE_APP_SPECIFIC_TOKENS', False):
msg = 'Application token support must be enabled to use External Application Token auth'
raise ConfigValidationException(msg)
# Ensure that direct login is disabled.
if config.get('FEATURE_DIRECT_LOGIN', True):
msg = 'Direct login must be disabled to use External Application Token auth'
raise ConfigValidationException(msg)

View file

@ -0,0 +1,30 @@
from bitbucket import BitBucket
from util.config.validators import BaseValidator, ConfigValidationException
class BitbucketTriggerValidator(BaseValidator):
name = "bitbucket-trigger"
@classmethod
def validate(cls, validator_context):
""" Validates the config for BitBucket. """
config = validator_context.config
trigger_config = config.get('BITBUCKET_TRIGGER_CONFIG')
if not trigger_config:
raise ConfigValidationException('Missing client ID and client secret')
if not trigger_config.get('CONSUMER_KEY'):
raise ConfigValidationException('Missing Consumer Key')
if not trigger_config.get('CONSUMER_SECRET'):
raise ConfigValidationException('Missing Consumer Secret')
key = trigger_config['CONSUMER_KEY']
secret = trigger_config['CONSUMER_SECRET']
callback_url = '%s/oauth1/bitbucket/callback/trigger/' % (validator_context.url_scheme_and_hostname.get_url())
bitbucket_client = BitBucket(key, secret, callback_url)
(result, _, _) = bitbucket_client.get_authorization_url()
if not result:
raise ConfigValidationException('Invalid consumer key or secret')

View file

@ -0,0 +1,20 @@
from peewee import OperationalError
from data.database import validate_database_precondition
from util.config.validators import BaseValidator, ConfigValidationException
class DatabaseValidator(BaseValidator):
name = "database"
@classmethod
def validate(cls, validator_context):
""" Validates connecting to the database. """
config = validator_context.config
try:
validate_database_precondition(config['DB_URI'], config.get('DB_CONNECTION_ARGS', {}))
except OperationalError as ex:
if ex.args and len(ex.args) > 1:
raise ConfigValidationException(ex.args[1])
else:
raise ex

View file

@ -0,0 +1,53 @@
from oauth.services.github import GithubOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
class BaseGitHubValidator(BaseValidator):
name = None
config_key = None
@classmethod
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a Github service. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
github_config = config.get(cls.config_key)
if not github_config:
raise ConfigValidationException('Missing GitHub client id and client secret')
endpoint = github_config.get('GITHUB_ENDPOINT')
if not endpoint:
raise ConfigValidationException('Missing GitHub Endpoint')
if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
raise ConfigValidationException('Github Endpoint must start with http:// or https://')
if not github_config.get('CLIENT_ID'):
raise ConfigValidationException('Missing Client ID')
if not github_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
if github_config.get('ORG_RESTRICT') and not github_config.get('ALLOWED_ORGANIZATIONS'):
raise ConfigValidationException('Organization restriction must have at least one allowed ' +
'organization')
oauth = GithubOAuthService(config, cls.config_key)
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')
if github_config.get('ALLOWED_ORGANIZATIONS'):
for org_id in github_config.get('ALLOWED_ORGANIZATIONS'):
if not oauth.validate_organization(org_id, client):
raise ConfigValidationException('Invalid organization: %s' % org_id)
class GitHubLoginValidator(BaseGitHubValidator):
name = "github-login"
config_key = "GITHUB_LOGIN_CONFIG"
class GitHubTriggerValidator(BaseGitHubValidator):
name = "github-trigger"
config_key = "GITHUB_TRIGGER_CONFIG"

View file

@ -0,0 +1,32 @@
from oauth.services.gitlab import GitLabOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
class GitLabTriggerValidator(BaseValidator):
name = "gitlab-trigger"
@classmethod
def validate(cls, validator_context):
""" Validates the OAuth credentials and API endpoint for a GitLab service. """
config = validator_context.config
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
client = validator_context.http_client
github_config = config.get('GITLAB_TRIGGER_CONFIG')
if not github_config:
raise ConfigValidationException('Missing GitLab client id and client secret')
endpoint = github_config.get('GITLAB_ENDPOINT')
if endpoint:
if endpoint.find('http://') != 0 and endpoint.find('https://') != 0:
raise ConfigValidationException('GitLab Endpoint must start with http:// or https://')
if not github_config.get('CLIENT_ID'):
raise ConfigValidationException('Missing Client ID')
if not github_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
oauth = GitLabOAuthService(config, 'GITLAB_TRIGGER_CONFIG')
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -0,0 +1,27 @@
from oauth.services.google import GoogleOAuthService
from util.config.validators import BaseValidator, ConfigValidationException
class GoogleLoginValidator(BaseValidator):
name = "google-login"
@classmethod
def validate(cls, validator_context):
""" Validates the Google Login client ID and secret. """
config = validator_context.config
client = validator_context.http_client
url_scheme_and_hostname = validator_context.url_scheme_and_hostname
google_login_config = config.get('GOOGLE_LOGIN_CONFIG')
if not google_login_config:
raise ConfigValidationException('Missing client ID and client secret')
if not google_login_config.get('CLIENT_ID'):
raise ConfigValidationException('Missing Client ID')
if not google_login_config.get('CLIENT_SECRET'):
raise ConfigValidationException('Missing Client Secret')
oauth = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
result = oauth.validate_client_id_and_secret(client, url_scheme_and_hostname)
if not result:
raise ConfigValidationException('Invalid client id or client secret')

View file

@ -0,0 +1,48 @@
import os
from data.users.externaljwt import ExternalJWTAuthN
from util.config.validators import BaseValidator, ConfigValidationException
class JWTAuthValidator(BaseValidator):
name = "jwt"
@classmethod
def validate(cls, validator_context, public_key_path=None):
""" Validates the JWT authentication system. """
config = validator_context.config
http_client = validator_context.http_client
jwt_auth_max = validator_context.jwt_auth_max
config_provider = validator_context.config_provider
if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT':
return
verify_endpoint = config.get('JWT_VERIFY_ENDPOINT')
query_endpoint = config.get('JWT_QUERY_ENDPOINT', None)
getuser_endpoint = config.get('JWT_GETUSER_ENDPOINT', None)
issuer = config.get('JWT_AUTH_ISSUER')
if not verify_endpoint:
raise ConfigValidationException('Missing JWT Verification endpoint')
if not issuer:
raise ConfigValidationException('Missing JWT Issuer ID')
override_config_directory = config_provider.get_config_dir_path()
# Try to instatiate the JWT authentication mechanism. This will raise an exception if
# the key cannot be found.
users = ExternalJWTAuthN(verify_endpoint, query_endpoint, getuser_endpoint, issuer,
override_config_directory,
http_client,
jwt_auth_max,
public_key_path=public_key_path,
requires_email=config.get('FEATURE_MAILING', True))
# Verify that we can reach the jwt server
(result, err_msg) = users.ping()
if not result:
msg = ('Verification of JWT failed: %s. \n\nWe cannot reach the JWT server' +
'OR JWT auth is misconfigured') % err_msg
raise ConfigValidationException(msg)

View file

@ -0,0 +1,44 @@
from util.config.validators import BaseValidator, ConfigValidationException
from data.users.keystone import get_keystone_users
class KeystoneValidator(BaseValidator):
name = "keystone"
@classmethod
def validate(cls, validator_context):
""" Validates the Keystone authentication system. """
config = validator_context.config
if config.get('AUTHENTICATION_TYPE', 'Database') != 'Keystone':
return
auth_url = config.get('KEYSTONE_AUTH_URL')
auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')
if not auth_url:
raise ConfigValidationException('Missing authentication URL')
if not admin_username:
raise ConfigValidationException('Missing admin username')
if not admin_password:
raise ConfigValidationException('Missing admin password')
if not admin_tenant:
raise ConfigValidationException('Missing admin tenant')
requires_email = config.get('FEATURE_MAILING', True)
users = get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
requires_email)
# Verify that the superuser exists. If not, raise an exception.
(result, err_msg) = users.at_least_one_user_exists()
if not result:
msg = ('Verification that users exist failed: %s. \n\nNo users exist ' +
'in the admin tenant/project ' +
'in the remote authentication system ' +
'OR Keystone auth is misconfigured.') % err_msg
raise ConfigValidationException(msg)

View file

@ -0,0 +1,68 @@
import os
import ldap
import subprocess
from data.users import LDAP_CERT_FILENAME
from data.users.externalldap import LDAPConnection, LDAPUsers
from util.config.validators import BaseValidator, ConfigValidationException
class LDAPValidator(BaseValidator):
name = "ldap"
@classmethod
def validate(cls, validator_context):
""" Validates the LDAP connection. """
config = validator_context.config
config_provider = validator_context.config_provider
init_scripts_location = validator_context.init_scripts_location
if config.get('AUTHENTICATION_TYPE', 'Database') != 'LDAP':
return
# If there is a custom LDAP certificate, then reinstall the certificates for the container.
if config_provider.volume_file_exists(LDAP_CERT_FILENAME):
subprocess.check_call([os.path.join(init_scripts_location, 'certs_install.sh')],
env={ 'QUAYCONFIG': config_provider.get_config_dir_path() })
# Note: raises ldap.INVALID_CREDENTIALS on failure
admin_dn = config.get('LDAP_ADMIN_DN')
admin_passwd = config.get('LDAP_ADMIN_PASSWD')
if not admin_dn:
raise ConfigValidationException('Missing Admin DN for LDAP configuration')
if not admin_passwd:
raise ConfigValidationException('Missing Admin Password for LDAP configuration')
ldap_uri = config.get('LDAP_URI', 'ldap://localhost')
if not ldap_uri.startswith('ldap://') and not ldap_uri.startswith('ldaps://'):
raise ConfigValidationException('LDAP URI must start with ldap:// or ldaps://')
allow_tls_fallback = config.get('LDAP_ALLOW_INSECURE_FALLBACK', False)
try:
with LDAPConnection(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback):
pass
except ldap.LDAPError as ex:
values = ex.args[0] if ex.args else {}
if not isinstance(values, dict):
raise ConfigValidationException(str(ex.args))
raise ConfigValidationException(values.get('desc', 'Unknown error'))
base_dn = config.get('LDAP_BASE_DN')
user_rdn = config.get('LDAP_USER_RDN', [])
uid_attr = config.get('LDAP_UID_ATTR', 'uid')
email_attr = config.get('LDAP_EMAIL_ATTR', 'mail')
requires_email = config.get('FEATURE_MAILING', True)
users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
allow_tls_fallback, requires_email=requires_email)
# Ensure at least one user exists to verify the connection is setup properly.
(result, err_msg) = users.at_least_one_user_exists()
if not result:
msg = ('Verification that users exist failed: %s. \n\nNo users exist ' +
'in the remote authentication system ' +
'OR LDAP auth is misconfigured.') % err_msg
raise ConfigValidationException(msg)

View file

@ -0,0 +1,36 @@
from oauth.loginmanager import OAuthLoginManager
from oauth.oidc import OIDCLoginService, DiscoveryFailureException
from util.config.validators import BaseValidator, ConfigValidationException
class OIDCLoginValidator(BaseValidator):
name = "oidc-login"
@classmethod
def validate(cls, validator_context):
config = validator_context.config
client = validator_context.http_client
login_manager = OAuthLoginManager(config, client=client)
for service in login_manager.services:
if not isinstance(service, OIDCLoginService):
continue
if service.config.get('OIDC_SERVER') is None:
msg = 'Missing OIDC_SERVER on OIDC service %s' % service.service_id()
raise ConfigValidationException(msg)
if service.config.get('CLIENT_ID') is None:
msg = 'Missing CLIENT_ID on OIDC service %s' % service.service_id()
raise ConfigValidationException(msg)
if service.config.get('CLIENT_SECRET') is None:
msg = 'Missing CLIENT_SECRET on OIDC service %s' % service.service_id()
raise ConfigValidationException(msg)
try:
if not service.validate():
msg = 'Could not validate OIDC service %s' % service.service_id()
raise ConfigValidationException(msg)
except DiscoveryFailureException as dfe:
msg = 'Could not validate OIDC service %s: %s' % (service.service_id(), dfe.message)
raise ConfigValidationException(msg)

View file

@ -0,0 +1,18 @@
import redis
from util.config.validators import BaseValidator, ConfigValidationException
class RedisValidator(BaseValidator):
name = "redis"
@classmethod
def validate(cls, validator_context):
""" Validates connecting to redis. """
config = validator_context.config
redis_config = config.get('BUILDLOGS_REDIS', {})
if not 'host' in redis_config:
raise ConfigValidationException('Missing redis hostname')
client = redis.StrictRedis(socket_connect_timeout=5, **redis_config)
client.ping()

View file

@ -0,0 +1,52 @@
import time
# from boot import setup_jwt_proxy
from util.secscan.api import SecurityScannerAPI
from util.config.validators import BaseValidator, ConfigValidationException
class SecurityScannerValidator(BaseValidator):
name = "security-scanner"
@classmethod
def validate(cls, validator_context):
""" Validates the configuration for talking to a Quay Security Scanner. """
config = validator_context.config
client = validator_context.http_client
feature_sec_scanner = validator_context.feature_sec_scanner
is_testing = validator_context.is_testing
server_hostname = validator_context.url_scheme_and_hostname.hostname
uri_creator = validator_context.uri_creator
if not feature_sec_scanner:
return
api = SecurityScannerAPI(config, None, server_hostname, client=client, skip_validation=True, uri_creator=uri_creator)
# if not is_testing:
# Generate a temporary Quay key to use for signing the outgoing requests.
# setup_jwt_proxy()
# We have to wait for JWT proxy to restart with the newly generated key.
max_tries = 5
response = None
last_exception = None
while max_tries > 0:
try:
response = api.ping()
last_exception = None
if response.status_code == 200:
return
except Exception as ex:
last_exception = ex
time.sleep(1)
max_tries = max_tries - 1
if last_exception is not None:
message = str(last_exception)
raise ConfigValidationException('Could not ping security scanner: %s' % message)
else:
message = 'Expected 200 status code, got %s: %s' % (response.status_code, response.text)
raise ConfigValidationException('Could not ping security scanner: %s' % message)

View file

@ -0,0 +1,22 @@
from StringIO import StringIO
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.signing import SIGNING_ENGINES
class SignerValidator(BaseValidator):
name = "signer"
@classmethod
def validate(cls, validator_context):
""" Validates the GPG public+private key pair used for signing converted ACIs. """
config = validator_context.config
config_provider = validator_context.config_provider
if config.get('SIGNING_ENGINE') is None:
return
if config['SIGNING_ENGINE'] not in SIGNING_ENGINES:
raise ConfigValidationException('Unknown signing engine: %s' % config['SIGNING_ENGINE'])
engine = SIGNING_ENGINES[config['SIGNING_ENGINE']](config, config_provider)
engine.detached_sign(StringIO('test string'))

View file

@ -0,0 +1,72 @@
from util.config.validators import BaseValidator, ConfigValidationException
from util.security.ssl import load_certificate, CertInvalidException, KeyInvalidException
SSL_FILENAMES = ['ssl.cert', 'ssl.key']
class SSLValidator(BaseValidator):
name = "ssl"
@classmethod
def validate(cls, validator_context):
""" Validates the SSL configuration (if enabled). """
config = validator_context.config
config_provider = validator_context.config_provider
# Skip if non-SSL.
if config.get('PREFERRED_URL_SCHEME', 'http') != 'https':
return
# Skip if externally terminated.
if config.get('EXTERNAL_TLS_TERMINATION', False) is True:
return
# Verify that we have all the required SSL files.
for filename in SSL_FILENAMES:
if not config_provider.volume_file_exists(filename):
raise ConfigValidationException('Missing required SSL file: %s' % filename)
# Read the contents of the SSL certificate.
with config_provider.get_volume_file(SSL_FILENAMES[0]) as f:
cert_contents = f.read()
# Validate the certificate.
try:
certificate = load_certificate(cert_contents)
except CertInvalidException as cie:
raise ConfigValidationException('Could not load SSL certificate: %s' % cie)
# Verify the certificate has not expired.
if certificate.expired:
raise ConfigValidationException('The specified SSL certificate has expired.')
# Verify the hostname matches the name in the certificate.
if not certificate.matches_name(_ssl_cn(config['SERVER_HOSTNAME'])):
msg = ('Supported names "%s" in SSL cert do not match server hostname "%s"' %
(', '.join(list(certificate.names)), _ssl_cn(config['SERVER_HOSTNAME'])))
raise ConfigValidationException(msg)
# Verify the private key against the certificate.
private_key_path = None
with config_provider.get_volume_file(SSL_FILENAMES[1]) as f:
private_key_path = f.name
if not private_key_path:
# Only in testing.
return
try:
certificate.validate_private_key(private_key_path)
except KeyInvalidException as kie:
raise ConfigValidationException('SSL private key failed to validate: %s' % kie)
def _ssl_cn(server_hostname):
""" Return the common name (fully qualified host name) from the SERVER_HOSTNAME. """
host_port = server_hostname.rsplit(':', 1)
# SERVER_HOSTNAME includes the port
if len(host_port) == 2:
if host_port[-1].isdigit():
return host_port[-2]
return server_hostname

View file

@ -0,0 +1,54 @@
from storage import get_storage_driver, TYPE_LOCAL_STORAGE
from util.config.validators import BaseValidator, ConfigValidationException
class StorageValidator(BaseValidator):
name = "registry-storage"
@classmethod
def validate(cls, validator_context):
""" Validates registry storage. """
config = validator_context.config
client = validator_context.http_client
ip_resolver = validator_context.ip_resolver
config_provider = validator_context.config_provider
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
providers = _get_storage_providers(config, ip_resolver, config_provider).items()
if not providers:
raise ConfigValidationException('Storage configuration required')
for name, (storage_type, driver) in providers:
# We can skip localstorage validation, since we can't guarantee that
# this will be the same machine Q.E. will run under
if storage_type == TYPE_LOCAL_STORAGE:
continue
try:
if replication_enabled and storage_type == 'LocalStorage':
raise ConfigValidationException('Locally mounted directory not supported ' +
'with storage replication')
# Run validation on the driver.
driver.validate(client)
# Run setup on the driver if the read/write succeeded.
driver.setup()
except Exception as ex:
msg = str(ex).strip().split("\n")[0]
raise ConfigValidationException('Invalid storage configuration: %s: %s' % (name, msg))
def _get_storage_providers(config, ip_resolver, config_provider):
storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG', {})
drivers = {}
try:
for name, parameters in storage_config.items():
driver = get_storage_driver(None, None, None, config_provider, ip_resolver, parameters)
drivers[name] = (parameters[0], driver)
except TypeError:
raise ConfigValidationException('Missing required parameter(s) for storage %s' % name)
return drivers

View file

@ -0,0 +1,31 @@
import logging
from util.config.validators import BaseValidator, ConfigValidationException
from util.timedeltastring import convert_to_timedelta
logger = logging.getLogger(__name__)
class TimeMachineValidator(BaseValidator):
name = "time-machine"
@classmethod
def validate(cls, validator_context):
config = validator_context.config
if not 'DEFAULT_TAG_EXPIRATION' in config:
# Old style config
return
try:
convert_to_timedelta(config['DEFAULT_TAG_EXPIRATION']).total_seconds()
except ValueError as ve:
raise ConfigValidationException('Invalid default expiration: %s' % ve.message)
if not config['DEFAULT_TAG_EXPIRATION'] in config.get('TAG_EXPIRATION_OPTIONS', []):
raise ConfigValidationException('Default expiration must be in expiration options set')
for ts in config.get('TAG_EXPIRATION_OPTIONS', []):
try:
convert_to_timedelta(ts)
except ValueError as ve:
raise ConfigValidationException('Invalid tag expiration option: %s' % ts)

View file

@ -0,0 +1,59 @@
import logging
from hashlib import sha1
from util.config.validators import BaseValidator, ConfigValidationException
from util.registry.torrent import jwt_from_infohash, TorrentConfiguration
logger = logging.getLogger(__name__)
class BittorrentValidator(BaseValidator):
name = "bittorrent"
@classmethod
def validate(cls, validator_context):
""" Validates the configuration for using BitTorrent for downloads. """
config = validator_context.config
client = validator_context.http_client
announce_url = config.get('BITTORRENT_ANNOUNCE_URL')
if not announce_url:
raise ConfigValidationException('Missing announce URL')
# Ensure that the tracker is reachable and accepts requests signed with a registry key.
params = {
'info_hash': sha1('test').digest(),
'peer_id': '-QUAY00-6wfG2wk6wWLc',
'uploaded': 0,
'downloaded': 0,
'left': 0,
'numwant': 0,
'port': 80,
}
torrent_config = TorrentConfiguration.for_testing(validator_context.instance_keys, announce_url,
validator_context.registry_title)
encoded_jwt = jwt_from_infohash(torrent_config, params['info_hash'])
params['jwt'] = encoded_jwt
resp = client.get(announce_url, timeout=5, params=params)
logger.debug('Got tracker response: %s: %s', resp.status_code, resp.text)
if resp.status_code == 404:
raise ConfigValidationException('Announce path not found; did you forget `/announce`?')
if resp.status_code == 500:
raise ConfigValidationException('Did not get expected response from Tracker; ' +
'please check your settings')
if resp.status_code == 200:
if 'invalid jwt' in resp.text:
raise ConfigValidationException('Could not authorize to Tracker; is your Tracker ' +
'properly configured?')
if 'failure reason' in resp.text:
raise ConfigValidationException('Could not validate signed announce request: ' + resp.text)
if 'go_goroutines' in resp.text:
raise ConfigValidationException('Could not validate signed announce request: ' +
'provided port is used for Prometheus')