Add config validation on startup (#2903)

* WIP

* Finish schema

Add three sections: security scanning, bittorrent support and feature flags.
This commit is contained in:
IvanCherepov 2017-12-01 10:46:39 -05:00 committed by GitHub
parent 1882545c69
commit c383ac1f9d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 632 additions and 1 deletions

View file

@ -1,6 +1,11 @@
import logging import logging
import yaml import yaml
from jsonschema import validate, ValidationError
from util.config.schema import CONFIG_SCHEMA
from util.license import LICENSE_FILENAME, LicenseDecodeError, decode_license from util.license import LICENSE_FILENAME, LicenseDecodeError, decode_license
@ -31,6 +36,15 @@ def import_yaml(config_obj, config_file):
if key.isupper(): if key.isupper():
config_obj[key] = c[key] config_obj[key] = c[key]
if config_obj.get('SETUP_COMPLETE', True):
try:
validate(config_obj, CONFIG_SCHEMA)
except ValidationError:
# TODO: Change this into a real error
logger.exception('Could not validate config schema')
else:
logger.debug('Skipping config schema validation because setup is not complete')
return config_obj return config_obj
@ -109,7 +123,7 @@ class BaseProvider(object):
indicating that this container requires a restart. indicating that this container requires a restart.
""" """
raise NotImplementedError raise NotImplementedError
def get_volume_path(self, directory, filename): def get_volume_path(self, directory, filename):
""" Helper for constructing file paths, which may differ between providers. For example, """ Helper for constructing file paths, which may differ between providers. For example,
kubernetes can't have subfolders in configmaps """ kubernetes can't have subfolders in configmaps """

617
util/config/schema.py Normal file
View file

@ -0,0 +1,617 @@
CONFIG_SCHEMA = {
'type': 'object',
'description': 'Schema for Quay configuration',
'required': [
'PREFERRED_URL_SCHEME',
'SERVER_HOSTNAME',
'DB_URI',
'AUTHENTICATION_TYPE',
'DISTRIBUTED_STORAGE_CONFIG',
'BUILDLOGS_REDIS',
'USER_EVENTS_REDIS',
'DISTRIBUTED_STORAGE_PREFERENCE',
'DEFAULT_TAG_EXPIRATION',
'TAG_EXPIRATION_OPTIONS',
],
'properties': {
# Hosting.
'PREFERRED_URL_SCHEME': {
'type': 'string',
'description': 'The URL scheme to use when hitting Quay. If Quay is behind SSL *at all*, this *must* be `https`',
'enum': ['http', 'https'],
'x-example': 'https',
},
'SERVER_HOSTNAME': {
'type': 'string',
'description': 'The URL at which Quay is accessible, without the scheme.',
'x-example': 'quay.io',
},
'EXTERNAL_TLS_TERMINATION': {
'type': 'boolean',
'description': 'If TLS is supported, but terminated at a layer before Quay, must be true.',
'x-example': True,
},
# User-visible configuration.
'REGISTRY_TITLE': {
'type': 'string',
'description': 'If specified, the long-form title for the registry. Defaults to `Quay Enterprise`.',
'x-example': 'Corp Container Service',
},
'REGISTRY_TITLE_SHORT': {
'type': 'string',
'description': 'If specified, the short-form title for the registry. Defaults to `Quay Enterprise`.',
'x-example': 'CCS',
},
'CONTACT_INFO': {
'type': 'array',
'minItems': 1,
'uniqueItems': True,
'description': 'If specified, contact information to display on the contact page. ' +
'If only a single piece of contact information is specified, the contact footer will link directly.',
'items': [
{
'type': 'string',
'pattern': '^mailto:(.)+$',
'x-example': 'mailto:support@quay.io',
'description': 'Adds a link to send an e-mail',
},
{
'type': 'string',
'pattern': '^irc://(.)+$',
'x-example': 'irc://chat.freenode.net:6665/quay',
'description': 'Adds a link to visit an IRC chat room',
},
{
'type': 'string',
'pattern': '^tel:(.)+$',
'x-example': 'tel:+1-888-930-3475',
'description': 'Adds a link to call a phone number',
},
{
'type': 'string',
'pattern': '^http(s)?://(.)+$',
'x-example': 'https://twitter.com/quayio',
'description': 'Adds a link to a defined URL',
},
],
},
# E-mail.
'FEATURE_MAILING': {
'type': 'boolean',
'description': 'Whether emails are enabled. Defaults to True',
'x-example': True,
},
'MAIL_SERVER': {
'type': 'string',
'description': 'The SMTP server to use for sending e-mails. Only required if FEATURE_MAILING is set to true.',
'x-example': 'smtp.somedomain.com',
},
'MAIL_USE_TLS': {
'type': 'boolean',
'description': 'If specified, whether to use TLS for sending e-mails.',
'x-example': True,
},
'MAIL_PORT': {
'type': 'number',
'description': 'The SMTP port to use. If not specified, defaults to 587.',
'x-example': 588,
},
'MAIL_USERNAME': {
'type': 'string',
'description': 'The SMTP username to use when sending e-mails.',
'x-example': 'myuser',
},
'MAIL_PASSWORD': {
'type': 'string',
'description': 'The SMTP password to use when sending e-mails.',
'x-example': 'mypassword',
},
'MAIL_DEFAULT_SENDER': {
'type': 'string',
'description': 'If specified, the e-mail address used as the `from` when Quay sends e-mails. If none, defaults to `support@quay.io`.',
'x-example': 'support@myco.com',
},
# Database.
'DB_URI': {
'type': 'string',
'description': 'The URI at which to access the database, including any credentials.',
'x-example': 'mysql+pymysql://username:password@dns.of.database/quay',
'x-reference': 'https://www.postgresql.org/docs/9.3/static/libpq-connect.html#AEN39495',
},
'ALLOW_PULLS_WITHOUT_STRICT_LOGGING': {
'type': 'boolean',
'description': 'If true, pulls in which the pull audit log entry cannot be written will ' +
'still succeed. Useful if the database can fallback into a read-only state ' +
'and it is desired for pulls to continue during that time. Defaults to False.',
'x-example': True,
},
# Storage.
'FEATURE_STORAGE_REPLICATION': {
'type': 'boolean',
'description': 'Whether to automatically replicate between storage engines. Defaults to False',
'x-example': False,
},
'FEATURE_PROXY_STORAGE': {
'type': 'boolean',
'description': 'Whether to proxy all direct download URLs in storage via the registry nginx. Defaults to False',
'x-example': False,
},
'MAXIMUM_LAYER_SIZE': {
'type': 'string',
'description': 'Maximum allowed size of an image layer. Defaults to 20G',
'x-example': '100G',
'pattern': '^[0-9]+(G|M)$',
},
'DISTRIBUTED_STORAGE_CONFIG': {
'type': 'object',
'description': 'Configuration for storage engine(s) to use in Quay. Each key is a unique ID' +
' for a storage engine, with the value being a tuple of the type and ' +
' configuration for that engine.',
'x-example': {
'local_storage': ['LocalStorage', {'storage_path': 'some/path/'}],
},
'items': {
'type': 'array',
},
},
'DISTRIBUTED_STORAGE_PREFERENCE': {
'type': 'array',
'description': 'The preferred storage engine(s) (by ID in DISTRIBUTED_STORAGE_CONFIG) to ' +
'use. A preferred engine means it is first checked for pullig and images are ' +
'pushed to it.',
'items': {
'type': 'string',
'uniqueItems': True,
},
'x-example': ['s3_us_east', 's3_us_west'],
},
'DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS': {
'type': 'array',
'description': 'The list of storage engine(s) (by ID in DISTRIBUTED_STORAGE_CONFIG) whose ' +
'images should be fully replicated, by default, to all other storage engines.',
'items': {
'type': 'string',
'uniqueItems': True,
},
'x-example': ['s3_us_east', 's3_us_west'],
},
# Authentication.
'AUTHENTICATION_TYPE': {
'type': 'string',
'description': 'The authentication engine to use for credential authentication.',
'x-example': 'Database',
'enum': ['Database', 'LDAP', 'JWT', 'Keystone', 'OIDC'],
},
'SUPER_USERS': {
'type': 'array',
'description': 'Quay usernames of those users to be granted superuser privileges',
'uniqueItems': True,
'items': {
'type': 'string',
},
},
'DIRECT_OAUTH_CLIENTID_WHITELIST': {
'type': 'array',
'description': 'A list of client IDs of *Quay-managed* applications that are allowed ' +
'to perform direct OAuth approval without user approval.',
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/direct-oauth.html',
'uniqueItems': True,
'items': {
'type': 'string',
},
},
# Redis.
'BUILDLOGS_REDIS': {
'type': 'object',
'description': 'Connection information for Redis for build logs caching',
'required': ['host'],
'properties': {
'host': {
'type': 'string',
'description': 'The hostname at which Redis is accessible',
'x-example': 'my.redis.cluster',
},
'port': {
'type': 'number',
'description': 'The port at which Redis is accessible',
'x-example': 1234,
},
'password': {
'type': 'string',
'description': 'The password to connect to the Redis instance',
'x-example': 'mypassword',
},
},
},
'USER_EVENTS_REDIS': {
'type': 'object',
'description': 'Connection information for Redis for user event handling',
'required': ['host'],
'properties': {
'host': {
'type': 'string',
'description': 'The hostname at which Redis is accessible',
'x-example': 'my.redis.cluster',
},
'port': {
'type': 'number',
'description': 'The port at which Redis is accessible',
'x-example': 1234,
},
'password': {
'type': 'string',
'description': 'The password to connect to the Redis instance',
'x-example': 'mypassword',
},
},
},
# OAuth configuration.
'GITHUB_LOGIN_CONFIG': {
'type': 'object',
'description': 'Configuration for using GitHub (Enterprise) as an external login provider',
'required': ['GITHUB_ENDPOINT', 'CLIENT_ID', 'CLIENT_SECRET'],
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-auth.html',
'properties': {
'GITHUB_ENDPOINT': {
'type': 'string',
'description': 'The endpoint of the GitHub (Enterprise) being hit',
'x-example': 'https://github.com/',
},
'API_ENDPOINT': {
'type': 'string',
'description': 'The endpoint of the GitHub (Enterprise) API to use. Must be overridden for github.com',
'x-example': 'https://api.github.com/',
},
'CLIENT_ID': {
'type': 'string',
'description': 'The registered client ID for this Quay instance; cannot be shared with GITHUB_TRIGGER_CONFIG',
'x-example': '0e8dbe15c4c7630b6780',
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html',
},
'CLIENT_SECRET': {
'type': 'string',
'description': 'The registered client secret for this Quay instance',
'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846',
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html',
},
'ORG_RESTRICT': {
'type': 'boolean',
'description': 'If true, only users within the organization whitelist can login using this provider',
'x-example': True,
},
'ALLOWED_ORGANIZATIONS': {
'type': 'array',
'description': 'The names of the GitHub (Enterprise) organizations whitelisted to work with the ORG_RESTRICT option',
'uniqueItems': True,
'items': {
'type': 'string',
},
},
},
},
'GITHUB_TRIGGER_CONFIG': {
'type': 'object',
'description': 'Configuration for using GitHub (Enterprise) for build triggers',
'required': ['GITHUB_ENDPOINT', 'CLIENT_ID', 'CLIENT_SECRET'],
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-build.html',
'properties': {
'GITHUB_ENDPOINT': {
'type': 'string',
'description': 'The endpoint of the GitHub (Enterprise) being hit',
'x-example': 'https://github.com/',
},
'API_ENDPOINT': {
'type': 'string',
'description': 'The endpoint of the GitHub (Enterprise) API to use. Must be overridden for github.com',
'x-example': 'https://api.github.com/',
},
'CLIENT_ID': {
'type': 'string',
'description': 'The registered client ID for this Quay instance; cannot be shared with GITHUB_LOGIN_CONFIG',
'x-example': '0e8dbe15c4c7630b6780',
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html',
},
'CLIENT_SECRET': {
'type': 'string',
'description': 'The registered client secret for this Quay instance',
'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846',
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html',
},
},
},
'GOOGLE_LOGIN_CONFIG': {
'type': 'object',
'description': 'Configuration for using Google for external authentication',
'required': ['CLIENT_ID', 'CLIENT_SECRET'],
'properties': {
'CLIENT_ID': {
'type': 'string',
'description': 'The registered client ID for this Quay instance',
'x-example': '0e8dbe15c4c7630b6780',
},
'CLIENT_SECRET': {
'type': 'string',
'description': 'The registered client secret for this Quay instance',
'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846',
},
},
},
'GITLAB_TRIGGER_CONFIG': {
'type': 'object',
'description': 'Configuration for using Gitlab (Enterprise) for external authentication',
'required': ['GITLAB_ENDPOINT', 'CLIENT_ID', 'CLIENT_SECRET'],
'properties': {
'GITLAB_ENDPOINT': {
'type': 'string',
'description': 'The endpoint at which Gitlab(Enterprise) is running',
'x-example': 'https://gitlab.com',
},
'CLIENT_ID': {
'type': 'string',
'description': 'The registered client ID for this Quay instance',
'x-example': '0e8dbe15c4c7630b6780',
},
'CLIENT_SECRET': {
'type': 'string',
'description': 'The registered client secret for this Quay instance',
'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846',
},
},
},
# Misc configuration.
'PUBLIC_NAMESPACES': {
'type': 'array',
'description': 'If a namespace is defined in the public namespace list, then it will appear on *all*' +
' user\'s repository list pages, regardless of whether that user is a member of the namespace.' +
' Typically, this is used by an enterprise customer in configuring a set of "well-known"' +
' namespaces.',
'uniqueItems': True,
'items': {
'type': 'string',
},
},
'AVATAR_KIND': {
'type': 'string',
'description': 'The types of avatars to display, either generated inline (local) or Gravatar (gravatar)',
'enum': ['local', 'gravatar'],
},
# Time machine and tag expiration settings.
'FEATURE_CHANGE_TAG_EXPIRATION': {
'type': 'boolean',
'description': 'Whether users and organizations are allowed to change the tag expiration for tags in their namespace. Defaults to True.',
'x-example': False,
},
'DEFAULT_TAG_EXPIRATION': {
'type': 'string',
'description': 'The default, configurable tag expiration time for time machine. Defaults to `2w`.',
'pattern': '^[0-9]+(w|m|d|h|s)$',
},
'TAG_EXPIRATION_OPTIONS': {
'type': 'array',
'description': 'The options that users can select for expiration of tags in their namespace (if enabled)',
'items': {
'type': 'string',
'pattern': '^[0-9]+(w|m|d|h|s)$',
},
},
# Team syncing.
'FEATURE_TEAM_SYNCING': {
'type': 'boolean',
'description': 'Whether to allow for team membership to be synced from a backing group in the authentication engine (LDAP or Keystone)',
'x-example': True,
},
'TEAM_RESYNC_STALE_TIME': {
'type': 'string',
'description': 'If team syncing is enabled for a team, how often to check its membership and resync if necessary (Default: 30m)',
'x-example': '2h',
'pattern': '^[0-9]+(w|m|d|h|s)$',
},
# Security scanning.
'FEATURE_SECURITY_SCANNER': {
'type': 'boolean',
'description': 'Whether to turn of/off the security scanner. Defaults to False',
'x-example': False,
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/security-scanning.html',
},
'FEATURE_SECURITY_NOTIFICATIONS': {
'type': 'boolean',
'description': 'If the security scanner is enabled, whether to turn of/off security notificaitons. Defaults to False',
'x-example': False,
},
'SECURITY_SCANNER_ENDPOINT' : {
'type': 'string',
'pattern': '^http(s)?://(.)+$',
'description': 'The endpoint for the security scanner',
'x-example': 'http://192.168.99.101:6060' ,
},
'SECURITY_SCANNER_INDEXING_INTERVAL': {
'type': 'number',
'description': 'The number of seconds between indexing intervals in the security scanner. Defaults to 30.',
'x-example': 30,
},
# Bittorrent support.
'FEATURE_BITTORRENT': {
'type': 'boolean',
'description': 'Whether to allow using Bittorrent-based pulls. Defaults to False',
'x-example': False,
'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/bittorrent.html',
},
'BITTORRENT_PIECE_SIZE': {
'type': 'number',
'description': 'The bittorent piece size to use. If not specified, defaults to 512 * 1024.',
'x-example': 512 * 1024,
},
'BITTORRENT_ANNOUNCE_URL': {
'type': 'string',
'pattern': '^http(s)?://(.)+$',
'description': 'The URL of the announce endpoint on the bittorrent tracker',
'x-example': 'https://localhost:6881/announce',
},
# Build
'FEATURE_GITHUB_BUILD': {
'type': 'boolean',
'description': 'Whether to support GitHub build triggers. Defaults to False',
'x-example': False,
},
'FEATURE_BITBUCKET_BUILD': {
'type': 'boolean',
'description': 'Whether to support Bitbucket build triggers. Defaults to False',
'x-example': False,
},
'FEATURE_GITLAB_BUILD': {
'type': 'boolean',
'description': 'Whether to support GitLab build triggers. Defaults to False',
'x-example': False,
},
'FEATURE_BUILD_SUPPORT': {
'type': 'boolean',
'description': 'Whether to support Dockerfile build. Defaults to True',
'x-example': True,
},
# Login
'FEATURE_GITHUB_LOGIN': {
'type': 'boolean',
'description': 'Whether GitHub login is supported. Defaults to False',
'x-example': False,
},
'FEATURE_GOOGLE_LOGIN': {
'type': 'boolean',
'description': 'Whether Google login is supported. Defaults to False',
'x-example': False,
},
# Feature Flag: Permanent Sessions.
'FEATURE_PERMANENT_SESSIONS': {
'type': 'boolean',
'description': 'Whether sessions are permanent. Defaults to True',
'x-example': True,
},
# Feature Flag: Super User Support.
'FEATURE_SUPER_USERS': {
'type': 'boolean',
'description': 'Whether super users are supported. Defaults to True',
'x-example': True,
},
# Feature Flag: Anonymous Users.
'FEATURE_ANONYMOUS_ACCESS': {
'type': 'boolean',
'description': ' Whether to allow anonymous users to browse and pull public repositories. Defaults to True',
'x-example': True,
},
# Feature Flag: User Creation.
'FEATURE_USER_CREATION': {
'type': 'boolean',
'description': 'Whether users can be created (by non-super users). Defaults to True',
'x-example': True,
},
# Feature Flag: Invite Only User Creation.
'FEATURE_INVITE_ONLY_USER_CREATION': {
'type': 'boolean',
'description': 'Whether users being created must be invited by another user. Defaults to False',
'x-example': False,
},
# Feature Flag: Encrypted Basic Auth.
'FEATURE_REQUIRE_ENCRYPTED_BASIC_AUTH': {
'type': 'boolean',
'description': 'Whether non-encrypted passwords (as opposed to encrypted tokens) can be used for basic auth. Defaults to False',
'x-example': False,
},
# Feature Flag: Direct Login.
'FEATURE_DIRECT_LOGIN': {
'type': 'boolean',
'description': 'Whether users can directly login to the UI. Defaults to True',
'x-example': True,
},
# Feature Flag: Advertising V2.
'FEATURE_ADVERTISE_V2': {
'type': 'boolean',
'description': 'Whether the v2/ endpoint is visible. Defaults to True',
'x-example': True,
},
# Feature Flag: Log Rotation.
'FEATURE_ACTION_LOG_ROTATION': {
'type': 'boolean',
'description': 'Whether or not to rotate old action logs to storage. Defaults to False',
'x-example': False,
},
# Feature Flag: ACI Conversion.
'FEATURE_ACI_CONVERSION': {
'type': 'boolean',
'description': 'Whether to enable conversion to ACIs. Defaults to False',
'x-example': False,
},
# Feature Flag: Library Support.
'FEATURE_LIBRARY_SUPPORT': {
'type': 'boolean',
'description': 'Whether to allow for "namespace-less" repositories when pulling and pushing from Docker. Defaults to True',
'x-example': True,
},
# Feature Flag: Require Team Invite.
'FEATURE_REQUIRE_TEAM_INVITE': {
'type': 'boolean',
'description': 'Whether to require invitations when adding a user to a team. Defaults to True',
'x-example': True,
},
# Feature Flag: Collecting and Supporting Metadata.
'FEATURE_USER_METADATA': {
'type': 'boolean',
'description': 'Whether to collect and support user metadata. Defaults to False',
'x-example': False,
},
# Feature Flag: Support App Registry.
'FEATURE_APP_REGISTRY': {
'type': 'boolean',
'description': 'Whether to enable support for App repositories. Defaults to False',
'x-example': False,
},
# Feature Flag: Public Reposiotires in _catalog Endpoint.
'FEATURE_PUBLIC_CATALOG': {
'type': 'boolean',
'description': 'If set to true, the _catalog endpoint returns public repositories. Otherwise, only private repositories can be returned. Defaults to False',
'x-example': False,
},
# Feature Flag: Reader Build Logs.
'FEATURE_READER_BUILD_LOGS': {
'type': 'boolean',
'description': 'If set to true, build logs may be read by those with read access to the repo, rather than only write access or admin access. Defaults to False',
'x-example': False,
},
# Feature Flag: Usernames Autocomplete.
'FEATURE_PARTIAL_USER_AUTOCOMPLETE': {
'type': 'boolean',
'description': 'If set to true, autocompletion will apply to partial usernames. Defaults to True',
'x-example': True,
},
},
}