diff --git a/util/config/provider/baseprovider.py b/util/config/provider/baseprovider.py index 663c64588..041de0de3 100644 --- a/util/config/provider/baseprovider.py +++ b/util/config/provider/baseprovider.py @@ -1,6 +1,11 @@ import logging import yaml + +from jsonschema import validate, ValidationError + + +from util.config.schema import CONFIG_SCHEMA from util.license import LICENSE_FILENAME, LicenseDecodeError, decode_license @@ -31,6 +36,15 @@ def import_yaml(config_obj, config_file): if key.isupper(): config_obj[key] = c[key] + if config_obj.get('SETUP_COMPLETE', True): + try: + validate(config_obj, CONFIG_SCHEMA) + except ValidationError: + # TODO: Change this into a real error + logger.exception('Could not validate config schema') + else: + logger.debug('Skipping config schema validation because setup is not complete') + return config_obj @@ -109,7 +123,7 @@ class BaseProvider(object): indicating that this container requires a restart. """ raise NotImplementedError - + def get_volume_path(self, directory, filename): """ Helper for constructing file paths, which may differ between providers. For example, kubernetes can't have subfolders in configmaps """ diff --git a/util/config/schema.py b/util/config/schema.py new file mode 100644 index 000000000..8af18dfa3 --- /dev/null +++ b/util/config/schema.py @@ -0,0 +1,617 @@ +CONFIG_SCHEMA = { + 'type': 'object', + 'description': 'Schema for Quay configuration', + 'required': [ + 'PREFERRED_URL_SCHEME', + 'SERVER_HOSTNAME', + 'DB_URI', + 'AUTHENTICATION_TYPE', + 'DISTRIBUTED_STORAGE_CONFIG', + 'BUILDLOGS_REDIS', + 'USER_EVENTS_REDIS', + 'DISTRIBUTED_STORAGE_PREFERENCE', + 'DEFAULT_TAG_EXPIRATION', + 'TAG_EXPIRATION_OPTIONS', + ], + 'properties': { + # Hosting. + 'PREFERRED_URL_SCHEME': { + 'type': 'string', + 'description': 'The URL scheme to use when hitting Quay. If Quay is behind SSL *at all*, this *must* be `https`', + 'enum': ['http', 'https'], + 'x-example': 'https', + }, + 'SERVER_HOSTNAME': { + 'type': 'string', + 'description': 'The URL at which Quay is accessible, without the scheme.', + 'x-example': 'quay.io', + }, + 'EXTERNAL_TLS_TERMINATION': { + 'type': 'boolean', + 'description': 'If TLS is supported, but terminated at a layer before Quay, must be true.', + 'x-example': True, + }, + + # User-visible configuration. + 'REGISTRY_TITLE': { + 'type': 'string', + 'description': 'If specified, the long-form title for the registry. Defaults to `Quay Enterprise`.', + 'x-example': 'Corp Container Service', + }, + 'REGISTRY_TITLE_SHORT': { + 'type': 'string', + 'description': 'If specified, the short-form title for the registry. Defaults to `Quay Enterprise`.', + 'x-example': 'CCS', + }, + 'CONTACT_INFO': { + 'type': 'array', + 'minItems': 1, + 'uniqueItems': True, + 'description': 'If specified, contact information to display on the contact page. ' + + 'If only a single piece of contact information is specified, the contact footer will link directly.', + 'items': [ + { + 'type': 'string', + 'pattern': '^mailto:(.)+$', + 'x-example': 'mailto:support@quay.io', + 'description': 'Adds a link to send an e-mail', + }, + { + 'type': 'string', + 'pattern': '^irc://(.)+$', + 'x-example': 'irc://chat.freenode.net:6665/quay', + 'description': 'Adds a link to visit an IRC chat room', + }, + { + 'type': 'string', + 'pattern': '^tel:(.)+$', + 'x-example': 'tel:+1-888-930-3475', + 'description': 'Adds a link to call a phone number', + }, + { + 'type': 'string', + 'pattern': '^http(s)?://(.)+$', + 'x-example': 'https://twitter.com/quayio', + 'description': 'Adds a link to a defined URL', + }, + ], + }, + + # E-mail. + 'FEATURE_MAILING': { + 'type': 'boolean', + 'description': 'Whether emails are enabled. Defaults to True', + 'x-example': True, + }, + 'MAIL_SERVER': { + 'type': 'string', + 'description': 'The SMTP server to use for sending e-mails. Only required if FEATURE_MAILING is set to true.', + 'x-example': 'smtp.somedomain.com', + }, + 'MAIL_USE_TLS': { + 'type': 'boolean', + 'description': 'If specified, whether to use TLS for sending e-mails.', + 'x-example': True, + }, + 'MAIL_PORT': { + 'type': 'number', + 'description': 'The SMTP port to use. If not specified, defaults to 587.', + 'x-example': 588, + }, + 'MAIL_USERNAME': { + 'type': 'string', + 'description': 'The SMTP username to use when sending e-mails.', + 'x-example': 'myuser', + }, + 'MAIL_PASSWORD': { + 'type': 'string', + 'description': 'The SMTP password to use when sending e-mails.', + 'x-example': 'mypassword', + }, + 'MAIL_DEFAULT_SENDER': { + 'type': 'string', + 'description': 'If specified, the e-mail address used as the `from` when Quay sends e-mails. If none, defaults to `support@quay.io`.', + 'x-example': 'support@myco.com', + }, + + # Database. + 'DB_URI': { + 'type': 'string', + 'description': 'The URI at which to access the database, including any credentials.', + 'x-example': 'mysql+pymysql://username:password@dns.of.database/quay', + 'x-reference': 'https://www.postgresql.org/docs/9.3/static/libpq-connect.html#AEN39495', + }, + 'ALLOW_PULLS_WITHOUT_STRICT_LOGGING': { + 'type': 'boolean', + 'description': 'If true, pulls in which the pull audit log entry cannot be written will ' + + 'still succeed. Useful if the database can fallback into a read-only state ' + + 'and it is desired for pulls to continue during that time. Defaults to False.', + 'x-example': True, + }, + + # Storage. + 'FEATURE_STORAGE_REPLICATION': { + 'type': 'boolean', + 'description': 'Whether to automatically replicate between storage engines. Defaults to False', + 'x-example': False, + }, + 'FEATURE_PROXY_STORAGE': { + 'type': 'boolean', + 'description': 'Whether to proxy all direct download URLs in storage via the registry nginx. Defaults to False', + 'x-example': False, + }, + 'MAXIMUM_LAYER_SIZE': { + 'type': 'string', + 'description': 'Maximum allowed size of an image layer. Defaults to 20G', + 'x-example': '100G', + 'pattern': '^[0-9]+(G|M)$', + }, + 'DISTRIBUTED_STORAGE_CONFIG': { + 'type': 'object', + 'description': 'Configuration for storage engine(s) to use in Quay. Each key is a unique ID' + + ' for a storage engine, with the value being a tuple of the type and ' + + ' configuration for that engine.', + 'x-example': { + 'local_storage': ['LocalStorage', {'storage_path': 'some/path/'}], + }, + 'items': { + 'type': 'array', + }, + }, + 'DISTRIBUTED_STORAGE_PREFERENCE': { + 'type': 'array', + 'description': 'The preferred storage engine(s) (by ID in DISTRIBUTED_STORAGE_CONFIG) to ' + + 'use. A preferred engine means it is first checked for pullig and images are ' + + 'pushed to it.', + 'items': { + 'type': 'string', + 'uniqueItems': True, + }, + 'x-example': ['s3_us_east', 's3_us_west'], + }, + 'DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS': { + 'type': 'array', + 'description': 'The list of storage engine(s) (by ID in DISTRIBUTED_STORAGE_CONFIG) whose ' + + 'images should be fully replicated, by default, to all other storage engines.', + 'items': { + 'type': 'string', + 'uniqueItems': True, + }, + 'x-example': ['s3_us_east', 's3_us_west'], + }, + + # Authentication. + 'AUTHENTICATION_TYPE': { + 'type': 'string', + 'description': 'The authentication engine to use for credential authentication.', + 'x-example': 'Database', + 'enum': ['Database', 'LDAP', 'JWT', 'Keystone', 'OIDC'], + }, + 'SUPER_USERS': { + 'type': 'array', + 'description': 'Quay usernames of those users to be granted superuser privileges', + 'uniqueItems': True, + 'items': { + 'type': 'string', + }, + }, + 'DIRECT_OAUTH_CLIENTID_WHITELIST': { + 'type': 'array', + 'description': 'A list of client IDs of *Quay-managed* applications that are allowed ' + + 'to perform direct OAuth approval without user approval.', + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/direct-oauth.html', + 'uniqueItems': True, + 'items': { + 'type': 'string', + }, + }, + + # Redis. + 'BUILDLOGS_REDIS': { + 'type': 'object', + 'description': 'Connection information for Redis for build logs caching', + 'required': ['host'], + 'properties': { + 'host': { + 'type': 'string', + 'description': 'The hostname at which Redis is accessible', + 'x-example': 'my.redis.cluster', + }, + 'port': { + 'type': 'number', + 'description': 'The port at which Redis is accessible', + 'x-example': 1234, + }, + 'password': { + 'type': 'string', + 'description': 'The password to connect to the Redis instance', + 'x-example': 'mypassword', + }, + }, + }, + 'USER_EVENTS_REDIS': { + 'type': 'object', + 'description': 'Connection information for Redis for user event handling', + 'required': ['host'], + 'properties': { + 'host': { + 'type': 'string', + 'description': 'The hostname at which Redis is accessible', + 'x-example': 'my.redis.cluster', + }, + 'port': { + 'type': 'number', + 'description': 'The port at which Redis is accessible', + 'x-example': 1234, + }, + 'password': { + 'type': 'string', + 'description': 'The password to connect to the Redis instance', + 'x-example': 'mypassword', + }, + }, + }, + + # OAuth configuration. + 'GITHUB_LOGIN_CONFIG': { + 'type': 'object', + 'description': 'Configuration for using GitHub (Enterprise) as an external login provider', + 'required': ['GITHUB_ENDPOINT', 'CLIENT_ID', 'CLIENT_SECRET'], + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-auth.html', + 'properties': { + 'GITHUB_ENDPOINT': { + 'type': 'string', + 'description': 'The endpoint of the GitHub (Enterprise) being hit', + 'x-example': 'https://github.com/', + }, + 'API_ENDPOINT': { + 'type': 'string', + 'description': 'The endpoint of the GitHub (Enterprise) API to use. Must be overridden for github.com', + 'x-example': 'https://api.github.com/', + }, + 'CLIENT_ID': { + 'type': 'string', + 'description': 'The registered client ID for this Quay instance; cannot be shared with GITHUB_TRIGGER_CONFIG', + 'x-example': '0e8dbe15c4c7630b6780', + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html', + }, + 'CLIENT_SECRET': { + 'type': 'string', + 'description': 'The registered client secret for this Quay instance', + 'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846', + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html', + }, + 'ORG_RESTRICT': { + 'type': 'boolean', + 'description': 'If true, only users within the organization whitelist can login using this provider', + 'x-example': True, + }, + 'ALLOWED_ORGANIZATIONS': { + 'type': 'array', + 'description': 'The names of the GitHub (Enterprise) organizations whitelisted to work with the ORG_RESTRICT option', + 'uniqueItems': True, + 'items': { + 'type': 'string', + }, + }, + }, + }, + 'GITHUB_TRIGGER_CONFIG': { + 'type': 'object', + 'description': 'Configuration for using GitHub (Enterprise) for build triggers', + 'required': ['GITHUB_ENDPOINT', 'CLIENT_ID', 'CLIENT_SECRET'], + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-build.html', + 'properties': { + 'GITHUB_ENDPOINT': { + 'type': 'string', + 'description': 'The endpoint of the GitHub (Enterprise) being hit', + 'x-example': 'https://github.com/', + }, + 'API_ENDPOINT': { + 'type': 'string', + 'description': 'The endpoint of the GitHub (Enterprise) API to use. Must be overridden for github.com', + 'x-example': 'https://api.github.com/', + }, + 'CLIENT_ID': { + 'type': 'string', + 'description': 'The registered client ID for this Quay instance; cannot be shared with GITHUB_LOGIN_CONFIG', + 'x-example': '0e8dbe15c4c7630b6780', + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html', + }, + 'CLIENT_SECRET': { + 'type': 'string', + 'description': 'The registered client secret for this Quay instance', + 'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846', + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/github-app.html', + }, + }, + }, + 'GOOGLE_LOGIN_CONFIG': { + 'type': 'object', + 'description': 'Configuration for using Google for external authentication', + 'required': ['CLIENT_ID', 'CLIENT_SECRET'], + 'properties': { + 'CLIENT_ID': { + 'type': 'string', + 'description': 'The registered client ID for this Quay instance', + 'x-example': '0e8dbe15c4c7630b6780', + }, + 'CLIENT_SECRET': { + 'type': 'string', + 'description': 'The registered client secret for this Quay instance', + 'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846', + }, + }, + }, + 'GITLAB_TRIGGER_CONFIG': { + 'type': 'object', + 'description': 'Configuration for using Gitlab (Enterprise) for external authentication', + 'required': ['GITLAB_ENDPOINT', 'CLIENT_ID', 'CLIENT_SECRET'], + 'properties': { + 'GITLAB_ENDPOINT': { + 'type': 'string', + 'description': 'The endpoint at which Gitlab(Enterprise) is running', + 'x-example': 'https://gitlab.com', + }, + 'CLIENT_ID': { + 'type': 'string', + 'description': 'The registered client ID for this Quay instance', + 'x-example': '0e8dbe15c4c7630b6780', + }, + 'CLIENT_SECRET': { + 'type': 'string', + 'description': 'The registered client secret for this Quay instance', + 'x-example': 'e4a58ddd3d7408b7aec109e85564a0d153d3e846', + }, + }, + }, + + # Misc configuration. + 'PUBLIC_NAMESPACES': { + 'type': 'array', + 'description': 'If a namespace is defined in the public namespace list, then it will appear on *all*' + + ' user\'s repository list pages, regardless of whether that user is a member of the namespace.' + + ' Typically, this is used by an enterprise customer in configuring a set of "well-known"' + + ' namespaces.', + 'uniqueItems': True, + 'items': { + 'type': 'string', + }, + }, + 'AVATAR_KIND': { + 'type': 'string', + 'description': 'The types of avatars to display, either generated inline (local) or Gravatar (gravatar)', + 'enum': ['local', 'gravatar'], + }, + + # Time machine and tag expiration settings. + 'FEATURE_CHANGE_TAG_EXPIRATION': { + 'type': 'boolean', + 'description': 'Whether users and organizations are allowed to change the tag expiration for tags in their namespace. Defaults to True.', + 'x-example': False, + }, + 'DEFAULT_TAG_EXPIRATION': { + 'type': 'string', + 'description': 'The default, configurable tag expiration time for time machine. Defaults to `2w`.', + 'pattern': '^[0-9]+(w|m|d|h|s)$', + }, + 'TAG_EXPIRATION_OPTIONS': { + 'type': 'array', + 'description': 'The options that users can select for expiration of tags in their namespace (if enabled)', + 'items': { + 'type': 'string', + 'pattern': '^[0-9]+(w|m|d|h|s)$', + }, + }, + + # Team syncing. + 'FEATURE_TEAM_SYNCING': { + 'type': 'boolean', + 'description': 'Whether to allow for team membership to be synced from a backing group in the authentication engine (LDAP or Keystone)', + 'x-example': True, + }, + 'TEAM_RESYNC_STALE_TIME': { + 'type': 'string', + 'description': 'If team syncing is enabled for a team, how often to check its membership and resync if necessary (Default: 30m)', + 'x-example': '2h', + 'pattern': '^[0-9]+(w|m|d|h|s)$', + }, + + # Security scanning. + 'FEATURE_SECURITY_SCANNER': { + 'type': 'boolean', + 'description': 'Whether to turn of/off the security scanner. Defaults to False', + 'x-example': False, + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/security-scanning.html', + }, + 'FEATURE_SECURITY_NOTIFICATIONS': { + 'type': 'boolean', + 'description': 'If the security scanner is enabled, whether to turn of/off security notificaitons. Defaults to False', + 'x-example': False, + }, + 'SECURITY_SCANNER_ENDPOINT' : { + 'type': 'string', + 'pattern': '^http(s)?://(.)+$', + 'description': 'The endpoint for the security scanner', + 'x-example': 'http://192.168.99.101:6060' , + }, + 'SECURITY_SCANNER_INDEXING_INTERVAL': { + 'type': 'number', + 'description': 'The number of seconds between indexing intervals in the security scanner. Defaults to 30.', + 'x-example': 30, + }, + + # Bittorrent support. + 'FEATURE_BITTORRENT': { + 'type': 'boolean', + 'description': 'Whether to allow using Bittorrent-based pulls. Defaults to False', + 'x-example': False, + 'x-reference': 'https://coreos.com/quay-enterprise/docs/latest/bittorrent.html', + }, + 'BITTORRENT_PIECE_SIZE': { + 'type': 'number', + 'description': 'The bittorent piece size to use. If not specified, defaults to 512 * 1024.', + 'x-example': 512 * 1024, + }, + 'BITTORRENT_ANNOUNCE_URL': { + 'type': 'string', + 'pattern': '^http(s)?://(.)+$', + 'description': 'The URL of the announce endpoint on the bittorrent tracker', + 'x-example': 'https://localhost:6881/announce', + }, + + # Build + 'FEATURE_GITHUB_BUILD': { + 'type': 'boolean', + 'description': 'Whether to support GitHub build triggers. Defaults to False', + 'x-example': False, + }, + 'FEATURE_BITBUCKET_BUILD': { + 'type': 'boolean', + 'description': 'Whether to support Bitbucket build triggers. Defaults to False', + 'x-example': False, + }, + 'FEATURE_GITLAB_BUILD': { + 'type': 'boolean', + 'description': 'Whether to support GitLab build triggers. Defaults to False', + 'x-example': False, + }, + 'FEATURE_BUILD_SUPPORT': { + 'type': 'boolean', + 'description': 'Whether to support Dockerfile build. Defaults to True', + 'x-example': True, + }, + + # Login + 'FEATURE_GITHUB_LOGIN': { + 'type': 'boolean', + 'description': 'Whether GitHub login is supported. Defaults to False', + 'x-example': False, + }, + 'FEATURE_GOOGLE_LOGIN': { + 'type': 'boolean', + 'description': 'Whether Google login is supported. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Permanent Sessions. + 'FEATURE_PERMANENT_SESSIONS': { + 'type': 'boolean', + 'description': 'Whether sessions are permanent. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Super User Support. + 'FEATURE_SUPER_USERS': { + 'type': 'boolean', + 'description': 'Whether super users are supported. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Anonymous Users. + 'FEATURE_ANONYMOUS_ACCESS': { + 'type': 'boolean', + 'description': ' Whether to allow anonymous users to browse and pull public repositories. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: User Creation. + 'FEATURE_USER_CREATION': { + 'type': 'boolean', + 'description': 'Whether users can be created (by non-super users). Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Invite Only User Creation. + 'FEATURE_INVITE_ONLY_USER_CREATION': { + 'type': 'boolean', + 'description': 'Whether users being created must be invited by another user. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Encrypted Basic Auth. + 'FEATURE_REQUIRE_ENCRYPTED_BASIC_AUTH': { + 'type': 'boolean', + 'description': 'Whether non-encrypted passwords (as opposed to encrypted tokens) can be used for basic auth. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Direct Login. + 'FEATURE_DIRECT_LOGIN': { + 'type': 'boolean', + 'description': 'Whether users can directly login to the UI. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Advertising V2. + 'FEATURE_ADVERTISE_V2': { + 'type': 'boolean', + 'description': 'Whether the v2/ endpoint is visible. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Log Rotation. + 'FEATURE_ACTION_LOG_ROTATION': { + 'type': 'boolean', + 'description': 'Whether or not to rotate old action logs to storage. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: ACI Conversion. + 'FEATURE_ACI_CONVERSION': { + 'type': 'boolean', + 'description': 'Whether to enable conversion to ACIs. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Library Support. + 'FEATURE_LIBRARY_SUPPORT': { + 'type': 'boolean', + 'description': 'Whether to allow for "namespace-less" repositories when pulling and pushing from Docker. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Require Team Invite. + 'FEATURE_REQUIRE_TEAM_INVITE': { + 'type': 'boolean', + 'description': 'Whether to require invitations when adding a user to a team. Defaults to True', + 'x-example': True, + }, + + # Feature Flag: Collecting and Supporting Metadata. + 'FEATURE_USER_METADATA': { + 'type': 'boolean', + 'description': 'Whether to collect and support user metadata. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Support App Registry. + 'FEATURE_APP_REGISTRY': { + 'type': 'boolean', + 'description': 'Whether to enable support for App repositories. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Public Reposiotires in _catalog Endpoint. + 'FEATURE_PUBLIC_CATALOG': { + 'type': 'boolean', + 'description': 'If set to true, the _catalog endpoint returns public repositories. Otherwise, only private repositories can be returned. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Reader Build Logs. + 'FEATURE_READER_BUILD_LOGS': { + 'type': 'boolean', + 'description': 'If set to true, build logs may be read by those with read access to the repo, rather than only write access or admin access. Defaults to False', + 'x-example': False, + }, + + # Feature Flag: Usernames Autocomplete. + 'FEATURE_PARTIAL_USER_AUTOCOMPLETE': { + 'type': 'boolean', + 'description': 'If set to true, autocompletion will apply to partial usernames. Defaults to True', + 'x-example': True, + }, + }, +} +