From 7619ab44e596d5f80a12f3c3002060081934b699 Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Mon, 25 Jun 2018 15:23:30 -0400 Subject: [PATCH 1/6] Revert inmemoryprov, skip local storage validation --- config_app/config_endpoints/api/suconfig.py | 15 ++-- .../config_endpoints/api/tar_config_loader.py | 6 +- config_app/config_util/config/__init__.py | 3 +- .../config_util/config/inmemoryprovider.py | 84 ------------------- .../config-setup-tool.html | 4 +- .../core-config-setup/config-setup-tool.html | 4 +- .../js/core-config-setup/core-config-setup.js | 8 +- util/config/validator.py | 24 +++--- util/config/validators/validate_storage.py | 2 +- 9 files changed, 35 insertions(+), 115 deletions(-) delete mode 100644 config_app/config_util/config/inmemoryprovider.py diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index ec4a84105..f7dc232b1 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -16,6 +16,7 @@ from data.runmigration import run_alembic_migration from util.config.configutil import add_enterprise_config_defaults from util.config.database import sync_database_with_config from util.config.validator import validate_service_for_config, ValidatorContext, is_valid_config_upload_filename +from util.config.validators import LocalStorageConfigValidationException logger = logging.getLogger(__name__) @@ -293,16 +294,14 @@ class SuperUserConfigValidate(ApiResource): # Note: This method is called to validate the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. Note that # this is also safe since this method does not access any information not given in the request. - if not config_provider.config_exists(): - config = request.get_json()['config'] - validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), - instance_keys=instance_keys, - ip_resolver=ip_resolver, - config_provider=config_provider) - return validate_service_for_config(service, validator_context) + config = request.get_json()['config'] + validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), + instance_keys=instance_keys, + ip_resolver=ip_resolver, + config_provider=config_provider) + return validate_service_for_config(service, validator_context) - abort(403) @resource('/v1/superuser/config/file/') diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index 63b57d214..e16b8a092 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -15,9 +15,9 @@ class TarConfigLoader(ApiResource): def put(self): """ Loads tarball config into the config provider """ input_stream = request.stream - tar_stream = tarfile.open(mode="r|gz", fileobj=input_stream) - - config_provider.load_from_tar_stream(tar_stream) + with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream: + # TODO: find a way to remove the contents of the directory on shutdown? + tar_stream.extractall(config_provider.config_volume) # now try to connect to the db provided in their config combined = dict(**app.config) diff --git a/config_app/config_util/config/__init__.py b/config_app/config_util/config/__init__.py index c344bb415..3735e4f66 100644 --- a/config_app/config_util/config/__init__.py +++ b/config_app/config_util/config/__init__.py @@ -1,6 +1,5 @@ from config_app.config_util.config.fileprovider import FileConfigProvider from config_app.config_util.config.testprovider import TestConfigProvider -from config_app.config_util.config.inmemoryprovider import InMemoryProvider def get_config_provider(config_volume, yaml_filename, py_filename, testing=False): @@ -9,4 +8,4 @@ def get_config_provider(config_volume, yaml_filename, py_filename, testing=False if testing: return TestConfigProvider() - return InMemoryProvider() + return FileConfigProvider(config_volume, yaml_filename, py_filename) diff --git a/config_app/config_util/config/inmemoryprovider.py b/config_app/config_util/config/inmemoryprovider.py deleted file mode 100644 index 7db5d1a89..000000000 --- a/config_app/config_util/config/inmemoryprovider.py +++ /dev/null @@ -1,84 +0,0 @@ -import logging -import yaml -import io -import os - -from config_app.config_util.config.baseprovider import BaseProvider - - -logger = logging.getLogger(__name__) - -CONFIG_FILENAME = 'config.yaml' - -class InMemoryProvider(BaseProvider): - def __init__(self): - self.files = {} - self.config = {} - self.was_loaded = False - - @property - def provider_id(self): - return 'memory' - - def update_app_config(self, app_config): - self.config = app_config - - def get_config(self): - return self.config - - def save_config(self, config_object): - self.config = config_object - self.was_loaded = True - - def config_exists(self): - return self.was_loaded - - def volume_exists(self): - return True - - def volume_file_exists(self, filename): - return any([name.startswith(filename) for name in self.files]) - - def get_volume_file(self, filename, mode='r'): - return io.BytesIO(self.files[filename]) - - def write_volume_file(self, filename, contents): - raise Exception('Not implemented yet') - - def remove_volume_file(self, filename): - raise Exception('Not implemented yet') - - def list_volume_directory(self, path): - def strip_directory(string): - if '/' in string: - return string[string.rfind('/') + 1:] - return string - - return [strip_directory(name) for name in self.files if name.startswith(path)] - - def save_volume_file(self, filename, flask_file): - self.files[filename] = flask_file.read() - - def requires_restart(self, app_config): - raise Exception('Not implemented yet') - - def get_volume_path(self, directory, filename): - return os.path.join(directory, filename) - - def load_from_tarball(self, tarfile): - for tarinfo in tarfile.getmembers(): - if tarinfo.isfile(): - self.files[tarinfo.name] = tarfile.extractfile(tarinfo.name).read() - - if self.files.has_key(CONFIG_FILENAME): - self.config = yaml.load(self.files.get(CONFIG_FILENAME)) - self.was_loaded = True - - def load_from_tar_stream(self, tarfile): - for tarinfo in tarfile: - if tarinfo.isfile(): - self.files[tarinfo.name] = tarfile.extractfile(tarinfo).read() - - if self.files.has_key(CONFIG_FILENAME): - self.config = yaml.load(self.files.get(CONFIG_FILENAME)) - self.was_loaded = True diff --git a/config_app/js/config-field-templates/config-setup-tool.html b/config_app/js/config-field-templates/config-setup-tool.html index ad86d3107..9c4dde631 100644 --- a/config_app/js/config-field-templates/config-setup-tool.html +++ b/config_app/js/config-field-templates/config-setup-tool.html @@ -1454,7 +1454,7 @@
- BitBucket Build Triggers + BitBucket Build Triggers
@@ -1496,7 +1496,7 @@
- GitLab Build Triggers + GitLab Build Triggers
diff --git a/config_app/js/core-config-setup/config-setup-tool.html b/config_app/js/core-config-setup/config-setup-tool.html index 9bec737fa..eed17afce 100644 --- a/config_app/js/core-config-setup/config-setup-tool.html +++ b/config_app/js/core-config-setup/config-setup-tool.html @@ -1455,7 +1455,7 @@
- BitBucket Build Triggers + BitBucket Build Triggers
@@ -1497,7 +1497,7 @@
- GitLab Build Triggers + GitLab Build Triggers
diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index 30126e43b..e55fb45e2 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -38,7 +38,13 @@ angular.module("quay-config") $scope.SERVICES = [ {'id': 'redis', 'title': 'Redis'}, - {'id': 'registry-storage', 'title': 'Registry Storage'}, + {'id': 'registry-storage', 'title': 'Registry Storage', 'condition': (config) => { + // We can skip validation if all of the storage locations are local, as we can't + // guarantee that this will be the same machine Q.E. will run under. Therefore, + // we just have a warning to the user that Q.E. won't start if the locations don't match + return Object.values(config.DISTRIBUTED_STORAGE_CONFIG) + .some(storageTuple => storageTuple[0] !== 'LocalStorage') + }}, {'id': 'time-machine', 'title': 'Time Machine'}, diff --git a/util/config/validator.py b/util/config/validator.py index f8f296a58..a0924c9e2 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -135,15 +135,15 @@ class ValidatorContext(object): url_scheme_and_hostname = URLSchemeAndHostname.from_app_config(app.config) return cls(config, - user_password, - client or app.config['HTTPCLIENT'], - app.app_context, - url_scheme_and_hostname, - app.config.get('JWT_AUTH_MAX_FRESH_S', 300), - app.config['REGISTRY_TITLE'], - ip_resolver, - instance_keys, - app.config.get('FEATURE_SECURITY_SCANNER', False), - app.config.get('TESTING', False), - get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), - config_provider) + user_password=user_password, + http_client=client or app.config['HTTPCLIENT'], + context=app.app_context, + url_scheme_and_hostname=url_scheme_and_hostname, + jwt_auth_max=app.config.get('JWT_AUTH_MAX_FRESH_S', 300), + registry_title=app.config['REGISTRY_TITLE'], + ip_resolver=ip_resolver, + feature_sec_scanner=app.config.get('FEATURE_SECURITY_SCANNER', False), + is_testing=app.config.get('TESTING', False), + uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), + config_provider=config_provider, + instance_keys=instance_keys) diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index 3e3de74ee..4aec66aae 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,5 +1,5 @@ from storage import get_storage_driver -from util.config.validators import BaseValidator, ConfigValidationException +from util.config.validators import BaseValidator, ConfigValidationException, LocalStorageConfigValidationException class StorageValidator(BaseValidator): From aa93d698b23c39d96cf48bdd5b7b3126b692ed8f Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Mon, 25 Jun 2018 17:40:59 -0400 Subject: [PATCH 2/6] Tarball the config and give it to the front end Download file as blob to avoid binary string encoding --- config_app/config_endpoints/api/suconfig.py | 5 - config_app/config_endpoints/api/superuser.py | 5 +- .../config_endpoints/api/tar_config_loader.py | 42 +- .../config-setup-tool.html | 1656 ----------------- .../core-config-setup/config-setup-tool.html | 4 +- .../js/core-config-setup/core-config-setup.js | 11 +- config_app/js/services/api-service.js | 18 +- util/config/validators/validate_storage.py | 2 +- 8 files changed, 66 insertions(+), 1677 deletions(-) delete mode 100644 config_app/js/config-field-templates/config-setup-tool.html diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index f7dc232b1..9ec395a05 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -1,7 +1,4 @@ import logging -import os -import subprocess -import signal from flask import abort, request @@ -14,9 +11,7 @@ from data.users import get_federated_service_name, get_users_handler from data.database import configure from data.runmigration import run_alembic_migration from util.config.configutil import add_enterprise_config_defaults -from util.config.database import sync_database_with_config from util.config.validator import validate_service_for_config, ValidatorContext, is_valid_config_upload_filename -from util.config.validators import LocalStorageConfigValidationException logger = logging.getLogger(__name__) diff --git a/config_app/config_endpoints/api/superuser.py b/config_app/config_endpoints/api/superuser.py index 0a4705d03..34bb9fba9 100644 --- a/config_app/config_endpoints/api/superuser.py +++ b/config_app/config_endpoints/api/superuser.py @@ -4,15 +4,16 @@ import os from flask import request, jsonify +from util.config.validator import EXTRA_CA_DIRECTORY + from config_app.config_endpoints.exception import InvalidRequest from config_app.config_endpoints.api import resource, ApiResource, nickname +from config_app.config_endpoints.api.superuser_models_pre_oci import pre_oci_model from config_app.config_util.ssl import load_certificate, CertInvalidException from config_app.c_app import app, config_provider -from config_app.config_endpoints.api.superuser_models_pre_oci import pre_oci_model logger = logging.getLogger(__name__) -EXTRA_CA_DIRECTORY = 'extra_ca_certs' @resource('/v1/superuser/customcerts/') diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index e16b8a092..f9b61e740 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -1,17 +1,53 @@ +import os import tarfile -from flask import request, make_response +from flask import request, make_response, send_file from data.database import configure +from util.config.validator import EXTRA_CA_DIRECTORY from config_app.c_app import app, config_provider from config_app.config_endpoints.api import resource, ApiResource, nickname @resource('/v1/configapp/tarconfig') class TarConfigLoader(ApiResource): - """ Resource for validating a block of configuration against an external service. """ + """ + Resource for dealing with configuration as a tarball, + including loading and generating functions + """ - @nickname('uploadTarballConfig') + @nickname('scGetConfigTarball') + def get(self): + config_path = config_provider.config_volume + + # remove the initial trailing / from the prefix path, and add the last dir one + tar_dir_prefix = config_path[1:] + '/' + + def tarinfo_filter(tarinfo): + # remove leading directory info + tarinfo.name = tarinfo.name.replace(tar_dir_prefix, '') + + # ignore any directory that isn't the specified extra ca one: + if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY: + return None + + return tarinfo + + # Remove the tar if it already exists so we don't write on top of existing tarball + if os.path.isfile('quay-config.tar.gz'): + os.remove('quay-config.tar.gz') + + tar = tarfile.open('quay-config.tar.gz', mode="w:gz") + + for name in os.listdir(config_path): + tar.add(os.path.join(config_path, name), filter=tarinfo_filter) + + tar.close() + + return send_file('quay-config.tar.gz', mimetype='application/gzip', + as_attachment=True, attachment_filename='quay-config.tar.gz') + + @nickname('scUploadTarballConfig') def put(self): """ Loads tarball config into the config provider """ input_stream = request.stream diff --git a/config_app/js/config-field-templates/config-setup-tool.html b/config_app/js/config-field-templates/config-setup-tool.html deleted file mode 100644 index 9c4dde631..000000000 --- a/config_app/js/config-field-templates/config-setup-tool.html +++ /dev/null @@ -1,1656 +0,0 @@ -
-
-
-
- - -
-
- Custom SSL Certificates -
-
-
-
-
- - -
-
- Basic Configuration -
-
- - - - - - - - - - -
Enterprise Logo URL: - -
- Enter the full URL to your company's logo. -
- 		- -
Contact Information: - -
- Information to show in the Contact Page. If none specified, CoreOS contact information - is displayed. -
-
-
-
- - -
-
- Server Configuration -
-
- - - - - - - - - -
Server Hostname: - -
- The HTTP host (and optionally the port number if a non-standard HTTP/HTTPS port) of the location - where the registry will be accessible on the network -
-
TLS: - - -
- Running without TLS should not be used for production workloads! -
- -
- Terminating TLS outside of Quay Enterprise can result in unusual behavior if the external load balancer is not - configured properly. This option is not recommended for simple setups. Please contact support - if you encounter problems while using this option. -
- -
- Enabling TLS also enables HTTP Strict Transport Security.
- This prevents downgrade attacks and cookie theft, but browsers will reject all future insecure connections on this hostname. -
- - - - - - - - - - -
Certificate: - -
- The certificate must be in PEM format. -
-
Private key: - -
-
- -
-
- - -
-
- Data Consistency Settings -
-
-
-

Relax constraints on consistency guarantees for specific operations - to enable higher performance and availability. -

-
- - - - -
-
- Allow repository pulls even if audit logging fails. -
- If enabled, failures to write to the audit log will fallback from - the database to the standard logger for registry pulls. -
-
-
-
-
- - -
-
- Time Machine -
-
-
-

Time machine keeps older copies of tags within a repository for the configured period - of time, after which they are garbage collected. This allows users to - revert tags to older images in case they accidentally pushed a broken image. It is - highly recommended to have time machine enabled, but it does take a bit more space - in storage. -

-
- - - - - - - - - - - - - - -
Allowed expiration periods: - -
- The expiration periods allowed for configuration. The default tag expiration *must* be in this list. -
-
Default expiration period: - -
- The default tag expiration period for all namespaces (users and organizations). Must be expressed in a duration string form: 30m, 1h, 1d, 2w. -
-
Allow users to select expiration: -
- Enable Expiration Configuration -
- If enabled, users will be able to select the tag expiration duration for the namespace(s) they - administrate, from the configured list of options. -
-
-
-
-
- - -
-
- redis -
-
-
-

A redis key-value store is required for real-time events and build logs.

-
- - - - - - - - - - - - - - -
Redis Hostname: - -
Redis port: - -
- Access to this port and hostname must be allowed from all hosts running - the enterprise registry -
-
Redis password: - -
-
-
- - -
-
- Registry Storage -
-
-
-

- Registry images can be stored either locally or in a remote storage system. - A remote storage system is required for high-availability systems. -

- -
- Enable Storage Replication -
- If enabled, replicates storage to other regions. See documentation for more information. -
-
- -
- - - - - - - - - - - - - - - - - - - - - -
Location ID: - -
- {{ sc.location }} -
-
- {{ storageConfigError[$index].location }} -
-
Remove
-
Set Default: -
- Replicate to storage engine by default -
-
Storage Engine: - - -
- {{ storageConfigError[$index].engine }} -
-
{{ field.title }}: - - - - {{ field.placeholder }} - - -
- -
-
- {{ field.help_text }} -
-
- See Documentation for more information -
-
-
- -
- Add Additional Storage Engine -
-
-
-
- - -
-
- Action Log Rotation and Archiving -
-
-
-

- All actions performed in are automatically logged. These logs are stored in a database table, which can become quite large. - Enabling log rotation and archiving will move all logs older than 30 days into storage. -

-
-
- Enable Action Log Rotation -
- - - - - - - - - - -
Storage location: - -
- The storage location in which to place archived action logs. Logs will only be archived to this single location. -
-
Storage path: - -
- The path under the configured storage engine in which to place the archived logs in JSON form. -
-
-
- - -
-
- Security Scanner -
-
-
-

If enabled, all images pushed to Quay will be scanned via the external security scanning service, with vulnerability information available in the UI and API, as well - as async notification support. -

-
- -
- Enable Security Scanning -
-
- A scanner compliant with the Quay Security Scanning API must be running to use this feature. Documentation on running Clair can be found at Running Clair Security Scanner. -
- - - - - - - - - - -
Authentication Key: - -
- The security scanning service requires an authorized service key to speak to Quay. Once setup, the key - can be managed in the Service Keys panel under the Super User Admin Panel. -
-
Security Scanner Endpoint: - -
- The HTTP URL at which the security scanner is running. -
-
- Is the security scanner behind a domain signed with a self-signed TLS certificate? If so, please make sure to register your SSL CA in the custom certificates panel above. -
-
-
-
- - -
-
- Application Registry -
-
-
-

If enabled, an additional registry API will be available for managing applications (Kubernetes manifests, Helm charts) via the App Registry specification. A great place to get started is to install the Helm Registry Plugin. -

- -
- Enable App Registry -
-
-
- - -
-
- BitTorrent-based download -
-
-
-

If enabled, all images in the registry can be downloaded using the quayctl tool via the BitTorrent protocol. A JWT-compatible BitTorrent tracker such as Chihaya must be run. -

- -
- Enable BitTorrent downloads -
- - - - - - -
Announce URL: - -
- The HTTP URL at which the torrents should be announced. A JWT-compatible tracker such as Chihaya must be run to ensure proper security. Documentation on running Chihaya with - this support can be found at Running Chihaya for Quay Enterprise. -
-
-
-
- - -
-
- rkt Conversion -
-
-
-

If enabled, all images in the registry can be fetched via rkt fetch or any other AppC discovery-compliant implementation.

-
- -
- Enable ACI Conversion -
- -
- Documentation on generating these keys can be found at Generating ACI Signing Keys. -
- - - - - - - - - - - - - - -
GPG2 Public Key File: - -
- The certificate must be in PEM format. -
-
GPG2 Private Key File: - -
GPG2 Private Key Name: - -
-
-
- - -
-
- E-mail -
-
-
-

Valid e-mail server configuration is required for notification e-mails and the ability of - users to reset their passwords.

-
- -
- Enable E-mails -
- - - - - - - - - - - - - - - - - - - - - - - -
SMTP Server: - > -
SMTP Server Port: - -
TLS: -
- Require TLS -
-
Mail Sender: - -
- E-mail address from which all e-mails are sent. If not specified, - support@quay.io will be used. -
-
Authentication: -
- Requires Authentication -
- - - - - - - - - - -
Username: - -
Password: - -
-
-
-
- - -
-
- Internal Authentication -
-
-
-

- Authentication for the registry can be handled by either the registry itself, LDAP, Keystone, or external JWT endpoint. -

-

- Additional external authentication providers (such as GitHub) can be used in addition for login into the UI. -

-
- -
-
- It is highly recommended to require encrypted client passwords. External passwords used in the Docker client will be stored in plaintext! - Enable this requirement now. -
- -
- Note: The "Require Encrypted Client Passwords" feature is currently enabled which will - prevent passwords from being saved as plaintext by the Docker client. -
-
- - - - - - - - - - - - - - - - - - - -
Authentication: - -
Team synchronization: -
- Enable Team Synchronization Support -
-
- If enabled, organization administrators who are also superusers can set teams to have their membership synchronized with a backing group in {{ config.AUTHENTICATION_TYPE }}. -
-
Resynchronization duration: - -
- The duration before a team must be re-synchronized. Must be expressed in a duration string form: 30m, 1h, 1d. -
-
Self-service team syncing setup: -
If enabled, this feature will allow *any organization administrator* to read the membership of any {{ config.AUTHENTICATION_TYPE }} group.
-
- Allow non-superusers to enable and manage team syncing -
-
- If enabled, non-superusers will be able to enable and manage team sycning on teams under organizations in which they are administrators. -
-
- - - - - - - - - - - - - - - - - - - - - - - -
Keystone API Version: - -
Keystone Authentication URL: - -
- The URL (starting with http or https) of the Keystone Server endpoint for auth. -
-
Keystone Administrator Username: - -
- The username for the Keystone admin. -
-
Keystone Administrator Password: - -
- The password for the Keystone admin. -
-
Keystone Administrator Tenant: - -
- The tenant (project/group) that contains the administrator user. -
-
- - -
- JSON Web Token authentication allows your organization to provide an HTTP endpoint that - verifies user credentials on behalf of . -
- Documentation - on the API required can be found here: https://github.com/coreos/jwt-auth-example. -
- - - - - - - - - - - - - - - - - - - - - - -
Authentication Issuer: - -
- The id of the issuer signing the JWT token. Must be unique to your organization. -
-
Public Key: - -
- A certificate containing the public key portion of the key pair used to sign - the JSON Web Tokens. This file must be in PEM format. -
-
User Verification Endpoint: - -
- The URL (starting with http or https) on the JWT authentication server for verifying username and password credentials. -
- -
- Credentials will be sent in the Authorization header as Basic Auth, and this endpoint should return 200 OK on success (or a 4** otherwise). -
-
User Query Endpoint: - -
- The URL (starting with http or https) on the JWT authentication server for looking up - users based on a prefix query. This is optional. -
- -
- The prefix query will be sent as a query parameter with name query. -
-
User Lookup Endpoint: - -
- The URL (starting with http or https) on the JWT authentication server for looking up - a user by username or email address. -
- -
- The username or email address will be sent as a query parameter with name username. -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
LDAP URI: - -
- The full LDAP URI, including the ldap:// or ldaps:// prefix. -
-
Base DN: - -
- A Distinguished Name path which forms the base path for looking up all LDAP records. -
-
- Example: dc=my,dc=domain,dc=com -
-
User Relative DN: - -
- A Distinguished Name path which forms the base path for looking up all user LDAP records, - relative to the Base DN defined above. -
-
- Example: ou=employees -
-
Secondary User Relative DNs: - -
- A list of Distinguished Name path(s) which forms the secondary base path(s) for - looking up all user LDAP records, relative to the Base DN defined above. These path(s) - will be tried if the user is not found via the primary relative DN. -
-
- Example: [ou=employees] -
-
Administrator DN: -
- The Distinguished Name for the Administrator account. This account must be able to login and view the records for all user accounts. -
-
- Example: uid=admin,ou=employees,dc=my,dc=domain,dc=com -
-
Administrator DN Password: -
- Note: This will be stored in - plaintext inside the config.yaml, so setting up a dedicated account or using - a password hash is highly recommended. -
- -
- The password for the Administrator DN. -
-
UID Attribute: - -
- The name of the property field in your LDAP user records that stores your - users' username. Typically "uid". -
-
Mail Attribute: - -
- The name of the property field in your LDAP user records that stores your - users' e-mail address(es). Typically "mail". -
-
Custom TLS Certificate: - -
- If specified, the certificate (in PEM format) for the LDAP TLS connection. -
-
Allow insecure: -
- Allow fallback to non-TLS connections -
-
- If enabled, LDAP will fallback to insecure non-TLS connections if TLS does not succeed. -
-
-
-
- -
-
- External Authorization (OAuth) -
-
- -
-
- GitHub (Enterprise) Authentication -
-
-
-

- If enabled, users can use GitHub or GitHub Enterprise to authenticate to the registry. -

-

- Note: A registered GitHub (Enterprise) OAuth application is required. - View instructions on how to - - Create an OAuth Application in GitHub - -

-
- -
- Enable GitHub Authentication -
- -
- Warning: This provider is not bound to your {{ config.AUTHENTICATION_TYPE }} authentication. Logging in via this provider will create a -only user, which is not the recommended approach. It is highly recommended to choose a "Binding Field" below. -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
GitHub: - -
GitHub Endpoint: - - -
- The GitHub Enterprise endpoint. Must start with http:// or https://. -
-
OAuth Client ID: - - -
OAuth Client Secret: - - -
Organization Filtering: -
- Restrict By Organization Membership -
- -
- If enabled, only members of specified GitHub - Enterprise organizations will be allowed to login via GitHub - Enterprise. -
- - - -
Binding Field: - -
- If selected, when a user logs in via this provider, they will be automatically bound to their user in {{ config.AUTHENTICATION_TYPE }} by matching the selected field from the provider to the associated user in {{ config.AUTHENTICATION_TYPE }}. -
-
- For example, selecting Subject here with a backing authentication system of LDAP means that a user logging in via this provider will also be bound to their user in LDAP by username. -
-
- If none selected, a user unique to will be created on initial login with this provider. This is not the recommended setup. -
-
-
-
- - -
-
- Google Authentication -
-
-
-

- If enabled, users can use Google to authenticate to the registry. -

-

- Note: A registered Google OAuth application is required. - Visit the - - Google Developer Console - - to register an application. -

-
- -
- Enable Google Authentication -
- -
- Warning: This provider is not bound to your {{ config.AUTHENTICATION_TYPE }} authentication. Logging in via this provider will create a -only user, which is not the recommended approach. It is highly recommended to choose a "Binding Field" below. -
- - - - - - - - - - - - - - -
OAuth Client ID: - - -
OAuth Client Secret: - - -
Binding Field: - -
- If selected, when a user logs in via this provider, they will be automatically bound to their user in {{ config.AUTHENTICATION_TYPE }} by matching the selected field from the provider to the associated user in {{ config.AUTHENTICATION_TYPE }}. -
-
- For example, selecting Subject here with a backing authentication system of LDAP means that a user logging in via this provider will also be bound to their user in LDAP by username. -
-
- If none selected, a user unique to will be created on initial login with this provider. This is not the recommended setup. -
-
-
-
- - -
-
- - {{ config[provider]['SERVICE_NAME'] || (getOIDCProviderId(provider) + ' Authentication') }} - (Delete) -
-
-
- Warning: This OIDC provider is not bound to your {{ config.AUTHENTICATION_TYPE }} authentication. Logging in via this provider will create a -only user, which is not the recommended approach. It is highly recommended to choose a "Binding Field" below. -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Service ID: - {{ getOIDCProviderId(provider) }} -
OIDC Server: - - -
- The URL of an OIDC-compliant server. -
-
Client ID: - -
Client Secret: - -
Service Name: - - -
- The user friendly name to display for the service on the login page. -
-
Service Icon (optional): - - -
- If specified, the icon to display for this login service on the login page. Can be either a URL to an icon or a CSS class name from Font Awesome -
-
Binding Field: - -
- If selected, when a user logs in via this OIDC provider, they will be automatically bound to their user in {{ config.AUTHENTICATION_TYPE }} by matching the selected field from the OIDC provider to the associated user in {{ config.AUTHENTICATION_TYPE }}. -
-
- For example, selecting Subject here with a backing authentication system of LDAP means that a user logging in via this OIDC provider will also be bound to their user in LDAP by username. -
-
- If none selected, a user unique to will be created on initial login with this OIDC provider. This is not the recommended setup. -
-
Login Scopes: - -
- If specified, the scopes to send to the OIDC provider when performing the login flow. Note that, if specified, these scopes will - override those set by default, so this list must include a scope for OpenID Connect - (typically the openid scope) or this provider will fail. -
-
-
-

Callback URLs for this service:

-
    -
  • {{ mapped.TLS_SETTING == 'none' ? 'http' : 'https' }}://{{ config.SERVER_HOSTNAME || '(configure server hostname)' }}/oauth2/{{ getOIDCProviderId(provider).toLowerCase() }}/callback
    • -
  • {{ mapped.TLS_SETTING == 'none' ? 'http' : 'https' }}://{{ config.SERVER_HOSTNAME || '(configure server hostname)' }}/oauth2/{{ getOIDCProviderId(provider).toLowerCase() }}/callback/attach
    • -
  • {{ mapped.TLS_SETTING == 'none' ? 'http' : 'https' }}://{{ config.SERVER_HOSTNAME || '(configure server hostname)' }}/oauth2/{{ getOIDCProviderId(provider).toLowerCase() }}/callback/cli
    • -
-
-
-
- - - Add OIDC Provider - What is OIDC? -
-
- - -
-
- Access Settings -
-
-
-

Various settings around access and authentication to the registry.

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Basic Credentials Login: -
- Login to User Interface via credentials -
-
-
- Login to User Interface via credentials must be enabled. Click here to enable. -
-
- Login to User Interface via credentials is enabled (requires at least one OIDC provider to disable) -
-
-
- If enabled, users will be able to login to the user interface via their username and password credentials. -
-
- If disabled, users will only be able to login to the user interface via one of the configured External Authentication providers. -
-
External Application tokens -
- Allow external application tokens -
-
- If enabled, users will be able to generate external application tokens for use on the Docker and rkt CLI. Note - that these tokens will not be required unless "App Token" is chosen as the Internal Authentication method above. -
-
External application token expiration - -
- The expiration time for user generated external application tokens. If none, tokens will never expire. -
-
Anonymous Access: -
- Enable Anonymous Access -
-
- If enabled, public repositories and search can be accessed by anyone that can - reach the registry, even if they are not authenticated. Disable to only allow - authenticated users to view and pull "public" resources. -
-
User Creation: -
- Enable Open User Creation -
-
- If enabled, user accounts can be created by anyone (unless restricted below to invited users). - Users can always be created in the users panel in this superuser tool, even if this feature is disabled. -
-
Invite-only User Creation: -
- Enable Invite-only User Creation -
-
- If enabled, user accounts can only be created when a user has been invited, by e-mail address, to join a team. - Users can always be created in the users panel in this superuser tool, even if this feature is enabled. -
-
Encrypted Client Password: -
- Require Encrypted Client Passwords -
-
- If enabled, users will not be able to login from the Docker command - line with a non-encrypted password and must generate an encrypted - password to use. -
-
- This feature is highly recommended for setups with external authentication, as Docker currently stores passwords in plaintext on user's machines. -
-
Prefix username autocompletion: -
- Allow prefix username autocompletion -
-
- If disabled, autocompletion for users will only match on exact usernames. -
-
Team Invitations: -
- Require Team Invitations -
-
- If enabled, when adding a new user to a team, they will receive an invitation to join the team, with the option to decline. - Otherwise, users will be immediately part of a team when added by a team administrator. -
-
-
-
- - -
-
- Dockerfile Build Support -
-
-
- If enabled, users can submit Dockerfiles to be built and pushed by . -
- -
- Enable Dockerfile Build -
- -
- Note: Build workers are required for this feature. - See Adding Build Workers for instructions on how to setup build workers. -
-
-
- - -
-
- GitHub (Enterprise) Build Triggers -
-
-
-

- If enabled, users can setup GitHub or GitHub Enterprise triggers to invoke Registry builds. -

-

- Note: A registered GitHub (Enterprise) OAuth application (separate from GitHub Authentication) is required. - View instructions on how to - - Create an OAuth Application in GitHub - -

-
- -
- Enable GitHub Triggers -
- - - - - - - - - - - - - - - - - - -
GitHub: - -
GitHub Endpoint: - - -
- The GitHub Enterprise endpoint. Must start with http:// or https://. -
-
OAuth Client ID: - - -
OAuth Client Secret: - - -
-
-
- - -
-
- BitBucket Build Triggers -
-
-
-

- If enabled, users can setup BitBucket triggers to invoke Registry builds. -

-

- Note: A registered BitBucket OAuth application is required. - View instructions on how to - - Create an OAuth Application in BitBucket - -

-
- -
- Enable BitBucket Triggers -
- - - - - - - - - - -
OAuth Consumer Key: - - -
OAuth Consumer Secret: - - -
-
-
- - -
-
- GitLab Build Triggers -
-
-
-

- If enabled, users can setup GitLab triggers to invoke Registry builds. -

-

- Note: A registered GitLab OAuth application is required. - Visit the - - GitLab applications admin panel - - to create a new application. -

-

The callback URL to use is:   - {{ config.PREFERRED_URL_SCHEME || 'http' }}://{{ config.SERVER_HOSTNAME || 'localhost' }}/oauth2/gitlab/callback/trigger -

-
- -
- Enable GitLab Triggers -
- - - - - - - - - - - - - - - - - - -
GitLab: - -
GitLab Endpoint: - - -
- The GitLab Enterprise endpoint. Must start with http:// or https://. -
-
Application Id: - - -
Secret: - - -
-
-
- - - - -
- - -
- - - - -
-
diff --git a/config_app/js/core-config-setup/config-setup-tool.html b/config_app/js/core-config-setup/config-setup-tool.html index eed17afce..a246e46e8 100644 --- a/config_app/js/core-config-setup/config-setup-tool.html +++ b/config_app/js/core-config-setup/config-setup-tool.html @@ -1630,9 +1630,9 @@
diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index e55fb45e2..1cf5ea7f9 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -413,12 +413,12 @@ angular.module("quay-config") } }; - $scope.saveConfiguration = function() { + $scope.generateConfigTarball = function() { $scope.savingConfiguration = true; // Make sure to note that fully verified setup is completed. We use this as a signal // in the setup tool. - $scope.config['SETUP_COMPLETE'] = true; + // $scope.config['SETUP_COMPLETE'] = true; var data = { 'config': $scope.config, @@ -432,9 +432,14 @@ angular.module("quay-config") authPassword = null; }); - ApiService.scUpdateConfig(data).then(function(resp) { + // We need to set the response type to 'blob', to ensure it's never encoded as a string + // (string encoded binary data can be difficult to transform with js) + // and to make it easier to save (FileSaver expects a blob) + ApiService.scGetConfigTarball(null, null, null, null, true).then(function(resp) { authPassword = null; + FileSaver.saveAs(resp, 'quay-config.tar.gz'); + $scope.savingConfiguration = false; $scope.mapped.$hasChanges = false; diff --git a/config_app/js/services/api-service.js b/config_app/js/services/api-service.js index 5163f1cba..2a36bf35b 100644 --- a/config_app/js/services/api-service.js +++ b/config_app/js/services/api-service.js @@ -212,12 +212,20 @@ angular.module('quay-config').factory('ApiService', ['Restangular', '$q', 'UtilS var urlPath = path['x-path']; // Add the operation itself. - apiService[operationName] = function(opt_options, opt_parameters, opt_background, opt_forceget) { + apiService[operationName] = function(opt_options, opt_parameters, opt_background, opt_forceget, opt_blobresp) { var one = Restangular.one(buildUrl(urlPath, opt_parameters)); - if (opt_background) { - one.withHttpConfig({ - 'ignoreLoadingBar': true - }); + + if (opt_background || opt_blobresp) { + let httpConfig = {}; + + if (opt_background) { + httpConfig['ignoreLoadingBar'] = true; + } + if (opt_blobresp) { + httpConfig['responseType'] = 'blob'; + } + + one.withHttpConfig(httpConfig); } var opObj = one[opt_forceget ? 'get' : 'custom' + method.toUpperCase()](opt_options); diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index 4aec66aae..3e3de74ee 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,5 +1,5 @@ from storage import get_storage_driver -from util.config.validators import BaseValidator, ConfigValidationException, LocalStorageConfigValidationException +from util.config.validators import BaseValidator, ConfigValidationException class StorageValidator(BaseValidator): From 2d0a599aabf7f7eca2b69ee6ebf98725e70b9f42 Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Wed, 27 Jun 2018 13:49:54 -0400 Subject: [PATCH 3/6] Create download modal following setup completion --- config_app/config_endpoints/api/suconfig.py | 60 +++++++++---------- .../config_endpoints/api/tar_config_loader.py | 5 +- .../config-setup-app.component.html | 3 +- .../config-setup-app.component.ts | 15 ++++- .../download-tarball-modal.component.html | 42 +++++++++++++ .../download-tarball-modal.component.ts | 34 +++++++++++ .../download-tarball-modal.css | 6 ++ config_app/js/config-app.module.ts | 2 + .../core-config-setup/config-setup-tool.html | 5 +- .../js/core-config-setup/core-config-setup.js | 15 ++--- config_app/js/setup/setup.component.js | 3 +- config_app/js/setup/setup.html | 4 +- 12 files changed, 142 insertions(+), 52 deletions(-) create mode 100644 config_app/js/components/download-tarball-modal/download-tarball-modal.component.html create mode 100644 config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts create mode 100644 config_app/js/components/download-tarball-modal/download-tarball-modal.css diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index 9ec395a05..fa8233a0b 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -65,44 +65,40 @@ class SuperUserConfig(ApiResource): """ Updates the config override file. """ # Note: This method is called to set the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. - if not config_provider.config_exists(): - config_object = request.get_json()['config'] - hostname = request.get_json()['hostname'] + config_object = request.get_json()['config'] + hostname = request.get_json()['hostname'] - # Add any enterprise defaults missing from the config. - add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname) + # Add any enterprise defaults missing from the config. + add_enterprise_config_defaults(config_object, app.config['SECRET_KEY'], hostname) - # Write the configuration changes to the config override file. - config_provider.save_config(config_object) + # Write the configuration changes to the config override file. + config_provider.save_config(config_object) + + # If the authentication system is federated, link the superuser account to the + # the authentication system chosen. + service_name = get_federated_service_name(config_object['AUTHENTICATION_TYPE']) + if service_name is not None: + current_user = get_authenticated_user() + if current_user is None: + abort(401) - # If the authentication system is federated, link the superuser account to the - # the authentication system chosen. service_name = get_federated_service_name(config_object['AUTHENTICATION_TYPE']) - if service_name is not None: - current_user = get_authenticated_user() - if current_user is None: - abort(401) + if not model.has_federated_login(current_user.username, service_name): + # Verify the user's credentials and retrieve the user's external username+email. + handler = get_users_handler(config_object, config_provider, OVERRIDE_CONFIG_DIRECTORY) + (result, err_msg) = handler.verify_credentials(current_user.username, + request.get_json().get('password', '')) + if not result: + logger.error('Could not save configuration due to external auth failure: %s', err_msg) + abort(400) - service_name = get_federated_service_name(config_object['AUTHENTICATION_TYPE']) - if not model.has_federated_login(current_user.username, service_name): - # Verify the user's credentials and retrieve the user's external username+email. - handler = get_users_handler(config_object, config_provider, OVERRIDE_CONFIG_DIRECTORY) - (result, err_msg) = handler.verify_credentials(current_user.username, - request.get_json().get('password', '')) - if not result: - logger.error('Could not save configuration due to external auth failure: %s', err_msg) - abort(400) - - # Link the existing user to the external user. - model.attach_federated_login(current_user.username, service_name, result.username) - - return { - 'exists': True, - 'config': config_object - } - - abort(403) + # Link the existing user to the external user. + model.attach_federated_login(current_user.username, service_name, result.username) + return { + 'exists': True, + 'config': config_object + } @resource('/v1/superuser/registrystatus') diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index f9b61e740..033a7a5f2 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -37,15 +37,14 @@ class TarConfigLoader(ApiResource): if os.path.isfile('quay-config.tar.gz'): os.remove('quay-config.tar.gz') - tar = tarfile.open('quay-config.tar.gz', mode="w:gz") + tar = tarfile.open('quay-config.tar.gz', mode="w|gz") for name in os.listdir(config_path): tar.add(os.path.join(config_path, name), filter=tarinfo_filter) tar.close() - return send_file('quay-config.tar.gz', mimetype='application/gzip', - as_attachment=True, attachment_filename='quay-config.tar.gz') + return send_file('quay-config.tar.gz', mimetype='application/gzip') @nickname('scUploadTarballConfig') def put(self): diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.html b/config_app/js/components/config-setup-app/config-setup-app.component.html index d1ea58f31..1cd0dd93c 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.html +++ b/config_app/js/components/config-setup-app/config-setup-app.component.html @@ -22,5 +22,6 @@
-
+
+ diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.ts b/config_app/js/components/config-setup-app/config-setup-app.component.ts index e22ca57c0..b453d7ef4 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.ts +++ b/config_app/js/components/config-setup-app/config-setup-app.component.ts @@ -1,4 +1,4 @@ -import { Input, Component, Inject } from 'ng-metadata/core'; +import { Component } from 'ng-metadata/core'; const templateUrl = require('./config-setup-app.component.html'); /** @@ -9,7 +9,13 @@ const templateUrl = require('./config-setup-app.component.html'); templateUrl: templateUrl, }) export class ConfigSetupAppComponent { - private state: 'choice' | 'setup' | 'load'; + private state + : 'choice' + | 'setup' + | 'load' + | 'download'; + + private loadedConfig = false; constructor() { this.state = 'choice'; @@ -21,9 +27,14 @@ export class ConfigSetupAppComponent { private chooseLoad(): void { this.state = 'load'; + this.loadedConfig = true; } private configLoaded(): void { this.state = 'setup'; } + + private setupCompleted(): void { + this.state = 'download'; + } } diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html new file mode 100644 index 000000000..89898455f --- /dev/null +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html @@ -0,0 +1,42 @@ +
+ +
diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts new file mode 100644 index 000000000..0d586a92f --- /dev/null +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts @@ -0,0 +1,34 @@ +import { Input, Component, Inject } from 'ng-metadata/core'; +const templateUrl = require('./download-tarball-modal.component.html'); +const styleUrl = require('./download-tarball-modal.css'); + +declare const FileSaver: any; + +/** + * Initial Screen and Choice in the Config App + */ +@Component({ + selector: 'download-tarball-modal', + templateUrl: templateUrl, + styleUrls: [ styleUrl ], +}) +export class DownloadTarballModalComponent { + @Input('<') public loadedConfig; + + constructor(@Inject('ApiService') private ApiService) { + + } + + private downloadTarball() { + const errorDisplay: Function = this.ApiService.errorDisplay( + 'Could not save configuration. Please report this error.' + ); + + // We need to set the response type to 'blob', to ensure it's never encoded as a string + // (string encoded binary data can be difficult to transform with js) + // and to make it easier to save (FileSaver expects a blob) + this.ApiService.scGetConfigTarball(null, null, null, null, true).then(function(resp) { + FileSaver.saveAs(resp, 'quay-config.tar.gz'); + }, errorDisplay); + } +} diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.css b/config_app/js/components/download-tarball-modal/download-tarball-modal.css new file mode 100644 index 000000000..3f68a1a7a --- /dev/null +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.css @@ -0,0 +1,6 @@ +.modal__warning-box { + background-color: #ddd; + padding: 15px; + border-radius: 5px; + margin-top: 15px; +} diff --git a/config_app/js/config-app.module.ts b/config_app/js/config-app.module.ts index ca90cd045..f5aa0c532 100644 --- a/config_app/js/config-app.module.ts +++ b/config_app/js/config-app.module.ts @@ -2,6 +2,7 @@ import { NgModule } from 'ng-metadata/core'; import * as restangular from 'restangular'; import { ConfigSetupAppComponent } from './components/config-setup-app/config-setup-app.component'; +import { DownloadTarballModalComponent } from './components/download-tarball-modal/download-tarball-modal.component'; import { LoadConfigComponent } from './components/load-config/load-config.component'; const quayDependencies: string[] = [ @@ -42,6 +43,7 @@ function provideConfig($provide: ng.auto.IProvideService, imports: [ DependencyConfig ], declarations: [ ConfigSetupAppComponent, + DownloadTarballModalComponent, LoadConfigComponent, ], providers: [] diff --git a/config_app/js/core-config-setup/config-setup-tool.html b/config_app/js/core-config-setup/config-setup-tool.html index a246e46e8..a1fb25e75 100644 --- a/config_app/js/core-config-setup/config-setup-tool.html +++ b/config_app/js/core-config-setup/config-setup-tool.html @@ -1,7 +1,6 @@
-
@@ -1630,9 +1629,9 @@
diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index 1cf5ea7f9..d8e7c0edc 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -27,7 +27,8 @@ angular.module("quay-config") restrict: 'C', scope: { 'isActive': '=isActive', - 'configurationSaved': '&configurationSaved' + 'configurationSaved': '&configurationSaved', + 'setupCompleted': '&setupCompleted', }, controller: function($rootScope, $scope, $element, $timeout, ApiService) { var authPassword = null; @@ -413,7 +414,7 @@ angular.module("quay-config") } }; - $scope.generateConfigTarball = function() { + $scope.saveConfiguration = function() { $scope.savingConfiguration = true; // Make sure to note that fully verified setup is completed. We use this as a signal @@ -432,20 +433,16 @@ angular.module("quay-config") authPassword = null; }); - // We need to set the response type to 'blob', to ensure it's never encoded as a string - // (string encoded binary data can be difficult to transform with js) - // and to make it easier to save (FileSaver expects a blob) - ApiService.scGetConfigTarball(null, null, null, null, true).then(function(resp) { + ApiService.scUpdateConfig(data).then(function(resp) { authPassword = null; - FileSaver.saveAs(resp, 'quay-config.tar.gz'); - $scope.savingConfiguration = false; $scope.mapped.$hasChanges = false; $('#validateAndSaveModal').modal('hide'); - $scope.configurationSaved({'config': $scope.config}); + // $scope.configurationSaved({'config': $scope.config}); + $scope.setupCompleted(); }, errorDisplay); }; diff --git a/config_app/js/setup/setup.component.js b/config_app/js/setup/setup.component.js index b8fe72bd3..1d13dc924 100644 --- a/config_app/js/setup/setup.component.js +++ b/config_app/js/setup/setup.component.js @@ -15,7 +15,8 @@ const templateUrl = require('./setup.html'); restrict: 'C', scope: { 'isActive': '=isActive', - 'configurationSaved': '&configurationSaved' + 'configurationSaved': '&configurationSaved', + 'setupCompleted': '&setupCompleted', }, controller: SetupCtrl, }; diff --git a/config_app/js/setup/setup.html b/config_app/js/setup/setup.html index 93cc87a87..e3b2fed37 100644 --- a/config_app/js/setup/setup.html +++ b/config_app/js/setup/setup.html @@ -25,7 +25,9 @@
+ configuration-saved="configurationSaved(config)" + setup-completed="setupCompleted()" + >
From db757edcd2ece75a501970c6e1d7db959cc61f8c Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Thu, 28 Jun 2018 13:45:26 -0400 Subject: [PATCH 4/6] Create transient config provider, temp dir logic Allows us to have a new config provider for each setup, with no overlap of the directories used, and automatic cleanup of those directories. --- config_app/config_endpoints/api/suconfig.py | 14 -------- .../config_endpoints/api/tar_config_loader.py | 32 +++++++++++++------ .../config/TransientDirectoryProvider.py | 30 +++++++++++++++++ config_app/config_util/config/__init__.py | 3 +- .../config-setup-app.component.ts | 12 +++++-- .../download-tarball-modal.component.ts | 2 +- .../js/core-config-setup/core-config-setup.js | 5 --- config_app/js/services/api-service.js | 8 ++--- requirements.txt | 1 + 9 files changed, 70 insertions(+), 37 deletions(-) create mode 100644 config_app/config_util/config/TransientDirectoryProvider.py diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index fa8233a0b..9e17701ab 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -110,20 +110,6 @@ class SuperUserRegistryStatus(ApiResource): def get(self): """ Returns the status of the registry. """ - # If we have SETUP_COMPLETE, then we're ready to go! - if app.config.get('SETUP_COMPLETE', False): - return { - 'provider_id': config_provider.provider_id, - 'requires_restart': config_provider.requires_restart(app.config), - 'status': 'ready' - } - - # If there is no conf/stack volume, then report that status. - if not config_provider.volume_exists(): - return { - 'status': 'missing-config-dir' - } - # If there is no config file, we need to setup the database. if not config_provider.config_exists(): return { diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index 033a7a5f2..011de5531 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -1,4 +1,5 @@ import os +import tempfile import tarfile from flask import request, make_response, send_file @@ -9,6 +10,19 @@ from util.config.validator import EXTRA_CA_DIRECTORY from config_app.c_app import app, config_provider from config_app.config_endpoints.api import resource, ApiResource, nickname +@resource('/v1/configapp/initialization') +class ConfigInitialization(ApiResource): + """ + Resource for dealing with any initialization logic for the config app + """ + + @nickname('scStartNewConfig') + def get(self): + config_provider.new_config_dir() + + return make_response('OK') + + @resource('/v1/configapp/tarconfig') class TarConfigLoader(ApiResource): """ @@ -18,7 +32,7 @@ class TarConfigLoader(ApiResource): @nickname('scGetConfigTarball') def get(self): - config_path = config_provider.config_volume + config_path = config_provider.get_config_dir_path() # remove the initial trailing / from the prefix path, and add the last dir one tar_dir_prefix = config_path[1:] + '/' @@ -33,28 +47,28 @@ class TarConfigLoader(ApiResource): return tarinfo - # Remove the tar if it already exists so we don't write on top of existing tarball - if os.path.isfile('quay-config.tar.gz'): - os.remove('quay-config.tar.gz') + temp = tempfile.NamedTemporaryFile() - tar = tarfile.open('quay-config.tar.gz', mode="w|gz") + tar = tarfile.open(temp.name, mode="w|gz") for name in os.listdir(config_path): tar.add(os.path.join(config_path, name), filter=tarinfo_filter) tar.close() - return send_file('quay-config.tar.gz', mimetype='application/gzip') + return send_file(temp.name, mimetype='application/gzip') @nickname('scUploadTarballConfig') def put(self): """ Loads tarball config into the config provider """ + # Generate a new empty dir to load the config into + config_provider.new_config_dir() + input_stream = request.stream with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream: - # TODO: find a way to remove the contents of the directory on shutdown? - tar_stream.extractall(config_provider.config_volume) + tar_stream.extractall(config_provider.get_config_dir_path()) - # now try to connect to the db provided in their config + # now try to connect to the db provided in their config to validate it works combined = dict(**app.config) combined.update(config_provider.get_config()) diff --git a/config_app/config_util/config/TransientDirectoryProvider.py b/config_app/config_util/config/TransientDirectoryProvider.py new file mode 100644 index 000000000..2e0509e74 --- /dev/null +++ b/config_app/config_util/config/TransientDirectoryProvider.py @@ -0,0 +1,30 @@ +import os +from backports.tempfile import TemporaryDirectory + +from config_app.config_util.config.fileprovider import FileConfigProvider + +class TransientDirectoryProvider(FileConfigProvider): + """ Implementation of the config provider that reads and writes the data + from/to the file system, only using temporary directories, + deleting old dirs and creating new ones as requested. + """ + def __init__(self, config_volume, yaml_filename, py_filename): + # Create a temp directory that will be cleaned up when we change the config path + # This should ensure we have no "pollution" of different configs: + # no uploaded config should ever affect subsequent config modifications/creations + temp_dir = TemporaryDirectory() + self.temp_dir = temp_dir + super(TransientDirectoryProvider, self).__init__(temp_dir.name, yaml_filename, py_filename) + + def new_config_dir(self): + """ + Update the path with a new temporary directory, deleting the old one in the process + """ + temp_dir = TemporaryDirectory() + + self.config_volume = temp_dir.name + self.temp_dir = temp_dir + self.yaml_path = os.path.join(temp_dir.name, self.yaml_filename) + + def get_config_dir_path(self): + return self.config_volume diff --git a/config_app/config_util/config/__init__.py b/config_app/config_util/config/__init__.py index 3735e4f66..b9edeba3a 100644 --- a/config_app/config_util/config/__init__.py +++ b/config_app/config_util/config/__init__.py @@ -1,5 +1,6 @@ from config_app.config_util.config.fileprovider import FileConfigProvider from config_app.config_util.config.testprovider import TestConfigProvider +from config_app.config_util.config.TransientDirectoryProvider import TransientDirectoryProvider def get_config_provider(config_volume, yaml_filename, py_filename, testing=False): @@ -8,4 +9,4 @@ def get_config_provider(config_volume, yaml_filename, py_filename, testing=False if testing: return TestConfigProvider() - return FileConfigProvider(config_volume, yaml_filename, py_filename) + return TransientDirectoryProvider(config_volume, yaml_filename, py_filename) diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.ts b/config_app/js/components/config-setup-app/config-setup-app.component.ts index b453d7ef4..550741d60 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.ts +++ b/config_app/js/components/config-setup-app/config-setup-app.component.ts @@ -1,4 +1,4 @@ -import { Component } from 'ng-metadata/core'; +import { Component, Inject } from 'ng-metadata/core'; const templateUrl = require('./config-setup-app.component.html'); /** @@ -17,12 +17,18 @@ export class ConfigSetupAppComponent { private loadedConfig = false; - constructor() { + constructor(@Inject('ApiService') private apiService) { this.state = 'choice'; } private chooseSetup(): void { - this.state = 'setup'; + this.apiService.scStartNewConfig() + .then(() => { + this.state = 'setup'; + }) + .catch(this.apiService.errorDisplay( + 'Could not initialize new setup. Please report this error' + )); } private chooseLoad(): void { diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts index 0d586a92f..943c0c015 100644 --- a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.ts @@ -27,7 +27,7 @@ export class DownloadTarballModalComponent { // We need to set the response type to 'blob', to ensure it's never encoded as a string // (string encoded binary data can be difficult to transform with js) // and to make it easier to save (FileSaver expects a blob) - this.ApiService.scGetConfigTarball(null, null, null, null, true).then(function(resp) { + this.ApiService.scGetConfigTarball(null, null, null, null, 'blob').then(function(resp) { FileSaver.saveAs(resp, 'quay-config.tar.gz'); }, errorDisplay); } diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index d8e7c0edc..21fc8aa38 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -417,10 +417,6 @@ angular.module("quay-config") $scope.saveConfiguration = function() { $scope.savingConfiguration = true; - // Make sure to note that fully verified setup is completed. We use this as a signal - // in the setup tool. - // $scope.config['SETUP_COMPLETE'] = true; - var data = { 'config': $scope.config, 'hostname': window.location.host, @@ -441,7 +437,6 @@ angular.module("quay-config") $('#validateAndSaveModal').modal('hide'); - // $scope.configurationSaved({'config': $scope.config}); $scope.setupCompleted(); }, errorDisplay); }; diff --git a/config_app/js/services/api-service.js b/config_app/js/services/api-service.js index 2a36bf35b..814e25a45 100644 --- a/config_app/js/services/api-service.js +++ b/config_app/js/services/api-service.js @@ -212,17 +212,17 @@ angular.module('quay-config').factory('ApiService', ['Restangular', '$q', 'UtilS var urlPath = path['x-path']; // Add the operation itself. - apiService[operationName] = function(opt_options, opt_parameters, opt_background, opt_forceget, opt_blobresp) { + apiService[operationName] = function(opt_options, opt_parameters, opt_background, opt_forceget, opt_responseType) { var one = Restangular.one(buildUrl(urlPath, opt_parameters)); - if (opt_background || opt_blobresp) { + if (opt_background || opt_responseType) { let httpConfig = {}; if (opt_background) { httpConfig['ignoreLoadingBar'] = true; } - if (opt_blobresp) { - httpConfig['responseType'] = 'blob'; + if (opt_responseType) { + httpConfig['responseType'] = opt_responseType; } one.withHttpConfig(httpConfig); diff --git a/requirements.txt b/requirements.txt index 44ccab610..8c3c4e236 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ azure-storage-blob==1.1.0 azure-storage-common==1.1.0 azure-storage-nspkg==3.0.0 Babel==2.5.3 +backports.tempfile==1.0 beautifulsoup4==4.6.0 bencode==1.0 bintrees==2.0.7 From d7ffb543336b53edfad8203ccd9ae7d4cba63ad9 Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Thu, 28 Jun 2018 16:56:33 -0400 Subject: [PATCH 5/6] Move tar filter to file, add tests for it --- config_app/config_endpoints/api/suconfig.py | 6 +++- .../config_endpoints/api/tar_config_loader.py | 16 ++-------- config_app/config_util/tar.py | 14 +++++++++ config_app/config_util/test/test_tar.py | 29 +++++++++++++++++++ .../download-tarball-modal.component.html | 14 +++++---- .../download-tarball-modal.css | 11 +++++-- .../js/core-config-setup/core-config-setup.js | 12 ++++---- util/config/validator.py | 9 ++++-- util/config/validators/validate_storage.py | 4 +++ 9 files changed, 83 insertions(+), 32 deletions(-) create mode 100644 config_app/config_util/tar.py create mode 100644 config_app/config_util/test/test_tar.py diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index 9e17701ab..97bd2a39d 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -271,11 +271,15 @@ class SuperUserConfigValidate(ApiResource): # Note: This method is called to validate the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. Note that # this is also safe since this method does not access any information not given in the request. + + # We can skip localstorage validation, since we can't guarantee that this will be the same machine + # Q.E. will run under config = request.get_json()['config'] validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), instance_keys=instance_keys, ip_resolver=ip_resolver, - config_provider=config_provider) + config_provider=config_provider, + skip_localstorage_validation=True) return validate_service_for_config(service, validator_context) diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index 011de5531..9cde0f68f 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -5,10 +5,10 @@ import tarfile from flask import request, make_response, send_file from data.database import configure -from util.config.validator import EXTRA_CA_DIRECTORY from config_app.c_app import app, config_provider from config_app.config_endpoints.api import resource, ApiResource, nickname +from config_app.config_util.tar import tarinfo_filter_partial @resource('/v1/configapp/initialization') class ConfigInitialization(ApiResource): @@ -17,7 +17,7 @@ class ConfigInitialization(ApiResource): """ @nickname('scStartNewConfig') - def get(self): + def post(self): config_provider.new_config_dir() return make_response('OK') @@ -37,22 +37,12 @@ class TarConfigLoader(ApiResource): # remove the initial trailing / from the prefix path, and add the last dir one tar_dir_prefix = config_path[1:] + '/' - def tarinfo_filter(tarinfo): - # remove leading directory info - tarinfo.name = tarinfo.name.replace(tar_dir_prefix, '') - - # ignore any directory that isn't the specified extra ca one: - if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY: - return None - - return tarinfo - temp = tempfile.NamedTemporaryFile() tar = tarfile.open(temp.name, mode="w|gz") for name in os.listdir(config_path): - tar.add(os.path.join(config_path, name), filter=tarinfo_filter) + tar.add(os.path.join(config_path, name), filter=tarinfo_filter_partial(tar_dir_prefix)) tar.close() diff --git a/config_app/config_util/tar.py b/config_app/config_util/tar.py new file mode 100644 index 000000000..379bdfae1 --- /dev/null +++ b/config_app/config_util/tar.py @@ -0,0 +1,14 @@ +from util.config.validator import EXTRA_CA_DIRECTORY + +def tarinfo_filter_partial(prefix): + def tarinfo_filter(tarinfo): + # remove leading directory info + tarinfo.name = tarinfo.name.replace(prefix, '') + + # ignore any directory that isn't the specified extra ca one: + if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY: + return None + + return tarinfo + + return tarinfo_filter diff --git a/config_app/config_util/test/test_tar.py b/config_app/config_util/test/test_tar.py new file mode 100644 index 000000000..432501e82 --- /dev/null +++ b/config_app/config_util/test/test_tar.py @@ -0,0 +1,29 @@ +import pytest + +from config_app.config_util.tar import tarinfo_filter_partial + +from util.config.validator import EXTRA_CA_DIRECTORY + +from test.fixtures import * + +class MockTarInfo: + def __init__(self, name, isdir): + self.name = name + self.isdir = lambda: isdir + + def __eq__(self, other): + return other is not None and self.name == other.name + +@pytest.mark.parametrize('prefix,tarinfo,expected', [ + # It should handle simple files + ('Users/sam/', MockTarInfo('Users/sam/config.yaml', False), MockTarInfo('config.yaml', False)), + # It should allow the extra CA dir + ('Users/sam/', MockTarInfo('Users/sam/%s' % EXTRA_CA_DIRECTORY, True), MockTarInfo('%s' % EXTRA_CA_DIRECTORY, True)), + # it should allow a file in that extra dir + ('Users/sam/', MockTarInfo('Users/sam/%s/cert.crt' % EXTRA_CA_DIRECTORY, False), MockTarInfo('%s/cert.crt' % EXTRA_CA_DIRECTORY, False)), + # it should not allow a directory that isn't the CA dir + ('Users/sam/', MockTarInfo('Users/sam/dirignore', True), None), +]) +def test_tarinfo_filter(prefix, tarinfo, expected): + partial = tarinfo_filter_partial(prefix) + assert partial(tarinfo) == expected diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html index 89898455f..dfc95c578 100644 --- a/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.component.html @@ -8,15 +8,17 @@

Download Configuration

-
+
Please download your updated configuration. To deploy these changes to your Quay Enterprise instances, please see the docs.
- Warning: - Your configuration and certificates are kept unencrypted. Please keep this file secure. +
+ Warning: + Your configuration and certificates are kept unencrypted. Please keep this file secure. +
@@ -25,8 +27,10 @@ see the docs.
- Warning: - Your configuration and certificates are kept unencrypted. Please keep this file secure. +
+ Warning: + Your configuration and certificates are kept unencrypted. Please keep this file secure. +
diff --git a/config_app/js/components/download-tarball-modal/download-tarball-modal.css b/config_app/js/components/download-tarball-modal/download-tarball-modal.css index 3f68a1a7a..eada2902c 100644 --- a/config_app/js/components/download-tarball-modal/download-tarball-modal.css +++ b/config_app/js/components/download-tarball-modal/download-tarball-modal.css @@ -1,6 +1,11 @@ -.modal__warning-box { - background-color: #ddd; +.co-dialog .modal-body.download-tarball-modal { padding: 15px; - border-radius: 5px; +} + +.modal__warning-box { margin-top: 15px; } + +.modal__warning-box .co-alert.co-alert-warning::before { + font-family:Font Awesome\ 5 Free; +} diff --git a/config_app/js/core-config-setup/core-config-setup.js b/config_app/js/core-config-setup/core-config-setup.js index 21fc8aa38..73841712a 100644 --- a/config_app/js/core-config-setup/core-config-setup.js +++ b/config_app/js/core-config-setup/core-config-setup.js @@ -39,13 +39,7 @@ angular.module("quay-config") $scope.SERVICES = [ {'id': 'redis', 'title': 'Redis'}, - {'id': 'registry-storage', 'title': 'Registry Storage', 'condition': (config) => { - // We can skip validation if all of the storage locations are local, as we can't - // guarantee that this will be the same machine Q.E. will run under. Therefore, - // we just have a warning to the user that Q.E. won't start if the locations don't match - return Object.values(config.DISTRIBUTED_STORAGE_CONFIG) - .some(storageTuple => storageTuple[0] !== 'LocalStorage') - }}, + {'id': 'registry-storage', 'title': 'Registry Storage'}, {'id': 'time-machine', 'title': 'Time Machine'}, @@ -417,6 +411,10 @@ angular.module("quay-config") $scope.saveConfiguration = function() { $scope.savingConfiguration = true; + // Make sure to note that fully verified setup is completed. We use this as a signal + // in the setup tool. + $scope.config['SETUP_COMPLETE'] = true; + var data = { 'config': $scope.config, 'hostname': window.location.host, diff --git a/util/config/validator.py b/util/config/validator.py index a0924c9e2..444d8340a 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -102,7 +102,8 @@ class ValidatorContext(object): def __init__(self, config, user_password=None, http_client=None, context=None, url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None, feature_sec_scanner=False, is_testing=False, - uri_creator=None, config_provider=None, instance_keys=None): + uri_creator=None, config_provider=None, instance_keys=None, + skip_localstorage_validation=False): self.config = config self.user = get_authenticated_user() self.user_password = user_password @@ -117,10 +118,11 @@ class ValidatorContext(object): self.uri_creator = uri_creator self.config_provider = config_provider self.instance_keys = instance_keys + self.skip_localstorage_validation = skip_localstorage_validation @classmethod def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None, - config_provider=None): + config_provider=None, skip_localstorage_validation=False): """ Creates a ValidatorContext from an app config, with a given config to validate :param app: the Flask app to pull configuration information from @@ -146,4 +148,5 @@ class ValidatorContext(object): is_testing=app.config.get('TESTING', False), uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), config_provider=config_provider, - instance_keys=instance_keys) + instance_keys=instance_keys, + skip_localstorage_validation=skip_localstorage_validation) diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index 3e3de74ee..366405739 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -12,6 +12,7 @@ class StorageValidator(BaseValidator): client = validator_context.http_client ip_resolver = validator_context.ip_resolver config_provider = validator_context.config_provider + skip_localstorage_validation = validator_context.skip_localstorage_validation replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) @@ -20,6 +21,9 @@ class StorageValidator(BaseValidator): raise ConfigValidationException('Storage configuration required') for name, (storage_type, driver) in providers: + if skip_localstorage_validation and storage_type == 'LocalStorage': + continue + try: if replication_enabled and storage_type == 'LocalStorage': raise ConfigValidationException('Locally mounted directory not supported ' + From 14fefea38f933c4f5ea6f9ccbc8c906a864bb19e Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Thu, 28 Jun 2018 19:42:08 -0400 Subject: [PATCH 6/6] Revert local validation context, extract another util --- config_app/config_endpoints/api/suconfig.py | 6 +----- config_app/config_endpoints/api/tar_config_loader.py | 12 ++---------- config_app/config_util/tar.py | 6 ++++++ storage/__init__.py | 2 ++ util/config/validator.py | 9 +++------ util/config/validators/validate_storage.py | 7 ++++--- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/config_app/config_endpoints/api/suconfig.py b/config_app/config_endpoints/api/suconfig.py index 97bd2a39d..9e17701ab 100644 --- a/config_app/config_endpoints/api/suconfig.py +++ b/config_app/config_endpoints/api/suconfig.py @@ -271,15 +271,11 @@ class SuperUserConfigValidate(ApiResource): # Note: This method is called to validate the database configuration before super users exists, # so we also allow it to be called if there is no valid registry configuration setup. Note that # this is also safe since this method does not access any information not given in the request. - - # We can skip localstorage validation, since we can't guarantee that this will be the same machine - # Q.E. will run under config = request.get_json()['config'] validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''), instance_keys=instance_keys, ip_resolver=ip_resolver, - config_provider=config_provider, - skip_localstorage_validation=True) + config_provider=config_provider) return validate_service_for_config(service, validator_context) diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index 9cde0f68f..de2eb50e1 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -8,7 +8,7 @@ from data.database import configure from config_app.c_app import app, config_provider from config_app.config_endpoints.api import resource, ApiResource, nickname -from config_app.config_util.tar import tarinfo_filter_partial +from config_app.config_util.tar import tarinfo_filter_partial, strip_absolute_path_and_add_trailing_dir @resource('/v1/configapp/initialization') class ConfigInitialization(ApiResource): @@ -19,7 +19,6 @@ class ConfigInitialization(ApiResource): @nickname('scStartNewConfig') def post(self): config_provider.new_config_dir() - return make_response('OK') @@ -33,19 +32,14 @@ class TarConfigLoader(ApiResource): @nickname('scGetConfigTarball') def get(self): config_path = config_provider.get_config_dir_path() - - # remove the initial trailing / from the prefix path, and add the last dir one - tar_dir_prefix = config_path[1:] + '/' - + tar_dir_prefix = strip_absolute_path_and_add_trailing_dir(config_path) temp = tempfile.NamedTemporaryFile() tar = tarfile.open(temp.name, mode="w|gz") - for name in os.listdir(config_path): tar.add(os.path.join(config_path, name), filter=tarinfo_filter_partial(tar_dir_prefix)) tar.close() - return send_file(temp.name, mimetype='application/gzip') @nickname('scUploadTarballConfig') @@ -53,7 +47,6 @@ class TarConfigLoader(ApiResource): """ Loads tarball config into the config provider """ # Generate a new empty dir to load the config into config_provider.new_config_dir() - input_stream = request.stream with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream: tar_stream.extractall(config_provider.get_config_dir_path()) @@ -61,7 +54,6 @@ class TarConfigLoader(ApiResource): # now try to connect to the db provided in their config to validate it works combined = dict(**app.config) combined.update(config_provider.get_config()) - configure(combined) return make_response('OK') diff --git a/config_app/config_util/tar.py b/config_app/config_util/tar.py index 379bdfae1..55930ce9b 100644 --- a/config_app/config_util/tar.py +++ b/config_app/config_util/tar.py @@ -1,5 +1,11 @@ from util.config.validator import EXTRA_CA_DIRECTORY +def strip_absolute_path_and_add_trailing_dir(path): + """ + Removes the initial trailing / from the prefix path, and add the last dir one + """ + return path[1:] + '/' + def tarinfo_filter_partial(prefix): def tarinfo_filter(tarinfo): # remove leading directory info diff --git a/storage/__init__.py b/storage/__init__.py index e48681973..3479ecc38 100644 --- a/storage/__init__.py +++ b/storage/__init__.py @@ -7,6 +7,8 @@ from storage.azurestorage import AzureStorage from storage.downloadproxy import DownloadProxy from util.ipresolver import NoopIPResolver +TYPE_LOCAL_STORAGE = 'LocalStorage' + STORAGE_DRIVER_CLASSES = { 'LocalStorage': LocalStorage, 'S3Storage': S3Storage, diff --git a/util/config/validator.py b/util/config/validator.py index 444d8340a..a0924c9e2 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -102,8 +102,7 @@ class ValidatorContext(object): def __init__(self, config, user_password=None, http_client=None, context=None, url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None, ip_resolver=None, feature_sec_scanner=False, is_testing=False, - uri_creator=None, config_provider=None, instance_keys=None, - skip_localstorage_validation=False): + uri_creator=None, config_provider=None, instance_keys=None): self.config = config self.user = get_authenticated_user() self.user_password = user_password @@ -118,11 +117,10 @@ class ValidatorContext(object): self.uri_creator = uri_creator self.config_provider = config_provider self.instance_keys = instance_keys - self.skip_localstorage_validation = skip_localstorage_validation @classmethod def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None, - config_provider=None, skip_localstorage_validation=False): + config_provider=None): """ Creates a ValidatorContext from an app config, with a given config to validate :param app: the Flask app to pull configuration information from @@ -148,5 +146,4 @@ class ValidatorContext(object): is_testing=app.config.get('TESTING', False), uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname), config_provider=config_provider, - instance_keys=instance_keys, - skip_localstorage_validation=skip_localstorage_validation) + instance_keys=instance_keys) diff --git a/util/config/validators/validate_storage.py b/util/config/validators/validate_storage.py index 366405739..3fe6b4324 100644 --- a/util/config/validators/validate_storage.py +++ b/util/config/validators/validate_storage.py @@ -1,4 +1,4 @@ -from storage import get_storage_driver +from storage import get_storage_driver, TYPE_LOCAL_STORAGE from util.config.validators import BaseValidator, ConfigValidationException @@ -12,7 +12,6 @@ class StorageValidator(BaseValidator): client = validator_context.http_client ip_resolver = validator_context.ip_resolver config_provider = validator_context.config_provider - skip_localstorage_validation = validator_context.skip_localstorage_validation replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False) @@ -21,7 +20,9 @@ class StorageValidator(BaseValidator): raise ConfigValidationException('Storage configuration required') for name, (storage_type, driver) in providers: - if skip_localstorage_validation and storage_type == 'LocalStorage': + # We can skip localstorage validation, since we can't guarantee that + # this will be the same machine Q.E. will run under + if storage_type == TYPE_LOCAL_STORAGE: continue try: