Move tar filter to file, add tests for it
This commit is contained in:
parent
db757edcd2
commit
d7ffb54333
9 changed files with 83 additions and 32 deletions
|
@ -271,11 +271,15 @@ class SuperUserConfigValidate(ApiResource):
|
||||||
# Note: This method is called to validate the database configuration before super users exists,
|
# Note: This method is called to validate the database configuration before super users exists,
|
||||||
# so we also allow it to be called if there is no valid registry configuration setup. Note that
|
# so we also allow it to be called if there is no valid registry configuration setup. Note that
|
||||||
# this is also safe since this method does not access any information not given in the request.
|
# this is also safe since this method does not access any information not given in the request.
|
||||||
|
|
||||||
|
# We can skip localstorage validation, since we can't guarantee that this will be the same machine
|
||||||
|
# Q.E. will run under
|
||||||
config = request.get_json()['config']
|
config = request.get_json()['config']
|
||||||
validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
|
validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
|
||||||
instance_keys=instance_keys,
|
instance_keys=instance_keys,
|
||||||
ip_resolver=ip_resolver,
|
ip_resolver=ip_resolver,
|
||||||
config_provider=config_provider)
|
config_provider=config_provider,
|
||||||
|
skip_localstorage_validation=True)
|
||||||
|
|
||||||
return validate_service_for_config(service, validator_context)
|
return validate_service_for_config(service, validator_context)
|
||||||
|
|
||||||
|
|
|
@ -5,10 +5,10 @@ import tarfile
|
||||||
from flask import request, make_response, send_file
|
from flask import request, make_response, send_file
|
||||||
|
|
||||||
from data.database import configure
|
from data.database import configure
|
||||||
from util.config.validator import EXTRA_CA_DIRECTORY
|
|
||||||
|
|
||||||
from config_app.c_app import app, config_provider
|
from config_app.c_app import app, config_provider
|
||||||
from config_app.config_endpoints.api import resource, ApiResource, nickname
|
from config_app.config_endpoints.api import resource, ApiResource, nickname
|
||||||
|
from config_app.config_util.tar import tarinfo_filter_partial
|
||||||
|
|
||||||
@resource('/v1/configapp/initialization')
|
@resource('/v1/configapp/initialization')
|
||||||
class ConfigInitialization(ApiResource):
|
class ConfigInitialization(ApiResource):
|
||||||
|
@ -17,7 +17,7 @@ class ConfigInitialization(ApiResource):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@nickname('scStartNewConfig')
|
@nickname('scStartNewConfig')
|
||||||
def get(self):
|
def post(self):
|
||||||
config_provider.new_config_dir()
|
config_provider.new_config_dir()
|
||||||
|
|
||||||
return make_response('OK')
|
return make_response('OK')
|
||||||
|
@ -37,22 +37,12 @@ class TarConfigLoader(ApiResource):
|
||||||
# remove the initial trailing / from the prefix path, and add the last dir one
|
# remove the initial trailing / from the prefix path, and add the last dir one
|
||||||
tar_dir_prefix = config_path[1:] + '/'
|
tar_dir_prefix = config_path[1:] + '/'
|
||||||
|
|
||||||
def tarinfo_filter(tarinfo):
|
|
||||||
# remove leading directory info
|
|
||||||
tarinfo.name = tarinfo.name.replace(tar_dir_prefix, '')
|
|
||||||
|
|
||||||
# ignore any directory that isn't the specified extra ca one:
|
|
||||||
if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return tarinfo
|
|
||||||
|
|
||||||
temp = tempfile.NamedTemporaryFile()
|
temp = tempfile.NamedTemporaryFile()
|
||||||
|
|
||||||
tar = tarfile.open(temp.name, mode="w|gz")
|
tar = tarfile.open(temp.name, mode="w|gz")
|
||||||
|
|
||||||
for name in os.listdir(config_path):
|
for name in os.listdir(config_path):
|
||||||
tar.add(os.path.join(config_path, name), filter=tarinfo_filter)
|
tar.add(os.path.join(config_path, name), filter=tarinfo_filter_partial(tar_dir_prefix))
|
||||||
|
|
||||||
tar.close()
|
tar.close()
|
||||||
|
|
||||||
|
|
14
config_app/config_util/tar.py
Normal file
14
config_app/config_util/tar.py
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
from util.config.validator import EXTRA_CA_DIRECTORY
|
||||||
|
|
||||||
|
def tarinfo_filter_partial(prefix):
|
||||||
|
def tarinfo_filter(tarinfo):
|
||||||
|
# remove leading directory info
|
||||||
|
tarinfo.name = tarinfo.name.replace(prefix, '')
|
||||||
|
|
||||||
|
# ignore any directory that isn't the specified extra ca one:
|
||||||
|
if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY:
|
||||||
|
return None
|
||||||
|
|
||||||
|
return tarinfo
|
||||||
|
|
||||||
|
return tarinfo_filter
|
29
config_app/config_util/test/test_tar.py
Normal file
29
config_app/config_util/test/test_tar.py
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from config_app.config_util.tar import tarinfo_filter_partial
|
||||||
|
|
||||||
|
from util.config.validator import EXTRA_CA_DIRECTORY
|
||||||
|
|
||||||
|
from test.fixtures import *
|
||||||
|
|
||||||
|
class MockTarInfo:
|
||||||
|
def __init__(self, name, isdir):
|
||||||
|
self.name = name
|
||||||
|
self.isdir = lambda: isdir
|
||||||
|
|
||||||
|
def __eq__(self, other):
|
||||||
|
return other is not None and self.name == other.name
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('prefix,tarinfo,expected', [
|
||||||
|
# It should handle simple files
|
||||||
|
('Users/sam/', MockTarInfo('Users/sam/config.yaml', False), MockTarInfo('config.yaml', False)),
|
||||||
|
# It should allow the extra CA dir
|
||||||
|
('Users/sam/', MockTarInfo('Users/sam/%s' % EXTRA_CA_DIRECTORY, True), MockTarInfo('%s' % EXTRA_CA_DIRECTORY, True)),
|
||||||
|
# it should allow a file in that extra dir
|
||||||
|
('Users/sam/', MockTarInfo('Users/sam/%s/cert.crt' % EXTRA_CA_DIRECTORY, False), MockTarInfo('%s/cert.crt' % EXTRA_CA_DIRECTORY, False)),
|
||||||
|
# it should not allow a directory that isn't the CA dir
|
||||||
|
('Users/sam/', MockTarInfo('Users/sam/dirignore', True), None),
|
||||||
|
])
|
||||||
|
def test_tarinfo_filter(prefix, tarinfo, expected):
|
||||||
|
partial = tarinfo_filter_partial(prefix)
|
||||||
|
assert partial(tarinfo) == expected
|
|
@ -8,28 +8,32 @@
|
||||||
<h4 class="modal-title"><span>Download Configuration</span></h4>
|
<h4 class="modal-title"><span>Download Configuration</span></h4>
|
||||||
</div>
|
</div>
|
||||||
<!-- Body -->
|
<!-- Body -->
|
||||||
<div class="modal-body">
|
<div class="modal-body download-tarball-modal">
|
||||||
<div ng-if="$ctrl.loadedConfig">
|
<div ng-if="$ctrl.loadedConfig">
|
||||||
Please download your updated configuration. To deploy these changes to your Quay Enterprise instances, please
|
Please download your updated configuration. To deploy these changes to your Quay Enterprise instances, please
|
||||||
<a target="_blank" href="https://coreos.com/quay-enterprise/docs/latest/initial-setup.html">
|
<a target="_blank" href="https://coreos.com/quay-enterprise/docs/latest/initial-setup.html">
|
||||||
see the docs.
|
see the docs.
|
||||||
</a>
|
</a>
|
||||||
<div class="modal__warning-box">
|
<div class="modal__warning-box">
|
||||||
<i class="fas fa-exclamation-triangle" style="margin-right: 10px;"></i><strong>Warning:</strong>
|
<div class="fas co-alert co-alert-warning">
|
||||||
|
<strong>Warning: </strong>
|
||||||
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
|
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
<div ng-if="!$ctrl.loadedConfig">
|
<div ng-if="!$ctrl.loadedConfig">
|
||||||
Please download your new configuration. For more information, and next steps, please
|
Please download your new configuration. For more information, and next steps, please
|
||||||
<a target="_blank" href="https://coreos.com/quay-enterprise/docs/latest/initial-setup.html">
|
<a target="_blank" href="https://coreos.com/quay-enterprise/docs/latest/initial-setup.html">
|
||||||
see the docs.
|
see the docs.
|
||||||
</a>
|
</a>
|
||||||
<div class="modal__warning-box">
|
<div class="modal__warning-box">
|
||||||
<i class="fas fa-exclamation-triangle" style="margin-right: 10px;"></i><strong>Warning: </strong>
|
<div class="fas co-alert co-alert-warning">
|
||||||
|
<strong>Warning: </strong>
|
||||||
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
|
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
<div class="modal-footer">
|
<div class="modal-footer">
|
||||||
<button class="btn btn-primary"
|
<button class="btn btn-primary"
|
||||||
ng-click="$ctrl.downloadTarball()">
|
ng-click="$ctrl.downloadTarball()">
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
.modal__warning-box {
|
.co-dialog .modal-body.download-tarball-modal {
|
||||||
background-color: #ddd;
|
|
||||||
padding: 15px;
|
padding: 15px;
|
||||||
border-radius: 5px;
|
}
|
||||||
|
|
||||||
|
.modal__warning-box {
|
||||||
margin-top: 15px;
|
margin-top: 15px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.modal__warning-box .co-alert.co-alert-warning::before {
|
||||||
|
font-family:Font Awesome\ 5 Free;
|
||||||
|
}
|
||||||
|
|
|
@ -39,13 +39,7 @@ angular.module("quay-config")
|
||||||
$scope.SERVICES = [
|
$scope.SERVICES = [
|
||||||
{'id': 'redis', 'title': 'Redis'},
|
{'id': 'redis', 'title': 'Redis'},
|
||||||
|
|
||||||
{'id': 'registry-storage', 'title': 'Registry Storage', 'condition': (config) => {
|
{'id': 'registry-storage', 'title': 'Registry Storage'},
|
||||||
// We can skip validation if all of the storage locations are local, as we can't
|
|
||||||
// guarantee that this will be the same machine Q.E. will run under. Therefore,
|
|
||||||
// we just have a warning to the user that Q.E. won't start if the locations don't match
|
|
||||||
return Object.values(config.DISTRIBUTED_STORAGE_CONFIG)
|
|
||||||
.some(storageTuple => storageTuple[0] !== 'LocalStorage')
|
|
||||||
}},
|
|
||||||
|
|
||||||
{'id': 'time-machine', 'title': 'Time Machine'},
|
{'id': 'time-machine', 'title': 'Time Machine'},
|
||||||
|
|
||||||
|
@ -417,6 +411,10 @@ angular.module("quay-config")
|
||||||
$scope.saveConfiguration = function() {
|
$scope.saveConfiguration = function() {
|
||||||
$scope.savingConfiguration = true;
|
$scope.savingConfiguration = true;
|
||||||
|
|
||||||
|
// Make sure to note that fully verified setup is completed. We use this as a signal
|
||||||
|
// in the setup tool.
|
||||||
|
$scope.config['SETUP_COMPLETE'] = true;
|
||||||
|
|
||||||
var data = {
|
var data = {
|
||||||
'config': $scope.config,
|
'config': $scope.config,
|
||||||
'hostname': window.location.host,
|
'hostname': window.location.host,
|
||||||
|
|
|
@ -102,7 +102,8 @@ class ValidatorContext(object):
|
||||||
def __init__(self, config, user_password=None, http_client=None, context=None,
|
def __init__(self, config, user_password=None, http_client=None, context=None,
|
||||||
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
|
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
|
||||||
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
|
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
|
||||||
uri_creator=None, config_provider=None, instance_keys=None):
|
uri_creator=None, config_provider=None, instance_keys=None,
|
||||||
|
skip_localstorage_validation=False):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.user = get_authenticated_user()
|
self.user = get_authenticated_user()
|
||||||
self.user_password = user_password
|
self.user_password = user_password
|
||||||
|
@ -117,10 +118,11 @@ class ValidatorContext(object):
|
||||||
self.uri_creator = uri_creator
|
self.uri_creator = uri_creator
|
||||||
self.config_provider = config_provider
|
self.config_provider = config_provider
|
||||||
self.instance_keys = instance_keys
|
self.instance_keys = instance_keys
|
||||||
|
self.skip_localstorage_validation = skip_localstorage_validation
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None,
|
def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None,
|
||||||
config_provider=None):
|
config_provider=None, skip_localstorage_validation=False):
|
||||||
"""
|
"""
|
||||||
Creates a ValidatorContext from an app config, with a given config to validate
|
Creates a ValidatorContext from an app config, with a given config to validate
|
||||||
:param app: the Flask app to pull configuration information from
|
:param app: the Flask app to pull configuration information from
|
||||||
|
@ -146,4 +148,5 @@ class ValidatorContext(object):
|
||||||
is_testing=app.config.get('TESTING', False),
|
is_testing=app.config.get('TESTING', False),
|
||||||
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
|
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
|
||||||
config_provider=config_provider,
|
config_provider=config_provider,
|
||||||
instance_keys=instance_keys)
|
instance_keys=instance_keys,
|
||||||
|
skip_localstorage_validation=skip_localstorage_validation)
|
||||||
|
|
|
@ -12,6 +12,7 @@ class StorageValidator(BaseValidator):
|
||||||
client = validator_context.http_client
|
client = validator_context.http_client
|
||||||
ip_resolver = validator_context.ip_resolver
|
ip_resolver = validator_context.ip_resolver
|
||||||
config_provider = validator_context.config_provider
|
config_provider = validator_context.config_provider
|
||||||
|
skip_localstorage_validation = validator_context.skip_localstorage_validation
|
||||||
|
|
||||||
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
|
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
|
||||||
|
|
||||||
|
@ -20,6 +21,9 @@ class StorageValidator(BaseValidator):
|
||||||
raise ConfigValidationException('Storage configuration required')
|
raise ConfigValidationException('Storage configuration required')
|
||||||
|
|
||||||
for name, (storage_type, driver) in providers:
|
for name, (storage_type, driver) in providers:
|
||||||
|
if skip_localstorage_validation and storage_type == 'LocalStorage':
|
||||||
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if replication_enabled and storage_type == 'LocalStorage':
|
if replication_enabled and storage_type == 'LocalStorage':
|
||||||
raise ConfigValidationException('Locally mounted directory not supported ' +
|
raise ConfigValidationException('Locally mounted directory not supported ' +
|
||||||
|
|
Reference in a new issue