vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Move tar filter to file, add tests for it

Browse source
This commit is contained in:
Sam Chow 2018-06-28 16:56:33 -04:00
parent db757edcd2
commit d7ffb54333
9 changed files with 83 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config_app/config_endpoints/api/suconfig.py
View file

@ -271,11 +271,15 @@ class SuperUserConfigValidate(ApiResource):
# Note: This method is called to validate the database configuration before super users exists,
# so we also allow it to be called if there is no valid registry configuration setup. Note that
# this is also safe since this method does not access any information not given in the request.
# We can skip localstorage validation, since we can't guarantee that this will be the same machine
# Q.E. will run under
config = request.get_json()['config']
validator_context = ValidatorContext.from_app(app, config, request.get_json().get('password', ''),
instance_keys=instance_keys,
ip_resolver=ip_resolver,
config_provider=config_provider)
config_provider=config_provider,
skip_localstorage_validation=True)
return validate_service_for_config(service, validator_context)

16
config_app/config_endpoints/api/tar_config_loader.py
View file

@ -5,10 +5,10 @@ import tarfile
from flask import request, make_response, send_file
from data.database import configure
from util.config.validator import EXTRA_CA_DIRECTORY
from config_app.c_app import app, config_provider
from config_app.config_endpoints.api import resource, ApiResource, nickname
from config_app.config_util.tar import tarinfo_filter_partial
@resource('/v1/configapp/initialization')
class ConfigInitialization(ApiResource):
@ -17,7 +17,7 @@ class ConfigInitialization(ApiResource):
"""
@nickname('scStartNewConfig')
def get(self):
def post(self):
config_provider.new_config_dir()
return make_response('OK')
@ -37,22 +37,12 @@ class TarConfigLoader(ApiResource):
# remove the initial trailing / from the prefix path, and add the last dir one
tar_dir_prefix = config_path[1:] + '/'
def tarinfo_filter(tarinfo):
# remove leading directory info
tarinfo.name = tarinfo.name.replace(tar_dir_prefix, '')
# ignore any directory that isn't the specified extra ca one:
if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY:
return None
return tarinfo
temp = tempfile.NamedTemporaryFile()
tar = tarfile.open(temp.name, mode="w|gz")
for name in os.listdir(config_path):
tar.add(os.path.join(config_path, name), filter=tarinfo_filter)
tar.add(os.path.join(config_path, name), filter=tarinfo_filter_partial(tar_dir_prefix))
tar.close()

14
config_app/config_util/tar.py Normal file
View file

@ -0,0 +1,14 @@
from util.config.validator import EXTRA_CA_DIRECTORY
def tarinfo_filter_partial(prefix):
def tarinfo_filter(tarinfo):
# remove leading directory info
tarinfo.name = tarinfo.name.replace(prefix, '')
# ignore any directory that isn't the specified extra ca one:
if tarinfo.isdir() and not tarinfo.name == EXTRA_CA_DIRECTORY:
return None
return tarinfo
return tarinfo_filter

29
config_app/config_util/test/test_tar.py Normal file
View file

@ -0,0 +1,29 @@
import pytest
from config_app.config_util.tar import tarinfo_filter_partial
from util.config.validator import EXTRA_CA_DIRECTORY
from test.fixtures import *
class MockTarInfo:
def __init__(self, name, isdir):
self.name = name
self.isdir = lambda: isdir
def __eq__(self, other):
return other is not None and self.name == other.name
@pytest.mark.parametrize('prefix,tarinfo,expected', [
# It should handle simple files
('Users/sam/', MockTarInfo('Users/sam/config.yaml', False), MockTarInfo('config.yaml', False)),
# It should allow the extra CA dir
('Users/sam/', MockTarInfo('Users/sam/%s' % EXTRA_CA_DIRECTORY, True), MockTarInfo('%s' % EXTRA_CA_DIRECTORY, True)),
# it should allow a file in that extra dir
('Users/sam/', MockTarInfo('Users/sam/%s/cert.crt' % EXTRA_CA_DIRECTORY, False), MockTarInfo('%s/cert.crt' % EXTRA_CA_DIRECTORY, False)),
# it should not allow a directory that isn't the CA dir
('Users/sam/', MockTarInfo('Users/sam/dirignore', True), None),
])
def test_tarinfo_filter(prefix, tarinfo, expected):
partial = tarinfo_filter_partial(prefix)
assert partial(tarinfo) == expected

14
config_app/js/components/download-tarball-modal/download-tarball-modal.component.html
View file

@ -8,15 +8,17 @@
<h4 class="modal-title"><span>Download Configuration</span></h4>
</div>
<!-- Body -->
<div class="modal-body">
<div class="modal-body download-tarball-modal">
<div ng-if="$ctrl.loadedConfig">
Please download your updated configuration. To deploy these changes to your Quay Enterprise instances, please
<a target="_blank" href="https://coreos.com/quay-enterprise/docs/latest/initial-setup.html">
see the docs.
</a>
<div class="modal__warning-box">
<i class="fas fa-exclamation-triangle" style="margin-right: 10px;"></i><strong>Warning:</strong>
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
<div class="fas co-alert co-alert-warning">
<strong>Warning: </strong>
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
</div>
</div>
</div>
<div ng-if="!$ctrl.loadedConfig">
@ -25,8 +27,10 @@
see the docs.
</a>
<div class="modal__warning-box">
<i class="fas fa-exclamation-triangle" style="margin-right: 10px;"></i><strong>Warning: </strong>
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
<div class="fas co-alert co-alert-warning">
<strong>Warning: </strong>
Your configuration and certificates are kept <i>unencrypted</i>. Please keep this file secure.
</div>
</div>
</div>
</div>

11
config_app/js/components/download-tarball-modal/download-tarball-modal.css
View file

@ -1,6 +1,11 @@
.modal__warning-box {
background-color: #ddd;
.co-dialog .modal-body.download-tarball-modal {
padding: 15px;
border-radius: 5px;
}
.modal__warning-box {
margin-top: 15px;
}
.modal__warning-box .co-alert.co-alert-warning::before {
font-family:Font Awesome\ 5 Free;
}

12
config_app/js/core-config-setup/core-config-setup.js
View file

@ -39,13 +39,7 @@ angular.module("quay-config")
$scope.SERVICES = [
{'id': 'redis', 'title': 'Redis'},
{'id': 'registry-storage', 'title': 'Registry Storage', 'condition': (config) => {
// We can skip validation if all of the storage locations are local, as we can't
// guarantee that this will be the same machine Q.E. will run under. Therefore,
// we just have a warning to the user that Q.E. won't start if the locations don't match
return Object.values(config.DISTRIBUTED_STORAGE_CONFIG)
.some(storageTuple => storageTuple[0] !== 'LocalStorage')
}},
{'id': 'registry-storage', 'title': 'Registry Storage'},
{'id': 'time-machine', 'title': 'Time Machine'},
@ -417,6 +411,10 @@ angular.module("quay-config")
$scope.saveConfiguration = function() {
$scope.savingConfiguration = true;
// Make sure to note that fully verified setup is completed. We use this as a signal
// in the setup tool.
$scope.config['SETUP_COMPLETE'] = true;
var data = {
'config': $scope.config,
'hostname': window.location.host,

9
util/config/validator.py
View file

@ -102,7 +102,8 @@ class ValidatorContext(object):
def __init__(self, config, user_password=None, http_client=None, context=None,
url_scheme_and_hostname=None, jwt_auth_max=None, registry_title=None,
ip_resolver=None, feature_sec_scanner=False, is_testing=False,
uri_creator=None, config_provider=None, instance_keys=None):
uri_creator=None, config_provider=None, instance_keys=None,
skip_localstorage_validation=False):
self.config = config
self.user = get_authenticated_user()
self.user_password = user_password
@ -117,10 +118,11 @@ class ValidatorContext(object):
self.uri_creator = uri_creator
self.config_provider = config_provider
self.instance_keys = instance_keys
self.skip_localstorage_validation = skip_localstorage_validation
@classmethod
def from_app(cls, app, config, user_password, ip_resolver, instance_keys, client=None,
config_provider=None):
config_provider=None, skip_localstorage_validation=False):
"""
Creates a ValidatorContext from an app config, with a given config to validate
:param app: the Flask app to pull configuration information from
@ -146,4 +148,5 @@ class ValidatorContext(object):
is_testing=app.config.get('TESTING', False),
uri_creator=get_blob_download_uri_getter(app.test_request_context('/'), url_scheme_and_hostname),
config_provider=config_provider,
instance_keys=instance_keys)
instance_keys=instance_keys,
skip_localstorage_validation=skip_localstorage_validation)

4
util/config/validators/validate_storage.py
View file

@ -12,6 +12,7 @@ class StorageValidator(BaseValidator):
client = validator_context.http_client
ip_resolver = validator_context.ip_resolver
config_provider = validator_context.config_provider
skip_localstorage_validation = validator_context.skip_localstorage_validation
replication_enabled = config.get('FEATURE_STORAGE_REPLICATION', False)
@ -20,6 +21,9 @@ class StorageValidator(BaseValidator):
raise ConfigValidationException('Storage configuration required')
for name, (storage_type, driver) in providers:
if skip_localstorage_validation and storage_type == 'LocalStorage':
continue
try:
if replication_enabled and storage_type == 'LocalStorage':
raise ConfigValidationException('Locally mounted directory not supported ' +