diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index ae85f6d48..9edbffd75 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -232,6 +232,9 @@ ng-selected="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name] == value">{{ value }} +
+ {{ field.help_text }} +
See Documentation for more information
diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 889e13341..e3d5c3483 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -90,14 +90,23 @@ angular.module("core-config-setup", ['angularFileUpload']) 'SwiftStorage': [ {'name': 'auth_version', 'title': 'Swift Version', 'kind': 'option', 'values': [1, 2]}, - {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'}, - {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'}, + {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': 'http://swiftdomain/auth/v1.0', 'kind': 'text'}, + {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text', + 'help_text': 'The swift container for all objects. Must already exist inside Swift.'}, + {'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'}, - {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'}, - {'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'}, + {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text', + 'help_text': 'Note: For Swift V1, this is "username:password" (-U on the CLI).'}, + {'name': 'swift_password', 'title': 'Key/Password', 'placeholder': 'secretkeyhere', 'kind': 'text', + 'help_text': 'Note: For Swift V1, this is the API token (-K on the CLI).'}, {'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true}, + + {'name': 'temp_url_key', 'title': 'Temp URL Key (optional)', 'placholder': 'key-here', 'kind': 'text', 'optional': true, + 'help_url': 'https://coreos.com/products/enterprise-registry/docs/latest/swift-temp-url.html', + 'help_text': 'If enabled, will allow for faster pulls directly from Swift.'}, + {'name': 'os_options', 'title': 'OS Options', 'kind': 'map', 'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']} ] diff --git a/storage/basestorage.py b/storage/basestorage.py index e085a5a08..9406fffec 100644 --- a/storage/basestorage.py +++ b/storage/basestorage.py @@ -58,8 +58,9 @@ class BaseStorage(StoragePaths): """ Called to perform any storage system setup. """ pass - def validate(self): - """ Called to perform any custom storage system validation. """ + def validate(self, client): + """ Called to perform any custom storage system validation. The client is an HTTP + client to use for any external calls. """ pass def get_direct_download_url(self, path, expires_in=60, requires_cors=False): diff --git a/storage/local.py b/storage/local.py index b2cbc458c..333c9724d 100644 --- a/storage/local.py +++ b/storage/local.py @@ -144,7 +144,7 @@ class LocalStorage(BaseStorageV2): else: logger.debug('Content already exists at path: %s', final_path_abs) - def validate(self): + def validate(self, client): # Load the set of disk mounts. try: mounts = psutil.disk_partitions(all=True) diff --git a/storage/swift.py b/storage/swift.py index 42023c3c0..928715fb4 100644 --- a/storage/swift.py +++ b/storage/swift.py @@ -2,8 +2,12 @@ from swiftclient.client import Connection, ClientException from storage.basestorage import BaseStorage from util.registry.generatorfile import GeneratorFile - +from urlparse import urlparse from random import SystemRandom +from hashlib import sha1 +from time import time + +import hmac import string import logging @@ -12,7 +16,8 @@ logger = logging.getLogger(__name__) class SwiftStorage(BaseStorage): def __init__(self, swift_container, storage_path, auth_url, swift_user, - swift_password, auth_version=None, os_options=None, ca_cert_path=None): + swift_password, auth_version=None, os_options=None, ca_cert_path=None, + temp_url_key=None): self._swift_container = swift_container self._storage_path = storage_path @@ -22,6 +27,8 @@ class SwiftStorage(BaseStorage): self._swift_user = swift_user self._swift_password = swift_password + self._temp_url_key = temp_url_key + try: self._auth_version = int(auth_version or '2') except ValueError: @@ -51,8 +58,12 @@ class SwiftStorage(BaseStorage): return path - def _normalize_path(self, path=None): - path = self._storage_path + (path or '') + def _normalize_path(self, object_path=None): + path = self._storage_path + if not path.endswith('/'): + path = path + '/' + + path = path + (object_path or '') # Openstack does not like paths starting with '/' and we always normalize # to remove trailing '/' @@ -116,15 +127,66 @@ class SwiftStorage(BaseStorage): logger.exception('Could not head object: %s', path) return None - def get_direct_download_url(self, path, expires_in=60, requires_cors=False): + def get_direct_download_url(self, object_path, expires_in=60, requires_cors=False): if requires_cors: return None - # TODO(jschorr): This method is not strictly necessary but would result in faster operations - # when using this storage engine. However, the implementation (as seen in the link below) - # is not clean, so we punt on this for now. - # http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html - return None + # Reference: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html + if not self._temp_url_key: + return None + + # Retrieve the auth details for the connection. + try: + object_url_value, _ = self._get_connection().get_auth() + except ClientException: + logger.exception('Got client exception when trying to load Swift auth') + return None + + object_url = urlparse(object_url_value) + scheme = object_url.scheme + path = object_url.path + hostname = object_url.netloc + + if not path.endswith('/'): + path = path + '/' + + object_path = self._normalize_path(object_path) + + # Generate the signed HMAC body. + method = 'GET' + expires = int(time() + expires_in) + full_path = '%s%s/%s' % (path, self._swift_container, object_path) + + hmac_body = '%s\n%s\n%s' % (method, expires, full_path) + sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest() + + surl = '{scheme}://{host}{full_path}?temp_url_sig={sig}&temp_url_expires={expires}' + return surl.format(scheme=scheme, host=hostname, full_path=full_path, sig=sig, expires=expires) + + def validate(self, client): + if self._temp_url_key: + # Add a file to test direct download. + self.put_content('dd_path', 'testing 3456') + + # Generate a direct download URL. + dd_url = self.get_direct_download_url('dd_path') + + if not dd_url: + self.remove('dd_path') + raise Exception('Could not validate direct download URL; the token may be invalid.') + + # Try to retrieve the direct download URL. + response = client.get(dd_url, timeout=2) + + # Remove the test file. + self.remove('dd_path') + + if response.status_code != 200: + logger.debug('Direct download failure: %s => %s with body %s', dd_url, + response.status_code, response.text) + + msg = 'Direct download URL failed with status code %s. Please check your temp-url-key.' + raise Exception(msg % response.status_code) def get_content(self, path): return self._get_object(path) diff --git a/util/config/validator.py b/util/config/validator.py index e66cb341d..d4dc1cd39 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -83,7 +83,7 @@ def _validate_registry_storage(config, _): driver = get_storage_provider(config) # Run custom validation on the driver. - driver.validate() + driver.validate(app.config['HTTPCLIENT']) # Put and remove a temporary file to make sure the normal storage paths work. driver.put_content('_verify', 'testing 123')