Merge pull request #484 from coreos-inc/swifturl

Add support for direct download in Swift storage engine

static/directives/config/config-setup-tool.html

@@ -232,6 +232,9 @@
                             ng-selected="config.DISTRIBUTED_STORAGE_CONFIG.local[1][field.name] == value">{{ value }}</option>
                   </select>
                 </div>
+                <div class="help-text" ng-if="field.help_text">
+                  {{ field.help_text }}
+                </div>
                 <div class="help-text" ng-if="field.help_url">
                   See <a href="{{ field.help_url }}" target="_blank">Documentation</a> for more information
                 </div>

static/js/core-config-setup.js

@@ -90,14 +90,23 @@ angular.module("core-config-setup", ['angularFileUpload'])
 
           'SwiftStorage': [
             {'name': 'auth_version', 'title': 'Swift Version', 'kind': 'option', 'values': [1, 2]},
-            {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': '', 'kind': 'text'},
+            {'name': 'auth_url', 'title': 'Swift Auth URL', 'placeholder': 'http://swiftdomain/auth/v1.0', 'kind': 'text'},
-            {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text'},
+            {'name': 'swift_container', 'title': 'Swift Container Name', 'placeholder': 'mycontainer', 'kind': 'text',
+             'help_text': 'The swift container for all objects. Must already exist inside Swift.'},
+
             {'name': 'storage_path', 'title': 'Storage Path', 'placeholder': '/path/inside/container', 'kind': 'text'},
 
-            {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text'},
+            {'name': 'swift_user', 'title': 'Username', 'placeholder': 'accesskeyhere', 'kind': 'text',
-            {'name': 'swift_password', 'title': 'Password/Key', 'placeholder': 'secretkeyhere', 'kind': 'text'},
+             'help_text': 'Note: For Swift V1, this is "username:password" (-U on the CLI).'},
+            {'name': 'swift_password', 'title': 'Key/Password', 'placeholder': 'secretkeyhere', 'kind': 'text',
+             'help_text': 'Note: For Swift V1, this is the API token (-K on the CLI).'},
 
             {'name': 'ca_cert_path', 'title': 'CA Cert Filename', 'placeholder': 'conf/stack/swift.cert', 'kind': 'text', 'optional': true},
+
+            {'name': 'temp_url_key', 'title': 'Temp URL Key (optional)', 'placholder': 'key-here', 'kind': 'text', 'optional': true,
+             'help_url': 'https://coreos.com/products/enterprise-registry/docs/latest/swift-temp-url.html',
+             'help_text': 'If enabled, will allow for faster pulls directly from Swift.'},
+
             {'name': 'os_options', 'title': 'OS Options', 'kind': 'map',
              'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name']}
           ]

storage/basestorage.py

@@ -58,8 +58,9 @@ class BaseStorage(StoragePaths):
     """ Called to perform any storage system setup. """
     pass
 
-  def validate(self):
+  def validate(self, client):
-    """ Called to perform any custom storage system validation. """
+    """ Called to perform any custom storage system validation. The client is an HTTP
+        client to use for any external calls. """
     pass
 
   def get_direct_download_url(self, path, expires_in=60, requires_cors=False):

storage/local.py

@@ -144,7 +144,7 @@ class LocalStorage(BaseStorageV2):
     else:
       logger.debug('Content already exists at path: %s', final_path_abs)
 
-  def validate(self):
+  def validate(self, client):
     # Load the set of disk mounts.
     try:
       mounts = psutil.disk_partitions(all=True)

storage/swift.py

@@ -2,8 +2,12 @@
 from swiftclient.client import Connection, ClientException
 from storage.basestorage import BaseStorage
 from util.registry.generatorfile import GeneratorFile
-
+from urlparse import urlparse
 from random import SystemRandom
+from hashlib import sha1
+from time import time
+
+import hmac
 import string
 import logging
 
@@ -12,7 +16,8 @@ logger = logging.getLogger(__name__)
 
 class SwiftStorage(BaseStorage):
   def __init__(self, swift_container, storage_path, auth_url, swift_user,
-               swift_password, auth_version=None, os_options=None, ca_cert_path=None):
+               swift_password, auth_version=None, os_options=None, ca_cert_path=None,
+               temp_url_key=None):
     self._swift_container = swift_container
     self._storage_path = storage_path
 
@@ -22,6 +27,8 @@ class SwiftStorage(BaseStorage):
     self._swift_user = swift_user
     self._swift_password = swift_password
 
+    self._temp_url_key = temp_url_key
+
     try:
       self._auth_version = int(auth_version or '2')
     except ValueError:
@@ -51,8 +58,12 @@ class SwiftStorage(BaseStorage):
 
     return path
 
-  def _normalize_path(self, path=None):
+  def _normalize_path(self, object_path=None):
-    path = self._storage_path + (path or '')
+    path = self._storage_path
+    if not path.endswith('/'):
+      path = path + '/'
+
+    path = path + (object_path or '')
 
     # Openstack does not like paths starting with '/' and we always normalize
     # to remove trailing '/'
@@ -116,15 +127,66 @@ class SwiftStorage(BaseStorage):
       logger.exception('Could not head object: %s', path)
       return None
 
-  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
+  def get_direct_download_url(self, object_path, expires_in=60, requires_cors=False):
     if requires_cors:
       return None
 
-    # TODO(jschorr): This method is not strictly necessary but would result in faster operations
+    # Reference: http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html
-    # when using this storage engine. However, the implementation (as seen in the link below)
+    if not self._temp_url_key:
-    # is not clean, so we punt on this for now.
+      return None
-    # http://docs.openstack.org/juno/config-reference/content/object-storage-tempurl.html
+
-    return None
+    # Retrieve the auth details for the connection.
+    try:
+      object_url_value, _ = self._get_connection().get_auth()
+    except ClientException:
+      logger.exception('Got client exception when trying to load Swift auth')
+      return None
+
+    object_url = urlparse(object_url_value)
+    scheme = object_url.scheme
+    path = object_url.path
+    hostname = object_url.netloc
+
+    if not path.endswith('/'):
+      path = path + '/'
+
+    object_path = self._normalize_path(object_path)
+
+    # Generate the signed HMAC body.
+    method = 'GET'
+    expires = int(time() + expires_in)
+    full_path = '%s%s/%s' % (path, self._swift_container, object_path)
+
+    hmac_body = '%s\n%s\n%s' % (method, expires, full_path)
+    sig = hmac.new(self._temp_url_key.encode('utf-8'), hmac_body.encode('utf-8'), sha1).hexdigest()
+
+    surl = '{scheme}://{host}{full_path}?temp_url_sig={sig}&temp_url_expires={expires}'
+    return surl.format(scheme=scheme, host=hostname, full_path=full_path, sig=sig, expires=expires)
+
+  def validate(self, client):
+    if self._temp_url_key:
+      # Add a file to test direct download.
+      self.put_content('dd_path', 'testing 3456')
+
+      # Generate a direct download URL.
+      dd_url = self.get_direct_download_url('dd_path')
+
+      if not dd_url:
+        self.remove('dd_path')
+        raise Exception('Could not validate direct download URL; the token may be invalid.')
+
+      # Try to retrieve the direct download URL.
+      response = client.get(dd_url, timeout=2)
+
+      # Remove the test file.
+      self.remove('dd_path')
+
+      if response.status_code != 200:
+        logger.debug('Direct download failure: %s => %s with body %s', dd_url,
+                     response.status_code, response.text)
+
+        msg = 'Direct download URL failed with status code %s. Please check your temp-url-key.'
+        raise Exception(msg % response.status_code)
 
   def get_content(self, path):
     return self._get_object(path)

util/config/validator.py

@@ -83,7 +83,7 @@ def _validate_registry_storage(config, _):
   driver = get_storage_provider(config)
 
   # Run custom validation on the driver.
-  driver.validate()
+  driver.validate(app.config['HTTPCLIENT'])
 
   # Put and remove a temporary file to make sure the normal storage paths work.
   driver.put_content('_verify', 'testing 123')