Merge pull request #2973 from coreos-inc/joseph.schorr/QS-116/cloudfront-storage

Add support for configuring cloudfront storage

endpoints/api/suconfig.py

@@ -20,7 +20,7 @@ from endpoints.api import (ApiResource, nickname, resource, internal_only, show_
 from endpoints.common import common_login
 from util.config.configutil import add_enterprise_config_defaults
 from util.config.database import sync_database_with_config
-from util.config.validator import validate_service_for_config, CONFIG_FILENAMES
+from util.config.validator import validate_service_for_config, is_valid_config_upload_filename
 from util.license import decode_license, LicenseDecodeError
 
 import features
@@ -319,12 +319,12 @@ class SuperUserConfigFile(ApiResource):
   @verify_not_prod
   def get(self, filename):
     """ Returns whether the configuration file with the given name exists. """
-    if not filename in CONFIG_FILENAMES:
+    if not is_valid_config_upload_filename(filename):
       abort(404)
 
     if SuperUserPermission().can():
       return {
-       'exists': config_provider.volume_file_exists(filename)
+        'exists': config_provider.volume_file_exists(filename)
       }
 
     abort(403)
@@ -333,7 +333,7 @@ class SuperUserConfigFile(ApiResource):
   @verify_not_prod
   def post(self, filename):
     """ Updates the configuration file with the given name. """
-    if not filename in CONFIG_FILENAMES:
+    if not is_valid_config_upload_filename(filename):
       abort(404)
 
     # Note: This method can be called before the configuration exists

static/directives/config/config-setup-tool.html

@@ -300,6 +300,7 @@
                     <option value="GoogleCloudStorage">Google Cloud Storage</option>
                     <option value="RadosGWStorage">Ceph Object Gateway (RADOS)</option>
                     <option value="SwiftStorage">OpenStack Storage (Swift)</option>
+                    <option value="CloudFrontedS3Storage">CloudFront + Amazon S3</option>
                   </select>
 
                   <div class="co-alert co-alert-danger" ng-if="storageConfigError[$index].engine">
@@ -327,6 +328,11 @@
                         ng-if="field.kind == 'bool'">
                         {{ field.placeholder }}
                   </span>
+                  <span class="config-file-field"
+                        filename="{{ sc.location + '-' + field.filesuffix }}"
+                        binding="sc.data[1][field.name]"
+                        has-file="hasfile['contact-' + sc.location + '-' + field.filesuffix]"
+                        ng-if="field.kind == 'file'"></span>
                   <div ng-if="field.kind == 'option'">
                     <select class="form-control" ng-model="sc.data[1][field.name]">
                       <option ng-repeat="value in field.values" value="{{ value }}"
@@ -512,7 +518,7 @@
               <span class="config-file-field" filename="signing-public.gpg" has-file="hasfile.gpgSigningPublic"></span>
               <div class="help-text">
                 The certificate must be in PEM format.
-              </div
+              </div>
             </td>
           </tr>
           <tr>

static/js/core-config-setup.js

@@ -147,7 +147,20 @@ angular.module("core-config-setup", ['angularFileUpload'])
             {'name': 'os_options', 'title': 'OS Options', 'kind': 'map',
              'keys': ['tenant_id', 'auth_token', 'service_type', 'endpoint_type', 'tenant_name', 'object_storage_url', 'region_name',
                       'project_id', 'project_name', 'project_domain_name', 'user_domain_name', 'user_domain_id']}
-          ]
+          ],
+
+          'CloudFrontedS3Storage': [
+            {'name': 's3_bucket', 'title': 'S3 Bucket',  'placeholder': 'my-cool-bucket', 'kind': 'text'},
+            {'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'},
+            {'name': 's3_access_key', 'title': 'AWS Access Key (optional if using IAM)', 'placeholder': 'accesskeyhere', 'kind': 'text', 'optional': true},
+            {'name': 's3_secret_key', 'title': 'AWS Secret Key (optional if using IAM)', 'placeholder': 'secretkeyhere', 'kind': 'text', 'optional': true},
+            {'name': 'host', 'title': 'S3 Host (optional)', 'placeholder': 's3.amazonaws.com', 'kind': 'text', 'optional': true},
+            {'name': 'port', 'title': 'S3 Port (optional)', 'placeholder': '443', 'kind': 'text', 'pattern': '^[0-9]+$', 'optional': true},
+
+            {'name': 'cloudfront_distribution_domain', 'title': 'CloudFront Distribution Domain Name', 'placeholder': 'somesubdomain.cloudfront.net', 'pattern': '^([0-9a-zA-Z]+\\.)+[0-9a-zA-Z]+$', 'kind': 'text'},
+            {'name': 'cloudfront_key_id', 'title': 'CloudFront Key ID', 'placeholder': 'APKATHISISAKEYID', 'kind': 'text'},
+            {'name': 'cloudfront_privatekey_filename', 'title': 'CloudFront Private Key', 'filesuffix': 'cloudfront-signing-key.pem', 'kind': 'file'},
+          ],
         };
 
         $scope.enableFeature = function(config, feature) {
@@ -914,14 +927,20 @@ angular.module("core-config-setup", ['angularFileUpload'])
       scope: {
         'filename': '@filename',
         'skipCheckFile': '@skipCheckFile',
-        'hasFile': '=hasFile'
+        'hasFile': '=hasFile',
+        'binding': '=?binding'
       },
       controller: function($scope, $element, Restangular, $upload) {
         $scope.hasFile = false;
 
+        var setHasFile = function(hasFile) {
+          $scope.hasFile = hasFile;
+          $scope.binding = hasFile ? $scope.filename : null;
+        };
+
         $scope.onFileSelect = function(files) {
           if (files.length < 1) {
-            $scope.hasFile = false;
+            setHasFile(false);
             return;
           }
 
@@ -935,17 +954,17 @@ angular.module("core-config-setup", ['angularFileUpload'])
             $scope.uploadProgress = parseInt(100.0 * evt.loaded / evt.total);
             if ($scope.uploadProgress == 100) {
               $scope.uploadProgress = null;
-              $scope.hasFile = true;
+              setHasFile(true);
             }
           }).success(function(data, status, headers, config) {
             $scope.uploadProgress = null;
-            $scope.hasFile = true;
+            setHasFile(true);
           });
         };
 
         var loadStatus = function(filename) {
           Restangular.one('superuser/config/file/' + filename).get().then(function(resp) {
-            $scope.hasFile = resp['exists'];
+            setHasFile(false);
           });
         };
 

storage/cloud.py

@@ -619,7 +619,7 @@ class CloudFrontedS3Storage(S3Storage):
       return super(CloudFrontedS3Storage, self).get_direct_download_url(path, request_ip,
                                                                         expires_in, requires_cors,
                                                                         head)
-      
+
     resolved_ip_info = None
     logger.debug('Got direct download request for path "%s" with IP "%s"', path, request_ip)
     if request_ip is not None:
@@ -651,7 +651,7 @@ class CloudFrontedS3Storage(S3Storage):
       signer = private_key.signer(padding.PKCS1v15(), hashes.SHA1())
       signer.update(message)
       return signer.finalize()
-    
+
     return handler
 
   @lru_cache(maxsize=1)
@@ -663,8 +663,8 @@ class CloudFrontedS3Storage(S3Storage):
       return None
 
     with self._context.config_provider.get_volume_file(cloudfront_privatekey_filename) as key_file:
-        return serialization.load_pem_private_key(
+      return serialization.load_pem_private_key(
-            key_file.read(),
+        key_file.read(),
-            password=None,
+        password=None,
-            backend=default_backend()
+        backend=default_backend()
-        )
+      )

util/config/test/test_validator.py

@@ -0,0 +1,32 @@
+import pytest
+
+from util.config.validator import is_valid_config_upload_filename
+from util.config.validator import CONFIG_FILENAMES, CONFIG_FILE_SUFFIXES
+
+def test_valid_config_upload_filenames():
+  for filename in CONFIG_FILENAMES:
+    assert is_valid_config_upload_filename(filename)
+
+  for suffix in CONFIG_FILE_SUFFIXES:
+    assert is_valid_config_upload_filename('foo' + suffix)
+    assert not is_valid_config_upload_filename(suffix + 'foo')
+
+
+@pytest.mark.parametrize('filename, expect_valid', [
+  ('', False),
+  ('foo', False),
+  ('config.yaml', False),
+
+  ('ssl.cert', True),
+  ('ssl.key', True),
+
+  ('ssl.crt', False),
+
+  ('foobar-cloudfront-signing-key.pem', True),
+  ('foobaz-cloudfront-signing-key.pem', True),
+  ('barbaz-cloudfront-signing-key.pem', True),
+
+  ('barbaz-cloudfront-signing-key.pem.bak', False),
+])
+def test_is_valid_config_upload_filename(filename, expect_valid):
+  assert is_valid_config_upload_filename(filename) == expect_valid

util/config/validator.py

@@ -38,6 +38,7 @@ ACI_CERT_FILENAMES = ['signing-public.gpg', 'signing-private.gpg']
 LDAP_FILENAMES = [LDAP_CERT_FILENAME]
 CONFIG_FILENAMES = (SSL_FILENAMES + DB_SSL_FILENAMES + JWT_FILENAMES + ACI_CERT_FILENAMES +
                     LDAP_FILENAMES)
+CONFIG_FILE_SUFFIXES = ['-cloudfront-signing-key.pem']
 EXTRA_CA_DIRECTORY = 'extra_ca_certs'
 
 VALIDATORS = {
@@ -83,3 +84,13 @@ def validate_service_for_config(service, config, password=None):
       'status': False,
       'reason': str(ex)
     }
+
+
+def is_valid_config_upload_filename(filename):
+  """ Returns true if and only if the given filename is one which is supported for upload
+      from the configuration UI tool.
+  """
+  if filename in CONFIG_FILENAMES:
+    return True
+
+  return any([filename.endswith(suffix) for suffix in CONFIG_FILE_SUFFIXES])