From 14b93f72ff460b606e8656b2a18d9af4f0aa01b8 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 11 Aug 2016 17:17:36 -0400 Subject: [PATCH] Make S3 access key and secret key optional, enabling IAM. If not specified, then boto will fallback to reading the credentials from IAM if on an EC2 machine. This should be safe as the validator will still ensure the credentials work if not specified. Fixes #1707 --- static/js/core-config-setup.js | 4 ++-- storage/cloud.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 1d41ac031..4d9f5b138 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -76,10 +76,10 @@ angular.module("core-config-setup", ['angularFileUpload']) ], 'S3Storage': [ - {'name': 's3_access_key', 'title': 'AWS Access Key', 'placeholder': 'accesskeyhere', 'kind': 'text'}, - {'name': 's3_secret_key', 'title': 'AWS Secret Key', 'placeholder': 'secretkeyhere', 'kind': 'text'}, {'name': 's3_bucket', 'title': 'S3 Bucket', 'placeholder': 'my-cool-bucket', 'kind': 'text'}, {'name': 'storage_path', 'title': 'Storage Directory', 'placeholder': '/path/inside/bucket', 'kind': 'text'}, + {'name': 's3_access_key', 'title': 'AWS Access Key (optional if using IAM)', 'placeholder': 'accesskeyhere', 'kind': 'text', 'optional': true}, + {'name': 's3_secret_key', 'title': 'AWS Secret Key (optional if using IAM)', 'placeholder': 'secretkeyhere', 'kind': 'text', 'optional': true}, {'name': 'host', 'title': 'S3 Host (optional)', 'placeholder': 's3.amazonaws.com', 'kind': 'text', 'optional': true} ], diff --git a/storage/cloud.py b/storage/cloud.py index 6f7d185c4..a9049ea58 100644 --- a/storage/cloud.py +++ b/storage/cloud.py @@ -47,7 +47,7 @@ class StreamReadKeyAsFile(BufferedIOBase): class _CloudStorage(BaseStorageV2): def __init__(self, metric_queue, connection_class, key_class, connect_kwargs, upload_params, - storage_path, access_key, secret_key, bucket_name): + storage_path, bucket_name, access_key=None, secret_key=None): super(_CloudStorage, self).__init__() self.automatic_chunk_size = 5 * 1024 * 1024 @@ -258,6 +258,7 @@ class _CloudStorage(BaseStorageV2): # First try to copy directly via boto, but only if the storages are the # same type, with the same access information. if (self.__class__ == destination.__class__ and + self._access_key and self._secret_key and self._access_key == destination._access_key and self._secret_key == destination._secret_key): logger.debug('Copying file from %s to %s via a direct boto copy', self._cloud_bucket, @@ -397,7 +398,7 @@ class _CloudStorage(BaseStorageV2): class S3Storage(_CloudStorage): - def __init__(self, metric_queue, storage_path, s3_access_key, s3_secret_key, s3_bucket, + def __init__(self, metric_queue, storage_path, s3_bucket, s3_access_key=None, s3_secret_key=None, host=None): upload_params = { 'encrypt_key': True, @@ -409,8 +410,9 @@ class S3Storage(_CloudStorage): connect_kwargs['host'] = host super(S3Storage, self).__init__(metric_queue, boto.s3.connection.S3Connection, boto.s3.key.Key, - connect_kwargs, upload_params, storage_path, s3_access_key, - s3_secret_key, s3_bucket) + connect_kwargs, upload_params, storage_path, s3_bucket, + access_key=s3_access_key or None, + secret_key=s3_secret_key or None) def setup(self): self.get_cloud_bucket().set_cors_xml(""" @@ -437,7 +439,7 @@ class GoogleCloudStorage(_CloudStorage): connect_kwargs = {} super(GoogleCloudStorage, self).__init__(metric_queue, boto.gs.connection.GSConnection, boto.gs.key.Key, connect_kwargs, upload_params, - storage_path, access_key, secret_key, bucket_name) + storage_path, bucket_name, access_key, secret_key) def setup(self): self.get_cloud_bucket().set_cors_xml(""" @@ -502,7 +504,7 @@ class RadosGWStorage(_CloudStorage): } super(RadosGWStorage, self).__init__(metric_queue, boto.s3.connection.S3Connection, boto.s3.key.Key, connect_kwargs, upload_params, - storage_path, access_key, secret_key, bucket_name) + storage_path, bucket_name, access_key, secret_key) # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624 def get_direct_download_url(self, path, expires_in=60, requires_cors=False):