Add a new RadosGW storage engine. Allow engines to distinguish not only between those that can support direct uploads and downloads, but those that support doing it through the browser. Rename resumeable->resumable.

storage/cloud.py

@@ -35,8 +35,8 @@ class StreamReadKeyAsFile(object):
 
 
 class _CloudStorage(BaseStorage):
-  def __init__(self, connection_class, key_class, upload_params, storage_path, access_key,
-               secret_key, bucket_name):
+  def __init__(self, connection_class, key_class, connect_kwargs, upload_params, storage_path,
+               access_key, secret_key, bucket_name):
     self._initialized = False
     self._bucket_name = bucket_name
     self._access_key = access_key
@@ -45,12 +45,14 @@ class _CloudStorage(BaseStorage):
     self._connection_class = connection_class
     self._key_class = key_class
     self._upload_params = upload_params
+    self._connect_kwargs = connect_kwargs
     self._cloud_conn = None
     self._cloud_bucket = None
 
   def _initialize_cloud_conn(self):
     if not self._initialized:
-      self._cloud_conn = self._connection_class(self._access_key, self._secret_key)
+      self._cloud_conn = self._connection_class(self._access_key, self._secret_key,
+                                                **self._connect_kwargs)
       self._cloud_bucket = self._cloud_conn.get_bucket(self._bucket_name)
       self._initialized = True
 
@@ -87,15 +89,22 @@ class _CloudStorage(BaseStorage):
     key.set_contents_from_string(content, **self._upload_params)
     return path
 
-  def get_supports_resumeable_downloads(self):
+  def get_supports_resumable_downloads(self):
     return True
 
-  def get_direct_download_url(self, path, expires_in=60):
+  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
     self._initialize_cloud_conn()
     path = self._init_path(path)
     k = self._key_class(self._cloud_bucket, path)
     return k.generate_url(expires_in)
 
+  def get_direct_upload_url(self, path, mime_type, requires_cors=True):
+    self._initialize_cloud_conn()
+    path = self._init_path(path)
+    key = self._key_class(self._cloud_bucket, path)
+    url = key.generate_url(300, 'PUT', headers={'Content-Type': mime_type}, encrypt_key=True)
+    return url
+
   def stream_read(self, path):
     self._initialize_cloud_conn()
     path = self._init_path(path)
@@ -179,21 +188,32 @@ class _CloudStorage(BaseStorage):
     for key in self._cloud_bucket.list(prefix=path):
       key.delete()
 
+  def get_checksum(self, path):
+    self._initialize_cloud_conn()
+    path = self._init_path(path)
+    key = self._key_class(self._cloud_bucket, path)
+    k = self._cloud_bucket.lookup(key)
+    return k.etag[1:-1][:7]
+
 
 class S3Storage(_CloudStorage):
   def __init__(self, storage_path, s3_access_key, s3_secret_key, s3_bucket):
     upload_params = {
       'encrypt_key': True,
     }
+    connect_kwargs = {}
     super(S3Storage, self).__init__(boto.s3.connection.S3Connection, boto.s3.key.Key,
-                                    upload_params, storage_path, s3_access_key, s3_secret_key,
-                                    s3_bucket)
+                                    connect_kwargs, upload_params, storage_path, s3_access_key,
+                                    s3_secret_key, s3_bucket)
 
 
 class GoogleCloudStorage(_CloudStorage):
   def __init__(self, storage_path, access_key, secret_key, bucket_name):
-    super(GoogleCloudStorage, self).__init__(boto.gs.connection.GSConnection, boto.gs.key.Key, {},
-                                             storage_path, access_key, secret_key, bucket_name)
+    upload_params = {}
+    connect_kwargs = {}
+    super(GoogleCloudStorage, self).__init__(boto.gs.connection.GSConnection, boto.gs.key.Key,
+                                             connect_kwargs, upload_params, storage_path,
+                                             access_key, secret_key, bucket_name)
 
   def stream_write(self, path, fp):
     # Minimum size of upload part size on S3 is 5MB
@@ -201,3 +221,30 @@ class GoogleCloudStorage(_CloudStorage):
     path = self._init_path(path)
     key = self._key_class(self._cloud_bucket, path)
     key.set_contents_from_stream(fp)
+
+
+class RadosGWStorage(_CloudStorage):
+  def __init__(self, hostname, is_secure, storage_path, access_key, secret_key, bucket_name):
+    upload_params = {}
+    connect_kwargs = {
+      'host': hostname,
+      'is_secure': is_secure,
+      'calling_format': boto.s3.connection.OrdinaryCallingFormat(),
+    }
+    super(RadosGWStorage, self).__init__(boto.s3.connection.S3Connection, boto.s3.key.Key,
+                                         connect_kwargs, upload_params, storage_path, access_key,
+                                         secret_key, bucket_name)
+
+  # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624
+  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
+    if requires_cors:
+      return None
+
+    return super(RadosGWStorage, self).get_direct_download_url(path, expires_in, requires_cors)
+
+  # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624
+  def get_direct_upload_url(self, path, mime_type, requires_cors=True):
+    if requires_cors:
+      return None
+
+    return super(RadosGWStorage, self).get_direct_upload_url(path, mime_type, requires_cors)