Add a new RadosGW storage engine. Allow engines to distinguish not only between those that can support direct uploads and downloads, but those that support doing it through the browser. Rename resumeable->resumable.

storage/__init__.py

@@ -1,5 +1,5 @@
 from storage.local import LocalStorage
-from storage.cloud import S3Storage, GoogleCloudStorage
+from storage.cloud import S3Storage, GoogleCloudStorage, RadosGWStorage
 from storage.fakestorage import FakeStorage
 from storage.distributedstorage import DistributedStorage
 
@@ -8,6 +8,7 @@ STORAGE_DRIVER_CLASSES = {
   'LocalStorage': LocalStorage,
   'S3Storage': S3Storage,
   'GoogleCloudStorage': GoogleCloudStorage,
+  'RadosGWStorage': RadosGWStorage,
 }
 
 

storage/basestorage.py

@@ -54,10 +54,13 @@ class BaseStorage(StoragePaths):
   # Set the IO buffer to 64kB
   buffer_size = 64 * 1024
 
-  def get_direct_download_url(self, path, expires_in=60):
+  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
     return None
 
-  def get_supports_resumeable_downloads(self):
+  def get_direct_upload_url(self, path, mime_type, requires_cors=True):
+    return None
+
+  def get_supports_resumable_downloads(self):
     return False
 
   def get_content(self, path):
@@ -83,3 +86,6 @@ class BaseStorage(StoragePaths):
 
   def remove(self, path):
     raise NotImplementedError
+
+  def get_checksum(self, path):
+    raise NotImplementedError

storage/cloud.py

@@ -35,8 +35,8 @@ class StreamReadKeyAsFile(object):
 
 
 class _CloudStorage(BaseStorage):
-  def __init__(self, connection_class, key_class, upload_params, storage_path, access_key,
-               secret_key, bucket_name):
+  def __init__(self, connection_class, key_class, connect_kwargs, upload_params, storage_path,
+               access_key, secret_key, bucket_name):
     self._initialized = False
     self._bucket_name = bucket_name
     self._access_key = access_key
@@ -45,12 +45,14 @@ class _CloudStorage(BaseStorage):
     self._connection_class = connection_class
     self._key_class = key_class
     self._upload_params = upload_params
+    self._connect_kwargs = connect_kwargs
     self._cloud_conn = None
     self._cloud_bucket = None
 
   def _initialize_cloud_conn(self):
     if not self._initialized:
-      self._cloud_conn = self._connection_class(self._access_key, self._secret_key)
+      self._cloud_conn = self._connection_class(self._access_key, self._secret_key,
+                                                **self._connect_kwargs)
       self._cloud_bucket = self._cloud_conn.get_bucket(self._bucket_name)
       self._initialized = True
 
@@ -87,15 +89,22 @@ class _CloudStorage(BaseStorage):
     key.set_contents_from_string(content, **self._upload_params)
     return path
 
-  def get_supports_resumeable_downloads(self):
+  def get_supports_resumable_downloads(self):
     return True
 
-  def get_direct_download_url(self, path, expires_in=60):
+  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
     self._initialize_cloud_conn()
     path = self._init_path(path)
     k = self._key_class(self._cloud_bucket, path)
     return k.generate_url(expires_in)
 
+  def get_direct_upload_url(self, path, mime_type, requires_cors=True):
+    self._initialize_cloud_conn()
+    path = self._init_path(path)
+    key = self._key_class(self._cloud_bucket, path)
+    url = key.generate_url(300, 'PUT', headers={'Content-Type': mime_type}, encrypt_key=True)
+    return url
+
   def stream_read(self, path):
     self._initialize_cloud_conn()
     path = self._init_path(path)
@@ -179,21 +188,32 @@ class _CloudStorage(BaseStorage):
     for key in self._cloud_bucket.list(prefix=path):
       key.delete()
 
+  def get_checksum(self, path):
+    self._initialize_cloud_conn()
+    path = self._init_path(path)
+    key = self._key_class(self._cloud_bucket, path)
+    k = self._cloud_bucket.lookup(key)
+    return k.etag[1:-1][:7]
+
 
 class S3Storage(_CloudStorage):
   def __init__(self, storage_path, s3_access_key, s3_secret_key, s3_bucket):
     upload_params = {
       'encrypt_key': True,
     }
+    connect_kwargs = {}
     super(S3Storage, self).__init__(boto.s3.connection.S3Connection, boto.s3.key.Key,
-                                    upload_params, storage_path, s3_access_key, s3_secret_key,
-                                    s3_bucket)
+                                    connect_kwargs, upload_params, storage_path, s3_access_key,
+                                    s3_secret_key, s3_bucket)
 
 
 class GoogleCloudStorage(_CloudStorage):
   def __init__(self, storage_path, access_key, secret_key, bucket_name):
-    super(GoogleCloudStorage, self).__init__(boto.gs.connection.GSConnection, boto.gs.key.Key, {},
-                                             storage_path, access_key, secret_key, bucket_name)
+    upload_params = {}
+    connect_kwargs = {}
+    super(GoogleCloudStorage, self).__init__(boto.gs.connection.GSConnection, boto.gs.key.Key,
+                                             connect_kwargs, upload_params, storage_path,
+                                             access_key, secret_key, bucket_name)
 
   def stream_write(self, path, fp):
     # Minimum size of upload part size on S3 is 5MB
@@ -201,3 +221,30 @@ class GoogleCloudStorage(_CloudStorage):
     path = self._init_path(path)
     key = self._key_class(self._cloud_bucket, path)
     key.set_contents_from_stream(fp)
+
+
+class RadosGWStorage(_CloudStorage):
+  def __init__(self, hostname, is_secure, storage_path, access_key, secret_key, bucket_name):
+    upload_params = {}
+    connect_kwargs = {
+      'host': hostname,
+      'is_secure': is_secure,
+      'calling_format': boto.s3.connection.OrdinaryCallingFormat(),
+    }
+    super(RadosGWStorage, self).__init__(boto.s3.connection.S3Connection, boto.s3.key.Key,
+                                         connect_kwargs, upload_params, storage_path, access_key,
+                                         secret_key, bucket_name)
+
+  # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624
+  def get_direct_download_url(self, path, expires_in=60, requires_cors=False):
+    if requires_cors:
+      return None
+
+    return super(RadosGWStorage, self).get_direct_download_url(path, expires_in, requires_cors)
+
+  # TODO remove when radosgw supports cors: http://tracker.ceph.com/issues/8718#change-38624
+  def get_direct_upload_url(self, path, mime_type, requires_cors=True):
+    if requires_cors:
+      return None
+
+    return super(RadosGWStorage, self).get_direct_upload_url(path, mime_type, requires_cors)

storage/distributedstorage.py

@@ -31,6 +31,7 @@ class DistributedStorage(StoragePaths):
     self.preferred_locations = list(preferred_locations)
 
   get_direct_download_url = _location_aware(BaseStorage.get_direct_download_url)
+  get_direct_upload_url = _location_aware(BaseStorage.get_direct_upload_url)  
   get_content = _location_aware(BaseStorage.get_content)
   put_content = _location_aware(BaseStorage.put_content)
   stream_read = _location_aware(BaseStorage.stream_read)
@@ -39,4 +40,5 @@ class DistributedStorage(StoragePaths):
   list_directory = _location_aware(BaseStorage.list_directory)
   exists = _location_aware(BaseStorage.exists)
   remove = _location_aware(BaseStorage.remove)
-  get_supports_resumeable_downloads = _location_aware(BaseStorage.get_supports_resumeable_downloads)
+  get_checksum = _location_aware(BaseStorage.get_checksum)
+  get_supports_resumable_downloads = _location_aware(BaseStorage.get_supports_resumable_downloads)

storage/fakestorage.py

@@ -22,3 +22,6 @@ class FakeStorage(BaseStorage):
 
   def exists(self, path):
     return False
+
+  def get_checksum(self, path):
+    return 'abcdefg'

storage/local.py

@@ -1,4 +1,3 @@
-
 import os
 import shutil
 
@@ -80,3 +79,14 @@ class LocalStorage(BaseStorage):
       os.remove(path)
     except OSError:
       pass
+
+  def get_checksum(self, path):
+    path = self._init_path(path)
+    sha_hash = hashlib.sha256()
+    with open(path, 'r') as to_hash:
+      while True:
+        buf = to_hash.read(self.buffer_size)
+        if not buf:
+          break
+        sha_hash.update(buf)
+    return sha_hash.hexdigest()[:7]