vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add automatic storage replication

Browse source 
Adds a worker to automatically replicate data between storages and update the database accordingly
This commit is contained in:
Joseph Schorr 2015-06-28 13:29:22 +03:00 • committed by Joseph Schorr
parent c693afca6a
commit 724b1607d7
18 changed files with 259 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

16
storage/distributedstorage.py
View file

@ -26,9 +26,15 @@ def _location_aware(unbound_func):
class DistributedStorage(StoragePaths):
def __init__(self, storages, preferred_locations=[]):
def __init__(self, storages, preferred_locations=[], default_locations=[]):
self._storages = dict(storages)
self.preferred_locations = list(preferred_locations)
self.default_locations = list(default_locations)
@property
def locations(self):
""" Returns the names of the locations supported. """
return list(self._storages.keys())
get_direct_download_url = _location_aware(BaseStorage.get_direct_download_url)
get_direct_upload_url = _location_aware(BaseStorage.get_direct_upload_url)
@ -42,6 +48,14 @@ class DistributedStorage(StoragePaths):
remove = _location_aware(BaseStorage.remove)
get_checksum = _location_aware(BaseStorage.get_checksum)
get_supports_resumable_downloads = _location_aware(BaseStorage.get_supports_resumable_downloads)
initiate_chunked_upload = _location_aware(BaseStorageV2.initiate_chunked_upload)
stream_upload_chunk = _location_aware(BaseStorageV2.stream_upload_chunk)
complete_chunked_upload = _location_aware(BaseStorageV2.complete_chunked_upload)
def copy_between(self, path, source_location, destination_location):
""" Copies a file between the source location and the destination location. """
source_storage = self._storages[source_location]
destination_storage = self._storages[destination_location]
source_storage.copy_to(destination_storage, path)