vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fix storage replication for CAS and add tests

Browse source
This commit is contained in:
Joseph Schorr 2016-07-11 14:33:29 -04:00
parent 3044f8ecbd
commit 5cd793331e
3 changed files with 86 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

26
workers/storagereplication.py
View file

@ -6,7 +6,7 @@ from app import app, storage, image_replication_queue
from data.database import CloseForLongOperation
from data import model
from storage.basestorage import StoragePaths
from workers.queueworker import QueueWorker
from workers.queueworker import QueueWorker, WorkerUnhealthyException
logger = logging.getLogger(__name__)
@ -18,7 +18,11 @@ class StorageReplicationWorker(QueueWorker):
logger.debug('Starting replication of image storage %s', storage_uuid)
namespace = model.user.get_namespace_user_by_user_id(job_details['namespace_user_id'])
if not self.replicate_storage(namespace, storage_uuid):
raise WorkerUnhealthyException()
def replicate_storage(self, namespace, storage_uuid):
# Lookup the namespace and its associated regions.
if not namespace:
logger.debug('Unknown namespace: %s', namespace)
@ -48,24 +52,26 @@ class StorageReplicationWorker(QueueWorker):
for location in locations_missing:
logger.debug('Copying image storage %s to location %s', partial_storage.uuid, location)
# Copy the various paths.
paths = [storage_paths.v1_image_layer_path]
# Copy the binary data.
path_to_copy = model.storage.get_layer_path(partial_storage)
copied = False
try:
for path_builder in paths:
current_path = path_builder(partial_storage.uuid)
if storage.exists([existing_location], path_to_copy):
with CloseForLongOperation(app.config):
storage.copy_between(current_path, existing_location, location)
storage.copy_between(path_to_copy, existing_location, location)
copied = True
except:
logger.exception('Exception when copying path %s of image storage %s to location %s',
current_path, partial_storage.uuid, location)
path_to_copy, partial_storage.uuid, location)
return False
# Create the storage location record for the storage now that the copies have
# completed.
model.storage.add_storage_placement(partial_storage, location)
logger.debug('Finished copy of image storage %s to location %s',
partial_storage.uuid, location)
if copied:
model.storage.add_storage_placement(partial_storage, location)
logger.debug('Finished copy of image storage %s to location %s',
partial_storage.uuid, location)
logger.debug('Completed replication of image storage %s to locations %s',
partial_storage.uuid, locations_missing)