cherry pick

workers/test/test_storagereplication.py

@@ -0,0 +1,171 @@
+import hashlib
+
+import pytest
+
+from data import model, database
+from storage.basestorage import StoragePaths
+from storage.fakestorage import FakeStorage
+from storage.distributedstorage import DistributedStorage
+from workers.storagereplication import (StorageReplicationWorker, JobException,
+                                        WorkerUnhealthyException)
+
+from test.fixtures import *
+
+
+@pytest.fixture()
+def storage_user(app):
+  user = model.user.get_user('devtable')
+  database.UserRegion.create(user=user,
+                             location=database.ImageStorageLocation.get(name='local_us'))
+  database.UserRegion.create(user=user,
+                             location=database.ImageStorageLocation.get(name='local_eu'))
+  return user
+
+
+@pytest.fixture()
+def storage_paths():
+  return StoragePaths()
+
+
+@pytest.fixture()
+def replication_worker():
+  return StorageReplicationWorker(None)
+
+
+@pytest.fixture()
+def storage():
+  return DistributedStorage({'local_us': FakeStorage('local'), 'local_eu': FakeStorage('local')},
+                            ['local_us'])
+
+
+def test_storage_replication_v1(storage_user, storage_paths, replication_worker, storage, app):
+  # Add a storage entry with a V1 path.
+  v1_storage = model.storage.create_v1_storage('local_us')
+  content_path = storage_paths.v1_image_layer_path(v1_storage.uuid)
+  storage.put_content(['local_us'], content_path, 'some content')
+
+  # Call replicate on it and verify it replicates.
+  replication_worker.replicate_storage(storage_user, v1_storage.uuid, storage)
+
+  # Ensure that the data was replicated to the other "region".
+  assert storage.get_content(['local_eu'], content_path) == 'some content'
+
+  locations = model.storage.get_storage_locations(v1_storage.uuid)
+  assert len(locations) == 2
+
+
+def test_storage_replication_cas(storage_user, storage_paths, replication_worker, storage, app):
+  # Add a storage entry with a CAS path.
+  content_checksum = 'sha256:' + hashlib.sha256('some content').hexdigest()
+  cas_storage = database.ImageStorage.create(cas_path=True, content_checksum=content_checksum)
+
+  location = database.ImageStorageLocation.get(name='local_us')
+  database.ImageStoragePlacement.create(storage=cas_storage, location=location)
+
+  content_path = storage_paths.blob_path(cas_storage.content_checksum)
+  storage.put_content(['local_us'], content_path, 'some content')
+
+  # Call replicate on it and verify it replicates.
+  replication_worker.replicate_storage(storage_user, cas_storage.uuid, storage)
+
+  # Ensure that the data was replicated to the other "region".
+  assert storage.get_content(['local_eu'], content_path) == 'some content'
+
+  locations = model.storage.get_storage_locations(cas_storage.uuid)
+  assert len(locations) == 2
+
+
+def test_storage_replication_missing_base(storage_user, storage_paths, replication_worker, storage,
+                                          app):
+  # Add a storage entry with a CAS path.
+  content_checksum = 'sha256:' + hashlib.sha256('some content').hexdigest()
+  cas_storage = database.ImageStorage.create(cas_path=True, content_checksum=content_checksum)
+
+  location = database.ImageStorageLocation.get(name='local_us')
+  database.ImageStoragePlacement.create(storage=cas_storage, location=location)
+
+  # Attempt to replicate storage. This should fail because the layer is missing from the base
+  # storage.
+  with pytest.raises(JobException):
+    replication_worker.replicate_storage(storage_user, cas_storage.uuid, storage,
+                                         backoff_check=False)
+
+  # Ensure the storage location count remains 1. This is technically inaccurate, but that's okay
+  # as we still require at least one location per storage.
+  locations = model.storage.get_storage_locations(cas_storage.uuid)
+  assert len(locations) == 1
+
+
+def test_storage_replication_copy_error(storage_user, storage_paths, replication_worker, storage,
+                                        app):
+  # Add a storage entry with a CAS path.
+  content_checksum = 'sha256:' + hashlib.sha256('some content').hexdigest()
+  cas_storage = database.ImageStorage.create(cas_path=True, content_checksum=content_checksum)
+
+  location = database.ImageStorageLocation.get(name='local_us')
+  database.ImageStoragePlacement.create(storage=cas_storage, location=location)
+
+  content_path = storage_paths.blob_path(cas_storage.content_checksum)
+  storage.put_content(['local_us'], content_path, 'some content')
+
+  # Tell storage to break copying.
+  storage.put_content(['local_us'], 'break_copying', 'true')
+
+  # Attempt to replicate storage. This should fail because the write fails.
+  with pytest.raises(JobException):
+    replication_worker.replicate_storage(storage_user, cas_storage.uuid, storage,
+                                         backoff_check=False)
+
+  # Ensure the storage location count remains 1.
+  locations = model.storage.get_storage_locations(cas_storage.uuid)
+  assert len(locations) == 1
+
+
+def test_storage_replication_copy_didnot_copy(storage_user, storage_paths, replication_worker,
+                                              storage, app):
+  # Add a storage entry with a CAS path.
+  content_checksum = 'sha256:' + hashlib.sha256('some content').hexdigest()
+  cas_storage = database.ImageStorage.create(cas_path=True, content_checksum=content_checksum)
+
+  location = database.ImageStorageLocation.get(name='local_us')
+  database.ImageStoragePlacement.create(storage=cas_storage, location=location)
+
+  content_path = storage_paths.blob_path(cas_storage.content_checksum)
+  storage.put_content(['local_us'], content_path, 'some content')
+
+  # Tell storage to fake copying (i.e. not actually copy the data).
+  storage.put_content(['local_us'], 'fake_copying', 'true')
+
+  # Attempt to replicate storage. This should fail because the copy doesn't actually do the copy.
+  with pytest.raises(JobException):
+    replication_worker.replicate_storage(storage_user, cas_storage.uuid, storage,
+                                         backoff_check=False)
+
+  # Ensure the storage location count remains 1.
+  locations = model.storage.get_storage_locations(cas_storage.uuid)
+  assert len(locations) == 1
+
+
+def test_storage_replication_copy_unhandled_exception(storage_user, storage_paths,
+                                                      replication_worker, storage, app):
+  # Add a storage entry with a CAS path.
+  content_checksum = 'sha256:' + hashlib.sha256('some content').hexdigest()
+  cas_storage = database.ImageStorage.create(cas_path=True, content_checksum=content_checksum)
+
+  location = database.ImageStorageLocation.get(name='local_us')
+  database.ImageStoragePlacement.create(storage=cas_storage, location=location)
+
+  content_path = storage_paths.blob_path(cas_storage.content_checksum)
+  storage.put_content(['local_us'], content_path, 'some content')
+
+  # Tell storage to raise an exception when copying.
+  storage.put_content(['local_us'], 'except_copying', 'true')
+
+  # Attempt to replicate storage. This should fail because the copy raises an unhandled exception.
+  with pytest.raises(WorkerUnhealthyException):
+    replication_worker.replicate_storage(storage_user, cas_storage.uuid, storage,
+                                         backoff_check=False)
+
+  # Ensure the storage location count remains 1.
+  locations = model.storage.get_storage_locations(cas_storage.uuid)
+  assert len(locations) == 1