Check the returned ETag in Swift when streaming data

This ensures that if Swift mis-writes the data, we know immediately and can fail

storage/test/test_swift.py

@@ -1,5 +1,6 @@
 import io
 import pytest
+import hashlib
 
 from collections import defaultdict
 from mock import MagicMock
@@ -25,24 +26,28 @@ class MockSwiftStorage(SwiftStorage):
     return self._connection
 
 class FakeSwiftStorage(SwiftStorage):
-  def __init__(self, *args, **kwargs):
+  def __init__(self, fail_checksum=False, *args, **kwargs):
     super(FakeSwiftStorage, self).__init__(*args, **kwargs)
-    self._connection = FakeSwift()
+    self._connection = FakeSwift(fail_checksum=fail_checksum)
 
   def _get_connection(self):
     return self._connection
 
 
 class FakeSwift(object):
-  def __init__(self):
+  def __init__(self, fail_checksum=False):
     self.containers = defaultdict(dict)
+    self.fail_checksum = fail_checksum
 
   def head_object(self, container, path):
     return self.containers[container].get(path)
 
   def put_object(self, container, path, content, chunk_size=None, content_type=None, headers=None):
     if not isinstance(content, str):
-      content = content.read()
+      if hasattr(content, 'read'):
+        content = content.read()
+      else:
+        content = ''.join(content)
 
     self.containers[container][path] = {
       'content': content,
@@ -51,6 +56,10 @@ class FakeSwift(object):
       'headers': headers,
     }
 
+    digest = hashlib.md5()
+    digest.update(content)
+    return digest.hexdigest() if not self.fail_checksum else 'invalid'
+
   def get_object(self, container, path, resp_chunk_size=None):
     data = self.containers[container].get(path, {})
     if 'X-Object-Manifest' in data['headers']:
@@ -124,6 +133,13 @@ def test_stream_read_write():
   assert swift.get_content('somepath') == 'some content here'
   assert ''.join(list(swift.stream_read('somepath'))) == 'some content here'
 
+def test_stream_read_write_invalid_checksum():
+  swift = FakeSwiftStorage(fail_checksum=True, **base_args)
+  assert not swift.exists('somepath')
+
+  with pytest.raises(IOError):
+    swift.stream_write('somepath', io.BytesIO('some content here'))
+
 def test_remove():
   swift = FakeSwiftStorage(**base_args)
   assert not swift.exists('somepath')