Check the returned ETag in Swift when streaming data

This ensures that if Swift mis-writes the data, we know immediately and can fail

storage/swift.py

@@ -11,7 +11,7 @@ import json
 from _pyio import BufferedReader
 from uuid import uuid4
 
-from swiftclient.client import Connection, ClientException
+from swiftclient.client import Connection, ClientException, ReadableToIterable
 from urlparse import urlparse
 from random import SystemRandom
 from hashlib import sha1
@@ -108,10 +108,14 @@ class SwiftStorage(BaseStorage):
     if content_encoding is not None:
       headers['Content-Encoding'] = content_encoding
 
+    is_filelike = hasattr(content, 'read')
+    if is_filelike:
+      content = ReadableToIterable(content, md5=True)
+
     try:
-      self._get_connection().put_object(self._swift_container, path, content,
-                                        chunk_size=chunk, content_type=content_type,
-                                        headers=headers)
+      etag = self._get_connection().put_object(self._swift_container, path, content,
+                                               chunk_size=chunk, content_type=content_type,
+                                               headers=headers)
     except ClientException:
       # We re-raise client exception here so that validation of config during setup can see
       # the client exception messages.
@@ -120,6 +124,16 @@ class SwiftStorage(BaseStorage):
       logger.exception('Could not put object at path %s: %s', path, ex)
       raise IOError("Could not put content: %s" % path)
 
+    # If we wrapped the content in a ReadableToIterable, compare its MD5 to the etag returned. If
+    # they don't match, raise an IOError indicating a write failure.
+    if is_filelike:
+      if etag != content.get_md5sum():
+        logger.error('Got mismatch in md5 etag for path %s: Expected %s, but server has %s', path,
+                     content.get_md5sum(), etag)
+        raise IOError('upload verification failed for path {0}:'
+                      'md5 mismatch, local {1} != remote {2}'
+                      .format(path, content.get_md5sum(), etag))
+
   def _head_object(self, path):
     path = self._normalize_path(path)
     try: