initial import for Open Source 🎉

storage/test/test_azure.py

@@ -0,0 +1,217 @@
+import base64
+import md5
+import pytest
+import io
+
+from contextlib import contextmanager
+from urlparse import parse_qs, urlparse
+from httmock import urlmatch, HTTMock
+from xml.dom import minidom
+
+from azure.storage.blob import BlockBlobService
+
+from storage.azurestorage import AzureStorage
+
+@contextmanager
+def fake_azure_storage(files=None):
+  service = BlockBlobService(is_emulated=True)
+  endpoint = service.primary_endpoint.split('/')
+  container_name = 'somecontainer'
+  files = files if files is not None else {}
+
+  container_prefix = '/' + endpoint[1] + '/' + container_name
+
+  @urlmatch(netloc=endpoint[0], path=container_prefix + '$')
+  def get_container(url, request):
+    return {'status_code': 200, 'content': '{}'}
+
+  @urlmatch(netloc=endpoint[0], path=container_prefix + '/.+')
+  def container_file(url, request):
+    filename = url.path[len(container_prefix)+1:]
+
+    if request.method == 'GET' or request.method == 'HEAD':
+      return {
+        'status_code': 200 if filename in files else 404,
+        'content': files.get(filename),
+        'headers': {
+          'ETag': 'foobar',
+        },
+      }
+
+    if request.method == 'DELETE':
+      files.pop(filename)
+      return {
+        'status_code': 201,
+        'content': '',
+        'headers': {
+          'ETag': 'foobar',
+        },
+      }
+      
+
+    if request.method == 'PUT':
+      query_params = parse_qs(url.query)
+      if query_params.get('comp') == ['properties']:
+        return {
+          'status_code': 201,
+          'content': '{}',
+          'headers': {
+            'x-ms-request-server-encrypted': "false",
+            'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
+          }
+        }
+
+      if query_params.get('comp') == ['block']:
+        block_id = query_params['blockid'][0]
+        files[filename] = files.get(filename) or {}
+        files[filename][block_id] = request.body
+        return {
+          'status_code': 201,
+          'content': '{}',
+          'headers': {
+            'Content-MD5': base64.b64encode(md5.new(request.body).digest()),
+            'ETag': 'foo',
+            'x-ms-request-server-encrypted': "false",
+            'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
+          }
+        }
+      
+      if query_params.get('comp') == ['blocklist']:
+        parsed = minidom.parseString(request.body)
+        latest = parsed.getElementsByTagName('Latest')
+        combined = []
+        for latest_block in latest:
+          combined.append(files[filename][latest_block.childNodes[0].data])
+
+        files[filename] = ''.join(combined)
+        return {
+          'status_code': 201,
+          'content': '{}',
+          'headers': {
+            'Content-MD5': base64.b64encode(md5.new(files[filename]).digest()),
+            'ETag': 'foo',
+            'x-ms-request-server-encrypted': "false",
+            'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
+          }
+        }
+
+      if request.headers.get('x-ms-copy-source'):
+        copy_source = request.headers['x-ms-copy-source']
+        copy_path = urlparse(copy_source).path[len(container_prefix) + 1:]
+        files[filename] = files[copy_path]
+        return {
+          'status_code': 201,
+          'content': '{}',
+          'headers': {
+            'x-ms-request-server-encrypted': "false",
+            'x-ms-copy-status': 'success',
+            'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
+          }
+        }
+
+      files[filename] = request.body
+
+      return {
+        'status_code': 201,
+        'content': '{}',
+        'headers': {
+          'Content-MD5': base64.b64encode(md5.new(request.body).digest()),
+          'ETag': 'foo',
+          'x-ms-request-server-encrypted': "false",
+          'last-modified': 'Wed, 21 Oct 2015 07:28:00 GMT',
+        }
+      }
+
+    return {'status_code': 405, 'content': ''}
+
+  @urlmatch(netloc=endpoint[0], path='.+')
+  def catchall(url, request):
+    return {'status_code': 405, 'content': ''}
+
+  with HTTMock(get_container, container_file, catchall):
+    yield AzureStorage(None, 'somecontainer', '', 'someaccount', is_emulated=True)
+    
+def test_validate():
+  with fake_azure_storage() as s:
+    s.validate(None)
+
+def test_basics():
+  with fake_azure_storage() as s:
+    s.put_content('hello', 'hello world')
+    assert s.exists('hello')
+    assert s.get_content('hello') == 'hello world'
+    assert s.get_checksum('hello')
+    assert ''.join(list(s.stream_read('hello'))) == 'hello world'
+    assert s.stream_read_file('hello').read() == 'hello world'
+
+    s.remove('hello')
+    assert not s.exists('hello')
+
+def test_does_not_exist():
+  with fake_azure_storage() as s:
+    assert not s.exists('hello')
+
+    with pytest.raises(IOError):
+      s.get_content('hello')
+
+    with pytest.raises(IOError):
+      s.get_checksum('hello')
+
+    with pytest.raises(IOError):
+      list(s.stream_read('hello'))
+
+    with pytest.raises(IOError):
+      s.stream_read_file('hello')
+
+def test_stream_write():
+  fp = io.BytesIO()
+  fp.write('hello world!')
+  fp.seek(0)
+
+  with fake_azure_storage() as s:
+    s.stream_write('hello', fp)
+
+    assert s.get_content('hello') == 'hello world!'
+
+@pytest.mark.parametrize('chunk_size', [
+  (1),
+  (5),
+  (10),
+])
+def test_chunked_uploading(chunk_size):
+  with fake_azure_storage() as s:
+    string_data = 'hello world!'
+    chunks = [string_data[index:index+chunk_size] for index in range(0, len(string_data), chunk_size)]
+
+    uuid, metadata = s.initiate_chunked_upload()
+    start_index = 0
+    
+    for chunk in chunks:
+      fp = io.BytesIO()
+      fp.write(chunk)
+      fp.seek(0)
+
+      total_bytes_written, metadata, error = s.stream_upload_chunk(uuid, start_index, -1, fp,
+                                                                   metadata)
+      assert total_bytes_written == len(chunk)
+      assert metadata
+      assert not error
+
+      start_index += total_bytes_written
+
+    s.complete_chunked_upload(uuid, 'chunked', metadata)
+    assert s.get_content('chunked') == string_data
+
+def test_get_direct_download_url():
+  with fake_azure_storage() as s:
+    s.put_content('hello', 'world')
+    assert 'sig' in s.get_direct_download_url('hello')
+
+def test_copy_to():
+  files = {}
+
+  with fake_azure_storage(files=files) as s:
+    s.put_content('hello', 'hello world')
+    with fake_azure_storage(files=files) as s2:
+      s.copy_to(s2, 'hello')
+      assert s2.exists('hello')

storage/test/test_cloud_storage.py

@@ -0,0 +1,258 @@
+import os
+
+from StringIO import StringIO
+
+import pytest
+
+import moto
+import boto
+
+from moto import mock_s3_deprecated as mock_s3
+
+from storage import S3Storage, StorageContext
+from storage.cloud import _CloudStorage, _PartUploadMetadata
+from storage.cloud import _CHUNKS_KEY
+
+_TEST_CONTENT = os.urandom(1024)
+_TEST_BUCKET = 'some_bucket'
+_TEST_USER = 'someuser'
+_TEST_PASSWORD = 'somepassword'
+_TEST_PATH = 'some/cool/path'
+_TEST_CONTEXT = StorageContext('nyc', None, None, None, None)
+
+
+@pytest.fixture(scope='function')
+def storage_engine():
+  with mock_s3():
+    # Create a test bucket and put some test content.
+    boto.connect_s3().create_bucket(_TEST_BUCKET)
+    engine = S3Storage(_TEST_CONTEXT, 'some/path', _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
+    engine.put_content(_TEST_PATH, _TEST_CONTENT)
+
+    yield engine
+
+
+def test_basicop(storage_engine):
+  # Ensure the content exists.
+  assert storage_engine.exists(_TEST_PATH)
+
+  # Verify it can be retrieved.
+  assert storage_engine.get_content(_TEST_PATH) == _TEST_CONTENT
+
+  # Retrieve a checksum for the content.
+  storage_engine.get_checksum(_TEST_PATH)
+
+  # Remove the file.
+  storage_engine.remove(_TEST_PATH)
+
+  # Ensure it no longer exists.
+  with pytest.raises(IOError):
+    storage_engine.get_content(_TEST_PATH)
+
+  with pytest.raises(IOError):
+    storage_engine.get_checksum(_TEST_PATH)
+
+  assert not storage_engine.exists(_TEST_PATH)
+
+
+@pytest.mark.parametrize('bucket, username, password', [
+  pytest.param(_TEST_BUCKET, _TEST_USER, _TEST_PASSWORD, id='same credentials'),
+  pytest.param('another_bucket', 'blech', 'password', id='different credentials'),
+])
+def test_copy(bucket, username, password, storage_engine):
+  # Copy the content to another engine.
+  another_engine = S3Storage(_TEST_CONTEXT, 'another/path', _TEST_BUCKET, _TEST_USER,
+                             _TEST_PASSWORD)
+  boto.connect_s3().create_bucket('another_bucket')
+  storage_engine.copy_to(another_engine, _TEST_PATH)
+
+  # Verify it can be retrieved.
+  assert another_engine.get_content(_TEST_PATH) == _TEST_CONTENT
+
+
+def test_copy_with_error(storage_engine):
+  another_engine = S3Storage(_TEST_CONTEXT, 'another/path', 'anotherbucket', 'foo',
+                             'bar')
+
+  with pytest.raises(IOError):
+    storage_engine.copy_to(another_engine, _TEST_PATH)
+
+
+def test_stream_read(storage_engine):
+  # Read the streaming content.
+  data = ''.join(storage_engine.stream_read(_TEST_PATH))
+  assert data == _TEST_CONTENT
+
+
+def test_stream_read_file(storage_engine):
+  with storage_engine.stream_read_file(_TEST_PATH) as f:
+    assert f.read() == _TEST_CONTENT
+
+
+def test_stream_write(storage_engine):
+  new_data = os.urandom(4096)
+  storage_engine.stream_write(_TEST_PATH, StringIO(new_data), content_type='Cool/Type')
+  assert storage_engine.get_content(_TEST_PATH) == new_data
+
+
+def test_stream_write_error():
+  with mock_s3():
+    # Create an engine but not the bucket.
+    engine = S3Storage(_TEST_CONTEXT, 'some/path', _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
+
+    # Attempt to write to the uncreated bucket, which should raise an error.
+    with pytest.raises(IOError):
+      engine.stream_write(_TEST_PATH, StringIO('hello world'), content_type='Cool/Type')
+
+    assert not engine.exists(_TEST_PATH)
+
+
+@pytest.mark.parametrize('chunk_count', [
+  0,
+  1,
+  50,
+])
+@pytest.mark.parametrize('force_client_side', [
+  False,
+  True
+])
+def test_chunk_upload(storage_engine, chunk_count, force_client_side):
+  if chunk_count == 0 and force_client_side:
+    return
+
+  upload_id, metadata = storage_engine.initiate_chunked_upload()
+  final_data = ''
+
+  for index in range(0, chunk_count):
+    chunk_data = os.urandom(1024)
+    final_data = final_data + chunk_data
+    bytes_written, new_metadata, error = storage_engine.stream_upload_chunk(upload_id, 0,
+                                                                            len(chunk_data),
+                                                                            StringIO(chunk_data),
+                                                                            metadata)
+    metadata = new_metadata
+
+    assert bytes_written == len(chunk_data)
+    assert error is None
+    assert len(metadata[_CHUNKS_KEY]) == index + 1
+
+  # Complete the chunked upload.
+  storage_engine.complete_chunked_upload(upload_id, 'some/chunked/path', metadata,
+                                         force_client_side=force_client_side)
+
+  # Ensure the file contents are valid.
+  assert storage_engine.get_content('some/chunked/path') == final_data
+
+
+@pytest.mark.parametrize('chunk_count', [
+  0,
+  1,
+  50,
+])
+def test_cancel_chunked_upload(storage_engine, chunk_count):
+  upload_id, metadata = storage_engine.initiate_chunked_upload()
+
+  for _ in range(0, chunk_count):
+    chunk_data = os.urandom(1024)
+    _, new_metadata, _ = storage_engine.stream_upload_chunk(upload_id, 0,
+                                                            len(chunk_data),
+                                                            StringIO(chunk_data),
+                                                            metadata)
+    metadata = new_metadata
+
+  # Cancel the upload.
+  storage_engine.cancel_chunked_upload(upload_id, metadata)
+
+  # Ensure all chunks were deleted.
+  for chunk in metadata[_CHUNKS_KEY]:
+    assert not storage_engine.exists(chunk.path)
+
+
+def test_large_chunks_upload(storage_engine):
+  # Make the max chunk size much smaller for testing.
+  storage_engine.maximum_chunk_size = storage_engine.minimum_chunk_size * 2
+
+  upload_id, metadata = storage_engine.initiate_chunked_upload()
+
+  # Write a "super large" chunk, to ensure that it is broken into smaller chunks.
+  chunk_data = os.urandom(int(storage_engine.maximum_chunk_size * 2.5))
+  bytes_written, new_metadata, _ = storage_engine.stream_upload_chunk(upload_id, 0,
+                                                                      -1,
+                                                                      StringIO(chunk_data),
+                                                                      metadata)
+  assert len(chunk_data) == bytes_written
+
+  # Complete the chunked upload.
+  storage_engine.complete_chunked_upload(upload_id, 'some/chunked/path', new_metadata)
+
+  # Ensure the file contents are valid.
+  assert len(chunk_data) == len(storage_engine.get_content('some/chunked/path'))
+  assert storage_engine.get_content('some/chunked/path') == chunk_data
+
+
+def test_large_chunks_with_ragged_edge(storage_engine):
+  # Make the max chunk size much smaller for testing and force it to have a ragged edge.
+  storage_engine.maximum_chunk_size = storage_engine.minimum_chunk_size * 2 + 10
+
+  upload_id, metadata = storage_engine.initiate_chunked_upload()
+
+  # Write a few "super large" chunks, to ensure that it is broken into smaller chunks.
+  all_data = ''
+  for _ in range(0, 2):
+    chunk_data = os.urandom(int(storage_engine.maximum_chunk_size) + 20)
+    bytes_written, new_metadata, _ = storage_engine.stream_upload_chunk(upload_id, 0,
+                                                                        -1,
+                                                                        StringIO(chunk_data),
+                                                                        metadata)
+    assert len(chunk_data) == bytes_written
+    all_data = all_data + chunk_data
+    metadata = new_metadata
+
+  # Complete the chunked upload.
+  storage_engine.complete_chunked_upload(upload_id, 'some/chunked/path', new_metadata)
+
+  # Ensure the file contents are valid.
+  assert len(all_data) == len(storage_engine.get_content('some/chunked/path'))
+  assert storage_engine.get_content('some/chunked/path') == all_data
+
+
+@pytest.mark.parametrize('max_size, parts', [
+  (50, [
+    _PartUploadMetadata('foo', 0, 50),
+    _PartUploadMetadata('foo', 50, 50),
+  ]),
+
+  (40, [
+    _PartUploadMetadata('foo', 0, 25),
+    _PartUploadMetadata('foo', 25, 25),
+    _PartUploadMetadata('foo', 50, 25),
+    _PartUploadMetadata('foo', 75, 25)
+  ]),
+
+  (51, [
+    _PartUploadMetadata('foo', 0, 50),
+    _PartUploadMetadata('foo', 50, 50),
+  ]),
+
+  (49, [
+    _PartUploadMetadata('foo', 0, 25),
+    _PartUploadMetadata('foo', 25, 25),
+    _PartUploadMetadata('foo', 50, 25),
+    _PartUploadMetadata('foo', 75, 25),
+  ]),
+
+  (99, [
+    _PartUploadMetadata('foo', 0, 50),
+    _PartUploadMetadata('foo', 50, 50),
+  ]),
+
+  (100, [
+    _PartUploadMetadata('foo', 0, 100),
+  ]),
+])
+def test_rechunked(max_size, parts):
+  chunk = _PartUploadMetadata('foo', 0, 100)
+  rechunked = list(_CloudStorage._rechunk(chunk, max_size))
+  assert len(rechunked) == len(parts)
+  for index, chunk in enumerate(rechunked):
+    assert chunk == parts[index]

storage/test/test_cloudfront.py

@@ -0,0 +1,80 @@
+import pytest
+
+from contextlib import contextmanager
+from mock import patch
+from moto import mock_s3_deprecated as mock_s3
+import boto
+
+from app import config_provider
+from storage import CloudFrontedS3Storage, StorageContext
+from util.ipresolver import IPResolver
+from util.ipresolver.test.test_ipresolver import test_aws_ip, aws_ip_range_data, test_ip_range_cache
+from test.fixtures import *
+
+_TEST_CONTENT = os.urandom(1024)
+_TEST_BUCKET = 'some_bucket'
+_TEST_USER = 'someuser'
+_TEST_PASSWORD = 'somepassword'
+_TEST_PATH = 'some/cool/path'
+
+@pytest.fixture(params=[True, False])
+def ipranges_populated(request):
+  return request.param
+
+@pytest.fixture()
+def test_empty_ip_range_cache(empty_range_data):
+  sync_token = empty_range_data['syncToken']
+  all_amazon = IPResolver._parse_amazon_ranges(empty_range_data)
+  fake_cache = {
+    'sync_token': sync_token,
+  }
+  return fake_cache
+
+@pytest.fixture()
+def empty_range_data():
+  empty_range_data = {
+    'syncToken': 123456789,
+    'prefixes': [],
+  }
+  return empty_range_data
+
+@mock_s3
+def test_direct_download(test_aws_ip, test_empty_ip_range_cache, test_ip_range_cache, aws_ip_range_data, ipranges_populated, app):
+  ipresolver = IPResolver(app)
+  if ipranges_populated:
+    ipresolver.sync_token = test_ip_range_cache['sync_token'] if ipranges_populated else test_empty_ip_range_cache['sync_token']
+    ipresolver.amazon_ranges = test_ip_range_cache['all_amazon'] if ipranges_populated else test_empty_ip_range_cache['all_amazon']
+    context = StorageContext('nyc', None, None, config_provider, ipresolver)
+
+    # Create a test bucket and put some test content.
+    boto.connect_s3().create_bucket(_TEST_BUCKET)
+
+    engine = CloudFrontedS3Storage(context, 'cloudfrontdomain', 'keyid', 'test/data/test.pem', 'some/path',
+                                  _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
+    engine.put_content(_TEST_PATH, _TEST_CONTENT)
+    assert engine.exists(_TEST_PATH)
+
+    # Request a direct download URL for a request from a known AWS IP, and ensure we are returned an S3 URL.
+    assert 's3.amazonaws.com' in engine.get_direct_download_url(_TEST_PATH, test_aws_ip)
+
+    if ipranges_populated:
+      # Request a direct download URL for a request from a non-AWS IP, and ensure we are returned a CloudFront URL.
+      assert 'cloudfrontdomain' in engine.get_direct_download_url(_TEST_PATH, '1.2.3.4')
+    else:
+      # Request a direct download URL for a request from a non-AWS IP, but since IP Ranges isn't populated, we still
+      # get back an S3 URL.
+      assert 's3.amazonaws.com' in engine.get_direct_download_url(_TEST_PATH, '1.2.3.4')
+
+@mock_s3
+def test_direct_download_no_ip(test_aws_ip, aws_ip_range_data, ipranges_populated, app):
+  ipresolver = IPResolver(app)
+  context = StorageContext('nyc', None, None, config_provider, ipresolver)
+
+  # Create a test bucket and put some test content.
+  boto.connect_s3().create_bucket(_TEST_BUCKET)
+
+  engine = CloudFrontedS3Storage(context, 'cloudfrontdomain', 'keyid', 'test/data/test.pem', 'some/path',
+                                _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
+  engine.put_content(_TEST_PATH, _TEST_CONTENT)
+  assert engine.exists(_TEST_PATH)
+  assert 's3.amazonaws.com' in engine.get_direct_download_url(_TEST_PATH)

storage/test/test_storageproxy.py

@@ -0,0 +1,95 @@
+import os
+
+import pytest
+import requests
+
+from flask import Flask
+from flask_testing import LiveServerTestCase
+
+from storage import Storage
+from util.security.instancekeys import InstanceKeys
+
+from test.registry.liveserverfixture import *
+from test.fixtures import *
+
+
+@pytest.fixture(params=[True, False])
+def is_proxying_enabled(request):
+  return request.param
+
+
+@pytest.fixture()
+def server_executor(app):
+  def reload_app(server_hostname):
+    # Close any existing connection.
+    close_db_filter(None)
+
+    # Reload the database config.
+    app.config['SERVER_HOSTNAME'] = server_hostname[len('http://'):]
+    configure(app.config)
+    return 'OK'
+
+  executor = LiveServerExecutor()
+  executor.register('reload_app', reload_app)
+  return executor
+
+
+@pytest.fixture()
+def liveserver_app(app, server_executor, init_db_path, is_proxying_enabled):
+  server_executor.apply_blueprint_to_app(app)
+
+  if os.environ.get('DEBUG') == 'true':
+    app.config['DEBUG'] = True
+
+  app.config['TESTING'] = True
+  app.config['INSTANCE_SERVICE_KEY_KID_LOCATION'] = 'test/data/test.kid'
+  app.config['INSTANCE_SERVICE_KEY_LOCATION'] = 'test/data/test.pem'
+  app.config['INSTANCE_SERVICE_KEY_SERVICE'] = 'quay'
+
+  app.config['FEATURE_PROXY_STORAGE'] = is_proxying_enabled
+
+  app.config['DISTRIBUTED_STORAGE_CONFIG'] = {
+    'test': ['FakeStorage', {}],
+  }
+  app.config['DISTRIBUTED_STORAGE_PREFERENCE'] = ['test']
+  return app
+
+
+@pytest.fixture()
+def instance_keys(liveserver_app):
+  return InstanceKeys(liveserver_app)
+
+
+@pytest.fixture()
+def storage(liveserver_app, instance_keys):
+  return Storage(liveserver_app, instance_keys=instance_keys)
+
+
+@pytest.fixture()
+def app_reloader(liveserver, server_executor):
+  server_executor.on(liveserver).reload_app(liveserver.url)
+  yield
+
+
+@pytest.mark.skipif(os.environ.get('TEST_DATABASE_URI') is not None,
+                    reason="not supported for non SQLite testing")
+def test_storage_proxy_auth(storage, liveserver_app, liveserver_session, is_proxying_enabled,
+                            app_reloader):
+  # Activate direct download on the fake storage.
+  storage.put_content(['test'], 'supports_direct_download', 'true')
+
+  # Get the unwrapped URL.
+  direct_download_url = storage.get_direct_download_url(['test'], 'somepath')
+  proxy_index = direct_download_url.find('/_storage_proxy/')
+  if is_proxying_enabled:
+    assert proxy_index >= 0
+  else:
+    assert proxy_index == -1
+
+  # Ensure that auth returns the expected value.
+  headers = {
+    'X-Original-URI': direct_download_url[proxy_index:] if proxy_index else 'someurihere'
+  }
+
+  resp = liveserver_session.get('_storage_proxy_auth', headers=headers)
+  assert resp.status_code == (500 if not is_proxying_enabled else 200)

storage/test/test_swift.py

@@ -0,0 +1,327 @@
+import io
+import pytest
+import hashlib
+import copy
+
+from collections import defaultdict
+from mock import MagicMock, patch
+
+from storage import StorageContext
+from storage.swift import SwiftStorage, _EMPTY_SEGMENTS_KEY
+from swiftclient.client import ClientException
+
+base_args = {
+  'context': StorageContext('nyc', None, None, None, None),
+  'swift_container': 'container-name',
+  'storage_path': '/basepath',
+  'auth_url': 'https://auth.com',
+  'swift_user': 'root',
+  'swift_password': 'password',
+}
+
+class MockSwiftStorage(SwiftStorage):
+  def __init__(self, *args, **kwargs):
+    super(MockSwiftStorage, self).__init__(*args, **kwargs)
+    self._connection = MagicMock()
+
+  def _get_connection(self):
+    return self._connection
+
+class FakeSwiftStorage(SwiftStorage):
+  def __init__(self, fail_checksum=False,connection=None, *args, **kwargs):
+    super(FakeSwiftStorage, self).__init__(*args, **kwargs)
+    self._connection = connection or FakeSwift(fail_checksum=fail_checksum,
+                                               temp_url_key=kwargs.get('temp_url_key'))
+
+  def _get_connection(self):
+    return self._connection
+
+
+class FakeSwift(object):
+  def __init__(self, fail_checksum=False, temp_url_key=None):
+    self.containers = defaultdict(dict)
+    self.fail_checksum = fail_checksum
+    self.temp_url_key = temp_url_key
+
+  def get_auth(self):
+    if self.temp_url_key == 'exception':
+      raise ClientException('I failed!')
+      
+    return 'http://fake/swift', None
+
+  def head_object(self, container, path):
+    return self.containers.get(container, {}).get(path, {}).get('headers', None)
+
+  def copy_object(self, container, path, target):
+    pieces = target.split('/', 2)
+    _, content = self.get_object(container, path)
+    self.put_object(pieces[1], pieces[2], content)
+
+  def get_container(self, container, prefix=None, full_listing=None):
+    container_entries = self.containers[container]
+    objs = []
+    for path, data in list(container_entries.iteritems()):
+      if not prefix or path.startswith(prefix):
+        objs.append({
+          'name': path,
+          'bytes': len(data['content']),
+        })
+    return {}, objs
+
+  def put_object(self, container, path, content, chunk_size=None, content_type=None, headers=None):
+    if not isinstance(content, str):
+      if hasattr(content, 'read'):
+        content = content.read()
+      else:
+        content = ''.join(content)
+
+    self.containers[container][path] = {
+      'content': content,
+      'chunk_size': chunk_size,
+      'content_type': content_type,
+      'headers': headers or {'is': True},
+    }
+
+    digest = hashlib.md5()
+    digest.update(content)
+    return digest.hexdigest() if not self.fail_checksum else 'invalid'
+
+  def get_object(self, container, path, resp_chunk_size=None):
+    data = self.containers[container].get(path, {})
+    if 'X-Object-Manifest' in data['headers']:
+      new_contents = []
+      prefix = data['headers']['X-Object-Manifest']
+      for key, value in self.containers[container].iteritems():
+        if ('container-name/' + key).startswith(prefix):
+          new_contents.append((key, value['content']))
+
+      new_contents.sort(key=lambda value: value[0])
+
+      data = dict(data)
+      data['content'] = ''.join([nc[1] for nc in new_contents])
+      return bool(data), data.get('content')
+
+    return bool(data), data.get('content')
+
+  def delete_object(self, container, path):
+    self.containers[container].pop(path, None)
+
+
+class FakeQueue(object):
+  def __init__(self):
+    self.items = []
+
+  def get(self):
+    if not self.items:
+      return None
+
+    return self.items.pop()
+
+  def put(self, names, item, available_after=0):
+    self.items.append({
+      'names': names,
+      'item': item,
+      'available_after': available_after,
+    })
+
+def test_fixed_path_concat():
+  swift = MockSwiftStorage(**base_args)
+  swift.exists('object/path')
+  swift._get_connection().head_object.assert_called_with('container-name', 'basepath/object/path')
+
+def test_simple_path_concat():
+  simple_concat_args = dict(base_args)
+  simple_concat_args['simple_path_concat'] = True
+  swift = MockSwiftStorage(**simple_concat_args)
+  swift.exists('object/path')
+  swift._get_connection().head_object.assert_called_with('container-name', 'basepathobject/path')
+
+def test_delete_unknown_path():
+  swift = SwiftStorage(**base_args)
+  with pytest.raises(IOError):
+    swift.remove('someunknownpath')
+
+def test_simple_put_get():
+  swift = FakeSwiftStorage(**base_args)
+  assert not swift.exists('somepath')
+
+  swift.put_content('somepath', 'hello world!')
+  assert swift.exists('somepath')
+  assert swift.get_content('somepath') == 'hello world!'
+
+def test_stream_read_write():
+  swift = FakeSwiftStorage(**base_args)
+  assert not swift.exists('somepath')
+
+  swift.stream_write('somepath', io.BytesIO('some content here'))
+  assert swift.exists('somepath')
+  assert swift.get_content('somepath') == 'some content here'
+  assert ''.join(list(swift.stream_read('somepath'))) == 'some content here'
+
+def test_stream_read_write_invalid_checksum():
+  swift = FakeSwiftStorage(fail_checksum=True, **base_args)
+  assert not swift.exists('somepath')
+
+  with pytest.raises(IOError):
+    swift.stream_write('somepath', io.BytesIO('some content here'))
+
+def test_remove():
+  swift = FakeSwiftStorage(**base_args)
+  assert not swift.exists('somepath')
+
+  swift.put_content('somepath', 'hello world!')
+  assert swift.exists('somepath')
+
+  swift.remove('somepath')
+  assert not swift.exists('somepath')
+
+def test_copy_to():
+  swift = FakeSwiftStorage(**base_args)
+
+  modified_args = copy.deepcopy(base_args)
+  modified_args['swift_container'] = 'another_container'
+
+  another_swift = FakeSwiftStorage(connection=swift._connection, **modified_args)
+
+  swift.put_content('somepath', 'some content here')
+  swift.copy_to(another_swift, 'somepath')
+
+  assert swift.exists('somepath')
+  assert another_swift.exists('somepath')
+
+  assert swift.get_content('somepath') == 'some content here'
+  assert another_swift.get_content('somepath') == 'some content here'
+
+def test_copy_to_different():
+  swift = FakeSwiftStorage(**base_args)
+
+  modified_args = copy.deepcopy(base_args)
+  modified_args['swift_user'] = 'foobarbaz'
+  modified_args['swift_container'] = 'another_container'
+
+  another_swift = FakeSwiftStorage(**modified_args)
+
+  swift.put_content('somepath', 'some content here')
+  swift.copy_to(another_swift, 'somepath')
+
+  assert swift.exists('somepath')
+  assert another_swift.exists('somepath')
+
+  assert swift.get_content('somepath') == 'some content here'
+  assert another_swift.get_content('somepath') == 'some content here'
+
+def test_checksum():
+  swift = FakeSwiftStorage(**base_args)
+  swift.put_content('somepath', 'hello world!')
+  assert swift.get_checksum('somepath') is not None
+
+@pytest.mark.parametrize('read_until_end', [
+  (True,),
+  (False,),
+])
+@pytest.mark.parametrize('max_chunk_size', [
+  (10000000),
+  (10),
+  (5),
+  (2),
+  (1),
+])
+@pytest.mark.parametrize('chunks', [
+  (['this', 'is', 'some', 'chunked', 'data', '']),
+  (['this is a very large chunk of data', '']),
+  (['h', 'e', 'l', 'l', 'o', '']),
+])
+def test_chunked_upload(chunks, max_chunk_size, read_until_end):
+  swift = FakeSwiftStorage(**base_args)
+  uuid, metadata = swift.initiate_chunked_upload()
+
+  offset = 0
+
+  with patch('storage.swift._MAXIMUM_SEGMENT_SIZE', max_chunk_size):
+    for chunk in chunks:
+      chunk_length = len(chunk) if not read_until_end else -1
+      bytes_written, metadata, error = swift.stream_upload_chunk(uuid, offset, chunk_length,
+                                                                 io.BytesIO(chunk), metadata)
+      assert error is None
+      assert len(chunk) == bytes_written
+      offset += len(chunk)
+
+    swift.complete_chunked_upload(uuid, 'somepath', metadata)
+    assert swift.get_content('somepath') == ''.join(chunks)
+
+    # Ensure each of the segments exist.
+    for segment in metadata['segments']:
+      assert swift.exists(segment.path)
+
+    # Delete the file and ensure all of its segments were removed.
+    swift.remove('somepath')
+    assert not swift.exists('somepath')
+
+    for segment in metadata['segments']:
+      assert not swift.exists(segment.path)
+
+
+def test_cancel_chunked_upload():
+  chunk_cleanup_queue = FakeQueue()
+
+  args = dict(base_args)
+  args['context'] = StorageContext('nyc', None, chunk_cleanup_queue, None, None)
+
+  swift = FakeSwiftStorage(**args)
+  uuid, metadata = swift.initiate_chunked_upload()
+
+  chunks = ['this', 'is', 'some', 'chunked', 'data', '']
+  offset = 0
+  for chunk in chunks:
+    bytes_written, metadata, error = swift.stream_upload_chunk(uuid, offset, len(chunk),
+                                                               io.BytesIO(chunk), metadata)
+    assert error is None
+    assert len(chunk) == bytes_written
+    offset += len(chunk)
+
+  swift.cancel_chunked_upload(uuid, metadata)
+
+  found = chunk_cleanup_queue.get()
+  assert found is not None
+
+
+def test_empty_chunks_queued_for_deletion():
+  chunk_cleanup_queue = FakeQueue()
+  args = dict(base_args)
+  args['context'] = StorageContext('nyc', None, chunk_cleanup_queue, None, None)
+
+  swift = FakeSwiftStorage(**args)
+  uuid, metadata = swift.initiate_chunked_upload()
+
+  chunks = ['this', '', 'is', 'some', '', 'chunked', 'data', '']
+  offset = 0
+  for chunk in chunks:
+    length = len(chunk)
+    if length == 0:
+      length = 1
+
+    bytes_written, metadata, error = swift.stream_upload_chunk(uuid, offset, length,
+                                                               io.BytesIO(chunk), metadata)
+    assert error is None
+    assert len(chunk) == bytes_written
+    offset += len(chunk)
+
+  swift.complete_chunked_upload(uuid, 'somepath', metadata)
+  assert ''.join(chunks) == swift.get_content('somepath')
+
+  # Check the chunk deletion queue and ensure we have the last chunk queued.
+  found = chunk_cleanup_queue.get()
+  assert found is not None
+
+  found2 = chunk_cleanup_queue.get()
+  assert found2 is None
+
+@pytest.mark.parametrize('temp_url_key, expects_url', [
+  (None, False),
+  ('foobarbaz', True),
+  ('exception', False),
+])
+def test_get_direct_download_url(temp_url_key, expects_url):
+  swift = FakeSwiftStorage(temp_url_key=temp_url_key, **base_args)
+  swift.put_content('somepath', 'hello world!')
+  assert (swift.get_direct_download_url('somepath') is not None) == expects_url