Add CloudFrontedS3Storage, which redirects to CloudFront for non-S3 ips

storage/test/test_cloudfront.py

@@ -0,0 +1,56 @@
+import pytest
+
+from httmock import urlmatch, HTTMock
+from moto import mock_s3
+import boto
+
+from app import config_provider
+from storage import CloudFrontedS3Storage, StorageContext
+from util.ipresolver import IPResolver
+from util.ipresolver.test.test_ipresolver import http_client, test_aws_ip, aws_ip_range_handler
+from test.fixtures import *
+
+_TEST_CONTENT = os.urandom(1024)
+_TEST_BUCKET = 'some_bucket'
+_TEST_USER = 'someuser'
+_TEST_PASSWORD = 'somepassword'
+_TEST_PATH = 'some/cool/path'
+
+@pytest.fixture(params=[True, False])
+def ipranges_populated(request):
+  return request.param
+
+@pytest.fixture()
+def ipresolver(http_client, aws_ip_range_handler, ipranges_populated, app):
+  with HTTMock(aws_ip_range_handler):
+    ipresolver = IPResolver(app, client=http_client)
+    if ipranges_populated:
+      assert ipresolver._update_aws_ip_range()
+  
+    return ipresolver
+
+@pytest.fixture()
+def storage_context(ipresolver, app):
+  return StorageContext('nyc', None, None, config_provider, ipresolver)
+
+@mock_s3
+def test_direct_download(storage_context, test_aws_ip, ipranges_populated, app):
+  # Create a test bucket and put some test content.
+  boto.connect_s3().create_bucket(_TEST_BUCKET)
+
+  engine = CloudFrontedS3Storage(storage_context, 'cloudfrontdomain', 'keyid', 'test/data/test.pem', 'some/path',
+                                 _TEST_BUCKET, _TEST_USER, _TEST_PASSWORD)
+  engine.put_content(_TEST_PATH, _TEST_CONTENT)
+  assert engine.exists(_TEST_PATH)
+
+  # Request a direct download URL for a request from a known AWS IP, and ensure we are returned an S3 URL.
+  assert 's3.amazonaws.com' in engine.get_direct_download_url(_TEST_PATH, test_aws_ip)
+
+  if ipranges_populated:
+    # Request a direct download URL for a request from a non-AWS IP, and ensure we are returned a CloudFront URL.
+    assert 'cloudfrontdomain' in engine.get_direct_download_url(_TEST_PATH, '1.2.3.4')
+  else:
+    # Request a direct download URL for a request from a non-AWS IP, but since IP Ranges isn't populated, we still
+    # get back an S3 URL.
+    assert 's3.amazonaws.com' in engine.get_direct_download_url(_TEST_PATH, '1.2.3.4')
+

storage/test/test_swift.py

@@ -9,7 +9,7 @@ from storage import StorageContext
 from storage.swift import SwiftStorage
 
 base_args = {
-  'context': StorageContext('nyc', None, None),
+  'context': StorageContext('nyc', None, None, None, None),
   'swift_container': 'container-name',
   'storage_path': '/basepath',
   'auth_url': 'https://auth.com',
@@ -191,7 +191,7 @@ def test_cancel_chunked_upload():
 def test_empty_chunks_queued_for_deletion():
   chunk_cleanup_queue = FakeQueue()
   args = dict(base_args)
-  args['context'] = StorageContext('nyc', None, chunk_cleanup_queue)
+  args['context'] = StorageContext('nyc', None, chunk_cleanup_queue, None, None)
 
   swift = FakeSwiftStorage(**args)
   uuid, metadata = swift.initiate_chunked_upload()