Add CloudFrontedS3Storage, which redirects to CloudFront for non-S3 ips

util/ipresolver/__init__.py

@@ -1,67 +1,50 @@
 import logging
-import time
+import json
 
-from cachetools import lru_cache
+from cachetools import ttl_cache, lru_cache
 from collections import namedtuple, defaultdict
 from netaddr import IPNetwork, IPAddress, IPSet, AddrFormatError
-from threading import Thread
 
 import geoip2.database
 import geoip2.errors
 
-_AWS_IP_RANGES_URL = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
-_UPDATE_TIME = 60 * 60 * 24
-_RETRY_TIME = 60 * 60 * 5
-
 ResolvedLocation = namedtuple('ResolvedLocation', ['provider', 'region', 'service', 'sync_token'])
 
 logger = logging.getLogger(__name__)
 
-class IPResolver(Thread):
-  def __init__(self, app, client=None, *args, **kwargs):
-    super(IPResolver, self).__init__(*args, **kwargs)
-    self.daemon = True
-
+class IPResolver(object):
+  def __init__(self, app, *args, **kwargs):
     self.app = app
-    self.client = client or app.config['HTTPCLIENT']
-
-    self.location_function = None
-    self.sync_token = None
-
     self.geoip_db = geoip2.database.Reader('util/ipresolver/GeoLite2-Country.mmdb')
 
   def resolve_ip(self, ip_address):
     """ Attempts to return resolved information about the specified IP Address. If such an attempt fails,
         returns None.
     """
-    location_function = self.location_function
+    location_function = self._get_location_function()
     if not ip_address or not location_function:
       return None
     
     return location_function(ip_address)
 
-  def _update_aws_ip_range(self):
-    logger.debug('Starting download of AWS IP Range table from %s', _AWS_IP_RANGES_URL)
+  @ttl_cache(maxsize=1, ttl=600)
+  def _get_location_function(self):
     try:
-      response = self.client.get(_AWS_IP_RANGES_URL)
-      if response.status_code / 100 != 2:
-        logger.error('Non-200 response (%s) for AWS IP Range table request', response.status_code)
-        return False
-    except:
-      logger.exception('Could not download AWS IP range table')
-      return False
-    
-    # Check if the sync token is the same. If so, no updates are necessary.
-    if self.sync_token and response.json()['syncToken'] == self.sync_token:
-      logger.debug('No updates necessary')
-      return True
+      with open('util/ipresolver/ip-ranges.json', 'r') as f:
+        ip_range_json = json.loads(f.read())
+    except IOError:
+      logger.exception('Could not load IP Ranges')
+      return None
+    except ValueError:
+      logger.exception('Could not load IP Ranges')
+      return None
+    except TypeError:
+      logger.exception('Could not load IP Ranges')
+      return None
 
-    # Otherwise, update the range lookup function.
-    all_amazon, regions, services = IPResolver._parse_amazon_ranges(response.json())
-    self.sync_token = response.json()['syncToken']
-    self.location_function = IPResolver._build_location_function(self.sync_token, all_amazon, regions, services, self.geoip_db)
-    logger.debug('Successfully updated AWS IP range table with sync token: %s', self.sync_token)
-    return True
+    sync_token = ip_range_json['syncToken']
+    all_amazon, regions, services = IPResolver._parse_amazon_ranges(ip_range_json)
+    return IPResolver._build_location_function(sync_token, all_amazon, regions, services, self.geoip_db)
 
   @staticmethod
   def _build_location_function(sync_token, all_amazon, regions, country, country_db):
@@ -111,14 +94,3 @@ class IPResolver(Thread):
       services[service].add(cidr)
 
     return all_amazon, regions, services
-
-  def run(self):
-    while True:
-      logger.debug('Updating AWS IP database')
-      if not self._update_aws_ip_range():
-        logger.debug('Failed; sleeping for %s seconds', _RETRY_TIME)
-        time.sleep(_RETRY_TIME)
-        continue
-
-      logger.debug('Success; sleeping for %s seconds', _UPDATE_TIME)
-      time.sleep(_UPDATE_TIME)

util/ipresolver/test/test_ipresolver.py

@@ -17,6 +17,10 @@ def http_client():
   sess.mount('https://', adapter)
   return sess
 
+@pytest.fixture()
+def test_aws_ip():
+  return '10.0.0.1'
+
 @pytest.fixture()
 def aws_ip_range_handler():
   @urlmatch(netloc=r'ip-ranges.amazonaws.com')
@@ -35,16 +39,16 @@ def aws_ip_range_handler():
 
   return handler
 
-def test_unstarted(app, http_client):
+def test_unstarted(app, test_aws_ip, http_client):
   ipresolver = IPResolver(app, client=http_client)
-  assert ipresolver.resolve_ip('10.0.0.1') is None
+  assert ipresolver.resolve_ip(test_aws_ip) is None
 
-def test_resolved(aws_ip_range_handler, app, http_client):
+def test_resolved(aws_ip_range_handler, test_aws_ip, app, http_client):
   with HTTMock(aws_ip_range_handler):
     ipresolver = IPResolver(app, client=http_client)
     assert ipresolver._update_aws_ip_range()
 
-    assert ipresolver.resolve_ip('10.0.0.1') == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
+    assert ipresolver.resolve_ip(test_aws_ip) == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
     assert ipresolver.resolve_ip('10.0.0.2') == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
     assert ipresolver.resolve_ip('1.2.3.4') == ResolvedLocation(provider='internet', region=u'NA', service=u'US', sync_token=123456789)
     assert ipresolver.resolve_ip('127.0.0.1') == ResolvedLocation(provider='internet', region=None, service=None, sync_token=123456789)