Add worker to update ipresolver data files every few hours

util/ipresolver/__init__.py

@@ -29,7 +29,7 @@ def update_resolver_datafiles():
       logger.debug('Successfully wrote %s', filename)
 
 class IPResolver(object):
-  def __init__(self, app, *args, **kwargs):
+  def __init__(self, app):
     self.app = app
     self.geoip_db = geoip2.database.Reader('util/ipresolver/GeoLite2-Country.mmdb')
 
@@ -43,11 +43,10 @@ class IPResolver(object):
     
     return location_function(ip_address)
 
-  @ttl_cache(maxsize=1, ttl=600)
-  def _get_location_function(self):
+  def _get_aws_ip_ranges(self):
     try:
       with open('util/ipresolver/aws-ip-ranges.json', 'r') as f:
-        aws_ip_range_json = json.loads(f.read())
+        return json.loads(f.read())
     except IOError:
       logger.exception('Could not load AWS IP Ranges')
       return None
@@ -57,6 +56,12 @@ class IPResolver(object):
     except TypeError:
       logger.exception('Could not load AWS IP Ranges')
       return None
+    
+  @ttl_cache(maxsize=1, ttl=600)
+  def _get_location_function(self):
+    aws_ip_range_json = self._get_aws_ip_ranges()
+    if aws_ip_range_json is None:
+      return None
 
     sync_token = aws_ip_range_json['syncToken']
     all_amazon, regions, services = IPResolver._parse_amazon_ranges(aws_ip_range_json)

util/ipresolver/test/test_ipresolver.py

@@ -1,54 +1,40 @@
-import requests
 import pytest
-import json
 
-from httmock import urlmatch, HTTMock
+from mock import patch
 
-from config import build_requests_session
 from util.ipresolver import IPResolver, ResolvedLocation
 from test.fixtures import *
 
-@pytest.fixture(scope="module")
-def http_client():
-  sess = requests.Session()
-  adapter = requests.adapters.HTTPAdapter(pool_connections=100,
-                                          pool_maxsize=100)
-  sess.mount('http://', adapter)
-  sess.mount('https://', adapter)
-  return sess
-
 @pytest.fixture()
 def test_aws_ip():
   return '10.0.0.1'
 
 @pytest.fixture()
-def aws_ip_range_handler():
-  @urlmatch(netloc=r'ip-ranges.amazonaws.com')
-  def handler(_, request):
-    fake_range_doc = {
-      'syncToken': 123456789,
-      'prefixes': [
-        {
-          'ip_prefix': '10.0.0.0/8',
-          'region': 'GLOBAL',
-          'service': 'AMAZON',
-        } 
-      ],
-    }
-    return {'status_code': 200, 'content': json.dumps(fake_range_doc)}
+def aws_ip_range_data():
+  fake_range_doc = {
+    'syncToken': 123456789,
+    'prefixes': [
+      {
+        'ip_prefix': '10.0.0.0/8',
+        'region': 'GLOBAL',
+        'service': 'AMAZON',
+      } 
+    ],
+  }
+  return fake_range_doc
 
-  return handler
-
-def test_unstarted(app, test_aws_ip, http_client):
-  ipresolver = IPResolver(app, client=http_client)
+def test_unstarted(app, test_aws_ip):
+  ipresolver = IPResolver(app)
   assert ipresolver.resolve_ip(test_aws_ip) is None
 
-def test_resolved(aws_ip_range_handler, test_aws_ip, app, http_client):
-  with HTTMock(aws_ip_range_handler):
-    ipresolver = IPResolver(app, client=http_client)
-    assert ipresolver._update_aws_ip_range()
+def test_resolved(aws_ip_range_data, test_aws_ip, app,):
+    ipresolver = IPResolver(app)
 
-    assert ipresolver.resolve_ip(test_aws_ip) == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
-    assert ipresolver.resolve_ip('10.0.0.2') == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
-    assert ipresolver.resolve_ip('1.2.3.4') == ResolvedLocation(provider='internet', region=u'NA', service=u'US', sync_token=123456789)
-    assert ipresolver.resolve_ip('127.0.0.1') == ResolvedLocation(provider='internet', region=None, service=None, sync_token=123456789)
+    def get_data():
+      return aws_ip_range_data
+
+    with patch.object(ipresolver, '_get_aws_ip_ranges', get_data):
+      assert ipresolver.resolve_ip(test_aws_ip) == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
+      assert ipresolver.resolve_ip('10.0.0.2') == ResolvedLocation(provider='aws', region=u'GLOBAL', service=None, sync_token=123456789)
+      assert ipresolver.resolve_ip('1.2.3.4') == ResolvedLocation(provider='internet', region=u'NA', service=u'US', sync_token=123456789)
+      assert ipresolver.resolve_ip('127.0.0.1') == ResolvedLocation(provider='internet', region=None, service=None, sync_token=123456789)