vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add CloudFrontedS3Storage, which redirects to CloudFront for non-S3 ips

Browse source
This commit is contained in:
Joseph Schorr 2017-09-26 16:08:50 -04:00
parent 2d522764f7
commit 010dda2c52
14 changed files with 175 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

70
util/ipresolver/__init__.py
View file

@ -1,67 +1,50 @@
import logging
import time
import json
from cachetools import lru_cache
from cachetools import ttl_cache, lru_cache
from collections import namedtuple, defaultdict
from netaddr import IPNetwork, IPAddress, IPSet, AddrFormatError
from threading import Thread
import geoip2.database
import geoip2.errors
_AWS_IP_RANGES_URL = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
_UPDATE_TIME = 60 * 60 * 24
_RETRY_TIME = 60 * 60 * 5
ResolvedLocation = namedtuple('ResolvedLocation', ['provider', 'region', 'service', 'sync_token'])
logger = logging.getLogger(__name__)
class IPResolver(Thread):
def __init__(self, app, client=None, *args, **kwargs):
super(IPResolver, self).__init__(*args, **kwargs)
self.daemon = True
class IPResolver(object):
def __init__(self, app, *args, **kwargs):
self.app = app
self.client = client or app.config['HTTPCLIENT']
self.location_function = None
self.sync_token = None
self.geoip_db = geoip2.database.Reader('util/ipresolver/GeoLite2-Country.mmdb')
def resolve_ip(self, ip_address):
""" Attempts to return resolved information about the specified IP Address. If such an attempt fails,
returns None.
"""
location_function = self.location_function
location_function = self._get_location_function()
if not ip_address or not location_function:
return None
return location_function(ip_address)
def _update_aws_ip_range(self):
logger.debug('Starting download of AWS IP Range table from %s', _AWS_IP_RANGES_URL)
@ttl_cache(maxsize=1, ttl=600)
def _get_location_function(self):
try:
response = self.client.get(_AWS_IP_RANGES_URL)
if response.status_code / 100 != 2:
logger.error('Non-200 response (%s) for AWS IP Range table request', response.status_code)
return False
except:
logger.exception('Could not download AWS IP range table')
return False
# Check if the sync token is the same. If so, no updates are necessary.
if self.sync_token and response.json()['syncToken'] == self.sync_token:
logger.debug('No updates necessary')
return True
with open('util/ipresolver/ip-ranges.json', 'r') as f:
ip_range_json = json.loads(f.read())
except IOError:
logger.exception('Could not load IP Ranges')
return None
except ValueError:
logger.exception('Could not load IP Ranges')
return None
except TypeError:
logger.exception('Could not load IP Ranges')
return None
# Otherwise, update the range lookup function.
all_amazon, regions, services = IPResolver._parse_amazon_ranges(response.json())
self.sync_token = response.json()['syncToken']
self.location_function = IPResolver._build_location_function(self.sync_token, all_amazon, regions, services, self.geoip_db)
logger.debug('Successfully updated AWS IP range table with sync token: %s', self.sync_token)
return True
sync_token = ip_range_json['syncToken']
all_amazon, regions, services = IPResolver._parse_amazon_ranges(ip_range_json)
return IPResolver._build_location_function(sync_token, all_amazon, regions, services, self.geoip_db)
@staticmethod
def _build_location_function(sync_token, all_amazon, regions, country, country_db):
@ -111,14 +94,3 @@ class IPResolver(Thread):
services[service].add(cidr)
return all_amazon, regions, services
def run(self):
while True:
logger.debug('Updating AWS IP database')
if not self._update_aws_ip_range():
logger.debug('Failed; sleeping for %s seconds', _RETRY_TIME)
time.sleep(_RETRY_TIME)
continue
logger.debug('Success; sleeping for %s seconds', _UPDATE_TIME)
time.sleep(_UPDATE_TIME)