153 lines
4.7 KiB
Python
153 lines
4.7 KiB
Python
import logging
|
|
import json
|
|
import time
|
|
|
|
from collections import namedtuple
|
|
|
|
from threading import Thread, Lock
|
|
from abc import ABCMeta, abstractmethod
|
|
from six import add_metaclass
|
|
from cachetools.func import ttl_cache, lru_cache
|
|
from netaddr import IPNetwork, IPAddress, IPSet, AddrFormatError
|
|
|
|
import geoip2.database
|
|
import geoip2.errors
|
|
import requests
|
|
|
|
from util.abchelpers import nooper
|
|
|
|
ResolvedLocation = namedtuple('ResolvedLocation', ['provider', 'service', 'sync_token',
|
|
'country_iso_code'])
|
|
|
|
AWS_SERVICES = {'EC2', 'CODEBUILD'}
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _get_aws_ip_ranges():
|
|
try:
|
|
with open('util/ipresolver/aws-ip-ranges.json', 'r') as f:
|
|
return json.loads(f.read())
|
|
except IOError:
|
|
logger.exception('Could not load AWS IP Ranges')
|
|
return None
|
|
except ValueError:
|
|
logger.exception('Could not load AWS IP Ranges')
|
|
return None
|
|
except TypeError:
|
|
logger.exception('Could not load AWS IP Ranges')
|
|
return None
|
|
|
|
|
|
@add_metaclass(ABCMeta)
|
|
class IPResolverInterface(object):
|
|
""" Helper class for resolving information about an IP address. """
|
|
@abstractmethod
|
|
def resolve_ip(self, ip_address):
|
|
""" Attempts to return resolved information about the specified IP Address. If such an attempt
|
|
fails, returns None.
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def is_ip_possible_threat(self, ip_address):
|
|
""" Attempts to return whether the given IP address is a possible abuser or spammer.
|
|
Returns False if the IP address information could not be looked up.
|
|
"""
|
|
pass
|
|
|
|
|
|
@nooper
|
|
class NoopIPResolver(IPResolverInterface):
|
|
""" No-op version of the security scanner API. """
|
|
pass
|
|
|
|
|
|
class IPResolver(IPResolverInterface):
|
|
def __init__(self, app):
|
|
self.app = app
|
|
self.geoip_db = geoip2.database.Reader('util/ipresolver/GeoLite2-Country.mmdb')
|
|
self.amazon_ranges = None
|
|
self.sync_token = None
|
|
|
|
logger.info('Loading AWS IP ranges from disk')
|
|
aws_ip_ranges_data = _get_aws_ip_ranges()
|
|
if aws_ip_ranges_data is not None and aws_ip_ranges_data.get('syncToken'):
|
|
logger.info('Building AWS IP ranges')
|
|
self.amazon_ranges = IPResolver._parse_amazon_ranges(aws_ip_ranges_data)
|
|
self.sync_token = aws_ip_ranges_data['syncToken']
|
|
logger.info('Finished building AWS IP ranges')
|
|
|
|
@ttl_cache(maxsize=100, ttl=600)
|
|
def is_ip_possible_threat(self, ip_address):
|
|
if self.app.config.get('THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT') is None:
|
|
return False
|
|
|
|
if self.app.config.get('IP_DATA_API_KEY') is None:
|
|
return False
|
|
|
|
if not ip_address:
|
|
return False
|
|
|
|
api_key = self.app.config['IP_DATA_API_KEY']
|
|
|
|
try:
|
|
logger.debug('Requesting IP data for IP %s', ip_address)
|
|
r = requests.get('https://api.ipdata.co/%s/threat?api-key=%s' % (ip_address, api_key),
|
|
timeout=1)
|
|
if r.status_code != 200:
|
|
logger.debug('Got non-200 response for IP %s: %s', ip_address, r.status_code)
|
|
return False
|
|
|
|
logger.debug('Got IP data for IP %s: %s => %s', ip_address, r.status_code, r.json())
|
|
threat_data = r.json()
|
|
return threat_data.get('is_threat', False) or threat_data.get('is_bogon', False)
|
|
except requests.RequestException:
|
|
logger.exception('Got exception when trying to lookup IP Address')
|
|
except ValueError:
|
|
logger.exception('Got exception when trying to lookup IP Address')
|
|
except Exception:
|
|
logger.exception('Got exception when trying to lookup IP Address')
|
|
|
|
return False
|
|
|
|
def resolve_ip(self, ip_address):
|
|
""" Attempts to return resolved information about the specified IP Address. If such an attempt
|
|
fails, returns None.
|
|
"""
|
|
if not ip_address:
|
|
return None
|
|
|
|
try:
|
|
parsed_ip = IPAddress(ip_address)
|
|
except AddrFormatError:
|
|
return ResolvedLocation('invalid_ip', None, self.sync_token, None)
|
|
|
|
# Try geoip classification
|
|
try:
|
|
geoinfo = self.geoip_db.country(ip_address)
|
|
except geoip2.errors.AddressNotFoundError:
|
|
geoinfo = None
|
|
|
|
if self.amazon_ranges is None or parsed_ip not in self.amazon_ranges:
|
|
if geoinfo:
|
|
return ResolvedLocation(
|
|
'internet',
|
|
geoinfo.country.iso_code,
|
|
self.sync_token,
|
|
geoinfo.country.iso_code,
|
|
)
|
|
|
|
return ResolvedLocation('internet', None, self.sync_token, None)
|
|
|
|
return ResolvedLocation('aws', None, self.sync_token,
|
|
geoinfo.country.iso_code if geoinfo else None)
|
|
|
|
@staticmethod
|
|
def _parse_amazon_ranges(ranges):
|
|
all_amazon = IPSet()
|
|
for service_description in ranges['prefixes']:
|
|
if service_description['service'] in AWS_SERVICES:
|
|
all_amazon.add(IPNetwork(service_description['ip_prefix']))
|
|
|
|
return all_amazon
|