quay/util/ipresolver/__init__.py

import logging
import json

from collections import namedtuple, defaultdict

from abc import ABCMeta, abstractmethod
from six import add_metaclass
from cachetools import ttl_cache, lru_cache
from netaddr import IPNetwork, IPAddress, IPSet, AddrFormatError

import geoip2.database
import geoip2.errors
import requests

from util.abchelpers import nooper

ResolvedLocation = namedtuple('ResolvedLocation', ['provider', 'region', 'service', 'sync_token'])

logger = logging.getLogger(__name__)

_DATA_FILES = {'aws-ip-ranges.json': 'https://ip-ranges.amazonaws.com/ip-ranges.json'}

def update_resolver_datafiles():
  """ Performs an update of the data file(s) used by the IP Resolver. """
  for filename, url in _DATA_FILES.iteritems():
    logger.debug('Updating IP resolver data file "%s" from URL "%s"', filename, url)
    with open('util/ipresolver/%s' % filename, 'w') as f:
      response = requests.get(url)
      logger.debug('Got %s response for URL %s', response.status_code, url)
      if response.status_code / 2 != 100:
        raise Exception('Got non-2XX status code for URL %s: %s' % (url, response.status_code))

      f.write(response.text)
      logger.debug('Successfully wrote %s', filename)


@add_metaclass(ABCMeta)
class IPResolverInterface(object):
  """ Helper class for resolving information about an IP address. """
  @abstractmethod
  def resolve_ip(self, ip_address):
    """ Attempts to return resolved information about the specified IP Address. If such an attempt
        fails, returns None.
    """
    pass

  @abstractmethod
  def is_ip_possible_threat(self, ip_address):
    """ Attempts to return whether the given IP address is a possible abuser or spammer.
        Returns False if the IP address information could not be looked up.
    """
    pass


@nooper
class NoopIPResolver(IPResolverInterface):
  """ No-op version of the security scanner API. """
  pass


class IPResolver(IPResolverInterface):
  def __init__(self, app):
    self.app = app
    self.geoip_db = geoip2.database.Reader('util/ipresolver/GeoLite2-Country.mmdb')

  @ttl_cache(maxsize=100, ttl=600)
  def is_ip_possible_threat(self, ip_address):
    if self.app.config.get('THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT') is None:
      return False

    if not ip_address:
      return False

    try:
      logger.debug('Requesting IP data for IP %s', ip_address)
      r = requests.get('https://api.ipdata.co/%s/en' % ip_address, timeout=1)
      if r.status_code != 200:
        logger.debug('Got non-200 response for IP %s: %s', ip_address, r.status_code)
        return False

      logger.debug('Got IP data for IP %s: %s => %s', ip_address, r.status_code, r.json())
      threat_data = r.json().get('threat', {})
      return threat_data.get('is_threat', False) or threat_data.get('is_bogon', False)
    except requests.RequestException:
      logger.exception('Got exception when trying to lookup IP Address')
    except ValueError:
      logger.exception('Got exception when trying to lookup IP Address')
    except Exception:
      logger.exception('Got exception when trying to lookup IP Address')

    return False

  def resolve_ip(self, ip_address):
    """ Attempts to return resolved information about the specified IP Address. If such an attempt
        fails, returns None.
    """
    location_function = self._get_location_function()
    if not ip_address or not location_function:
      return None

    return location_function(ip_address)

  def _get_aws_ip_ranges(self):
    try:
      with open('util/ipresolver/aws-ip-ranges.json', 'r') as f:
        return json.loads(f.read())
    except IOError:
      logger.exception('Could not load AWS IP Ranges')
      return None
    except ValueError:
      logger.exception('Could not load AWS IP Ranges')
      return None
    except TypeError:
      logger.exception('Could not load AWS IP Ranges')
      return None

  @ttl_cache(maxsize=1, ttl=600)
  def _get_location_function(self):
    aws_ip_range_json = self._get_aws_ip_ranges()
    if aws_ip_range_json is None:
      return None

    sync_token = aws_ip_range_json['syncToken']
    all_amazon, regions, services = IPResolver._parse_amazon_ranges(aws_ip_range_json)
    return IPResolver._build_location_function(sync_token, all_amazon, regions, services,
                                               self.geoip_db)

  @staticmethod
  def _build_location_function(sync_token, all_amazon, regions, country, country_db):
    @lru_cache(maxsize=4096)
    def _get_location(ip_address):
      try:
        parsed_ip = IPAddress(ip_address)
      except AddrFormatError:
        return ResolvedLocation('invalid_ip', None, None, sync_token)

      if parsed_ip not in all_amazon:
        # Try geoip classification
        try:
          found = country_db.country(parsed_ip)
          return ResolvedLocation(
            'internet',
            found.continent.code,
            found.country.iso_code,
            sync_token,
          )
        except geoip2.errors.AddressNotFoundError:
          return ResolvedLocation('internet', None, None, sync_token)

      region = None

      for region_name, region_set in regions.items():
        if parsed_ip in region_set:
          region = region_name
          break

      return ResolvedLocation('aws', region, None, sync_token)
    return _get_location

  @staticmethod
  def _parse_amazon_ranges(ranges):
    all_amazon = IPSet()
    regions = defaultdict(IPSet)
    services = defaultdict(IPSet)

    for service_description in ranges['prefixes']:
      cidr = IPNetwork(service_description['ip_prefix'])
      service = service_description['service']
      region = service_description['region']

      all_amazon.add(cidr)
      regions[region].add(cidr)
      services[service].add(cidr)

    return all_amazon, regions, services