From 3309daa32e405b04b14ccf727641614c2ec6f9fa Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Fri, 20 Apr 2018 18:01:05 +0300 Subject: [PATCH] Add support for reduced initial build count for new possible abusing users If configured, we now check the IP address of the user signing up and, if they are a possible threat, we further reduce their number of allowed maximum builds to the configured value. --- config.py | 6 ++++- data/model/organization.py | 28 ++++++++++---------- data/model/user.py | 14 +++++++--- endpoints/api/organization.py | 7 +++-- endpoints/api/user.py | 4 ++- util/config/schema.py | 1 + util/ipresolver/__init__.py | 49 +++++++++++++++++++++++++++++------ 7 files changed, 81 insertions(+), 28 deletions(-) diff --git a/config.py b/config.py index f91637992..90191aca5 100644 --- a/config.py +++ b/config.py @@ -502,7 +502,7 @@ class DefaultConfig(ImmutableConfig): # The size of pages returned by the Docker V2 API. V2_PAGINATION_SIZE = 50 - + # If enabled, ensures that API calls are made with the X-Requested-With header # when called from a browser. BROWSER_API_CALLS_XHR_ONLY = True @@ -510,6 +510,10 @@ class DefaultConfig(ImmutableConfig): # If set to a non-None integer value, the default number of maximum builds for a namespace. DEFAULT_NAMESPACE_MAXIMUM_BUILD_COUNT = None + # If set to a non-None integer value, the default number of maximum builds for a namespace whose + # creator IP is deemed a threat. + THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT = None + # For Billing Support Only: The number of allowed builds on a namespace that has been billed # successfully. BILLED_NAMESPACE_MAXIMUM_BUILD_COUNT = None diff --git a/data/model/organization.py b/data/model/organization.py index 463604915..0087f49c0 100644 --- a/data/model/organization.py +++ b/data/model/organization.py @@ -5,22 +5,24 @@ from data.model import (user, team, DataModelException, InvalidOrganizationExcep InvalidUsernameException, db_transaction, _basequery) -def create_organization(name, email, creating_user, email_required=True): - try: - # Create the org - new_org = user.create_user_noverify(name, email, email_required=email_required) - new_org.organization = True - new_org.save() +def create_organization(name, email, creating_user, email_required=True, is_possible_abuser=False): + with db_transaction(): + try: + # Create the org + new_org = user.create_user_noverify(name, email, email_required=email_required, + is_possible_abuser=is_possible_abuser) + new_org.organization = True + new_org.save() - # Create a team for the owners - owners_team = team.create_team('owners', new_org, 'admin') + # Create a team for the owners + owners_team = team.create_team('owners', new_org, 'admin') - # Add the user who created the org to the owners team - team.add_user_to_team(creating_user, owners_team) + # Add the user who created the org to the owners team + team.add_user_to_team(creating_user, owners_team) - return new_org - except InvalidUsernameException as iue: - raise InvalidOrganizationException(iue.message) + return new_org + except InvalidUsernameException as iue: + raise InvalidOrganizationException(iue.message) def get_organization(name): diff --git a/data/model/user.py b/data/model/user.py index 7f98a99a9..c742824e9 100644 --- a/data/model/user.py +++ b/data/model/user.py @@ -37,12 +37,14 @@ def hash_password(password, salt=None): salt = salt or bcrypt.gensalt() return bcrypt.hashpw(password.encode('utf-8'), salt) -def create_user(username, password, email, auto_verify=False, email_required=True, prompts=tuple()): +def create_user(username, password, email, auto_verify=False, email_required=True, prompts=tuple(), + is_possible_abuser=False): """ Creates a regular user, if allowed. """ if not validate_password(password): raise InvalidPasswordException(INVALID_PASSWORD_MESSAGE) - created = create_user_noverify(username, email, email_required=email_required, prompts=prompts) + created = create_user_noverify(username, email, email_required=email_required, prompts=prompts, + is_possible_abuser=is_possible_abuser) created.password_hash = hash_password(password) created.verified = auto_verify created.save() @@ -50,7 +52,8 @@ def create_user(username, password, email, auto_verify=False, email_required=Tru return created -def create_user_noverify(username, email, email_required=True, prompts=tuple()): +def create_user_noverify(username, email, email_required=True, prompts=tuple(), + is_possible_abuser=False): if email_required: if not validate_email(email): raise InvalidEmailAddressException('Invalid email address: %s' % email) @@ -82,6 +85,11 @@ def create_user_noverify(username, email, email_required=True, prompts=tuple()): try: default_expr_s = _convert_to_s(config.app_config['DEFAULT_TAG_EXPIRATION']) default_max_builds = config.app_config.get('DEFAULT_NAMESPACE_MAXIMUM_BUILD_COUNT') + threat_max_builds = config.app_config.get('THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT') + + if is_possible_abuser and threat_max_builds is not None: + default_max_builds = threat_max_builds + new_user = User.create(username=username, email=email, removed_tag_expiration_s=default_expr_s, maximum_queued_builds_count=default_max_builds) for prompt in prompts: diff --git a/endpoints/api/organization.py b/endpoints/api/organization.py index 0ac57b7c6..f3efdc115 100644 --- a/endpoints/api/organization.py +++ b/endpoints/api/organization.py @@ -6,7 +6,8 @@ from flask import request import features -from app import billing as stripe, avatar, all_queues, authentication, namespace_gc_queue +from app import (billing as stripe, avatar, all_queues, authentication, namespace_gc_queue, + ip_resolver) from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error, related_user_resource, internal_only, require_user_admin, log_action, show_if, path_param, require_scope, require_fresh_login) @@ -111,9 +112,11 @@ class OrganizationList(ApiResource): if features.MAILING and not org_data.get('email'): raise request_error(message='Email address is required') + is_possible_abuser = ip_resolver.is_ip_possible_threat(request.remote_addr) try: model.organization.create_organization(org_data['name'], org_data.get('email'), user, - email_required=features.MAILING) + email_required=features.MAILING, + is_possible_abuser=is_possible_abuser) return 'Created', 201 except model.DataModelException as ex: raise request_error(exception=ex) diff --git a/endpoints/api/user.py b/endpoints/api/user.py index 7f86df918..de5e23850 100644 --- a/endpoints/api/user.py +++ b/endpoints/api/user.py @@ -12,7 +12,7 @@ from peewee import IntegrityError import features from app import (app, billing as stripe, authentication, avatar, user_analytics, all_queues, - oauth_login, namespace_gc_queue) + oauth_login, namespace_gc_queue, ip_resolver) from auth import scopes from auth.auth_context import get_authenticated_user @@ -455,12 +455,14 @@ class User(ApiResource): 'message': 'Are you a bot? If not, please revalidate the captcha.' }, 400 + is_possible_abuser = ip_resolver.is_ip_possible_threat(request.remote_addr) try: prompts = model.user.get_default_user_prompts(features) new_user = model.user.create_user(user_data['username'], user_data['password'], user_data.get('email'), auto_verify=not features.MAILING, email_required=features.MAILING, + is_possible_abuser=is_possible_abuser, prompts=prompts) email_address_confirmed = handle_invite_code(invite_code, new_user) diff --git a/util/config/schema.py b/util/config/schema.py index f37f0e766..b8ce47fa8 100644 --- a/util/config/schema.py +++ b/util/config/schema.py @@ -73,6 +73,7 @@ INTERNAL_ONLY_PROPERTIES = { 'SENTRY_PUBLIC_DSN', 'BILLED_NAMESPACE_MAXIMUM_BUILD_COUNT', + 'THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT', 'SECURITY_SCANNER_ENDPOINT_BATCH', 'SECURITY_SCANNER_API_TIMEOUT_SECONDS', diff --git a/util/ipresolver/__init__.py b/util/ipresolver/__init__.py index 8bd88c106..2adebbef5 100644 --- a/util/ipresolver/__init__.py +++ b/util/ipresolver/__init__.py @@ -1,16 +1,16 @@ import logging import json -import requests + +from collections import namedtuple, defaultdict from abc import ABCMeta, abstractmethod from six import add_metaclass - from cachetools import ttl_cache, lru_cache -from collections import namedtuple, defaultdict from netaddr import IPNetwork, IPAddress, IPSet, AddrFormatError import geoip2.database import geoip2.errors +import requests from util.abchelpers import nooper @@ -44,6 +44,13 @@ class IPResolverInterface(object): """ pass + @abstractmethod + def is_ip_possible_threat(self, ip_address): + """ Attempts to return whether the given IP address is a possible abuser or spammer. + Returns False if the IP address information could not be looked up. + """ + pass + @nooper class NoopIPResolver(IPResolverInterface): @@ -56,14 +63,39 @@ class IPResolver(IPResolverInterface): self.app = app self.geoip_db = geoip2.database.Reader('util/ipresolver/GeoLite2-Country.mmdb') + @ttl_cache(maxsize=100, ttl=600) + def is_ip_possible_threat(self, ip_address): + if self.app.config.get('THREAT_NAMESPACE_MAXIMUM_BUILD_COUNT') is None: + return False + + if not ip_address: + return False + + try: + logger.debug('Requesting IP data for IP %s', ip_address) + r = requests.get('https://api.ipdata.co/%s/en' % ip_address, timeout=1) + if r.status_code != 200: + logger.debug('Got non-200 response for IP %s: %s', ip_address, r.status_code) + return False + + logger.debug('Got IP data for IP %s: %s => %s', ip_address, r.status_code, r.json) + threat_data = r.json.get('threat', {}) + return threat_data.get('is_threat', False) or threat_data.get('is_bogon', False) + except requests.RequestException: + logger.exception('Got exception when trying to lookup IP Address') + except ValueError: + logger.exception('Got exception when trying to lookup IP Address') + + return False + def resolve_ip(self, ip_address): - """ Attempts to return resolved information about the specified IP Address. If such an attempt fails, - returns None. + """ Attempts to return resolved information about the specified IP Address. If such an attempt + fails, returns None. """ location_function = self._get_location_function() if not ip_address or not location_function: return None - + return location_function(ip_address) def _get_aws_ip_ranges(self): @@ -79,7 +111,7 @@ class IPResolver(IPResolverInterface): except TypeError: logger.exception('Could not load AWS IP Ranges') return None - + @ttl_cache(maxsize=1, ttl=600) def _get_location_function(self): aws_ip_range_json = self._get_aws_ip_ranges() @@ -88,7 +120,8 @@ class IPResolver(IPResolverInterface): sync_token = aws_ip_range_json['syncToken'] all_amazon, regions, services = IPResolver._parse_amazon_ranges(aws_ip_range_json) - return IPResolver._build_location_function(sync_token, all_amazon, regions, services, self.geoip_db) + return IPResolver._build_location_function(sync_token, all_amazon, regions, services, + self.geoip_db) @staticmethod def _build_location_function(sync_token, all_amazon, regions, country, country_db):