Lay foundation for truly dynamic external logins

Moves all the external login services into a set of classes that share as much code as possible. These services are then registered on both the client and server, allowing us in the followup change to dynamically register new handlers

oauth/base.py

@@ -0,0 +1,124 @@
+import logging
+
+logger = logging.getLogger(__name__)
+
+class OAuthExchangeCodeException(Exception):
+  """ Exception raised if a code exchange fails. """
+  pass
+
+class OAuthGetUserInfoException(Exception):
+  """ Exception raised if a call to get user information fails. """
+  pass
+
+
+class OAuthService(object):
+  """ A base class for defining an external service, exposed via OAuth. """
+  def __init__(self, config, key_name):
+    self.key_name = key_name
+    self.config = config.get(key_name) or {}
+
+  def service_id(self):
+    """ The internal ID for this service. Must match the URL portion for the service, e.g. `github`
+    """
+    raise NotImplementedError
+
+  def service_name(self):
+    """ The user-readable name for the service, e.g. `GitHub`"""
+    raise NotImplementedError
+
+  def token_endpoint(self):
+    """ The endpoint at which the OAuth code can be exchanged for a token. """
+    raise NotImplementedError
+
+  def user_endpoint(self):
+    """ The endpoint at which user information can be looked up. """
+    raise NotImplementedError
+
+  def validate_client_id_and_secret(self, http_client, app_config):
+    """ Performs validation of the client ID and secret, raising an exception on failure. """
+    raise NotImplementedError
+
+  def requires_form_encoding(self):
+    """ Returns True if form encoding is necessary for the exchange_code_for_token call. """
+    return False
+
+  def client_id(self):
+    return self.config.get('CLIENT_ID')
+
+  def client_secret(self):
+    return self.config.get('CLIENT_SECRET')
+
+  def get_redirect_uri(self, app_config, redirect_suffix=''):
+    return '%s://%s/oauth2/%s/callback%s' % (app_config['PREFERRED_URL_SCHEME'],
+                                             app_config['SERVER_HOSTNAME'],
+                                             self.service_name().lower(),
+                                             redirect_suffix)
+
+  def get_user_info(self, http_client, token):
+    token_param = {
+      'access_token': token,
+      'alt': 'json',
+    }
+
+    got_user = http_client.get(self.user_endpoint(), params=token_param)
+    if got_user.status_code // 100 != 2:
+      raise OAuthGetUserInfoException('Non-2XX response code for user_info call: %s' %
+                                      got_user.status_code)
+
+    user_info = got_user.json()
+    if user_info is None:
+      raise OAuthGetUserInfoException()
+
+    return user_info
+
+  def exchange_code_for_token(self, app_config, http_client, code, form_encode=False,
+                              redirect_suffix='', client_auth=False):
+    """ Exchanges an OAuth access code for the associated OAuth token. """
+    json_data = self._exchange_code(app_config, http_client, code, form_encode, redirect_suffix,
+                                    client_auth)
+
+    access_token = json_data.get('access_token', None)
+    if access_token is None:
+      logger.debug('Got successful get_access_token response %s', json_data)
+      raise OAuthExchangeCodeException('Missing `access_token` in OAuth response')
+
+    return access_token
+
+  def _exchange_code(self, app_config, http_client, code, form_encode=False, redirect_suffix='',
+                     client_auth=False):
+    payload = {
+      'code': code,
+      'grant_type': 'authorization_code',
+      'redirect_uri': self.get_redirect_uri(app_config, redirect_suffix)
+    }
+
+    headers = {
+      'Accept': 'application/json'
+    }
+
+    auth = None
+    if client_auth:
+      auth = (self.client_id(), self.client_secret())
+    else:
+      payload['client_id'] = self.client_id()
+      payload['client_secret'] = self.client_secret()
+
+    token_url = self.token_endpoint()
+    if form_encode:
+      get_access_token = http_client.post(token_url, data=payload, headers=headers, auth=auth)
+    else:
+      get_access_token = http_client.post(token_url, params=payload, headers=headers, auth=auth)
+
+    if get_access_token.status_code // 100 != 2:
+      logger.debug('Got get_access_token response %s', get_access_token.text)
+      raise OAuthExchangeCodeException('Got non-2XX response for code exchange: %s' %
+                                       get_access_token.status_code)
+
+    json_data = get_access_token.json()
+    if not json_data:
+      raise OAuthExchangeCodeException('Got non-JSON response for code exchange')
+
+    if 'error' in json_data:
+      raise OAuthExchangeCodeException(json_data.get('error_description', json_data['error']))
+
+    return json_data

oauth/login.py

@@ -0,0 +1,81 @@
+import logging
+
+import features
+
+from oauth.base import OAuthService, OAuthExchangeCodeException, OAuthGetUserInfoException
+
+logger = logging.getLogger(__name__)
+
+class OAuthLoginException(Exception):
+  """ Exception raised if a login operation fails. """
+  pass
+
+class OAuthLoginService(OAuthService):
+  """ A base class for defining an OAuth-compliant service that can be used for, amongst other
+      things, login and authentication. """
+
+  def get_login_service_id(self, user_info):
+    """ Returns the internal ID for the given user under this login service. """
+    raise NotImplementedError
+
+  def get_login_service_username(self, user_info):
+    """ Returns the username for the given user under this login service. """
+    raise NotImplementedError
+
+  def get_verified_user_email(self, app_config, http_client, token, user_info):
+    """ Returns the verified email address for the given user, if any or None if none. """
+    raise NotImplementedError
+
+  def get_icon(self):
+    """ Returns the icon to display for this login service. """
+    raise NotImplementedError
+
+  def get_login_scopes(self):
+    """ Returns the list of scopes for login for this service. """
+    raise NotImplementedError
+
+  def service_verify_user_info_for_login(self, app_config, http_client, token, user_info):
+    """ Performs service-specific verification of user information for login. On failure, a service
+        should raise a OAuthLoginService.
+    """
+    # By default, does nothing.
+    pass
+
+  def exchange_code_for_login(self, app_config, http_client, code, redirect_suffix):
+    """ Exchanges the given OAuth access code for user information on behalf of a user trying to
+        login or attach their account. Raises a OAuthLoginService exception on failure. Returns
+        a tuple consisting of (service_id, service_username, email)
+    """
+
+    # Retrieve the token for the OAuth code.
+    try:
+      token = self.exchange_code_for_token(app_config, http_client, code,
+                                           redirect_suffix=redirect_suffix,
+                                           form_encode=self.requires_form_encoding())
+    except OAuthExchangeCodeException as oce:
+      raise OAuthLoginException(oce.message)
+
+    # Retrieve the user's information with the token.
+    try:
+      user_info = self.get_user_info(http_client, token)
+    except OAuthGetUserInfoException as oge:
+      raise OAuthLoginException(oge.message)
+
+    if user_info.get('id', None) is None:
+      logger.debug('Got user info response %s', user_info)
+      raise OAuthLoginException('Missing `id` column in returned user information')
+
+    # Perform any custom verification for this login service.
+    self.service_verify_user_info_for_login(app_config, http_client, token, user_info)
+
+    # Retrieve the user's email address (if necessary).
+    email_address = self.get_verified_user_email(app_config, http_client, token, user_info)
+    if features.MAILING and email_address is None:
+      raise OAuthLoginException('A verified email address is required to login with this service')
+
+    service_user_id = self.get_login_service_id(user_info)
+    service_username = self.get_login_service_username(user_info)
+
+    logger.debug('Completed successful exchange for service %s: %s, %s, %s',
+                 self.service_id(), service_user_id, service_username, email_address)
+    return (service_user_id, service_username, email_address)

oauth/loginmanager.py

@@ -0,0 +1,19 @@
+import features
+
+from oauth.services.github import GithubOAuthService
+from oauth.services.google import GoogleOAuthService
+
+class OAuthLoginManager(object):
+  """ Helper class which manages all registered OAuth login services. """
+  def __init__(self, config):
+    self.services = []
+
+    # Register the endpoints for each of the OAuth login services.
+    # TODO(jschorr): make this dynamic.
+    if config.get('GITHUB_LOGIN_CONFIG') is not None and features.GITHUB_LOGIN:
+      github_service = GithubOAuthService(config, 'GITHUB_LOGIN_CONFIG')
+      self.services.append(github_service)
+
+    if config.get('GOOGLE_LOGIN_CONFIG') is not None and features.GOOGLE_LOGIN:
+      google_service = GoogleOAuthService(config, 'GOOGLE_LOGIN_CONFIG')
+      self.services.append(google_service)

oauth/oidc.py

@@ -0,0 +1,110 @@
+import time
+import json
+import logging
+import urlparse
+
+from cachetools import lru_cache
+from cachetools.ttl import TTLCache
+
+from util.oauth.base import OAuthService
+
+logger = logging.getLogger(__name__)
+
+
+def decode_user_jwt(token, oidc_provider):
+  try:
+    return decode(token, oidc_provider.get_public_key(), algorithms=['RS256'],
+                  audience=oidc_provider.client_id(),
+                  issuer=oidc_provider.issuer)
+  except InvalidTokenError:
+    # Public key may have expired. Try to retrieve an updated public key and use it to decode.
+    return decode(token, oidc_provider.get_public_key(force_refresh=True), algorithms=['RS256'],
+                  audience=oidc_provider.client_id(),
+                  issuer=oidc_provider.issuer)
+
+OIDC_WELLKNOWN = ".well-known/openid-configuration"
+PUBLIC_KEY_CACHE_TTL = 3600 # 1 hour
+
+class OIDCConfig(OAuthService):
+  def __init__(self, config, key_name):
+    super(OIDCConfig, self).__init__(config, key_name)
+
+    self._public_key_cache = TTLCache(1, PUBLIC_KEY_CACHE_TTL, missing=self._get_public_key)
+    self._config = config
+    self._http_client = config['HTTPCLIENT']
+
+  @lru_cache(maxsize=1)
+  def _oidc_config(self):
+    if self.config.get('OIDC_SERVER'):
+      return self._load_via_discovery(self._config.get('DEBUGGING', False))
+    else:
+      return {}
+
+  def _load_via_discovery(self, is_debugging):
+    oidc_server = self.config['OIDC_SERVER']
+    if not oidc_server.startswith('https://') and not is_debugging:
+      raise Exception('OIDC server must be accessed over SSL')
+
+    discovery_url = urlparse.urljoin(oidc_server, OIDC_WELLKNOWN)
+    discovery = self._http_client.get(discovery_url, timeout=5)
+
+    if discovery.status_code / 100 != 2:
+      raise Exception("Could not load OIDC discovery information")
+
+    try:
+      return json.loads(discovery.text)
+    except ValueError:
+      logger.exception('Could not parse OIDC discovery for url: %s', discovery_url)
+      raise Exception("Could not parse OIDC discovery information")
+
+  def authorize_endpoint(self):
+    return self._oidc_config().get('authorization_endpoint', '') + '?'
+
+  def token_endpoint(self):
+    return self._oidc_config().get('token_endpoint')
+
+  def user_endpoint(self):
+    return None
+
+  def validate_client_id_and_secret(self, http_client, app_config):
+    pass
+
+  def get_public_config(self):
+    return {
+      'CLIENT_ID': self.client_id(),
+      'AUTHORIZE_ENDPOINT': self.authorize_endpoint(),
+      'OIDC': True,
+    }
+
+  @property
+  def issuer(self):
+    return self.config.get('OIDC_ISSUER', self.config['OIDC_SERVER'])
+
+  def get_public_key(self, force_refresh=False):
+    """ Retrieves the public key for this handler. """
+    # If force_refresh is true, we expire all the items in the cache by setting the time to
+    # the current time + the expiration TTL.
+    if force_refresh:
+      self._public_key_cache.expire(time=time.time() + PUBLIC_KEY_CACHE_TTL)
+
+    # Retrieve the public key from the cache. If the cache does not contain the public key,
+    # it will internally call _get_public_key to retrieve it and then save it. The None is
+    # a random key chose to be stored in the cache, and could be anything.
+    return self._public_key_cache[None]
+
+  def _get_public_key(self, _):
+    """ Retrieves the public key for this handler. """
+    keys_url = self._oidc_config()['jwks_uri']
+
+    keys = KEYS()
+    keys.load_from_url(keys_url)
+
+    if not list(keys):
+      raise Exception('No keys provided by OIDC provider')
+
+    rsa_key = list(keys)[0]
+    rsa_key.deserialize()
+
+    # Reload the key so that we can give a key *instance* to PyJWT to work around its weird parsing
+    # issues.
+    return load_der_public_key(rsa_key.key.exportKey('DER'), backend=default_backend())

oauth/services/github.py

@@ -0,0 +1,176 @@
+import logging
+
+from oauth.login import OAuthLoginService, OAuthLoginException
+from util import slash_join
+
+logger = logging.getLogger(__name__)
+
+class GithubOAuthService(OAuthLoginService):
+  def __init__(self, config, key_name):
+    super(GithubOAuthService, self).__init__(config, key_name)
+
+  def service_id(self):
+    return 'github'
+
+  def service_name(self):
+    if self.is_enterprise():
+      return 'GitHub Enterprise'
+
+    return 'GitHub'
+
+  def get_icon(self):
+    return 'fa-github'
+
+  def get_login_scopes(self):
+    if self.config.get('ORG_RESTRICT'):
+      return ['user:email', 'read:org']
+
+    return ['user:email']
+
+  def allowed_organizations(self):
+    if not self.config.get('ORG_RESTRICT', False):
+      return None
+
+    allowed = self.config.get('ALLOWED_ORGANIZATIONS', None)
+    if allowed is None:
+      return None
+
+    return [org.lower() for org in allowed]
+
+  def get_public_url(self, suffix):
+    return slash_join(self._endpoint(), suffix)
+
+  def _endpoint(self):
+    return self.config.get('GITHUB_ENDPOINT', 'https://github.com')
+
+  def is_enterprise(self):
+    return self._api_endpoint().find('.github.com') < 0
+
+  def authorize_endpoint(self):
+    return slash_join(self._endpoint(), '/login/oauth/authorize')  + '?'
+
+  def token_endpoint(self):
+    return slash_join(self._endpoint(), '/login/oauth/access_token')
+
+  def _api_endpoint(self):
+    return self.config.get('API_ENDPOINT', slash_join(self._endpoint(), '/api/v3/'))
+
+  def api_endpoint(self):
+    endpoint = self._api_endpoint()
+    if endpoint.endswith('/'):
+      return endpoint[0:-1]
+
+    return endpoint
+
+  def user_endpoint(self):
+    return slash_join(self._api_endpoint(), 'user')
+
+  def email_endpoint(self):
+    return slash_join(self._api_endpoint(), 'user/emails')
+
+  def orgs_endpoint(self):
+    return slash_join(self._api_endpoint(), 'user/orgs')
+
+  def validate_client_id_and_secret(self, http_client, app_config):
+    # First: Verify that the github endpoint is actually Github by checking for the
+    # X-GitHub-Request-Id here.
+    api_endpoint = self._api_endpoint()
+    result = http_client.get(api_endpoint, auth=(self.client_id(), self.client_secret()), timeout=5)
+    if not 'X-GitHub-Request-Id' in result.headers:
+      raise Exception('Endpoint is not a Github (Enterprise) installation')
+
+    # Next: Verify the client ID and secret.
+    # Note: The following code is a hack until such time as Github officially adds an API endpoint
+    # for verifying a {client_id, client_secret} pair. That being said, this hack was given to us
+    # *by a Github Engineer*, so I think it is okay for the time being :)
+    #
+    # TODO(jschorr): Replace with the real API call once added.
+    #
+    # Hitting the endpoint applications/{client_id}/tokens/foo will result in the following
+    # behavior IF the client_id is given as the HTTP username and the client_secret as the HTTP
+    # password:
+    #   - If the {client_id, client_secret} pair is invalid in some way, we get a 401 error.
+    #   - If the pair is valid, then we get a 404 because the 'foo' token does not exists.
+    validate_endpoint = slash_join(api_endpoint, 'applications/%s/tokens/foo' % self.client_id())
+    result = http_client.get(validate_endpoint, auth=(self.client_id(), self.client_secret()),
+                             timeout=5)
+    return result.status_code == 404
+
+  def validate_organization(self, organization_id, http_client):
+    org_endpoint = slash_join(self._api_endpoint(), 'orgs/%s' % organization_id.lower())
+
+    result = http_client.get(org_endpoint,
+                             headers={'Accept': 'application/vnd.github.moondragon+json'},
+                             timeout=5)
+
+    return result.status_code == 200
+
+
+  def get_public_config(self):
+    return  {
+      'CLIENT_ID': self.client_id(),
+      'AUTHORIZE_ENDPOINT': self.authorize_endpoint(),
+      'GITHUB_ENDPOINT': self._endpoint(),
+      'ORG_RESTRICT': self.config.get('ORG_RESTRICT', False)
+    }
+
+  def get_login_service_id(self, user_info):
+    return user_info['id']
+
+  def get_login_service_username(self, user_info):
+    return user_info['login']
+
+  def get_verified_user_email(self, app_config, http_client, token, user_info):
+    v3_media_type = {
+      'Accept': 'application/vnd.github.v3'
+    }
+
+    token_param = {
+      'access_token': token,
+    }
+
+    # Find the e-mail address for the user: we will accept any email, but we prefer the primary
+    get_email = http_client.get(self.email_endpoint(), params=token_param, headers=v3_media_type)
+    if get_email.status_code // 100 != 2:
+      raise OAuthLoginException('Got non-2XX status code for emails endpoint: %s' %
+                                get_email.status_code)
+
+    verified_emails = [email for email in get_email.json() if email['verified']]
+    primary_emails = [email for email in get_email.json() if email['primary']]
+
+    # Special case: We don't care about whether an e-mail address is "verified" under GHE.
+    if self.is_enterprise() and not verified_emails:
+      verified_emails = primary_emails
+
+    allowed_emails = (primary_emails or verified_emails or [])
+    return allowed_emails[0]['email'] if len(allowed_emails) > 0 else None
+
+  def service_verify_user_info_for_login(self, app_config, http_client, token, user_info):
+    # Retrieve the user's orgnizations (if organization filtering is turned on)
+    if self.allowed_organizations() is None:
+      return
+
+    moondragon_media_type = {
+      'Accept': 'application/vnd.github.moondragon+json'
+    }
+
+    token_param = {
+      'access_token': token,
+    }
+
+    get_orgs = http_client.get(self.orgs_endpoint(), params=token_param,
+                               headers=moondragon_media_type)
+
+    if get_orgs.status_code // 100 != 2:
+      logger.debug('get_orgs response: %s', get_orgs.json())
+      raise OAuthLoginException('Got non-2XX response for org lookup: %s' %
+                                get_orgs.status_code)
+
+    organizations = set([org.get('login').lower() for org in get_orgs.json()])
+    matching_organizations = organizations & set(self.allowed_organizations())
+    if not matching_organizations:
+      logger.debug('Found organizations %s, but expected one of %s', organizations,
+                   self.allowed_organizations())
+      err = """You are not a member of an allowed GitHub organization.
+               Please contact your system administrator if you believe this is in error."""
+      raise OAuthLoginException(err)

oauth/services/gitlab.py

@@ -0,0 +1,56 @@
+from oauth.base import OAuthService
+from util import slash_join
+
+class GitLabOAuthService(OAuthService):
+  def __init__(self, config, key_name):
+    super(GitLabOAuthService, self).__init__(config, key_name)
+
+  def service_id(self):
+    return 'gitlab'
+
+  def service_name(self):
+    return 'GitLab'
+
+  def _endpoint(self):
+    return self.config.get('GITLAB_ENDPOINT', 'https://gitlab.com')
+
+  def user_endpoint(self):
+    raise NotImplementedError
+
+  def api_endpoint(self):
+    return self._endpoint()
+
+  def get_public_url(self, suffix):
+    return slash_join(self._endpoint(), suffix)
+
+  def authorize_endpoint(self):
+    return slash_join(self._endpoint(), '/oauth/authorize')
+
+  def token_endpoint(self):
+    return slash_join(self._endpoint(), '/oauth/token')
+
+  def validate_client_id_and_secret(self, http_client, app_config):
+    url = self.token_endpoint()
+    redirect_uri = self.get_redirect_uri(app_config, redirect_suffix='trigger')
+    data = {
+      'code': 'fakecode',
+      'client_id': self.client_id(),
+      'client_secret': self.client_secret(),
+      'grant_type': 'authorization_code',
+      'redirect_uri': redirect_uri
+    }
+
+    # We validate by checking the error code we receive from this call.
+    result = http_client.post(url, data=data, timeout=5)
+    value = result.json()
+    if not value:
+      return False
+
+    return value.get('error', '') != 'invalid_client'
+
+  def get_public_config(self):
+    return {
+      'CLIENT_ID': self.client_id(),
+      'AUTHORIZE_ENDPOINT': self.authorize_endpoint(),
+      'GITLAB_ENDPOINT': self._endpoint(),
+    }

oauth/services/google.py

@@ -0,0 +1,76 @@
+from oauth.login import OAuthLoginService
+
+def _get_email_username(email_address):
+  username = email_address
+  at = username.find('@')
+  if at > 0:
+    username = username[0:at]
+
+  return username
+
+class GoogleOAuthService(OAuthLoginService):
+  def __init__(self, config, key_name):
+    super(GoogleOAuthService, self).__init__(config, key_name)
+
+  def service_id(self):
+    return 'google'
+
+  def service_name(self):
+    return 'Google'
+
+  def get_icon(self):
+    return 'fa-google'
+
+  def get_login_scopes(self):
+    return ['openid', 'email']
+
+  def authorize_endpoint(self):
+    return 'https://accounts.google.com/o/oauth2/auth?response_type=code&'
+
+  def token_endpoint(self):
+    return 'https://accounts.google.com/o/oauth2/token'
+
+  def user_endpoint(self):
+    return 'https://www.googleapis.com/oauth2/v1/userinfo'
+
+  def requires_form_encoding(self):
+    return True
+
+  def validate_client_id_and_secret(self, http_client, app_config):
+    # To verify the Google client ID and secret, we hit the
+    # https://www.googleapis.com/oauth2/v3/token endpoint with an invalid request. If the client
+    # ID or secret are invalid, we get returned a 403 Unauthorized. Otherwise, we get returned
+    # another response code.
+    url = 'https://www.googleapis.com/oauth2/v3/token'
+    data = {
+      'code': 'fakecode',
+      'client_id': self.client_id(),
+      'client_secret': self.client_secret(),
+      'grant_type': 'authorization_code',
+      'redirect_uri': 'http://example.com'
+    }
+
+    result = http_client.post(url, data=data, timeout=5)
+    return result.status_code != 401
+
+  def get_public_config(self):
+    return  {
+      'CLIENT_ID': self.client_id(),
+      'AUTHORIZE_ENDPOINT': self.authorize_endpoint()
+    }
+
+  def get_login_service_id(self, user_info):
+    return user_info['id']
+
+  def get_login_service_username(self, user_info):
+    return _get_email_username(user_info['email'])
+
+  def get_verified_user_email(self, app_config, http_client, token, user_info):
+    if not user_info.get('verified_email', False):
+      return None
+
+    return user_info['email']
+
+  def service_verify_user_info_for_login(self, app_config, http_client, token, user_info):
+    # Nothing to do.
+    pass