Merge pull request #1905 from coreos-inc/external-auth-search

Add support for entity search against external auth users not yet linked
This commit is contained in:
josephschorr 2016-10-27 16:06:42 -04:00 committed by GitHub
commit 934cdecbd6
16 changed files with 817 additions and 100 deletions

View file

@ -499,8 +499,7 @@ def get_matching_user_namespaces(namespace_prefix, username, limit=10):
return _basequery.filter_to_repos_for_user(base_query, username).limit(limit)
def get_matching_users(username_prefix, robot_namespace=None,
organization=None):
def get_matching_users(username_prefix, robot_namespace=None, organization=None, limit=20):
user_search = _basequery.prefix_search(User.username, username_prefix)
direct_user_query = (user_search & (User.organization == False) & (User.robot == False))
@ -516,23 +515,24 @@ def get_matching_users(username_prefix, robot_namespace=None,
if organization:
query = (query
.select(User.username, User.email, User.robot, fn.Sum(Team.id))
.select(User.id, User.username, User.email, User.robot, fn.Sum(Team.id))
.join(TeamMember, JOIN_LEFT_OUTER)
.join(Team, JOIN_LEFT_OUTER, on=((Team.id == TeamMember.team) &
(Team.organization == organization))))
class MatchingUserResult(object):
def __init__(self, *args):
self.username = args[0]
self.email = args[1]
self.robot = args[2]
self.id = args[0]
self.username = args[1]
self.email = args[2]
self.robot = args[3]
if organization:
self.is_org_member = (args[3] != None)
else:
self.is_org_member = None
return (MatchingUserResult(*args) for args in query.tuples().limit(10))
return (MatchingUserResult(*args) for args in query.tuples().limit(limit))
def verify_user(username_or_email, password):
@ -749,3 +749,16 @@ def get_region_locations(user):
""" Returns the locations defined as preferred storage for the given user. """
query = UserRegion.select().join(ImageStorageLocation).where(UserRegion.user == user)
return set([region.location.name for region in query])
def get_federated_logins(user_ids, service_name):
""" Returns all federated logins for the given user ids under the given external service. """
if not user_ids:
return []
return (FederatedLogin
.select()
.join(User)
.switch(FederatedLogin)
.join(LoginService)
.where(FederatedLogin.user << user_ids,
LoginService.name == service_name))

View file

@ -9,7 +9,7 @@ from data import model
from data.users.database import DatabaseUsers
from data.users.externalldap import LDAPUsers
from data.users.externaljwt import ExternalJWTAuthN
from data.users.keystone import KeystoneUsers
from data.users.keystone import get_keystone_users
from util.security.aes import AESCipher
logger = logging.getLogger(__name__)
@ -54,17 +54,23 @@ def get_users_handler(config, config_provider, override_config_dir):
verify_url = config.get('JWT_VERIFY_ENDPOINT')
issuer = config.get('JWT_AUTH_ISSUER')
max_fresh_s = config.get('JWT_AUTH_MAX_FRESH_S', 300)
return ExternalJWTAuthN(verify_url, issuer, override_config_dir, config['HTTPCLIENT'],
max_fresh_s)
query_url = config.get('JWT_QUERY_ENDPOINT', None)
getuser_url = config.get('JWT_GETUSER_ENDPOINT', None)
return ExternalJWTAuthN(verify_url, query_url, getuser_url, issuer, override_config_dir,
config['HTTPCLIENT'], max_fresh_s)
if authentication_type == 'Keystone':
auth_url = config.get('KEYSTONE_AUTH_URL')
auth_version = int(config.get('KEYSTONE_AUTH_VERSION', 2))
timeout = config.get('KEYSTONE_AUTH_TIMEOUT')
keystone_admin_username = config.get('KEYSTONE_ADMIN_USERNAME')
keystone_admin_password = config.get('KEYSTONE_ADMIN_PASSWORD')
keystone_admin_tenant = config.get('KEYSTONE_ADMIN_TENANT')
return KeystoneUsers(auth_url, keystone_admin_username, keystone_admin_password,
keystone_admin_tenant, timeout)
return get_keystone_users(auth_version, auth_url, keystone_admin_username,
keystone_admin_password, keystone_admin_tenant, timeout)
raise RuntimeError('Unknown authentication type: %s' % authentication_type)
@ -139,6 +145,31 @@ class UserAuthentication(object):
return data.get('password', encrypted)
@property
def federated_service(self):
""" Returns the name of the federated service for the auth system. If none, should return None.
"""
return self.state.federated_service
def query_users(self, query, limit=20):
""" Performs a lookup against the user system for the specified query. The returned tuple
will be of the form (results, federated_login_id, err_msg). If the method is unsupported,
the results portion of the tuple will be None instead of empty list.
Note that this method can and will return results for users not yet found within the
database; it is the responsibility of the caller to call link_user if they need the
database row for the user system record.
Results will be in the form of objects's with username and email fields.
"""
return self.state.query_users(query, limit)
def link_user(self, username_or_email):
""" Returns a tuple containing the database user record linked to the given username/email
and any error that occurred when trying to link the user.
"""
return self.state.link_user(username_or_email)
def confirm_existing_user(self, username, password):
""" Verifies that the given password matches to the given DB username. Unlike
verify_credentials, this call first translates the DB user via the FederatedLogin table

View file

@ -1,6 +1,10 @@
from data import model
class DatabaseUsers(object):
@property
def federated_service(self):
return None
def verify_credentials(self, username_or_email, password):
""" Simply delegate to the model implementation. """
result = model.user.verify_user(username_or_email, password)
@ -16,3 +20,11 @@ class DatabaseUsers(object):
def confirm_existing_user(self, username, password):
return self.verify_credentials(username, password)
def link_user(self, username_or_email):
""" Never used since all users being added are already, by definition, in the database. """
return (None, 'Unsupported for this authentication system')
def query_users(self, query, limit):
""" No need to implement, as we already query for users directly in the database. """
return (None, '', '')

View file

@ -2,7 +2,7 @@ import logging
import json
import os
from data.users.federated import FederatedUsers, VerifiedCredentials
from data.users.federated import FederatedUsers, UserInformation
from util.security import jwtutil
@ -13,10 +13,13 @@ class ExternalJWTAuthN(FederatedUsers):
""" Delegates authentication to a REST endpoint that returns JWTs. """
PUBLIC_KEY_FILENAME = 'jwt-authn.cert'
def __init__(self, verify_url, issuer, override_config_dir, http_client, max_fresh_s,
public_key_path=None):
def __init__(self, verify_url, query_url, getuser_url, issuer, override_config_dir, http_client,
max_fresh_s, public_key_path=None):
super(ExternalJWTAuthN, self).__init__('jwtauthn')
self.verify_url = verify_url
self.query_url = query_url
self.getuser_url = getuser_url
self.issuer = issuer
self.client = http_client
self.max_fresh_s = max_fresh_s
@ -32,34 +35,80 @@ class ExternalJWTAuthN(FederatedUsers):
with open(public_key_path) as public_key_file:
self.public_key = public_key_file.read()
def verify_credentials(self, username_or_email, password):
result = self.client.get(self.verify_url, timeout=2, auth=(username_or_email, password))
def get_user(self, username_or_email):
if self.getuser_url is None:
return (None, 'No endpoint defined for retrieving user')
(payload, err_msg) = self._execute_call(self.getuser_url, 'quay.io/jwtauthn/getuser',
params=dict(username=username_or_email))
if err_msg is not None:
return (None, err_msg)
if not 'sub' in payload:
raise Exception('Missing sub field in JWT')
if not 'email' in payload:
raise Exception('Missing email field in JWT')
# Parse out the username and email.
user_info = UserInformation(username=payload['sub'], email=payload['email'], id=payload['sub'])
return (user_info, None)
def query_users(self, query, limit=20):
if self.query_url is None:
return (None, self.federated_service, 'No endpoint defined for querying users')
(payload, err_msg) = self._execute_call(self.query_url, 'quay.io/jwtauthn/query',
params=dict(query=query, limit=limit))
if err_msg is not None:
return (None, self.federated_service, err_msg)
query_results = []
for result in payload['results'][0:limit]:
user_info = UserInformation(username=result['username'], email=result['email'],
id=result['username'])
query_results.append(user_info)
return (query_results, self.federated_service, None)
def verify_credentials(self, username_or_email, password):
(payload, err_msg) = self._execute_call(self.verify_url, 'quay.io/jwtauthn',
auth=(username_or_email, password))
if err_msg is not None:
return (None, err_msg)
if not 'sub' in payload:
raise Exception('Missing sub field in JWT')
if not 'email' in payload:
raise Exception('Missing email field in JWT')
user_info = UserInformation(username=payload['sub'], email=payload['email'], id=payload['sub'])
return (user_info, None)
def _execute_call(self, url, aud, auth=None, params=None):
""" Executes a call to the external JWT auth provider. """
result = self.client.get(url, timeout=2, auth=auth, params=params)
if result.status_code != 200:
return (None, result.text or 'Invalid username or password')
return (None, result.text or 'Could not make JWT auth call')
try:
result_data = json.loads(result.text)
except ValueError:
raise Exception('Returned JWT Authentication body does not contain JSON')
raise Exception('Returned JWT body for url %s does not contain JSON', url)
# Load the JWT returned.
encoded = result_data.get('token', '')
exp_limit_options = jwtutil.exp_max_s_option(self.max_fresh_s)
try:
payload = jwtutil.decode(encoded, self.public_key, algorithms=['RS256'],
audience='quay.io/jwtauthn', issuer=self.issuer,
audience=aud, issuer=self.issuer,
options=exp_limit_options)
return (payload, None)
except jwtutil.InvalidTokenError:
logger.exception('Exception when decoding returned JWT')
return (None, 'Invalid username or password')
if not 'sub' in payload:
raise Exception('Missing username field in JWT')
if not 'email' in payload:
raise Exception('Missing email field in JWT')
# Parse out the username and email.
return (VerifiedCredentials(username=payload['sub'], email=payload['email']), None)
logger.exception('Exception when decoding returned JWT for url %s', url)
return (None, 'Exception when decoding returned JWT')

View file

@ -1,9 +1,10 @@
import ldap
import logging
import os
import itertools
from collections import namedtuple
from data.users.federated import FederatedUsers, VerifiedCredentials
from data.users.federated import FederatedUsers, UserInformation
logger = logging.getLogger(__name__)
@ -25,11 +26,11 @@ class LDAPConnection(object):
self._user_dn = user_dn
self._user_pw = user_pw
self._allow_tls_fallback = allow_tls_fallback
self._conn = None
def __enter__(self):
trace_level = 2 if os.environ.get('USERS_DEBUG') == '1' else 0
self._conn = ldap.initialize(self._ldap_uri, trace_level=trace_level)
self._conn.set_option(ldap.OPT_REFERRALS, 1)
@ -43,6 +44,10 @@ class LDAPConnection(object):
def __exit__(self, exc_type, value, tb):
self._conn.unbind_s()
def _take(n, iterable):
"Return first n items of the iterable as a list"
return list(itertools.islice(iterable, n))
class LDAPUsers(FederatedUsers):
_LDAPResult = namedtuple('LDAPResult', ['dn', 'attrs'])
@ -51,6 +56,7 @@ class LDAPUsers(FederatedUsers):
allow_tls_fallback=False, secondary_user_rdns=None):
super(LDAPUsers, self).__init__('ldap')
self._ldap = LDAPConnectionBuilder(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback)
self._ldap_uri = ldap_uri
self._uid_attr = uid_attr
@ -109,7 +115,10 @@ class LDAPUsers(FederatedUsers):
logger.exception('LDAP search exception')
return (None, 'Username not found')
def _ldap_user_search(self, username_or_email):
def _ldap_user_search(self, username_or_email, limit=20):
if not username_or_email:
return (None, 'Empty username/email')
# Verify the admin connection works first. We do this here to avoid wrapping
# the entire block in the INVALID CREDENTIALS check.
try:
@ -130,22 +139,72 @@ class LDAPUsers(FederatedUsers):
return (None, err_msg)
logger.debug('Found matching pairs: %s', pairs)
results = [LDAPUsers._LDAPResult(*pair) for pair in pairs]
results = [LDAPUsers._LDAPResult(*pair) for pair in _take(limit, pairs)]
# Filter out pairs without DNs. Some LDAP impls will return such
# pairs.
# Filter out pairs without DNs. Some LDAP impls will return such pairs.
with_dns = [result for result in results if result.dn]
if len(with_dns) < 1:
return (None, 'Username not found')
return (with_dns, None)
# If we have found a single pair, then return it.
if len(with_dns) == 1:
return (with_dns[0], None)
def _ldap_single_user_search(self, username_or_email):
with_dns, err_msg = self._ldap_user_search(username_or_email)
if err_msg is not None:
return (None, err_msg)
# Otherwise, there are multiple pairs with DNs, so find the one with the mail
# attribute (if any).
with_mail = [result for result in results if result.attrs.get(self._email_attr)]
return (with_mail[0] if with_mail else with_dns[0], None)
# Make sure we have at least one result.
if len(with_dns) < 1:
return (None, 'Username not found')
# If we have found a single pair, then return it.
if len(with_dns) == 1:
return (with_dns[0], None)
# Otherwise, there are multiple pairs with DNs, so find the one with the mail
# attribute (if any).
with_mail = [result for result in with_dns if result.attrs.get(self._email_attr)]
return (with_mail[0] if with_mail else with_dns[0], None)
def _credential_for_user(self, response):
if not response.get(self._uid_attr):
return (None, 'Missing uid field "%s" in user record' % self._uid_attr)
if not response.get(self._email_attr):
return (None, 'Missing mail field "%s" in user record' % self._email_attr)
username = response[self._uid_attr][0].decode('utf-8')
email = response[self._email_attr][0]
return (UserInformation(username=username, email=email, id=username), None)
def get_user(self, username_or_email):
""" Looks up a username or email in LDAP. """
logger.debug('Looking up LDAP username or email %s', username_or_email)
(found_user, err_msg) = self._ldap_single_user_search(username_or_email)
if err_msg is not None:
return (None, err_msg)
logger.debug('Found user for LDAP username or email %s', username_or_email)
_, found_response = found_user
return self._credential_for_user(found_response)
def query_users(self, query, limit=20):
""" Queries LDAP for matching users. """
if not query:
return (None, 'Empty query')
logger.debug('Got query %s with limit %s', query, limit)
(results, err_msg) = self._ldap_user_search(query + '*', limit=limit)
if err_msg is not None:
return (None, err_msg)
final_results = []
for result in results[0:limit]:
credentials, err_msg = self._credential_for_user(result.attrs)
if err_msg is not None:
continue
final_results.append(credentials)
logger.debug('For query %s found results %s', query, final_results)
return (final_results, None)
def verify_credentials(self, username_or_email, password):
""" Verify the credentials with LDAP. """
@ -153,7 +212,7 @@ class LDAPUsers(FederatedUsers):
if not password:
return (None, 'Anonymous binding not allowed')
(found_user, err_msg) = self._ldap_user_search(username_or_email)
(found_user, err_msg) = self._ldap_single_user_search(username_or_email)
if found_user is None:
return (None, err_msg)
@ -183,14 +242,4 @@ class LDAPUsers(FederatedUsers):
logger.exception('Invalid LDAP credentials')
return (None, 'Invalid password')
# Now check if we have a federated login for this user
if not found_response.get(self._uid_attr):
return (None, 'Missing uid field "%s" in user record' % self._uid_attr)
if not found_response.get(self._email_attr):
return (None, 'Missing mail field "%s" in user record' % self._email_attr)
username = found_response[self._uid_attr][0].decode('utf-8')
email = found_response[self._email_attr][0]
return (VerifiedCredentials(username=username, email=email), None)
return self._credential_for_user(found_response)

View file

@ -7,7 +7,7 @@ from util.validation import generate_valid_usernames
logger = logging.getLogger(__name__)
VerifiedCredentials = namedtuple('VerifiedCredentials', ['username', 'email'])
UserInformation = namedtuple('UserInformation', ['username', 'email', 'id'])
class FederatedUsers(object):
""" Base class for all federated users systems. """
@ -15,11 +15,26 @@ class FederatedUsers(object):
def __init__(self, federated_service):
self._federated_service = federated_service
@property
def federated_service(self):
return self._federated_service
def get_user(self, username_or_email):
""" Retrieves the user with the given username or email, returning a tuple containing
a UserInformation (if success) and the error message (on failure).
"""
raise NotImplementedError
def verify_credentials(self, username_or_email, password):
""" Verifies the given credentials against the backing federated service, returning
a tuple containing a VerifiedCredentials (if success) and the error message (if failed). """
a tuple containing a UserInformation (on success) and the error message (on failure).
"""
raise NotImplementedError
def query_users(self, query, limit=20):
""" If implemented, get_user must be implemented as well. """
return (None, 'Not supported')
def _get_federated_user(self, username, email):
db_user = model.user.verify_federated_login(self._federated_service, username)
if not db_user:
@ -34,7 +49,8 @@ class FederatedUsers(object):
return (None, 'Unable to pick a username. Please report this to your administrator.')
db_user = model.user.create_federated_user(valid_username, email, self._federated_service,
username, set_password_notification=False)
username,
set_password_notification=False)
else:
# Update the db attributes from the federated service.
db_user.email = email
@ -42,6 +58,13 @@ class FederatedUsers(object):
return (db_user, None)
def link_user(self, username_or_email):
(credentials, err_msg) = self.get_user(username_or_email)
if credentials is None:
return (None, err_msg)
return self._get_federated_user(credentials.username, credentials.email)
def verify_and_link_user(self, username_or_email, password):
""" Verifies the given credentials and, if valid, creates/links a database user to the
associated federated service.

View file

@ -1,19 +1,34 @@
import logging
import os
import itertools
from keystoneclient.v2_0 import client as kclient
from keystoneclient.v3 import client as kv3client
from keystoneclient.exceptions import AuthorizationFailure as KeystoneAuthorizationFailure
from keystoneclient.exceptions import Unauthorized as KeystoneUnauthorized
from data.users.federated import FederatedUsers, VerifiedCredentials
from data.users.federated import FederatedUsers, UserInformation
logger = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 10 # seconds
class KeystoneUsers(FederatedUsers):
""" Delegates authentication to OpenStack Keystone. """
def _take(n, iterable):
"Return first n items of the iterable as a list"
return list(itertools.islice(iterable, n))
def get_keystone_users(auth_version, auth_url, admin_username, admin_password, admin_tenant,
timeout=None):
if auth_version == 3:
return KeystoneV3Users(auth_url, admin_username, admin_password, admin_tenant, timeout)
else:
return KeystoneV2Users(auth_url, admin_username, admin_password, admin_tenant, timeout)
class KeystoneV2Users(FederatedUsers):
""" Delegates authentication to OpenStack Keystone V2. """
def __init__(self, auth_url, admin_username, admin_password, admin_tenant, timeout=None):
super(KeystoneUsers, self).__init__('keystone')
super(KeystoneV2Users, self).__init__('keystone')
self.auth_url = auth_url
self.admin_username = admin_username
self.admin_password = admin_password
@ -43,4 +58,75 @@ class KeystoneUsers(FederatedUsers):
logger.exception('Keystone unauthorized admin')
return (None, 'Keystone admin credentials are invalid: %s' % kut.message)
return (VerifiedCredentials(username=username_or_email, email=user.email), None)
return (UserInformation(username=username_or_email, email=user.email, id=user_id), None)
def query_users(self, query, limit=20):
return (None, 'Unsupported in Keystone V2')
def get_user(self, username_or_email):
return (None, 'Unsupported in Keystone V2')
class KeystoneV3Users(FederatedUsers):
""" Delegates authentication to OpenStack Keystone V3. """
def __init__(self, auth_url, admin_username, admin_password, admin_tenant, timeout=None):
super(KeystoneV3Users, self).__init__('keystone')
self.auth_url = auth_url
self.admin_username = admin_username
self.admin_password = admin_password
self.admin_tenant = admin_tenant
self.timeout = timeout or DEFAULT_TIMEOUT
self.debug = os.environ.get('USERS_DEBUG') == '1'
def verify_credentials(self, username_or_email, password):
try:
keystone_client = kv3client.Client(username=username_or_email, password=password,
auth_url=self.auth_url, timeout=self.timeout,
debug=self.debug)
user_id = keystone_client.user_id
user = keystone_client.users.get(user_id)
return (self._user_info(user), None)
except KeystoneAuthorizationFailure as kaf:
logger.exception('Keystone auth failure for user: %s', username_or_email)
return (None, kaf.message or 'Invalid username or password')
except KeystoneUnauthorized as kut:
logger.exception('Keystone unauthorized for user: %s', username_or_email)
return (None, kut.message or 'Invalid username or password')
def get_user(self, username_or_email):
users_found, err_msg = self.query_users(username_or_email)
if err_msg is not None:
return (None, err_msg)
if len(users_found) != 1:
return (None, 'Single user not found')
return (users_found[0], None)
@staticmethod
def _user_info(user):
# Because Keystone uses defined attributes...
email = user.email if hasattr(user, 'email') else ''
return UserInformation(user.name, email, user.id)
def query_users(self, query, limit=20):
if len(query) < 3:
return ([], None)
try:
keystone_client = kv3client.Client(username=self.admin_username, password=self.admin_password,
tenant_name=self.admin_tenant, auth_url=self.auth_url,
timeout=self.timeout, debug=self.debug)
found_users = list(_take(limit, keystone_client.users.list(name=query)))
logger.debug('For Keystone query %s found users: %s', query, found_users)
if not found_users:
return ([], None)
return ([self._user_info(user) for user in found_users], None)
except KeystoneAuthorizationFailure as kaf:
logger.exception('Keystone auth failure for admin user for query %s', query)
return (None, kaf.message or 'Invalid admin username or password')
except KeystoneUnauthorized as kut:
logger.exception('Keystone unauthorized for admin user for query %s', query)
return (None, kut.message or 'Invalid admin username or password')