From 8aac3fd86eaf56647b1f32ffe200cea3650025c6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 2 Jun 2015 18:19:22 -0400 Subject: [PATCH] Add support for an external JWT-based authentication system This authentication system hits two HTTP endpoints to check and verify the existence of users: Existance endpoint: GET http://endpoint/ with Authorization: Basic (username:) => Returns 200 if the username/email exists, 4** otherwise Verification endpoint: GET http://endpoint/ with Authorization: Basic (username:password) => Returns 200 and a signed JWT with the user's username and email address if the username+password validates, 4** otherwise with the body containing an optional error message The JWT produced by the endpoint must be issued with an issuer matching that configured in the config.yaml, and the audience must be "quay.io/jwtauthn". The JWT is signed using a private key and then validated on the Quay.io side with the associated public key, found as "jwt-authn.cert" in the conf/stack directory. --- ...ae_add_jwt_authentication_login_service.py | 28 ++++ data/users.py | 143 +++++++++++++--- endpoints/api/suconfig.py | 6 +- initdb.py | 1 + requirements-nover.txt | 3 +- requirements.txt | 1 + .../directives/config/config-setup-tool.html | 78 ++++++++- static/js/core-config-setup.js | 4 + test/test_jwt_auth.py | 156 ++++++++++++++++++ util/config/validator.py | 35 +++- 10 files changed, 417 insertions(+), 38 deletions(-) create mode 100644 data/migrations/versions/41f4587c84ae_add_jwt_authentication_login_service.py create mode 100644 test/test_jwt_auth.py diff --git a/data/migrations/versions/41f4587c84ae_add_jwt_authentication_login_service.py b/data/migrations/versions/41f4587c84ae_add_jwt_authentication_login_service.py new file mode 100644 index 000000000..f15875b83 --- /dev/null +++ b/data/migrations/versions/41f4587c84ae_add_jwt_authentication_login_service.py @@ -0,0 +1,28 @@ +"""Add JWT Authentication login service + +Revision ID: 41f4587c84ae +Revises: 1f116e06b68 +Create Date: 2015-06-02 16:13:02.636590 + +""" + +# revision identifiers, used by Alembic. +revision = '41f4587c84ae' +down_revision = '1f116e06b68' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + op.bulk_insert(tables.loginservice, + [ + {'id': 5, 'name':'jwtauthn'}, + ]) + + +def downgrade(tables): + op.execute( + (tables.loginservice.delete() + .where(tables.loginservice.c.name == op.inline_literal('jwtauthn'))) + ) diff --git a/data/users.py b/data/users.py index f528694a3..e3d4a7d62 100644 --- a/data/users.py +++ b/data/users.py @@ -5,11 +5,14 @@ import itertools import uuid import struct import os +import urllib +import jwt from util.aes import AESCipher from util.validation import generate_valid_usernames from data import model from collections import namedtuple +from datetime import datetime, timedelta logger = logging.getLogger(__name__) if os.environ.get('LDAP_DEBUG') == '1': @@ -20,6 +23,115 @@ if os.environ.get('LDAP_DEBUG') == '1': logger.addHandler(ch) + +def _get_federated_user(username, email, federated_service, create_new_user): + db_user = model.verify_federated_login(federated_service, username) + if not db_user: + if not create_new_user: + return (None, 'Invalid user') + + # We must create the user in our db + valid_username = None + for valid_username in generate_valid_usernames(username): + if model.is_username_unique(valid_username): + break + + if not valid_username: + logger.error('Unable to pick a username for user: %s', username) + return (None, 'Unable to pick a username. Please report this to your administrator.') + + db_user = model.create_federated_user(valid_username, email, federated_service, username, + set_password_notification=False) + else: + # Update the db attributes from ldap + db_user.email = email + db_user.save() + + return (db_user, None) + + +class JWTAuthUsers(object): + """ Delegates authentication to a REST endpoint that returns JWTs. """ + PUBLIC_KEY_FILENAME = 'jwt-authn.cert' + + def __init__(self, exists_url, verify_url, issuer, public_key_path=None): + from app import OVERRIDE_CONFIG_DIRECTORY + + self.verify_url = verify_url + self.exists_url = exists_url + self.issuer = issuer + + default_key_path = os.path.join(OVERRIDE_CONFIG_DIRECTORY, JWTAuthUsers.PUBLIC_KEY_FILENAME) + public_key_path = public_key_path or default_key_path + if not os.path.exists(public_key_path): + error_message = ('JWT Authentication public key file "%s" not found in directory %s' % + (JWTAuthUsers.PUBLIC_KEY_FILENAME, OVERRIDE_CONFIG_DIRECTORY)) + + raise Exception(error_message) + + with open(public_key_path) as public_key_file: + self.public_key = public_key_file.read() + + def verify_user(self, username_or_email, password, create_new_user=True): + from app import app + client = app.config['HTTPCLIENT'] + result = client.get(self.verify_url, timeout=2, auth=(username_or_email, password)) + + if result.status_code != 200: + return (None, result.text or 'Invalid username or password') + + try: + result_data = json.loads(result.text) + except ValueError: + raise Exception('Returned JWT Authentication body does not contain JSON') + + # Load the JWT returned. + encoded = result_data.get('token', '') + try: + payload = jwt.decode(encoded, self.public_key, algorithms=['RS256'], + audience='quay.io/jwtauthn', issuer=self.issuer) + except jwt.InvalidTokenError: + logger.exception('Exception when decoding returned JWT') + return (None, 'Invalid username or password') + + if not 'sub' in payload: + raise Exception('Missing username field in JWT') + + if not 'email' in payload: + raise Exception('Missing email field in JWT') + + if not 'exp' in payload: + raise Exception('Missing exp field in JWT') + + # Verify that the expiration is no more than 300 seconds in the future. + if datetime.fromtimestamp(payload['exp']) > datetime.utcnow() + timedelta(seconds=300): + logger.debug('Payload expiration is outside of the 300 second window: %s', payload['exp']) + return (None, 'Invalid username or password') + + # Parse out the username and email. + return _get_federated_user(payload['sub'], payload['email'], 'jwtauthn', create_new_user) + + def user_exists(self, username): + from app import app + client = app.config['HTTPCLIENT'] + result = client.get(self.exists_url, auth=(username, ''), timeout=2) + if result.status_code / 500 >= 1: + raise Exception('Internal Error when trying to check if user exists: %s' % result.text) + + return result.status_code == 200 + + def confirm_existing_user(self, username, password): + db_user = model.get_user(username) + if not db_user: + return (None, 'Invalid user') + + federated_login = model.lookup_federated_login(db_user, 'jwtauthn') + if not federated_login: + return (None, 'Invalid user') + + return self.verify_user(federated_login.service_ident, password, create_new_user=False) + + class DatabaseUsers(object): def verify_user(self, username_or_email, password): """ Simply delegate to the model implementation. """ @@ -189,30 +301,7 @@ class LDAPUsers(object): username = found_response[self._uid_attr][0].decode('utf-8') email = found_response[self._email_attr][0] - db_user = model.verify_federated_login('ldap', username) - - if not db_user: - if not create_new_user: - return (None, 'Invalid user') - - # We must create the user in our db - valid_username = None - for valid_username in generate_valid_usernames(username): - if model.is_username_unique(valid_username): - break - - if not valid_username: - logger.error('Unable to pick a username for user: %s', username) - return (None, 'Unable to pick a username. Please report this to your administrator.') - - db_user = model.create_federated_user(valid_username, email, 'ldap', username, - set_password_notification=False) - else: - # Update the db attributes from ldap - db_user.email = email - db_user.save() - - return (db_user, None) + return _get_federated_user(username, email, 'ldap', create_new_user) def user_exists(self, username): found_user = self._ldap_user_search(username) @@ -243,7 +332,11 @@ class UserAuthentication(object): email_attr = app.config.get('LDAP_EMAIL_ATTR', 'mail') users = LDAPUsers(ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr) - + elif authentication_type == 'JWT': + verify_url = app.config.get('JWT_VERIFY_ENDPOINT') + exists_url = app.config.get('JWT_EXISTS_ENDPOINT') + issuer = app.config.get('JWT_AUTH_ISSUER') + users = JWTAuthUsers(exists_url, verify_url, issuer) else: raise RuntimeError('Unknown authentication type: %s' % authentication_type) diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index b1152231c..4802dc99d 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -16,7 +16,7 @@ from auth.auth_context import get_authenticated_user from data.database import User from util.config.configutil import add_enterprise_config_defaults from util.config.provider import CannotWriteConfigException -from util.config.validator import validate_service_for_config, SSL_FILENAMES +from util.config.validator import validate_service_for_config, CONFIG_FILENAMES from data.runmigration import run_alembic_migration import features @@ -224,7 +224,7 @@ class SuperUserConfigFile(ApiResource): @verify_not_prod def get(self, filename): """ Returns whether the configuration file with the given name exists. """ - if not filename in SSL_FILENAMES: + if not filename in CONFIG_FILENAMES: abort(404) if SuperUserPermission().can(): @@ -238,7 +238,7 @@ class SuperUserConfigFile(ApiResource): @verify_not_prod def post(self, filename): """ Updates the configuration file with the given name. """ - if not filename in SSL_FILENAMES: + if not filename in CONFIG_FILENAMES: abort(404) if SuperUserPermission().can(): diff --git a/initdb.py b/initdb.py index b67d0440e..61193fb92 100644 --- a/initdb.py +++ b/initdb.py @@ -201,6 +201,7 @@ def initialize_database(): LoginService.create(name='github') LoginService.create(name='quayrobot') LoginService.create(name='ldap') + LoginService.create(name='jwtauthn') BuildTriggerService.create(name='github') BuildTriggerService.create(name='custom-git') diff --git a/requirements-nover.txt b/requirements-nover.txt index 8cbb82cf8..ad25b23d3 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -52,4 +52,5 @@ psutil stringscore python-swiftclient python-keystoneclient -Flask-Testing \ No newline at end of file +Flask-Testing +pyjwt \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4531f5227..74b748c7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ Pillow==2.8.1 PyMySQL==0.6.6 PyPDF2==1.24 PyYAML==3.11 +PyJWT==1.3.0 SQLAlchemy==1.0.3 WebOb==1.4.1 Werkzeug==0.10.4 diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 065d55134..932f3a9b9 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -58,8 +58,8 @@ line with a non-encrypted password and must generate an encrypted password to use. -
- This feature is highly recommended for setups with LDAP authentication, as Docker currently stores passwords in plaintext on user's machines. +
+ This feature is highly recommended for setups with external authentication, as Docker currently stores passwords in plaintext on user's machines.
@@ -316,19 +316,20 @@

- Authentication for the registry can be handled by either the registry itself or LDAP. - External authentication providers (such as GitHub) can be used on top of this choice. + Authentication for the registry can be handled by either the registry itself, LDAP or external JWT endpoint. +
+ Additional external authentication providers (such as GitHub) can be used on top of this choice.

-
- It is highly recommended to require encrypted client passwords. LDAP passwords used in the Docker client will be stored in plaintext! +
+ It is highly recommended to require encrypted client passwords. External passwords used in the Docker client will be stored in plaintext! Enable this requirement now.
-
+
Note: The "Require Encrypted Client Passwords" feature is currently enabled which will - prevent LDAP passwords from being saved as plaintext by the Docker client. + prevent passwords from being saved as plaintext by the Docker client.
@@ -338,11 +339,72 @@
+ +
+ JSON Web Token authentication allows your organization to provide an HTTP endpoint that + verifies user credentials on behalf of . +
+ Documentation + on the API required can be found here: https://coreos.com/docs/enterprise-registry/jwt-auth. +
+ + + + + + + + + + + + + + + + + +
User Verification Endpoint: + +
+ The URL (starting with http or https) on the JWT authentication server for verifying username and password credentials. +
+ +
+ Credentials will be sent in the Authorization header as Basic Auth, and this endpoint should return 200 OK on success (or a 4** otherwise). +
+
User Exists Endpoint: + +
+ The URL (starting with http or https) on the JWT authentication server for checking whether a username exists. +
+ +
+ The username will be sent in the Authorization header as Basic Auth, and this endpoint should return 200 OK on success (or a 4** otherwise). +
+
Authentication Issuer: + +
+ The id of the issuer signing the JWT token. Must be unique to your organization. +
+
Public Key: + +
+ A certificate containing the public key portion of the key pair used to sign + the JSON Web Tokens. This file must be in PEM format. +
+
+ + diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 93c07c7a6..1d9cf852d 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -27,6 +27,10 @@ angular.module("core-config-setup", ['angularFileUpload']) return config.AUTHENTICATION_TYPE == 'LDAP'; }}, + {'id': 'jwt', 'title': 'JWT Authentication', 'condition': function(config) { + return config.AUTHENTICATION_TYPE == 'JWT'; + }}, + {'id': 'mail', 'title': 'E-mail Support', 'condition': function(config) { return config.FEATURE_MAILING; }}, diff --git a/test/test_jwt_auth.py b/test/test_jwt_auth.py new file mode 100644 index 000000000..3ebbc3de4 --- /dev/null +++ b/test/test_jwt_auth.py @@ -0,0 +1,156 @@ +import unittest +import requests +import jwt +import base64 + +from app import app +from flask import Flask, abort, jsonify, request, make_response +from flask.ext.testing import LiveServerTestCase +from initdb import setup_database_for_testing, finished_database_for_testing +from data.users import JWTAuthUsers +from tempfile import NamedTemporaryFile +from Crypto.PublicKey import RSA +from datetime import datetime, timedelta + +class JWTAuthTestCase(LiveServerTestCase): + maxDiff = None + + @classmethod + def setUpClass(cls): + public_key = NamedTemporaryFile(delete=True) + + key = RSA.generate(1024) + private_key_data = key.exportKey('PEM') + + pubkey = key.publickey() + public_key.write(pubkey.exportKey('OpenSSH')) + public_key.seek(0) + + JWTAuthTestCase.public_key = public_key + JWTAuthTestCase.private_key_data = private_key_data + + def create_app(self): + users = [ + { 'name': 'cooluser', 'email': 'user@domain.com', 'password': 'password' }, + { 'name': 'some.neat.user', 'email': 'neat@domain.com', 'password': 'foobar'} + ] + + jwt_app = Flask('testjwt') + private_key = JWTAuthTestCase.private_key_data + + def _get_basic_auth(): + data = base64.b64decode(request.headers['Authorization'][len('Basic '):]) + return data.split(':', 1) + + @jwt_app.route('/user/exists', methods=['GET']) + def user_exists(): + username, _ = _get_basic_auth() + for user in users: + if user['name'] == username or user['email'] == username: + return 'OK' + + abort(404) + + @jwt_app.route('/user/verify', methods=['GET']) + def verify_user(): + username, password = _get_basic_auth() + + if username == 'disabled': + return make_response('User is currently disabled', 401) + + for user in users: + if user['name'] == username or user['email'] == username: + if password != user['password']: + return make_response('', 404) + + token_data = { + 'iss': 'authy', + 'aud': 'quay.io/jwtauthn', + 'nbf': datetime.utcnow(), + 'exp': datetime.utcnow() + timedelta(seconds=60), + 'sub': user['name'], + 'email': user['email'] + } + + encoded = jwt.encode(token_data, private_key, 'RS256') + return jsonify({ + 'token': encoded + }) + + return make_response('', 404) + + jwt_app.config['TESTING'] = True + jwt_app.config['DEBUG'] = True + return jwt_app + + + def setUp(self): + setup_database_for_testing(self) + self.app = app.test_client() + self.ctx = app.test_request_context() + self.ctx.__enter__() + + self.session = requests.Session() + + self.jwt_auth = JWTAuthUsers( + self.get_server_url() + '/user/exists', + self.get_server_url() + '/user/verify', + 'authy', JWTAuthTestCase.public_key.name) + + def tearDown(self): + finished_database_for_testing(self) + self.ctx.__exit__(True, None, None) + + def test_user_exists(self): + self.assertFalse(self.jwt_auth.user_exists('testuser')) + self.assertFalse(self.jwt_auth.user_exists('anotheruser')) + + self.assertTrue(self.jwt_auth.user_exists('cooluser')) + self.assertTrue(self.jwt_auth.user_exists('user@domain.com')) + + def test_verify_user(self): + result, error_message = self.jwt_auth.verify_user('invaliduser', 'foobar') + self.assertEquals('Invalid username or password', error_message) + self.assertIsNone(result) + + result, _ = self.jwt_auth.verify_user('cooluser', 'invalidpassword') + self.assertIsNone(result) + + result, _ = self.jwt_auth.verify_user('cooluser', 'password') + self.assertIsNotNone(result) + self.assertEquals('cooluser', result.username) + + result, _ = self.jwt_auth.verify_user('some.neat.user', 'foobar') + self.assertIsNotNone(result) + self.assertEquals('some_neat_user', result.username) + + def test_confirm_existing_user(self): + # Create the users in the DB. + result, _ = self.jwt_auth.verify_user('cooluser', 'password') + self.assertIsNotNone(result) + + result, _ = self.jwt_auth.verify_user('some.neat.user', 'foobar') + self.assertIsNotNone(result) + + # Confirm a user with the same internal and external username. + result, _ = self.jwt_auth.confirm_existing_user('cooluser', 'password') + self.assertIsNotNone(result) + self.assertEquals('cooluser', result.username) + + # Fail to confirm the *external* username, which should return nothing. + result, _ = self.jwt_auth.confirm_existing_user('some.neat.user', 'password') + self.assertIsNone(result) + + # Now confirm the internal username. + result, _ = self.jwt_auth.confirm_existing_user('some_neat_user', 'foobar') + self.assertIsNotNone(result) + self.assertEquals('some_neat_user', result.username) + + def test_disabled_user_custom_erorr(self): + result, error_message = self.jwt_auth.verify_user('disabled', 'password') + self.assertIsNone(result) + self.assertEquals('User is currently disabled', error_message) + + +if __name__ == '__main__': + unittest.main() diff --git a/util/config/validator.py b/util/config/validator.py index ab1ee696d..1fda3e911 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -7,7 +7,7 @@ import OpenSSL import logging from fnmatch import fnmatch -from data.users import LDAPConnection +from data.users import LDAPConnection, JWTAuthUsers from flask import Flask from flask.ext.mail import Mail, Message from data.database import validate_database_url, User @@ -20,6 +20,8 @@ from bitbucket import BitBucket logger = logging.getLogger(__name__) SSL_FILENAMES = ['ssl.cert', 'ssl.key'] +JWT_FILENAMES = ['jwt-authn.cert'] +CONFIG_FILENAMES = SSL_FILENAMES + JWT_FILENAMES def get_storage_provider(config): parameters = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).get('local', ['LocalStorage', {}]) @@ -303,6 +305,36 @@ def _validate_ldap(config): raise Exception(values.get('desc', 'Unknown error')) +def _validate_jwt(config): + """ Validates the JWT authentication system. """ + if config.get('AUTHENTICATION_TYPE', 'Database') != 'JWT': + return + + verify_endpoint = config.get('JWT_VERIFY_ENDPOINT') + exists_endpoint = config.get('JWT_EXISTS_ENDPOINT') + issuer = config.get('JWT_AUTH_ISSUER') + + if not verify_endpoint: + raise Exception('Missing JWT Verification endpoint') + + if not exists_endpoint: + raise Exception('Missing JWT Exists endpoint') + + if not issuer: + raise Exception('Missing JWT Issuer ID') + + # Try to instatiate the JWT authentication mechanism. This will raise an exception if + # the key cannot be found. + users = JWTAuthUsers(exists_endpoint, verify_endpoint, issuer) + + # Verify that the superuser exists. If not, raise an exception. + username = get_authenticated_user().username + result = users.user_exists(username) + if not result: + raise Exception('Verification of superuser %s failed. The user either does not exist ' + + 'in the remote authentication system OR JWT auth is misconfigured.' % username) + + _VALIDATORS = { 'database': _validate_database, 'redis': _validate_redis, @@ -315,4 +347,5 @@ _VALIDATORS = { 'google-login': _validate_google_login, 'ssl': _validate_ssl, 'ldap': _validate_ldap, + 'jwt': _validate_jwt, } \ No newline at end of file
LDAP URI: