diff --git a/app.py b/app.py index 214efb3b6..1fd0bc936 100644 --- a/app.py +++ b/app.py @@ -1,47 +1,51 @@ +import json import logging import os -import json from functools import partial + +from Crypto.PublicKey import RSA from flask import Flask, request, Request, _request_ctx_stack -from flask_principal import Principal from flask_login import LoginManager, UserMixin from flask_mail import Mail -from werkzeug.routing import BaseConverter +from flask_principal import Principal from jwkest.jwk import RSAKey -from Crypto.PublicKey import RSA +from werkzeug.routing import BaseConverter import features from avatars.avatars import Avatar -from storage import Storage -from data import model from data import database -from data.userfiles import Userfiles -from data.users import UserAuthentication +from data import model +from data.archivedlogs import LogArchive from data.billing import Billing from data.buildlogs import BuildLogs -from data.archivedlogs import LogArchive -from data.userevent import UserEventsBuilderModule from data.queue import WorkQueue, BuildMetricQueueReporter +from data.userevent import UserEventsBuilderModule +from data.userfiles import Userfiles +from data.users import UserAuthentication +from storage import Storage from util import get_app_url from util.saas.analytics import Analytics from util.saas.useranalytics import UserAnalytics from util.saas.exceptionlog import Sentry from util.names import urn_generator +from util.config.configutil import generate_secret_key from util.config.oauth import (GoogleOAuthConfig, GithubOAuthConfig, GitLabOAuthConfig, DexOAuthConfig) - -from util.security.signing import Signer -from util.security.instancekeys import InstanceKeys -from util.saas.cloudwatch import start_cloudwatch_sender from util.config.provider import get_config_provider -from util.config.configutil import generate_secret_key from util.config.superusermanager import SuperUserManager -from util.secscan.api import SecurityScannerAPI +from util.label_validator import LabelValidator +from util.license import LicenseValidator, LICENSE_FILENAME from util.metrics.metricqueue import MetricQueue from util.metrics.prometheus import PrometheusPlugin -from util.label_validator import LabelValidator +from util.names import urn_generator +from util.saas.analytics import Analytics +from util.saas.cloudwatch import start_cloudwatch_sender +from util.saas.exceptionlog import Sentry +from util.secscan.api import SecurityScannerAPI +from util.security.instancekeys import InstanceKeys +from util.security.signing import Signer OVERRIDE_CONFIG_DIRECTORY = 'conf/stack/' OVERRIDE_CONFIG_YAML_FILENAME = 'conf/stack/config.yaml' @@ -189,6 +193,9 @@ signer = Signer(app, config_provider) instance_keys = InstanceKeys(app) label_validator = LabelValidator(app) +license_validator = LicenseValidator(os.path.join(OVERRIDE_CONFIG_DIRECTORY, LICENSE_FILENAME)) +license_validator.start() + start_cloudwatch_sender(metric_queue, app) tf = app.config['DB_TRANSACTION_FACTORY'] diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index 502c048d4..3da059290 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -6,7 +6,8 @@ import signal from flask import abort from endpoints.api import (ApiResource, nickname, resource, internal_only, show_if, - require_fresh_login, request, validate_json_request, verify_not_prod) + require_fresh_login, request, validate_json_request, verify_not_prod, + InvalidRequest) from endpoints.common import common_login from app import app, config_provider, superusers, OVERRIDE_CONFIG_DIRECTORY @@ -18,6 +19,7 @@ from data.database import User from util.config.configutil import add_enterprise_config_defaults from util.config.database import sync_database_with_config from util.config.validator import validate_service_for_config, CONFIG_FILENAMES +from util.license import decode_license, LicenseError from data.runmigration import run_alembic_migration from data.users import get_federated_service_name, get_users_handler @@ -62,6 +64,12 @@ class SuperUserRegistryStatus(ApiResource): 'status': 'missing-config-dir' } + # If there is no license file, we need to ask the user to upload it. + if not config_provider.has_license_file(): + return { + 'status': 'upload-license' + } + # If there is no config file, we need to setup the database. if not config_provider.config_exists(): return { @@ -244,6 +252,50 @@ class SuperUserConfig(ApiResource): abort(403) +@resource('/v1/superuser/config/license') +@internal_only +@show_if(features.SUPER_USERS) +class SuperUserSetAndValidateLicense(ApiResource): + """ Resource for setting and validating a license. """ + schemas = { + 'ValidateLicense': { + 'type': 'object', + 'description': 'Validates and sets a license', + 'required': [ + 'license', + ], + 'properties': { + 'license': { + 'type': 'string' + }, + }, + }, + } + + @nickname('suSetAndValidateLicense') + @verify_not_prod + @validate_json_request('ValidateLicense') + def post(self): + """ Validates the given license contents and then saves it to the config volume. """ + if config_provider.has_license_file(): + abort(403) + + license_contents = request.get_json()['license'] + try: + decoded_license = decode_license(license_contents) + except LicenseError as le: + raise InvalidRequest(le.message) + + if decoded_license.is_expired: + raise InvalidRequest('License has expired') + + config_provider.save_license(license_contents) + return { + 'decoded': decoded_license.subscription, + 'success': True + } + + @resource('/v1/superuser/config/file/') @internal_only @show_if(features.SUPER_USERS) diff --git a/endpoints/api/superuser.py b/endpoints/api/superuser.py index baa5b1a86..d1fe52518 100644 --- a/endpoints/api/superuser.py +++ b/endpoints/api/superuser.py @@ -18,11 +18,12 @@ from auth.permissions import SuperUserPermission from endpoints.api import (ApiResource, nickname, resource, validate_json_request, internal_only, require_scope, show_if, parse_args, query_param, abort, require_fresh_login, path_param, verify_not_prod, - page_support, log_action) + page_support, log_action, InvalidRequest) from endpoints.api.logs import get_logs, get_aggregate_logs from data import model from data.database import ServiceKeyApprovalType from util.useremails import send_confirmation_email, send_recovery_email +from util.license import decode_license, LicenseError logger = logging.getLogger(__name__) @@ -819,3 +820,143 @@ class SuperUserServiceKeyApproval(ApiResource): return make_response('', 201) abort(403) + + +@resource('/v1/superuser/license') +@internal_only +@show_if(features.SUPER_USERS) +class SuperUserLicense(ApiResource): + """ Resource for getting and setting a license. """ + schemas = { + 'UpdateLicense': { + 'type': 'object', + 'description': 'Updates a license', + 'required': [ + 'license', + ], + 'properties': { + 'license': { + 'type': 'string' + }, + }, + }, + } + + @nickname('getLicense') + @require_fresh_login + @require_scope(scopes.SUPERUSER) + @verify_not_prod + def get(self): + """ Returns the current decoded license. """ + if SuperUserPermission().can(): + try: + decoded_license = config_provider.get_license() + except LicenseError as le: + raise InvalidRequest(le.message) + + if decoded_license.is_expired: + raise InvalidRequest('License has expired') + + return { + 'decoded': decoded_license.subscription, + 'success': True + } + + abort(403) + + @nickname('updateLicense') + @require_fresh_login + @require_scope(scopes.SUPERUSER) + @verify_not_prod + @validate_json_request('UpdateLicense') + def put(self): + """ Validates the given license contents and then saves it to the config volume. """ + if SuperUserPermission().can(): + license_contents = request.get_json()['license'] + try: + decoded_license = decode_license(license_contents) + except LicenseError as le: + raise InvalidRequest(le.message) + + if decoded_license.is_expired: + raise InvalidRequest('License has expired') + + config_provider.save_license(license_contents) + return { + 'decoded': decoded_license.subscription, + 'success': True + } + + abort(403) + + +@resource('/v1/messages') +@show_if(features.SUPER_USERS) +class SuperUserMessages(ApiResource): + """ Resource for getting a list of super user messages """ + + schemas = { + 'GetMessage': { + 'id': 'GetMessage', + 'type': 'object', + 'description': 'Messages that a super user has saved in the past', + 'properties': { + 'message': { + 'type': 'array', + 'description': 'A list of messages', + 'itemType': { + 'type': 'object', + 'properties': { + 'id': { + 'type': 'integer', + 'description': 'The message id', + }, + 'content': { + 'type': 'string', + 'description': 'The actual message', + }, + }, + }, + }, + }, + }, + 'CreateMessage': { + 'id': 'CreateMessage', + 'type': 'object', + 'description': 'Create a new message', + 'properties': { + 'message': { + 'type': 'object', + 'description': 'A single message', + 'properties': { + 'content': { + 'type': 'string', + 'description': 'The actual message', + }, + }, + }, + }, + } + } + + @nickname('getMessages') + def get(self): + """ Return a super users messages """ + return { + 'messages': [message_view(m) for m in model.message.get_messages()], + } + + @verify_not_prod + @nickname('createMessages') + @validate_json_request('CreateMessage') + @require_scope(scopes.SUPERUSER) + def post(self): + """ Create a message """ + if SuperUserPermission().can(): + model.message.create([request.get_json()['message']]) + return make_response('', 201) + abort(403) + + +def message_view(message): + return {'id': message.id, 'content': message.content} diff --git a/endpoints/v1/__init__.py b/endpoints/v1/__init__.py index 1e9715787..18ef430c4 100644 --- a/endpoints/v1/__init__.py +++ b/endpoints/v1/__init__.py @@ -1,14 +1,15 @@ from flask import Blueprint, make_response -from app import metric_queue +from app import metric_queue, license_validator from endpoints.decorators import anon_protect, anon_allowed from util.metrics.metricqueue import time_blueprint v1_bp = Blueprint('v1', __name__) - +license_validator.enforce_license_before_request(v1_bp) time_blueprint(v1_bp, metric_queue) + # Note: This is *not* part of the Docker index spec. This is here for our own health check, # since we have nginx handle the _ping below. @v1_bp.route('/_internal_ping') @@ -29,4 +30,4 @@ def ping(): from endpoints.v1 import index from endpoints.v1 import registry -from endpoints.v1 import tag \ No newline at end of file +from endpoints.v1 import tag diff --git a/endpoints/v2/__init__.py b/endpoints/v2/__init__.py index da20b9077..26150f875 100644 --- a/endpoints/v2/__init__.py +++ b/endpoints/v2/__init__.py @@ -9,7 +9,7 @@ from semantic_version import Spec import features -from app import app, metric_queue, get_app_url +from app import app, metric_queue, get_app_url, license_validator from auth.auth_context import get_grant_context from auth.permissions import (ReadRepositoryPermission, ModifyRepositoryPermission, AdministerRepositoryPermission) @@ -18,16 +18,32 @@ from data import model from endpoints.decorators import anon_protect, anon_allowed from endpoints.v2.errors import V2RegistryException, Unauthorized from util.http import abort -from util.registry.dockerver import docker_version from util.metrics.metricqueue import time_blueprint +from util.registry.dockerver import docker_version from util.pagination import encrypt_page_token, decrypt_page_token -logger = logging.getLogger(__name__) -v2_bp = Blueprint('v2', __name__) +logger = logging.getLogger(__name__) + + +v2_bp = Blueprint('v2', __name__) +license_validator.enforce_license_before_request(v2_bp) time_blueprint(v2_bp, metric_queue) +@v2_bp.app_errorhandler(V2RegistryException) +def handle_registry_v2_exception(error): + response = jsonify({ + 'errors': [error.as_dict()] + }) + + response.status_code = error.http_status_code + if response.status_code == 401: + response.headers.extend(get_auth_headers(repository=error.repository, scopes=error.scopes)) + logger.debug('sending response: %s', response.get_data()) + return response + + _MAX_RESULTS_PER_PAGE = 50 @@ -72,19 +88,6 @@ def paginate(limit_kwarg_name='limit', offset_kwarg_name='offset', return wrapper -@v2_bp.app_errorhandler(V2RegistryException) -def handle_registry_v2_exception(error): - response = jsonify({ - 'errors': [error.as_dict()] - }) - - response.status_code = error.http_status_code - if response.status_code == 401: - response.headers.extend(get_auth_headers(repository=error.repository, scopes=error.scopes)) - logger.debug('sending response: %s', response.get_data()) - return response - - def _require_repo_permission(permission_class, scopes=None, allow_public=False): def wrapper(func): @wraps(func) @@ -117,7 +120,6 @@ def get_input_stream(flask_request): return flask_request.stream -# TODO remove when v2 is deployed everywhere def route_show_if(value): def decorator(f): @wraps(f) @@ -129,6 +131,7 @@ def route_show_if(value): return decorated_function return decorator + @v2_bp.route('/') @route_show_if(features.ADVERTISE_V2) @process_registry_jwt_auth() diff --git a/endpoints/verbs/__init__.py b/endpoints/verbs/__init__.py index 7dddd0a8c..d7c8e248b 100644 --- a/endpoints/verbs/__init__.py +++ b/endpoints/verbs/__init__.py @@ -5,7 +5,7 @@ from flask import redirect, Blueprint, abort, send_file, make_response, request import features -from app import app, signer, storage, metric_queue +from app import app, signer, storage, metric_queue, license_validator from auth.auth_context import get_authenticated_user from auth.permissions import ReadRepositoryPermission from auth.process import process_auth @@ -25,9 +25,11 @@ from util.registry.torrent import (make_torrent, per_user_torrent_filename, publ PieceHasher) -verbs = Blueprint('verbs', __name__) logger = logging.getLogger(__name__) +verbs = Blueprint('verbs', __name__) +license_validator.enforce_license_before_request(verbs) + def _open_stream(formatter, namespace, repository, tag, derived_image_id, repo_image, handlers): """ diff --git a/static/css/core-ui.css b/static/css/core-ui.css index 26fb1b5cf..bd482b37e 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -567,6 +567,33 @@ a:focus { margin-right: 4px; } +.config-license-field-element textarea { + padding: 10px; + margin-bottom: 10px; + height: 250px; +} + +.config-license-field-element .license-status { + margin-bottom: 26px; +} + +.config-license-field-element table td:first-child { + width: 150px; + font-weight: bold; +} + +.config-license-field-element .fa { + margin-right: 6px; +} + +.config-license-field-element .license-valid h4 { + color: #2FC98E; +} + +.config-license-field-element .license-invalid h4 { + color: #D64456; +} + .co-checkbox { position: relative; } diff --git a/static/css/pages/setup.css b/static/css/pages/setup.css new file mode 100644 index 000000000..2ca87dbdc --- /dev/null +++ b/static/css/pages/setup.css @@ -0,0 +1,56 @@ +.initial-setup-modal .upload-license textarea { + border: 1px solid #eee !important; + transition: all ease-in-out 200ms; + resize: none; +} + +.initial-setup-modal .upload-license textarea { + padding: 10px; + margin-top: 20px; + margin-bottom: 10px; +} + +.initial-setup-modal .upload-license .validate-message { + display: inline-block; + margin-left: 10px; + margin-top: 10px; +} + +.initial-setup-modal .upload-license .license-invalid h5 { + font-size: 18px; + color: red; +} + +.initial-setup-modal .upload-license .license-invalid h6 { + margin-bottom: 10px; + font-size: 16px; +} + +.initial-setup-modal .upload-license .license-invalid .fa { + margin-right: 6px; +} + +.initial-setup-modal .license-valid h5 { + color: #2FC98E; + font-size: 16px; + margin-bottom: 16px; +} + +.initial-setup-modal .license-valid .fa { + margin-right: 6px; +} + +.initial-setup-modal .license-valid table { + margin-top: 40px; +} + +.initial-setup-modal .license-valid table td { + border: 0px; + padding: 4px; +} + +.initial-setup-modal .license-valid table td:first-child { + font-weight: bold; + max-width: 100px; + padding-right: 20px; +} diff --git a/static/directives/config/config-license-field.html b/static/directives/config/config-license-field.html new file mode 100644 index 000000000..9eb688156 --- /dev/null +++ b/static/directives/config/config-license-field.html @@ -0,0 +1,40 @@ +
+ + + +
+ +
+

License Valid

+ + + +
Product:{{ licenseDecoded.publicProductName || licenseDecoded.productName }}
Plan:{{ licenseDecoded.publicPlanName || licenseDecoded.planName }}
+
+ +
+

Validation Failed

+
{{ licenseError }}
+
+ + + +
+

+ Your license can be found under the "Raw Format" tab of your Quay Enterprise + subscription in the Tectonic Account. +

+ + + + + +
+ Validating License +
+
+
\ No newline at end of file diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index c86187488..d810ad1a7 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -3,6 +3,16 @@
+ +
+
+ License +
+
+
+
+
+
diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index c47ba2f77..19977d359 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -1246,5 +1246,73 @@ angular.module("core-config-setup", ['angularFileUpload']) } }; return directiveDefinitionObject; + }) + + .directive('configLicenseField', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/config/config-license-field.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + }, + controller: function($scope, $element, ApiService, UserService) { + $scope.state = 'loading-license'; + $scope.showingEditor = false; + $scope.requiredBox = ''; + + var loadLicense = function() { + ApiService.getLicense().then(function(resp) { + $scope.state = 'license-valid'; + $scope.showingEditor = false; + $scope.licenseDecoded = resp['decoded']; + $scope.requiredBox = 'filled'; + }, function(resp) { + $scope.licenseError = ApiService.getErrorMessage(resp); + $scope.state = 'license-error'; + $scope.showingEditor = true; + $scope.requiredBox = ''; + }); + }; + + UserService.updateUserIn($scope, function(user) { + if (!user.anonymous) { + loadLicense(); + } + }); + + $scope.showEditor = function($event) { + $event.preventDefault(); + $event.stopPropagation(); + + $scope.showingEditor = true; + }; + + $scope.validateAndUpdate = function($event) { + $event.preventDefault(); + $event.stopPropagation(); + + $scope.state = 'validating-license'; + + var data = { + 'license': $scope.licenseContents + }; + + ApiService.updateLicense(data).then(function(resp) { + $scope.state = 'license-valid'; + $scope.showingEditor = false; + $scope.licenseDecoded = resp['decoded']; + $scope.requiredBox = 'filled'; + }, function(resp) { + $scope.licenseError = ApiService.getErrorMessage(resp); + $scope.state = 'license-error'; + $scope.showingEditor = true; + $scope.requiredBox = ''; + }); + }; + } + }; + return directiveDefinitionObject; }); diff --git a/static/js/pages/setup.js b/static/js/pages/setup.js index cbd539c16..3759a9bf2 100644 --- a/static/js/pages/setup.js +++ b/static/js/pages/setup.js @@ -37,6 +37,15 @@ // The config.yaml exists but it is invalid. 'INVALID_CONFIG': 'config-invalid', + // License is being uploaded. + 'UPLOAD_LICENSE': 'upload-license', + + // License is being validated. + 'VALIDATING_LICENSE': 'upload-license-validating', + + // License is validated. + 'VALIDATED_LICENSE': 'upload-license-validated', + // DB is being configured. 'CONFIG_DB': 'config-db', @@ -95,7 +104,10 @@ $scope.currentConfig = null; $scope.currentState = { - 'hasDatabaseSSLCert': false + 'hasDatabaseSSLCert': false, + 'licenseContents': '', + 'licenseError': null, + 'licenseDecoded': null }; $scope.$watch('currentStep', function(currentStep) { @@ -121,6 +133,7 @@ case $scope.States.CREATE_SUPERUSER: case $scope.States.DB_RESTARTING: case $scope.States.CONFIG_DB: + case $scope.States.UPLOAD_LICENSE: case $scope.States.VALID_CONFIG: case $scope.States.READY: $('#setupModal').modal({ @@ -131,6 +144,27 @@ } }); + $scope.validateLicense = function() { + $scope.currentStep = $scope.States.VALIDATING_LICENSE; + + var data = { + 'license': $scope.currentState.licenseContents + }; + + ApiService.suSetAndValidateLicense(data).then(function(resp) { + $scope.currentStep = $scope.States.VALIDATED_LICENSE; + + $scope.currentState.licenseError = null; + $scope.currentState.licenseDecoded = resp['decoded']; + }, function(resp) { + $scope.currentStep = $scope.States.UPLOAD_LICENSE; + + $scope.currentState.licenseError = ApiService.getErrorMessage(resp); + $scope.currentState.licenseContents = ''; + $scope.currentState.licenseDecoded = null; + }); + }; + $scope.restartContainer = function(state) { $scope.currentStep = state; ContainerService.restartContainer(function() { @@ -166,6 +200,7 @@ var States = $scope.States; return [ + isStepFamily(step, States.UPLOAD_LICENSE), isStepFamily(step, States.CONFIG_DB), isStepFamily(step, States.DB_SETUP), isStep(step, States.DB_RESTARTING), @@ -191,6 +226,10 @@ return false; }; + $scope.beginSetup = function() { + $scope.currentStep = $scope.States.CONFIG_DB; + }; + $scope.showInvalidConfigDialog = function() { var message = "The config.yaml file found in conf/stack could not be parsed." var title = "Invalid configuration file"; diff --git a/static/partials/setup.html b/static/partials/setup.html index e28837b6e..e238ab846 100644 --- a/static/partials/setup.html +++ b/static/partials/setup.html @@ -9,12 +9,13 @@
- + + - - - + + + @@ -36,12 +37,13 @@
- + + - - - + + + @@ -128,6 +130,37 @@ The container must be restarted to apply the configuration changes.
+ +
+
License Validated
+ Your license has been validated and saved. Please press "Next" to continue setup of your Quay Enterprise installation. + + + +
Product:{{ currentState.licenseDecoded.publicProductName || currentState.licenseDecoded.productName }}
Plan:{{ currentState.licenseDecoded.publicPlanName || currentState.licenseDecoded.planName }}
+
+ + +
+

+ Quay Enterprise License +

+
+ Please provide your Quay Enterprise License. It can be found under the "Raw Format" tab + of your Quay Enterprise subscription in the Tectonic Account. +
+ +
+
Validation Failed
+
{{ currentState.licenseError }}
+ Please try copying your license from the Tectonic Account again. +
+
+
@@ -226,6 +259,29 @@ Database Validation Issue: {{ errors.DatabaseValidationError }}
+ +
+
+ + Validating License... +
+ + +
+ + +
+ +
+
diff --git a/test/test_api_security.py b/test/test_api_security.py index 426e7ea20..550707e55 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -51,8 +51,8 @@ from endpoints.api.superuser import (SuperUserLogs, SuperUserList, SuperUserMana SuperUserOrganizationManagement, SuperUserOrganizationList, SuperUserAggregateLogs, SuperUserServiceKeyManagement, SuperUserServiceKey, SuperUserServiceKeyApproval, - SuperUserTakeOwnership,) -from endpoints.api.globalmessages import (GlobalUserMessage, GlobalUserMessages,) + SuperUserTakeOwnership, SuperUserMessages, SuperUserLicense) +from endpoints.api.globalmessages import GlobalUserMessage, GlobalUserMessages from endpoints.api.secscan import RepositoryImageSecurity from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel @@ -4158,6 +4158,37 @@ class TestSuperUserList(ApiTestCase): self._run_test('GET', 200, 'devtable', None) +class TestSuperUserLicense(ApiTestCase): + def setUp(self): + ApiTestCase.setUp(self) + self._set_url(SuperUserLicense) + + def test_get_anonymous(self): + self._run_test('GET', 401, None, None) + + def test_get_freshuser(self): + self._run_test('GET', 403, 'freshuser', None) + + def test_get_reader(self): + self._run_test('GET', 403, 'reader', None) + + def test_get_devtable(self): + self._run_test('GET', 400, 'devtable', None) + + + def test_put_anonymous(self): + self._run_test('PUT', 401, None, {}) + + def test_put_freshuser(self): + self._run_test('PUT', 403, 'freshuser', {'license': ''}) + + def test_put_reader(self): + self._run_test('PUT', 403, 'reader', {'license': ''}) + + def test_put_devtable(self): + self._run_test('PUT', 400, 'devtable', {'license': ''}) + + class TestSuperUserManagement(ApiTestCase): def setUp(self): ApiTestCase.setUp(self) diff --git a/test/test_api_usage.py b/test/test_api_usage.py index ec1d3883f..f9d5e29cf 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -3740,13 +3740,20 @@ class TestSuperUserCreateInitialSuperUser(ApiTestCase): class TestSuperUserConfig(ApiTestCase): def test_get_status_update_config(self): - # With no config the status should be 'config-db'. + # With no config the status should be 'upload-license'. json = self.getJsonResponse(SuperUserRegistryStatus) - self.assertEquals('config-db', json['status']) + self.assertEquals('upload-license', json['status']) # And the config should 401. self.getResponse(SuperUserConfig, expected_code=401) + # Add a fake license file. + config_provider.save_license('something') + + # With no config but a license the status should be 'config-db'. + json = self.getJsonResponse(SuperUserRegistryStatus) + self.assertEquals('config-db', json['status']) + # Add some fake config. fake_config = { 'AUTHENTICATION_TYPE': 'Database', diff --git a/test/test_license.py b/test/test_license.py index 5c7c93089..9dc920d61 100644 --- a/test/test_license.py +++ b/test/test_license.py @@ -3,13 +3,13 @@ import unittest from datetime import datetime, timedelta import jwt +import json from Crypto.PublicKey import RSA from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.serialization import load_der_public_key -from util.config.provider.license import (decode_license, LICENSE_PRODUCT_NAME, - LicenseValidationError) +from util.license import decode_license, LICENSE_PRODUCT_NAME, LicenseValidationError class TestLicense(unittest.TestCase): @@ -22,10 +22,14 @@ class TestLicense(unittest.TestCase): return (public_key, private_key) def create_license(self, license_data): + jwt_data = { + 'license': json.dumps(license_data), + } + (public_key, private_key) = self.keys() # Encode the license with the JWT key. - encoded = jwt.encode(license_data, private_key, algorithm='RS256') + encoded = jwt.encode(jwt_data, private_key, algorithm='RS256') # Decode it into a license object. return decode_license(encoded, public_key_instance=public_key) @@ -53,7 +57,7 @@ class TestLicense(unittest.TestCase): if 'duration' in kwargs: sub['durationPeriod'] = kwargs['duration'] - license_data['subscriptions'] = [sub] + license_data['subscriptions'] = {'somesub': sub} decoded_license = self.create_license(license_data) return decoded_license @@ -83,15 +87,15 @@ class TestLicense(unittest.TestCase): self.assertTrue(license.is_expired) def test_monthly_license_valid(self): - license = self.get_license(timedelta(days=30), service_end=timedelta(days=10), duration='monthly') + license = self.get_license(timedelta(days=30), service_end=timedelta(days=10), duration='months') self.assertFalse(license.is_expired) def test_monthly_license_withingrace(self): - license = self.get_license(timedelta(days=30), service_end=timedelta(days=-10), duration='monthly') + license = self.get_license(timedelta(days=30), service_end=timedelta(days=-10), duration='months') self.assertFalse(license.is_expired) def test_monthly_license_outsidegrace(self): - license = self.get_license(timedelta(days=30), service_end=timedelta(days=-40), duration='monthly') + license = self.get_license(timedelta(days=30), service_end=timedelta(days=-40), duration='months') self.assertTrue(license.is_expired) def test_yearly_license_withingrace(self): @@ -107,18 +111,19 @@ class TestLicense(unittest.TestCase): self.assertFalse(license.is_expired) def test_validate_basic_license(self): - decoded = self.get_license(timedelta(days=30), entitlements={}) + decoded = self.get_license(timedelta(days=30), service_end=timedelta(days=40), + duration='months', entitlements={}) decoded.validate({'DISTRIBUTED_STORAGE_CONFIG': [{}]}) def test_validate_storage_entitlement_valid(self): - decoded = self.get_license(timedelta(days=30), entitlements={ + decoded = self.get_license(timedelta(days=30), service_end=timedelta(days=40), entitlements={ 'software.quay.regions': 2, }) decoded.validate({'DISTRIBUTED_STORAGE_CONFIG': [{}]}) def test_validate_storage_entitlement_invalid(self): - decoded = self.get_license(timedelta(days=30), entitlements={ + decoded = self.get_license(timedelta(days=30), service_end=timedelta(days=40), entitlements={ 'software.quay.regions': 1, }) diff --git a/test/test_suconfig_api.py b/test/test_suconfig_api.py index 6cbdcafa8..494866e80 100644 --- a/test/test_suconfig_api.py +++ b/test/test_suconfig_api.py @@ -21,7 +21,7 @@ class TestSuperUserRegistryStatus(ApiTestCase): def test_registry_status(self): with ConfigForTesting(): json = self.getJsonResponse(SuperUserRegistryStatus) - self.assertEquals('config-db', json['status']) + self.assertEquals('upload-license', json['status']) class TestSuperUserConfigFile(ApiTestCase): diff --git a/util/config/provider/baseprovider.py b/util/config/provider/baseprovider.py index 18dd13add..f9a48c67a 100644 --- a/util/config/provider/baseprovider.py +++ b/util/config/provider/baseprovider.py @@ -1,18 +1,22 @@ -import yaml import logging +import yaml + +from util.license import LICENSE_FILENAME, LicenseError, decode_license -from util.config.provider.license import LICENSE_FILENAME, LicenseError, decode_license logger = logging.getLogger(__name__) + class CannotWriteConfigException(Exception): """ Exception raised when the config cannot be written. """ pass + class SetupIncompleteException(Exception): """ Exception raised when attempting to verify config that has not yet been setup. """ pass + def import_yaml(config_obj, config_file): with open(config_file) as f: c = yaml.safe_load(f) @@ -33,6 +37,7 @@ def import_yaml(config_obj, config_file): def get_yaml(config_obj): return yaml.safe_dump(config_obj, encoding='utf-8', allow_unicode=True) + def export_yaml(config_obj, config_file): try: with open(config_file, 'w') as f: @@ -76,7 +81,11 @@ class BaseProvider(object): raise NotImplementedError def get_volume_file(self, filename, mode='r'): - """ Returns a Python file referring to the given name under the config override volumne. """ + """ Returns a Python file referring to the given name under the config override volume. """ + raise NotImplementedError + + def write_volume_file(self, filename, contents): + """ Writes the given contents to the config override volumne, with the given filename. """ raise NotImplementedError def save_volume_file(self, filename, flask_file): @@ -91,24 +100,29 @@ class BaseProvider(object): """ raise NotImplementedError - def validate_license(self, config): - """ Validates that the configuration matches the license file (if any). """ - if not config.get('SETUP_COMPLETE', False): - raise SetupIncompleteException() - - with self._get_license_file() as f: - license_file_contents = f.read() - - self.license = decode_license(license_file_contents) - self.license.validate(config) - def _get_license_file(self): """ Returns the contents of the license file. """ + if not self.has_license_file(): + msg = 'Could not find license file. Please make sure it is in your config volume.' + raise LicenseError(msg) + try: return self.get_volume_file(LICENSE_FILENAME) except IOError: msg = 'Could not open license file. Please make sure it is in your config volume.' raise LicenseError(msg) + def get_license(self): + """ Returns the decoded license, if any. """ + with self._get_license_file() as f: + license_file_contents = f.read() + return decode_license(license_file_contents) + def save_license(self, license_file_contents): + """ Saves the given contents as the license file. """ + self.write_volume_file(LICENSE_FILENAME, license_file_contents) + + def has_license_file(self): + """ Returns true if a license file was found in the config directory. """ + return self.volume_file_exists(LICENSE_FILENAME) diff --git a/util/config/provider/fileprovider.py b/util/config/provider/fileprovider.py index fe2b69df5..099c5c3fd 100644 --- a/util/config/provider/fileprovider.py +++ b/util/config/provider/fileprovider.py @@ -50,7 +50,14 @@ class FileConfigProvider(BaseProvider): return os.path.exists(os.path.join(self.config_volume, filename)) def get_volume_file(self, filename, mode='r'): - return open(os.path.join(self.config_volume, filename), mode) + return open(os.path.join(self.config_volume, filename), mode=mode) + + def write_volume_file(self, filename, contents): + filepath = os.path.join(self.config_volume, filename) + with open(filepath, mode='w') as f: + f.write(contents) + + return filepath def save_volume_file(self, filename, flask_file): filepath = os.path.join(self.config_volume, filename) diff --git a/util/config/provider/k8sprovider.py b/util/config/provider/k8sprovider.py index 5775e94dd..334cf4044 100644 --- a/util/config/provider/k8sprovider.py +++ b/util/config/provider/k8sprovider.py @@ -47,6 +47,14 @@ class KubernetesConfigProvider(FileConfigProvider): self._update_secret_file(self.yaml_filename, get_yaml(config_obj)) super(KubernetesConfigProvider, self).save_config(config_obj) + def write_volume_file(self, filename, contents): + super(KubernetesConfigProvider, self).write_volume_file(filename, contents) + + try: + self._update_secret_file(filename, contents) + except IOError as ioe: + raise CannotWriteConfigException(str(ioe)) + def save_volume_file(self, filename, flask_file): filepath = super(KubernetesConfigProvider, self).save_volume_file(filename, flask_file) diff --git a/util/config/provider/license.py b/util/config/provider/license.py deleted file mode 100644 index 5346137a4..000000000 --- a/util/config/provider/license.py +++ /dev/null @@ -1,136 +0,0 @@ -import logging - -from dateutil import parser -from datetime import datetime, timedelta -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.serialization import load_pem_public_key - -import jwt - -logger = logging.getLogger(__name__) - -TRIAL_GRACE_PERIOD = timedelta(7, 0) # 1 week -MONTHLY_GRACE_PERIOD = timedelta(30, 0) # 1 month -YEARLY_GRACE_PERIOD = timedelta(90, 0) # 3 months - -LICENSE_PRODUCT_NAME = "quay-enterprise" - -class LicenseError(Exception): - """ Exception raised if the license could not be read, decoded or has expired. """ - pass - - -class LicenseDecodeError(LicenseError): - """ Exception raised if the license could not be decoded. """ - pass - - -class LicenseValidationError(LicenseError): - """ Exception raised if the license could not be validated. """ - pass - - -def _get_date(decoded, field): - """ Retrieves the encoded date found at the given field under the decoded license block. """ - date_str = decoded.get(field) - if date_str: - return parser.parse(date_str).replace(tzinfo=None) - - return datetime.now() - timedelta(days=2) - - -class License(object): - """ License represents a fully decoded and validated (but potentially expired) license. """ - def __init__(self, decoded): - self.decoded = decoded - - def validate(self, config): - """ Validates the license and all its entitlements against the given config. """ - # Check that the license has not expired. - if self.is_expired: - raise LicenseValidationError('License has expired') - - # Check the maximum number of replication regions. - max_regions = min(self.decoded.get('entitlements', {}).get('software.quay.regions', 1), 1) - config_regions = len(config.get('DISTRIBUTED_STORAGE_CONFIG', [])) - if max_regions != -1 and config_regions > max_regions: - msg = '{} regions configured, but license file allows up to {}'.format(config_regions, - max_regions) - raise LicenseValidationError(msg) - - @property - def is_expired(self): - return self._get_expired(datetime.now()) - - def _get_expired(self, compare_date): - # Check if the license overall has expired. - expiration_date = _get_date(self.decoded, 'expirationDate') - if expiration_date <= compare_date: - logger.debug('License expired on %s', expiration_date) - return True - - # Check for any QE subscriptions. - for sub in self.decoded.get('subscriptions', []): - if sub.get('productName') != LICENSE_PRODUCT_NAME: - continue - - # Check for a trial-only license. - if sub.get('trialOnly', False): - trial_end_date = _get_date(sub, 'trialEnd') - logger.debug('Trial-only license expires on %s', trial_end_date) - return trial_end_date <= (compare_date - TRIAL_GRACE_PERIOD) - - # Check for a normal license that is in trial. - service_end_date = _get_date(sub, 'serviceEnd') - if sub.get('inTrial', False): - # If the subscription is in a trial, but not a trial only - # subscription, give 7 days after trial end to update license - # to one which has been paid (they've put in a credit card and it - # might auto convert, so we could assume it will auto-renew) - logger.debug('In-trial license expires on %s', service_end_date) - return service_end_date <= (compare_date - TRIAL_GRACE_PERIOD) - - # Otherwise, check the service expiration. - duration_period = sub.get('durationPeriod', 'monthly') - - # If the subscription is monthly, give 3 months grace period - if duration_period == "monthly": - logger.debug('Monthly license expires on %s', service_end_date) - return service_end_date <= (compare_date - MONTHLY_GRACE_PERIOD) - - if duration_period == "years": - logger.debug('Yearly license expires on %s', service_end_date) - return service_end_date <= (compare_date - YEARLY_GRACE_PERIOD) - - return True - - -LICENSE_FILENAME = 'license' - - -_PROD_LICENSE_PUBLIC_KEY_DATA = """ ------BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuCkRnkuqox3A0djgRnHR -e3U3jHrcbd5iUqdbfO/8E2TMbiByIy3NzUyJrMIzrTjdxTVIZF/ueaHLEtgaofUA -1X73OZlsaGyNVDFA2eGZRgyNrmfLFoxnN2KB+gEJ88nPkHZXY+4ncZBjVMKfHQEv -busC7xpnF7Diy2GxZKDZRnvjL4ZNrocdoeE0GuroWwebtck5Ea7LqzRxCJ5T3UWt -EozttOBQAqCmKxSDdtdw+CsK/uTfl6Yh9xCZUrCeh5taSOHOvU0ne/p3gM+AsjU4 -ScjObTKaSUOGen6aYFF5Bd6V/ucxHmcmJlycwNZOKGFpbhLU173/oBJ+okvDbJpN -qwIDAQAB ------END PUBLIC KEY----- -""" - - -_PROD_LICENSE_PUBLIC_KEY = load_pem_public_key(_PROD_LICENSE_PUBLIC_KEY_DATA, - backend=default_backend()) - -def decode_license(license_contents, public_key_instance=None): - """ Decodes the specified license contents, returning the decoded license. """ - license_public_key = public_key_instance or _PROD_LICENSE_PUBLIC_KEY - try: - decoded = jwt.decode(license_contents, key=license_public_key) - except jwt.exceptions.DecodeError as de: - logger.exception('Could not decode license file') - raise LicenseDecodeError('Could not decode license found: %s' % de.message) - - return License(decoded) diff --git a/util/config/provider/testprovider.py b/util/config/provider/testprovider.py index d607e2ff5..d08b13049 100644 --- a/util/config/provider/testprovider.py +++ b/util/config/provider/testprovider.py @@ -1,5 +1,5 @@ import json -from StringIO import StringIO +import io from util.config.provider.baseprovider import BaseProvider @@ -46,11 +46,14 @@ class TestConfigProvider(BaseProvider): def save_volume_file(self, filename, flask_file): self.files[filename] = '' + def write_volume_file(self, filename, contents): + self.files[filename] = contents + def get_volume_file(self, filename, mode='r'): if filename in REAL_FILES: return open(filename, mode=mode) - return StringIO(self.files[filename]) + return io.BytesIO(self.files[filename]) def requires_restart(self, app_config): return False diff --git a/util/license.py b/util/license.py new file mode 100644 index 000000000..1fd9f9fb5 --- /dev/null +++ b/util/license.py @@ -0,0 +1,242 @@ +import json +import logging +import multiprocessing +import time + +from ctypes import c_bool +from datetime import datetime, timedelta +from threading import Thread + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.serialization import load_pem_public_key +from dateutil import parser +from flask import make_response + +import jwt + + +logger = logging.getLogger(__name__) + + +TRIAL_GRACE_PERIOD = timedelta(7, 0) # 1 week +MONTHLY_GRACE_PERIOD = timedelta(30, 0) # 1 month +YEARLY_GRACE_PERIOD = timedelta(90, 0) # 3 months +LICENSE_PRODUCT_NAME = "quay-enterprise" +LICENSE_FILENAME = 'license' + + +class LicenseError(Exception): + """ Exception raised if the license could not be read, decoded or has expired. """ + pass + + +class LicenseDecodeError(LicenseError): + """ Exception raised if the license could not be decoded. """ + pass + + +class LicenseValidationError(LicenseError): + """ Exception raised if the license could not be validated. """ + pass + + +def _get_date(decoded, field): + """ Retrieves the encoded date found at the given field under the decoded license block. """ + date_str = decoded.get(field) + return parser.parse(date_str).replace(tzinfo=None) if date_str else None + + +class LicenseExpirationDate(object): + def __init__(self, title, expiration_date, grace_period=None): + self.title = title + self.expiration_date = expiration_date + self.grace_period = grace_period or timedelta(seconds=0) + + def check_expired(self, cutoff_date=None): + return self.expiration_and_grace <= (cutoff_date or datetime.now()) + + @property + def expiration_and_grace(self): + return self.expiration_date + self.grace_period + + def __str__(self): + return 'License expiration "%s" date %s with grace %s: %s' % (self.title, self.expiration_date, + self.grace_period, + self.check_expired()) + +class License(object): + """ License represents a fully decoded and validated (but potentially expired) license. """ + def __init__(self, decoded): + self.decoded = decoded + + @property + def subscription(self): + """ Returns the Quay Enterprise subscription, if any. """ + for sub in self.decoded.get('subscriptions', {}).values(): + if sub.get('productName') == LICENSE_PRODUCT_NAME: + return sub + + return None + + @property + def is_expired(self): + cutoff_date = datetime.now() + return bool([dt for dt in self._get_expiration_dates() if dt.check_expired(cutoff_date)]) + + def validate(self, config): + """ Validates the license and all its entitlements against the given config. """ + # Check that the license has not expired. + if self.is_expired: + raise LicenseValidationError('License has expired') + + # Check the maximum number of replication regions. + max_regions = min(self.decoded.get('entitlements', {}).get('software.quay.regions', 1), 1) + config_regions = len(config.get('DISTRIBUTED_STORAGE_CONFIG', [])) + if max_regions != -1 and config_regions > max_regions: + msg = '{} regions configured, but license file allows up to {}'.format(config_regions, + max_regions) + raise LicenseValidationError(msg) + + def _get_expiration_dates(self): + # Check if the license overall has expired. + expiration_date = _get_date(self.decoded, 'expirationDate') + if expiration_date is None: + yield LicenseExpirationDate('No valid Tectonic Account License', datetime.min) + return + + yield LicenseExpirationDate('Tectonic Account License', expiration_date) + + # Check for any QE subscriptions. + sub = self.subscription + if sub is None: + yield LicenseExpirationDate('No Quay Enterprise Subscription', datetime.min) + return + + # Check for a trial-only license. + if sub.get('trialOnly', False): + trial_end_date = _get_date(sub, 'trialEnd') + if trial_end_date is None: + yield LicenseExpirationDate('Invalid trial subscription', datetime.min) + else: + yield LicenseExpirationDate('Trial subscription', trial_end_date, TRIAL_GRACE_PERIOD) + + return + + # Check for a normal license that is in trial. + service_end_date = _get_date(sub, 'serviceEnd') + if service_end_date is None: + yield LicenseExpirationDate('No valid Quay Enterprise Subscription', datetime.min) + return + + if sub.get('inTrial', False): + # If the subscription is in a trial, but not a trial only + # subscription, give 7 days after trial end to update license + # to one which has been paid (they've put in a credit card and it + # might auto convert, so we could assume it will auto-renew) + yield LicenseExpirationDate('In-trial subscription', service_end_date, TRIAL_GRACE_PERIOD) + + # Otherwise, check the service expiration. + duration_period = sub.get('durationPeriod', 'months') + + # If the subscription is monthly, give 3 months grace period + if duration_period == "months": + yield LicenseExpirationDate('Monthly subscription', service_end_date, MONTHLY_GRACE_PERIOD) + + if duration_period == "years": + yield LicenseExpirationDate('Yearly subscription', service_end_date, YEARLY_GRACE_PERIOD) + + + +_PROD_LICENSE_PUBLIC_KEY_DATA = """ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuCkRnkuqox3A0djgRnHR +e3U3jHrcbd5iUqdbfO/8E2TMbiByIy3NzUyJrMIzrTjdxTVIZF/ueaHLEtgaofUA +1X73OZlsaGyNVDFA2eGZRgyNrmfLFoxnN2KB+gEJ88nPkHZXY+4ncZBjVMKfHQEv +busC7xpnF7Diy2GxZKDZRnvjL4ZNrocdoeE0GuroWwebtck5Ea7LqzRxCJ5T3UWt +EozttOBQAqCmKxSDdtdw+CsK/uTfl6Yh9xCZUrCeh5taSOHOvU0ne/p3gM+AsjU4 +ScjObTKaSUOGen6aYFF5Bd6V/ucxHmcmJlycwNZOKGFpbhLU173/oBJ+okvDbJpN +qwIDAQAB +-----END PUBLIC KEY----- +""" +_PROD_LICENSE_PUBLIC_KEY = load_pem_public_key(_PROD_LICENSE_PUBLIC_KEY_DATA, + backend=default_backend()) + +def decode_license(license_contents, public_key_instance=None): + """ Decodes the specified license contents, returning the decoded license. """ + license_public_key = public_key_instance or _PROD_LICENSE_PUBLIC_KEY + try: + jwt_data = jwt.decode(license_contents, key=license_public_key) + except jwt.exceptions.DecodeError as de: + logger.exception('Could not decode license file') + raise LicenseDecodeError('Could not decode license found: %s' % de.message) + + try: + decoded = json.loads(jwt_data.get('license', '{}')) + except ValueError as ve: + logger.exception('Could not decode license file') + raise LicenseDecodeError('Could not decode license found: %s' % ve.message) + + return License(decoded) + + +LICENSE_VALIDATION_INTERVAL = 3600 # seconds +LICENSE_VALIDATION_EXPIRED_INTERVAL = 60 # seconds + + +class LicenseValidator(Thread): + """ + LicenseValidator is a thread that asynchronously reloads and validates license files. + + This thread is meant to be run before registry gunicorn workers fork and uses shared memory as a + synchronization primitive. + """ + def __init__(self, license_path, *args, **kwargs): + self._license_path = license_path + + # multiprocessing.Value does not ensure consistent write-after-reads, but we don't need that. + self._license_is_expired = multiprocessing.Value(c_bool, True) + + super(LicenseValidator, self).__init__(*args, **kwargs) + self.daemon = True + + @property + def expired(self): + return self._license_is_expired.value + + def _check_expiration(self): + try: + with open(self._license_path) as f: + current_license = decode_license(f.read()) + is_expired = current_license.is_expired + logger.debug('updating license expiration to %s', is_expired) + self._license_is_expired.value = is_expired + except (IOError, LicenseError): + logger.exception('failed to validate license') + is_expired = True + self._license_is_expired.value = is_expired + + return is_expired + + def run(self): + logger.debug('Starting license validation thread') + while True: + expired = self._check_expiration() + sleep_time = LICENSE_VALIDATION_EXPIRED_INTERVAL if expired else LICENSE_VALIDATION_INTERVAL + logger.debug('waiting %d seconds before retrying to validate license', sleep_time) + time.sleep(sleep_time) + + def enforce_license_before_request(self, blueprint, response_func=None): + """ + Adds a pre-check to a Flask blueprint such that if the provided license_validator determines the + license has become invalid, the client will receive a HTTP 402 response. + """ + if response_func is None: + def _response_func(): + return make_response('License has expired.', 402) + response_func = _response_func + + def _enforce_license(): + if self.expired: + logger.debug('blocked interaction due to expired license') + return response_func() + blueprint.before_request(_enforce_license)