From 9695c98e5f3eaf37e539080f0cb3300772425bd1 Mon Sep 17 00:00:00 2001 From: Sam Chow Date: Mon, 27 Aug 2018 17:05:53 -0400 Subject: [PATCH] Add config ability to rollback changes on kube --- config_app/config_endpoints/api/__init__.py | 4 +- .../api/{kubeconfig.py => kube_endpoints.py} | 70 +++++++++++++++++-- .../config_endpoints/api/tar_config_loader.py | 2 + .../config/TransientDirectoryProvider.py | 48 ++++++------- config_app/config_util/config/__init__.py | 27 +++++++ config_app/config_util/k8saccessor.py | 22 +++++- .../config-tool-servicetoken-role.yml | 2 + .../config-setup-app.component.html | 2 +- .../config-setup-app.component.ts | 1 + .../cor-loader/cor-loader-inline.html | 5 ++ .../js/components/cor-loader/cor-loader.js | 17 ++++- .../kube-deploy-modal.component.html | 34 +++++---- .../kube-deploy-modal.component.ts | 47 ++++++++++--- .../kube-deploy-modal/kube-deploy-modal.css | 15 ++++ .../static/css/config-setup-app-component.css | 4 ++ 15 files changed, 237 insertions(+), 63 deletions(-) rename config_app/config_endpoints/api/{kubeconfig.py => kube_endpoints.py} (53%) create mode 100644 config_app/js/components/cor-loader/cor-loader-inline.html diff --git a/config_app/config_endpoints/api/__init__.py b/config_app/config_endpoints/api/__init__.py index 4f5b4c2f8..c80fc1c9c 100644 --- a/config_app/config_endpoints/api/__init__.py +++ b/config_app/config_endpoints/api/__init__.py @@ -150,10 +150,8 @@ def kubernetes_only(f): nickname = partial(add_method_metadata, 'nickname') - -import config_app.config_endpoints.api import config_app.config_endpoints.api.discovery -import config_app.config_endpoints.api.kubeconfig +import config_app.config_endpoints.api.kube_endpoints import config_app.config_endpoints.api.suconfig import config_app.config_endpoints.api.superuser import config_app.config_endpoints.api.tar_config_loader diff --git a/config_app/config_endpoints/api/kubeconfig.py b/config_app/config_endpoints/api/kube_endpoints.py similarity index 53% rename from config_app/config_endpoints/api/kubeconfig.py rename to config_app/config_endpoints/api/kube_endpoints.py index 556e36504..07ba03bd4 100644 --- a/config_app/config_endpoints/api/kubeconfig.py +++ b/config_app/config_endpoints/api/kube_endpoints.py @@ -1,10 +1,11 @@ -from flask import request +from flask import request, make_response +from config_app.config_util.config import get_config_as_kube_secret from data.database import configure from config_app.c_app import app, config_provider -from config_app.config_endpoints.api import resource, ApiResource, nickname, kubernetes_only -from config_app.config_util.k8saccessor import KubernetesAccessorSingleton +from config_app.config_endpoints.api import resource, ApiResource, nickname, kubernetes_only, validate_json_request +from config_app.config_util.k8saccessor import KubernetesAccessorSingleton, K8sApiException @resource('/v1/kubernetes/deployments/') @@ -32,11 +33,13 @@ class SuperUserKubernetesDeployment(ApiResource): return KubernetesAccessorSingleton.get_instance().get_qe_deployments() @kubernetes_only + @validate_json_request('ValidateDeploymentNames') @nickname('scCycleQEDeployments') def put(self): deployment_names = request.get_json()['deploymentNames'] return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names) + @resource('/v1/kubernetes/deployment//status') class QEDeploymentRolloutStatus(ApiResource): @kubernetes_only @@ -49,14 +52,66 @@ class QEDeploymentRolloutStatus(ApiResource): } -@resource('/v1/superuser/config/kubernetes') +@resource('/v1/kubernetes/deployments/rollback') +class QEDeploymentRollback(ApiResource): + """ Resource for rolling back deployments """ + schemas = { + 'ValidateDeploymentNames': { + 'type': 'object', + 'description': 'Validates deployment names for rolling back', + 'required': [ + 'deploymentNames' + ], + 'properties': { + 'deploymentNames': { + 'type': 'array', + 'description': 'The names of the deployments to rollback' + }, + }, + } + } + + @kubernetes_only + @nickname('scRollbackDeployments') + @validate_json_request('ValidateDeploymentNames') + def post(self): + """ + Returns the config to its original state and rolls back deployments + :return: + """ + deployment_names = request.get_json()['deploymentNames'] + + # To roll back a deployment, we must do 2 things: + # 1. Roll back the config secret to its old value (discarding changes we made in this session + # 2. Trigger a rollback to the previous revision, so that the pods will be restarted with + # the old config + old_secret = get_config_as_kube_secret(config_provider.get_old_config_dir()) + kube_accessor = KubernetesAccessorSingleton.get_instance() + kube_accessor.replace_qe_secret(old_secret) + + try: + for name in deployment_names: + kube_accessor.rollback_deployment(name) + except K8sApiException as e: + return make_response(e.message, 500) + + return make_response('Ok', 204) + + +@resource('/v1/kubernetes/config') class SuperUserKubernetesConfiguration(ApiResource): """ Resource for saving the config files to kubernetes secrets. """ @kubernetes_only @nickname('scDeployConfiguration') def post(self): - return config_provider.save_configuration_to_kubernetes() + try: + new_secret = get_config_as_kube_secret(config_provider.get_config_dir_path()) + KubernetesAccessorSingleton.get_instance().replace_qe_secret(new_secret) + except K8sApiException as e: + return make_response(e.message, 500) + + return make_response('Ok', 201) @resource('/v1/kubernetes/config/populate') @@ -68,7 +123,10 @@ class KubernetesConfigurationPopulator(ApiResource): def post(self): # Get a clean transient directory to write the config into config_provider.new_config_dir() - KubernetesAccessorSingleton.get_instance().save_secret_to_directory(config_provider.get_config_dir_path()) + + kube_accessor = KubernetesAccessorSingleton.get_instance() + kube_accessor.save_secret_to_directory(config_provider.get_config_dir_path()) + config_provider.create_copy_of_config_dir() # We update the db configuration to connect to their specified one # (Note, even if this DB isn't valid, it won't affect much in the config app, since we'll report an error, diff --git a/config_app/config_endpoints/api/tar_config_loader.py b/config_app/config_endpoints/api/tar_config_loader.py index 62f658cde..49e77b6a0 100644 --- a/config_app/config_endpoints/api/tar_config_loader.py +++ b/config_app/config_endpoints/api/tar_config_loader.py @@ -52,6 +52,8 @@ class TarConfigLoader(ApiResource): with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream: tar_stream.extractall(config_provider.get_config_dir_path()) + config_provider.create_copy_of_config_dir() + # now try to connect to the db provided in their config to validate it works combined = dict(**app.config) combined.update(config_provider.get_config()) diff --git a/config_app/config_util/config/TransientDirectoryProvider.py b/config_app/config_util/config/TransientDirectoryProvider.py index 7dab7dcf3..5ac685592 100644 --- a/config_app/config_util/config/TransientDirectoryProvider.py +++ b/config_app/config_util/config/TransientDirectoryProvider.py @@ -1,13 +1,11 @@ import os -import base64 +from shutil import copytree from backports.tempfile import TemporaryDirectory from config_app.config_util.config.fileprovider import FileConfigProvider -from config_app.config_util.k8saccessor import KubernetesAccessorSingleton -from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX - +OLD_CONFIG_SUBDIR = 'old/' class TransientDirectoryProvider(FileConfigProvider): """ Implementation of the config provider that reads and writes the data @@ -21,6 +19,7 @@ class TransientDirectoryProvider(FileConfigProvider): # no uploaded config should ever affect subsequent config modifications/creations temp_dir = TemporaryDirectory() self.temp_dir = temp_dir + self.old_config_dir = None super(TransientDirectoryProvider, self).__init__(temp_dir.name, yaml_filename, py_filename) @property @@ -38,29 +37,26 @@ class TransientDirectoryProvider(FileConfigProvider): self.temp_dir = temp_dir self.yaml_path = os.path.join(temp_dir.name, self.yaml_filename) + def create_copy_of_config_dir(self): + """ + Create a directory to store loaded/populated configuration (for rollback if necessary) + """ + if self.old_config_dir is not None: + self.old_config_dir.cleanup() + + temp_dir = TemporaryDirectory() + self.old_config_dir = temp_dir + + # Python 2.7's shutil.copy() doesn't allow for copying to existing directories, + # so when copying/reading to the old saved config, we have to talk to a subdirectory, + # and use the shutil.copytree() function + copytree(self.config_volume, os.path.join(temp_dir.name, OLD_CONFIG_SUBDIR)) + def get_config_dir_path(self): return self.config_volume - def save_configuration_to_kubernetes(self): - data = {} + def get_old_config_dir(self): + if self.old_config_dir is None: + raise Exception('Cannot return a configuration that was no old configuration') - # Kubernetes secrets don't have sub-directories, so for the extra_ca_certs dir - # we have to put the extra certs in with a prefix, and then one of our init scripts - # (02_get_kube_certs.sh) will expand the prefixed certs into the equivalent directory - # so that they'll be installed correctly on startup by the certs_install script - certs_dir = os.path.join(self.config_volume, EXTRA_CA_DIRECTORY) - if os.path.exists(certs_dir): - for extra_cert in os.listdir(certs_dir): - with open(os.path.join(certs_dir, extra_cert)) as f: - data[EXTRA_CA_DIRECTORY_PREFIX + extra_cert] = base64.b64encode(f.read()) - - - for name in os.listdir(self.config_volume): - file_path = os.path.join(self.config_volume, name) - if not os.path.isdir(file_path): - with open(file_path) as f: - data[name] = base64.b64encode(f.read()) - - KubernetesAccessorSingleton.get_instance().replace_qe_secret(data) - - return 200 + return os.path.join(self.old_config_dir.name, OLD_CONFIG_SUBDIR) diff --git a/config_app/config_util/config/__init__.py b/config_app/config_util/config/__init__.py index 0bfa348ac..d39d0ea1c 100644 --- a/config_app/config_util/config/__init__.py +++ b/config_app/config_util/config/__init__.py @@ -1,6 +1,10 @@ +import base64 +import os + from config_app.config_util.config.fileprovider import FileConfigProvider from config_app.config_util.config.testprovider import TestConfigProvider from config_app.config_util.config.TransientDirectoryProvider import TransientDirectoryProvider +from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX def get_config_provider(config_volume, yaml_filename, py_filename, testing=False): @@ -10,3 +14,26 @@ def get_config_provider(config_volume, yaml_filename, py_filename, testing=False return TestConfigProvider() return TransientDirectoryProvider(config_volume, yaml_filename, py_filename) + + +def get_config_as_kube_secret(config_path): + data = {} + + # Kubernetes secrets don't have sub-directories, so for the extra_ca_certs dir + # we have to put the extra certs in with a prefix, and then one of our init scripts + # (02_get_kube_certs.sh) will expand the prefixed certs into the equivalent directory + # so that they'll be installed correctly on startup by the certs_install script + certs_dir = os.path.join(config_path, EXTRA_CA_DIRECTORY) + if os.path.exists(certs_dir): + for extra_cert in os.listdir(certs_dir): + with open(os.path.join(certs_dir, extra_cert)) as f: + data[EXTRA_CA_DIRECTORY_PREFIX + extra_cert] = base64.b64encode(f.read()) + + + for name in os.listdir(config_path): + file_path = os.path.join(config_path, name) + if not os.path.isdir(file_path): + with open(file_path) as f: + data[name] = base64.b64encode(f.read()) + + return data diff --git a/config_app/config_util/k8saccessor.py b/config_app/config_util/k8saccessor.py index 90597adfd..46fe4b83f 100644 --- a/config_app/config_util/k8saccessor.py +++ b/config_app/config_util/k8saccessor.py @@ -21,6 +21,9 @@ QE_CONTAINER_NAME = 'quay-enterprise-app' # message is any string describing the state. DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message']) +class K8sApiException(Exception): + pass + def _deployment_rollout_status_message(deployment, deployment_name): """ @@ -225,11 +228,24 @@ class KubernetesAccessorSingleton(object): } }, api_prefix='apis/extensions/v1beta1', content_type='application/strategic-merge-patch+json')) - def _assert_success(self, response): - if response.status_code != 200: + def rollback_deployment(self, deployment_name): + deployment_rollback_url = 'namespaces/%s/deployments/%s/rollback' % ( + self.kube_config.qe_namespace, deployment_name + ) + + self._assert_success(self._execute_k8s_api('POST', deployment_rollback_url, { + 'name': deployment_name, + 'rollbackTo': { + # revision=0 makes the deployment rollout to the previous revision + 'revision': 0 + } + }, api_prefix='apis/extensions/v1beta1'), 201) + + def _assert_success(self, response, expected_code=200): + if response.status_code != expected_code: logger.error('Kubernetes API call failed with response: %s => %s', response.status_code, response.text) - raise Exception('Kubernetes API call failed: %s' % response.text) + raise K8sApiException('Kubernetes API call failed: %s' % response.text) def _update_secret_file(self, relative_file_path, value=None): if '/' in relative_file_path: diff --git a/config_app/docs/k8s_templates/config-tool-servicetoken-role.yml b/config_app/docs/k8s_templates/config-tool-servicetoken-role.yml index e645ae3f8..f11ee3281 100644 --- a/config_app/docs/k8s_templates/config-tool-servicetoken-role.yml +++ b/config_app/docs/k8s_templates/config-tool-servicetoken-role.yml @@ -24,7 +24,9 @@ rules: - "apps" resources: - deployments + - deployments/rollback verbs: + - create - get - list - patch diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.html b/config_app/js/components/config-setup-app/config-setup-app.component.html index 16dc0940f..58edfcac2 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.html +++ b/config_app/js/components/config-setup-app/config-setup-app.component.html @@ -58,4 +58,4 @@ is-kubernetes="$ctrl.kubeNamespace !== false" choose-deploy="$ctrl.chooseDeploy()"> - + diff --git a/config_app/js/components/config-setup-app/config-setup-app.component.ts b/config_app/js/components/config-setup-app/config-setup-app.component.ts index 3b05c304c..769974027 100644 --- a/config_app/js/components/config-setup-app/config-setup-app.component.ts +++ b/config_app/js/components/config-setup-app/config-setup-app.component.ts @@ -47,6 +47,7 @@ export class ConfigSetupAppComponent { this.apiService.scKubePopulateConfig() .then(() => { this.state = 'setup'; + this.loadedConfig = true; }) .catch(err => { this.apiService.errorDisplay( diff --git a/config_app/js/components/cor-loader/cor-loader-inline.html b/config_app/js/components/cor-loader/cor-loader-inline.html new file mode 100644 index 000000000..3a2c42c1d --- /dev/null +++ b/config_app/js/components/cor-loader/cor-loader-inline.html @@ -0,0 +1,5 @@ +
+
+
+
+
diff --git a/config_app/js/components/cor-loader/cor-loader.js b/config_app/js/components/cor-loader/cor-loader.js index e58c82230..17c090666 100644 --- a/config_app/js/components/cor-loader/cor-loader.js +++ b/config_app/js/components/cor-loader/cor-loader.js @@ -1,9 +1,22 @@ -const templateUrl = require('./cor-loader.html'); +const loaderUrl = require('./cor-loader.html'); +const inlineUrl = require('./cor-loader-inline.html'); angular.module('quay-config') .directive('corLoader', function() { var directiveDefinitionObject = { - templateUrl, + templateUrl: loaderUrl, + replace: true, + restrict: 'C', + scope: { + }, + controller: function($rootScope, $scope, $element) { + } + }; + return directiveDefinitionObject; + }) + .directive('corLoaderInline', function() { + var directiveDefinitionObject = { + templateUrl: inlineUrl, replace: true, restrict: 'C', scope: { diff --git a/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html b/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html index 162097072..2b4cd24d7 100644 --- a/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html +++ b/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html @@ -2,7 +2,7 @@ - +
+ Successfully rolled back changes. Please try deploying again with your configuration later. +
+ + + + \ No newline at end of file diff --git a/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts b/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts index 35c513d4f..fdeb1ed3f 100644 --- a/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts +++ b/config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts @@ -1,5 +1,5 @@ -import {Component, Inject, OnDestroy } from 'ng-metadata/core'; -import {AngularPollChannel, PollHandle} from "../../services/services.types"; +import { Input, Component, Inject, OnDestroy } from 'ng-metadata/core'; +import { AngularPollChannel, PollHandle } from "../../services/services.types"; const templateUrl = require('./kube-deploy-modal.component.html'); const styleUrl = require('./kube-deploy-modal.css'); @@ -24,22 +24,25 @@ const DEPLOYMENT_POLL_SLEEPTIME = 5000; /* 5 seconds */ styleUrls: [ styleUrl ], }) export class KubeDeployModalComponent implements OnDestroy { + @Input('<') public loadedConfig; private state : 'loadingDeployments' | 'readyToDeploy' | 'deployingConfiguration' | 'cyclingDeployments' | 'deployed' - | 'error'; + | 'error' + | 'rolledBackWarning' = 'loadingDeployments'; private errorMessage: string; - private offerRollback: boolean; private deploymentsStatus: DeploymentStatus[] = []; private deploymentsCycled: number = 0; private onDestroyListeners: Function[] = []; + private rollingBackStatus + : 'none' + | 'offer' + | 'rolling' = 'none'; constructor(@Inject('ApiService') private ApiService, @Inject('AngularPollChannel') private AngularPollChannel: AngularPollChannel) { - this.state = 'loadingDeployments'; - ApiService.scGetNumDeployments().then(resp => { this.deploymentsStatus = resp.items.map(dep => ({ name: dep.metadata.name, numPods: dep.spec.replicas })); this.state = 'readyToDeploy'; @@ -80,7 +83,7 @@ export class KubeDeployModalComponent implements OnDestroy { watchDeployments(): void { this.deploymentsStatus.forEach(deployment => { - const pollChannel = this.AngularPollChannel.create( { + const pollChannel = this.AngularPollChannel.create({ // Have to mock the scope object for the poll channel since we're calling into angular1 code // We register the onDestroy function to be called later when this object is destroyed '$on': (_, onDestruction) => { this.onDestroyListeners.push(onDestruction) } @@ -113,14 +116,38 @@ export class KubeDeployModalComponent implements OnDestroy { continue_callback(false); deployment.message = deploymentRollout.message; this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`; - this.offerRollback = true; + + // Only offer rollback if we loaded/populated a config. (Can't rollback an initial setup) + if (this.loadedConfig) { + this.rollingBackStatus = 'offer'; + this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`; + } } }).catch(err => { continue_callback(false); this.state = 'error'; - this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`; - this.offerRollback = true; + this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}\ + Would you like to rollback the deployment to its previous state?`; + // Only offer rollback if we loaded/populated a config. (Can't rollback an initial setup) + if (this.loadedConfig) { + this.rollingBackStatus = 'offer'; + this.errorMessage = `Could not get deployment information for: ${deployment}`; + } }); } } + + rollbackDeployments(): void { + this.rollingBackStatus = 'rolling'; + const deploymentNames: string[] = this.deploymentsStatus.map(dep => dep.name); + + this.ApiService.scRollbackDeployments({ deploymentNames }).then(() => { + this.state = 'rolledBackWarning'; + this.rollingBackStatus = 'none'; + }).catch(err => { + this.rollingBackStatus = 'none'; + this.state = 'error'; + this.errorMessage = `Could not cycle the deployments back to their previous states. Please contact support: ${err.toString()}`; + }) + } } \ No newline at end of file diff --git a/config_app/js/components/kube-deploy-modal/kube-deploy-modal.css b/config_app/js/components/kube-deploy-modal/kube-deploy-modal.css index ffa76ed37..8841807e8 100644 --- a/config_app/js/components/kube-deploy-modal/kube-deploy-modal.css +++ b/config_app/js/components/kube-deploy-modal/kube-deploy-modal.css @@ -29,3 +29,18 @@ margin-top: 20px; margin-bottom: 10px; } + +.kube-deploy-modal .rollback { + margin-left: auto; +} + +.kube-deploy-modal .co-alert.co-alert-warning { + padding: 25px; + display: flex; + margin-bottom: 0; +} + +.kube-deploy-modal .co-alert.co-alert-warning:before { + position: static; + padding-right: 15px; +} diff --git a/config_app/static/css/config-setup-app-component.css b/config_app/static/css/config-setup-app-component.css index 6cf0d85bc..2d41544d9 100644 --- a/config_app/static/css/config-setup-app-component.css +++ b/config_app/static/css/config-setup-app-component.css @@ -62,6 +62,10 @@ position: static; } +.co-alert.co-alert-danger:after { + /* Ignore the exclamation mark, it also messes with spacing elements */ + content: none; +} /* Fixes the transition to font awesome 5 */ .quay-config-app .co-alert.co-alert-warning::before {