Add config ability to rollback changes on kube

config_app/config_endpoints/api/__init__.py

@@ -150,10 +150,8 @@ def kubernetes_only(f):
 nickname = partial(add_method_metadata, 'nickname')
 
 
-
-import config_app.config_endpoints.api
 import config_app.config_endpoints.api.discovery
-import config_app.config_endpoints.api.kubeconfig
+import config_app.config_endpoints.api.kube_endpoints
 import config_app.config_endpoints.api.suconfig
 import config_app.config_endpoints.api.superuser
 import config_app.config_endpoints.api.tar_config_loader

config_app/config_endpoints/api/kube_endpoints.py

@@ -1,10 +1,11 @@
-from flask import request
+from flask import request, make_response
 
+from config_app.config_util.config import get_config_as_kube_secret
 from data.database import configure
 
 from config_app.c_app import app, config_provider
-from config_app.config_endpoints.api import resource, ApiResource, nickname, kubernetes_only
+from config_app.config_endpoints.api import resource, ApiResource, nickname, kubernetes_only, validate_json_request
-from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
+from config_app.config_util.k8saccessor import KubernetesAccessorSingleton, K8sApiException
 
 
 @resource('/v1/kubernetes/deployments/')
@@ -32,11 +33,13 @@ class SuperUserKubernetesDeployment(ApiResource):
     return KubernetesAccessorSingleton.get_instance().get_qe_deployments()
 
   @kubernetes_only
+  @validate_json_request('ValidateDeploymentNames')
   @nickname('scCycleQEDeployments')
   def put(self):
     deployment_names = request.get_json()['deploymentNames']
     return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names)
 
+
 @resource('/v1/kubernetes/deployment/<deployment>/status')
 class QEDeploymentRolloutStatus(ApiResource):
   @kubernetes_only
@@ -49,14 +52,66 @@ class QEDeploymentRolloutStatus(ApiResource):
     }
 
 
-@resource('/v1/superuser/config/kubernetes')
+@resource('/v1/kubernetes/deployments/rollback')
+class QEDeploymentRollback(ApiResource):
+  """ Resource for rolling back deployments """
+  schemas = {
+    'ValidateDeploymentNames': {
+      'type': 'object',
+      'description': 'Validates deployment names for rolling back',
+      'required': [
+        'deploymentNames'
+      ],
+      'properties': {
+        'deploymentNames': {
+          'type': 'array',
+          'description': 'The names of the deployments to rollback'
+        },
+      },
+    }
+  }
+
+  @kubernetes_only
+  @nickname('scRollbackDeployments')
+  @validate_json_request('ValidateDeploymentNames')
+  def post(self):
+    """
+    Returns the config to its original state and rolls back deployments
+    :return:
+    """
+    deployment_names = request.get_json()['deploymentNames']
+
+    # To roll back a deployment, we must do 2 things:
+    # 1. Roll back the config secret to its old value (discarding changes we made in this session
+    # 2. Trigger a rollback to the previous revision, so that the pods will be restarted with
+    # the old config
+    old_secret = get_config_as_kube_secret(config_provider.get_old_config_dir())
+    kube_accessor = KubernetesAccessorSingleton.get_instance()
+    kube_accessor.replace_qe_secret(old_secret)
+
+    try:
+      for name in deployment_names:
+        kube_accessor.rollback_deployment(name)
+    except K8sApiException as e:
+      return make_response(e.message, 500)
+
+    return make_response('Ok', 204)
+
+
+@resource('/v1/kubernetes/config')
 class SuperUserKubernetesConfiguration(ApiResource):
   """ Resource for saving the config files to kubernetes secrets. """
 
   @kubernetes_only
   @nickname('scDeployConfiguration')
   def post(self):
-    return config_provider.save_configuration_to_kubernetes()
+    try:
+      new_secret = get_config_as_kube_secret(config_provider.get_config_dir_path())
+      KubernetesAccessorSingleton.get_instance().replace_qe_secret(new_secret)
+    except K8sApiException as e:
+      return make_response(e.message, 500)
+
+    return make_response('Ok', 201)
 
 
 @resource('/v1/kubernetes/config/populate')
@@ -68,7 +123,10 @@ class KubernetesConfigurationPopulator(ApiResource):
   def post(self):
     # Get a clean transient directory to write the config into
     config_provider.new_config_dir()
-    KubernetesAccessorSingleton.get_instance().save_secret_to_directory(config_provider.get_config_dir_path())
+
+    kube_accessor = KubernetesAccessorSingleton.get_instance()
+    kube_accessor.save_secret_to_directory(config_provider.get_config_dir_path())
+    config_provider.create_copy_of_config_dir()
 
     # We update the db configuration to connect to their specified one
     # (Note, even if this DB isn't valid, it won't affect much in the config app, since we'll report an error,

config_app/config_endpoints/api/tar_config_loader.py

@@ -52,6 +52,8 @@ class TarConfigLoader(ApiResource):
     with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream:
       tar_stream.extractall(config_provider.get_config_dir_path())
 
+    config_provider.create_copy_of_config_dir()
+
     # now try to connect to the db provided in their config to validate it works
     combined = dict(**app.config)
     combined.update(config_provider.get_config())

config_app/config_util/config/TransientDirectoryProvider.py

@@ -1,13 +1,11 @@
 import os
-import base64
 
+from shutil import copytree
 from backports.tempfile import TemporaryDirectory
 
 from config_app.config_util.config.fileprovider import FileConfigProvider
-from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
-from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
-
 
+OLD_CONFIG_SUBDIR = 'old/'
 
 class TransientDirectoryProvider(FileConfigProvider):
   """ Implementation of the config provider that reads and writes the data
@@ -21,6 +19,7 @@ class TransientDirectoryProvider(FileConfigProvider):
     # no uploaded config should ever affect subsequent config modifications/creations
     temp_dir = TemporaryDirectory()
     self.temp_dir = temp_dir
+    self.old_config_dir = None
     super(TransientDirectoryProvider, self).__init__(temp_dir.name, yaml_filename, py_filename)
 
   @property
@@ -38,29 +37,26 @@ class TransientDirectoryProvider(FileConfigProvider):
     self.temp_dir = temp_dir
     self.yaml_path = os.path.join(temp_dir.name, self.yaml_filename)
 
+  def create_copy_of_config_dir(self):
+    """
+    Create a directory to store loaded/populated configuration (for rollback if necessary)
+    """
+    if self.old_config_dir is not None:
+      self.old_config_dir.cleanup()
+
+    temp_dir = TemporaryDirectory()
+    self.old_config_dir = temp_dir
+
+    # Python 2.7's shutil.copy() doesn't allow for copying to existing directories,
+    # so when copying/reading to the old saved config, we have to talk to a subdirectory,
+    # and use the shutil.copytree() function
+    copytree(self.config_volume, os.path.join(temp_dir.name, OLD_CONFIG_SUBDIR))
+
   def get_config_dir_path(self):
     return self.config_volume
 
-  def save_configuration_to_kubernetes(self):
+  def get_old_config_dir(self):
-    data = {}
+    if self.old_config_dir is None:
+      raise Exception('Cannot return a configuration that  was no old configuration')
 
-    # Kubernetes secrets don't have sub-directories, so for the extra_ca_certs dir
+    return os.path.join(self.old_config_dir.name, OLD_CONFIG_SUBDIR)
-    # we have to put the extra certs in with a prefix, and then one of our init scripts
-    # (02_get_kube_certs.sh) will expand the prefixed certs into the equivalent directory
-    # so that they'll be installed correctly on startup by the certs_install script
-    certs_dir = os.path.join(self.config_volume, EXTRA_CA_DIRECTORY)
-    if os.path.exists(certs_dir):
-      for extra_cert in os.listdir(certs_dir):
-        with open(os.path.join(certs_dir, extra_cert)) as f:
-          data[EXTRA_CA_DIRECTORY_PREFIX + extra_cert] = base64.b64encode(f.read())
-
-
-    for name in os.listdir(self.config_volume):
-      file_path = os.path.join(self.config_volume, name)
-      if not os.path.isdir(file_path):
-        with open(file_path) as f:
-          data[name] = base64.b64encode(f.read())
-
-    KubernetesAccessorSingleton.get_instance().replace_qe_secret(data)
-
-    return 200

config_app/config_util/config/__init__.py

@@ -1,6 +1,10 @@
+import base64
+import os
+
 from config_app.config_util.config.fileprovider import FileConfigProvider
 from config_app.config_util.config.testprovider import TestConfigProvider
 from config_app.config_util.config.TransientDirectoryProvider import TransientDirectoryProvider
+from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
 
 
 def get_config_provider(config_volume, yaml_filename, py_filename, testing=False):
@@ -10,3 +14,26 @@ def get_config_provider(config_volume, yaml_filename, py_filename, testing=False
     return TestConfigProvider()
 
   return TransientDirectoryProvider(config_volume, yaml_filename, py_filename)
+
+
+def get_config_as_kube_secret(config_path):
+  data = {}
+
+  # Kubernetes secrets don't have sub-directories, so for the extra_ca_certs dir
+  # we have to put the extra certs in with a prefix, and then one of our init scripts
+  # (02_get_kube_certs.sh) will expand the prefixed certs into the equivalent directory
+  # so that they'll be installed correctly on startup by the certs_install script
+  certs_dir = os.path.join(config_path,  EXTRA_CA_DIRECTORY)
+  if os.path.exists(certs_dir):
+    for extra_cert in os.listdir(certs_dir):
+      with open(os.path.join(certs_dir, extra_cert)) as f:
+        data[EXTRA_CA_DIRECTORY_PREFIX + extra_cert] = base64.b64encode(f.read())
+
+
+  for name in os.listdir(config_path):
+    file_path = os.path.join(config_path, name)
+    if not os.path.isdir(file_path):
+      with open(file_path) as f:
+        data[name] = base64.b64encode(f.read())
+
+  return data

config_app/config_util/k8saccessor.py

@@ -21,6 +21,9 @@ QE_CONTAINER_NAME = 'quay-enterprise-app'
 # message is any string describing the state.
 DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message'])
 
+class K8sApiException(Exception):
+  pass
+
 
 def _deployment_rollout_status_message(deployment, deployment_name):
   """
@@ -225,11 +228,24 @@ class KubernetesAccessorSingleton(object):
         }
       }, api_prefix='apis/extensions/v1beta1', content_type='application/strategic-merge-patch+json'))
 
-  def _assert_success(self, response):
+  def rollback_deployment(self, deployment_name):
-    if response.status_code != 200:
+    deployment_rollback_url = 'namespaces/%s/deployments/%s/rollback' % (
+      self.kube_config.qe_namespace, deployment_name
+    )
+
+    self._assert_success(self._execute_k8s_api('POST', deployment_rollback_url, {
+      'name': deployment_name,
+      'rollbackTo': {
+        # revision=0 makes the deployment rollout to the previous revision
+        'revision': 0
+      }
+    }, api_prefix='apis/extensions/v1beta1'), 201)
+
+  def _assert_success(self, response, expected_code=200):
+    if response.status_code != expected_code:
       logger.error('Kubernetes API call failed with response: %s => %s', response.status_code,
                    response.text)
-      raise Exception('Kubernetes API call failed: %s' % response.text)
+      raise K8sApiException('Kubernetes API call failed: %s' % response.text)
 
   def _update_secret_file(self, relative_file_path, value=None):
     if '/' in relative_file_path:

config_app/docs/k8s_templates/config-tool-servicetoken-role.yml

@@ -24,7 +24,9 @@ rules:
   - "apps"
   resources:
   - deployments
+  - deployments/rollback
   verbs:
+  - create
   - get
   - list
   - patch

config_app/js/components/config-setup-app/config-setup-app.component.html

@@ -58,4 +58,4 @@
         is-kubernetes="$ctrl.kubeNamespace !== false"
         choose-deploy="$ctrl.chooseDeploy()">
 </download-tarball-modal>
-<kube-deploy-modal ng-if="$ctrl.state === 'deploy'"></kube-deploy-modal>
+<kube-deploy-modal ng-if="$ctrl.state === 'deploy'" loaded-config="$ctrl.loadedConfig"></kube-deploy-modal>

config_app/js/components/config-setup-app/config-setup-app.component.ts

@@ -47,6 +47,7 @@ export class ConfigSetupAppComponent {
         this.apiService.scKubePopulateConfig()
             .then(() => {
                 this.state = 'setup';
+                this.loadedConfig = true;
             })
             .catch(err => {
                 this.apiService.errorDisplay(

config_app/js/components/cor-loader/cor-loader-inline.html

@@ -0,0 +1,5 @@
+<div class="co-m-inline-loader co-an-fade-in-out">
+  <div class="co-m-loader-dot__one"></div>
+  <div class="co-m-loader-dot__two"></div>
+  <div class="co-m-loader-dot__three"></div>
+</div>

config_app/js/components/cor-loader/cor-loader.js

@@ -1,9 +1,22 @@
-const templateUrl = require('./cor-loader.html');
+const loaderUrl = require('./cor-loader.html');
+const inlineUrl = require('./cor-loader-inline.html');
 
 angular.module('quay-config')
     .directive('corLoader', function() {
         var directiveDefinitionObject = {
-            templateUrl,
+            templateUrl: loaderUrl,
+            replace: true,
+            restrict: 'C',
+            scope: {
+            },
+            controller: function($rootScope, $scope, $element) {
+            }
+        };
+        return directiveDefinitionObject;
+    })
+    .directive('corLoaderInline', function() {
+        var directiveDefinitionObject = {
+            templateUrl: inlineUrl,
             replace: true,
             restrict: 'C',
             scope: {

config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html

@@ -2,7 +2,7 @@
     <div class="co-dialog modal fade initial-setup-modal in" id="setupModal" style="display: block;">
         <div class="modal-backdrop fade in" style="height: 1000px;"></div>
         <div class="modal-dialog fade in">
-            <div class="modal-content">
+            <div class="modal-content kube-deploy-modal">
                 <!-- Header -->
                 <div class="modal-header">
                     <span class="cor-step-bar">
@@ -37,11 +37,13 @@
                         </div>
                         <div ng-if="$ctrl.state === 'cyclingDeployments'">
                             <div class="cor-loader"></div>
-                            Cycling deployments...
+                            <span class="kube-deploy-modal__list-header">Cycling deployments...</span>
+                            <ul class="kube-deploy-modal__list">
                                 <li class="kube-deploy-modal__list-item" ng-repeat="deployment in $ctrl.deploymentsStatus">
                                     <i class="fa ci-k8s-logo"></i>
                                     <code>{{deployment.name}}</code>: {{deployment.message || 'Waiting for deployment information...'}}
                                 </li>
+                            </ul>
                         </div>
                     </div>
                 </div>
@@ -51,11 +53,19 @@
                 </div>
                 <div ng-if="$ctrl.state === 'error'" class="modal-footer alert alert-danger">
                     {{ $ctrl.errorMessage }}
-                    <div ng-if="$ctrl.offerRollback">
+                    <div ng-if="$ctrl.rollingBackStatus !== 'none'" class="rollback">
-                        // todo
+                        <button ng-if="$ctrl.rollingBackStatus === 'offer'" class="btn btn-default" ng-click="$ctrl.rollbackDeployments()">
+                            <i class="fas fa-history" style="margin-right: 10px;"></i>
+                            Rollback deployments
+                        </button>
+                        <div ng-if="$ctrl.rollingBackStatus === 'rolling'" class="cor-loader-inline"></div>
+                    </div>
+                </div>
+                <div ng-if="$ctrl.state === 'rolledBackWarning'" class="modal-footer co-alert co-alert-warning">
+                    Successfully rolled back changes. Please try deploying again with your configuration later.
                 </div>
             </div>
         </div><!-- /.modal-content -->
     </div><!-- /.modal-dialog -->
-    </div>
+</div>
 </div>

config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts

@@ -1,5 +1,5 @@
-import {Component, Inject, OnDestroy } from 'ng-metadata/core';
+import { Input, Component, Inject, OnDestroy } from 'ng-metadata/core';
-import {AngularPollChannel, PollHandle} from "../../services/services.types";
+import { AngularPollChannel, PollHandle } from "../../services/services.types";
 const templateUrl = require('./kube-deploy-modal.component.html');
 const styleUrl = require('./kube-deploy-modal.css');
 
@@ -24,22 +24,25 @@ const DEPLOYMENT_POLL_SLEEPTIME = 5000; /* 5 seconds */
     styleUrls: [ styleUrl ],
 })
 export class KubeDeployModalComponent implements OnDestroy {
+    @Input('<') public loadedConfig;
     private state
         : 'loadingDeployments'
         | 'readyToDeploy'
         | 'deployingConfiguration'
         | 'cyclingDeployments'
         | 'deployed'
-        | 'error';
+        | 'error'
+        | 'rolledBackWarning' = 'loadingDeployments';
     private errorMessage: string;
-    private offerRollback: boolean;
     private deploymentsStatus: DeploymentStatus[] = [];
     private deploymentsCycled: number = 0;
     private onDestroyListeners: Function[] = [];
+    private rollingBackStatus
+        : 'none'
+        | 'offer'
+        | 'rolling' = 'none';
 
     constructor(@Inject('ApiService') private ApiService, @Inject('AngularPollChannel') private AngularPollChannel: AngularPollChannel) {
-        this.state = 'loadingDeployments';
-
         ApiService.scGetNumDeployments().then(resp => {
             this.deploymentsStatus = resp.items.map(dep => ({ name: dep.metadata.name, numPods: dep.spec.replicas }));
             this.state = 'readyToDeploy';
@@ -80,7 +83,7 @@ export class KubeDeployModalComponent implements OnDestroy {
 
     watchDeployments(): void {
         this.deploymentsStatus.forEach(deployment => {
-            const pollChannel = this.AngularPollChannel.create( {
+            const pollChannel = this.AngularPollChannel.create({
                 // Have to mock the scope object for the poll channel since we're calling into angular1 code
                 // We register the onDestroy function to be called later when this object is destroyed
                 '$on': (_, onDestruction) => { this.onDestroyListeners.push(onDestruction) }
@@ -113,14 +116,38 @@ export class KubeDeployModalComponent implements OnDestroy {
                     continue_callback(false);
                     deployment.message = deploymentRollout.message;
                     this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`;
-                    this.offerRollback = true;
+
+                    // Only offer rollback if we loaded/populated a config. (Can't rollback an initial setup)
+                    if (this.loadedConfig) {
+                        this.rollingBackStatus = 'offer';
+                        this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`;
+                    }
                 }
             }).catch(err => {
                 continue_callback(false);
                 this.state = 'error';
-                this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
+                this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}\
-                this.offerRollback = true;
+                                     Would you like to rollback the deployment to its previous state?`;
+                // Only offer rollback if we loaded/populated a config. (Can't rollback an initial setup)
+                if (this.loadedConfig) {
+                    this.rollingBackStatus = 'offer';
+                    this.errorMessage = `Could not get deployment information for: ${deployment}`;
+                }
             });
         }
     }
+
+    rollbackDeployments(): void {
+        this.rollingBackStatus = 'rolling';
+        const deploymentNames: string[] = this.deploymentsStatus.map(dep => dep.name);
+
+        this.ApiService.scRollbackDeployments({ deploymentNames }).then(() => {
+            this.state = 'rolledBackWarning';
+            this.rollingBackStatus = 'none';
+        }).catch(err => {
+            this.rollingBackStatus = 'none';
+            this.state = 'error';
+            this.errorMessage = `Could not cycle the deployments back to their previous states. Please contact support: ${err.toString()}`;
+        })
+    }
 }

config_app/js/components/kube-deploy-modal/kube-deploy-modal.css

@@ -29,3 +29,18 @@
     margin-top: 20px;
     margin-bottom: 10px;
 }
+
+.kube-deploy-modal .rollback {
+    margin-left: auto;
+}
+
+.kube-deploy-modal .co-alert.co-alert-warning {
+    padding: 25px;
+    display: flex;
+    margin-bottom: 0;
+}
+
+.kube-deploy-modal .co-alert.co-alert-warning:before {
+    position: static;
+    padding-right: 15px;
+}

config_app/static/css/config-setup-app-component.css

@@ -62,6 +62,10 @@
     position: static;
 }
 
+.co-alert.co-alert-danger:after {
+    /* Ignore the exclamation mark, it also messes with spacing elements */
+    content: none;
+}
 
 /* Fixes the transition to font awesome 5 */
 .quay-config-app .co-alert.co-alert-warning::before {