Merge pull request #3230 from quay/project/rollback-deploy

Add ability to rollback config to config tool (kube deployments)
This commit is contained in:
Sam Chow 2018-08-30 10:17:43 -04:00 committed by GitHub
commit e711c1efe6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 385 additions and 63 deletions

View file

@ -150,10 +150,8 @@ def kubernetes_only(f):
nickname = partial(add_method_metadata, 'nickname')
import config_app.config_endpoints.api
import config_app.config_endpoints.api.discovery
import config_app.config_endpoints.api.kubeconfig
import config_app.config_endpoints.api.kube_endpoints
import config_app.config_endpoints.api.suconfig
import config_app.config_endpoints.api.superuser
import config_app.config_endpoints.api.tar_config_loader

View file

@ -1,11 +1,15 @@
from flask import request
import logging
from flask import request, make_response
from config_app.config_util.config import get_config_as_kube_secret
from data.database import configure
from config_app.c_app import app, config_provider
from config_app.config_endpoints.api import resource, ApiResource, nickname, kubernetes_only
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
from config_app.config_endpoints.api import resource, ApiResource, nickname, kubernetes_only, validate_json_request
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton, K8sApiException
logger = logging.getLogger(__name__)
@resource('/v1/kubernetes/deployments/')
class SuperUserKubernetesDeployment(ApiResource):
@ -32,11 +36,13 @@ class SuperUserKubernetesDeployment(ApiResource):
return KubernetesAccessorSingleton.get_instance().get_qe_deployments()
@kubernetes_only
@validate_json_request('ValidateDeploymentNames')
@nickname('scCycleQEDeployments')
def put(self):
deployment_names = request.get_json()['deploymentNames']
return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names)
@resource('/v1/kubernetes/deployment/<deployment>/status')
class QEDeploymentRolloutStatus(ApiResource):
@kubernetes_only
@ -49,14 +55,68 @@ class QEDeploymentRolloutStatus(ApiResource):
}
@resource('/v1/superuser/config/kubernetes')
@resource('/v1/kubernetes/deployments/rollback')
class QEDeploymentRollback(ApiResource):
""" Resource for rolling back deployments """
schemas = {
'ValidateDeploymentNames': {
'type': 'object',
'description': 'Validates deployment names for rolling back',
'required': [
'deploymentNames'
],
'properties': {
'deploymentNames': {
'type': 'array',
'description': 'The names of the deployments to rollback'
},
},
}
}
@kubernetes_only
@nickname('scRollbackDeployments')
@validate_json_request('ValidateDeploymentNames')
def post(self):
"""
Returns the config to its original state and rolls back deployments
:return:
"""
deployment_names = request.get_json()['deploymentNames']
# To roll back a deployment, we must do 2 things:
# 1. Roll back the config secret to its old value (discarding changes we made in this session)
# 2. Trigger a rollback to the previous revision, so that the pods will be restarted with
# the old config
old_secret = get_config_as_kube_secret(config_provider.get_old_config_dir())
kube_accessor = KubernetesAccessorSingleton.get_instance()
kube_accessor.replace_qe_secret(old_secret)
try:
for name in deployment_names:
kube_accessor.rollback_deployment(name)
except K8sApiException as e:
logger.exception('Failed to rollback deployment.')
return make_response(e.message, 503)
return make_response('Ok', 204)
@resource('/v1/kubernetes/config')
class SuperUserKubernetesConfiguration(ApiResource):
""" Resource for saving the config files to kubernetes secrets. """
@kubernetes_only
@nickname('scDeployConfiguration')
def post(self):
return config_provider.save_configuration_to_kubernetes()
try:
new_secret = get_config_as_kube_secret(config_provider.get_config_dir_path())
KubernetesAccessorSingleton.get_instance().replace_qe_secret(new_secret)
except K8sApiException as e:
logger.exception('Failed to deploy qe config secret to kubernetes.')
return make_response(e.message, 503)
return make_response('Ok', 201)
@resource('/v1/kubernetes/config/populate')
@ -68,7 +128,10 @@ class KubernetesConfigurationPopulator(ApiResource):
def post(self):
# Get a clean transient directory to write the config into
config_provider.new_config_dir()
KubernetesAccessorSingleton.get_instance().save_secret_to_directory(config_provider.get_config_dir_path())
kube_accessor = KubernetesAccessorSingleton.get_instance()
kube_accessor.save_secret_to_directory(config_provider.get_config_dir_path())
config_provider.create_copy_of_config_dir()
# We update the db configuration to connect to their specified one
# (Note, even if this DB isn't valid, it won't affect much in the config app, since we'll report an error,

View file

@ -52,6 +52,8 @@ class TarConfigLoader(ApiResource):
with tarfile.open(mode="r|gz", fileobj=input_stream) as tar_stream:
tar_stream.extractall(config_provider.get_config_dir_path())
config_provider.create_copy_of_config_dir()
# now try to connect to the db provided in their config to validate it works
combined = dict(**app.config)
combined.update(config_provider.get_config())

View file

@ -1,13 +1,11 @@
import os
import base64
from shutil import copytree
from backports.tempfile import TemporaryDirectory
from config_app.config_util.config.fileprovider import FileConfigProvider
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
OLD_CONFIG_SUBDIR = 'old/'
class TransientDirectoryProvider(FileConfigProvider):
""" Implementation of the config provider that reads and writes the data
@ -21,6 +19,7 @@ class TransientDirectoryProvider(FileConfigProvider):
# no uploaded config should ever affect subsequent config modifications/creations
temp_dir = TemporaryDirectory()
self.temp_dir = temp_dir
self.old_config_dir = None
super(TransientDirectoryProvider, self).__init__(temp_dir.name, yaml_filename, py_filename)
@property
@ -38,29 +37,26 @@ class TransientDirectoryProvider(FileConfigProvider):
self.temp_dir = temp_dir
self.yaml_path = os.path.join(temp_dir.name, self.yaml_filename)
def create_copy_of_config_dir(self):
"""
Create a directory to store loaded/populated configuration (for rollback if necessary)
"""
if self.old_config_dir is not None:
self.old_config_dir.cleanup()
temp_dir = TemporaryDirectory()
self.old_config_dir = temp_dir
# Python 2.7's shutil.copy() doesn't allow for copying to existing directories,
# so when copying/reading to the old saved config, we have to talk to a subdirectory,
# and use the shutil.copytree() function
copytree(self.config_volume, os.path.join(temp_dir.name, OLD_CONFIG_SUBDIR))
def get_config_dir_path(self):
return self.config_volume
def save_configuration_to_kubernetes(self):
data = {}
def get_old_config_dir(self):
if self.old_config_dir is None:
raise Exception('Cannot return a configuration that was no old configuration')
# Kubernetes secrets don't have sub-directories, so for the extra_ca_certs dir
# we have to put the extra certs in with a prefix, and then one of our init scripts
# (02_get_kube_certs.sh) will expand the prefixed certs into the equivalent directory
# so that they'll be installed correctly on startup by the certs_install script
certs_dir = os.path.join(self.config_volume, EXTRA_CA_DIRECTORY)
if os.path.exists(certs_dir):
for extra_cert in os.listdir(certs_dir):
with open(os.path.join(certs_dir, extra_cert)) as f:
data[EXTRA_CA_DIRECTORY_PREFIX + extra_cert] = base64.b64encode(f.read())
for name in os.listdir(self.config_volume):
file_path = os.path.join(self.config_volume, name)
if not os.path.isdir(file_path):
with open(file_path) as f:
data[name] = base64.b64encode(f.read())
KubernetesAccessorSingleton.get_instance().replace_qe_secret(data)
return 200
return os.path.join(self.old_config_dir.name, OLD_CONFIG_SUBDIR)

View file

@ -1,6 +1,10 @@
import base64
import os
from config_app.config_util.config.fileprovider import FileConfigProvider
from config_app.config_util.config.testprovider import TestConfigProvider
from config_app.config_util.config.TransientDirectoryProvider import TransientDirectoryProvider
from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
def get_config_provider(config_volume, yaml_filename, py_filename, testing=False):
@ -10,3 +14,26 @@ def get_config_provider(config_volume, yaml_filename, py_filename, testing=False
return TestConfigProvider()
return TransientDirectoryProvider(config_volume, yaml_filename, py_filename)
def get_config_as_kube_secret(config_path):
data = {}
# Kubernetes secrets don't have sub-directories, so for the extra_ca_certs dir
# we have to put the extra certs in with a prefix, and then one of our init scripts
# (02_get_kube_certs.sh) will expand the prefixed certs into the equivalent directory
# so that they'll be installed correctly on startup by the certs_install script
certs_dir = os.path.join(config_path, EXTRA_CA_DIRECTORY)
if os.path.exists(certs_dir):
for extra_cert in os.listdir(certs_dir):
with open(os.path.join(certs_dir, extra_cert)) as f:
data[EXTRA_CA_DIRECTORY_PREFIX + extra_cert] = base64.b64encode(f.read())
for name in os.listdir(config_path):
file_path = os.path.join(config_path, name)
if not os.path.isdir(file_path):
with open(file_path) as f:
data[name] = base64.b64encode(f.read())
return data

View file

@ -0,0 +1,75 @@
import pytest
import os
import base64
from backports.tempfile import TemporaryDirectory
from config_app.config_util.config import get_config_as_kube_secret
from util.config.validator import EXTRA_CA_DIRECTORY
def _create_temp_file_structure(file_structure):
temp_dir = TemporaryDirectory()
for filename, data in file_structure.iteritems():
if filename == EXTRA_CA_DIRECTORY:
extra_ca_dir_path = os.path.join(temp_dir.name, EXTRA_CA_DIRECTORY)
os.mkdir(extra_ca_dir_path)
for name, cert_value in data:
with open(os.path.join(extra_ca_dir_path, name), 'w') as f:
f.write(cert_value)
else:
with open(os.path.join(temp_dir.name, filename), 'w') as f:
f.write(data)
return temp_dir
@pytest.mark.parametrize('file_structure, expected_secret', [
pytest.param({
'config.yaml': 'test:true',
},
{
'config.yaml': 'dGVzdDp0cnVl',
}, id='just a config value'),
pytest.param({
'config.yaml': 'test:true',
'otherfile.ext': 'im a file'
},
{
'config.yaml': 'dGVzdDp0cnVl',
'otherfile.ext': base64.b64encode('im a file')
}, id='config and another file'),
pytest.param({
'config.yaml': 'test:true',
'extra_ca_certs': [
('cert.crt', 'im a cert!'),
]
},
{
'config.yaml': 'dGVzdDp0cnVl',
'extra_ca_certs_cert.crt': base64.b64encode('im a cert!'),
}, id='config and an extra cert'),
pytest.param({
'config.yaml': 'test:true',
'otherfile.ext': 'im a file',
'extra_ca_certs': [
('cert.crt', 'im a cert!'),
('another.crt', 'im a different cert!'),
]
},
{
'config.yaml': 'dGVzdDp0cnVl',
'otherfile.ext': base64.b64encode('im a file'),
'extra_ca_certs_cert.crt': base64.b64encode('im a cert!'),
'extra_ca_certs_another.crt': base64.b64encode('im a different cert!'),
}, id='config, files, and extra certs!'),
])
def test_get_config_as_kube_secret(file_structure, expected_secret):
temp_dir = _create_temp_file_structure(file_structure)
secret = get_config_as_kube_secret(temp_dir.name)
assert secret == expected_secret
temp_dir.cleanup()

View file

@ -0,0 +1,68 @@
import pytest
import os
from config_app.config_util.config.TransientDirectoryProvider import TransientDirectoryProvider
@pytest.mark.parametrize('files_to_write, operations, expected_new_dir', [
pytest.param({
'config.yaml': 'a config',
}, ([], [], []), {
'config.yaml': 'a config',
}, id='just a config'),
pytest.param({
'config.yaml': 'a config',
'oldfile': 'hmmm'
}, ([], [], ['oldfile']), {
'config.yaml': 'a config',
}, id='delete a file'),
pytest.param({
'config.yaml': 'a config',
'oldfile': 'hmmm'
}, ([('newfile', 'asdf')], [], ['oldfile']), {
'config.yaml': 'a config',
'newfile': 'asdf'
}, id='delete and add a file'),
pytest.param({
'config.yaml': 'a config',
'somefile': 'before'
}, ([('newfile', 'asdf')], [('somefile', 'after')], []), {
'config.yaml': 'a config',
'newfile': 'asdf',
'somefile': 'after',
}, id='add new files and change files'),
])
def test_transient_dir_copy_config_dir(files_to_write, operations, expected_new_dir):
config_provider = TransientDirectoryProvider('', '', '')
for name, data in files_to_write.iteritems():
config_provider.write_volume_file(name, data)
config_provider.create_copy_of_config_dir()
for create in operations[0]:
(name, data) = create
config_provider.write_volume_file(name, data)
for update in operations[1]:
(name, data) = update
config_provider.write_volume_file(name, data)
for delete in operations[2]:
config_provider.remove_volume_file(delete)
# check that the new directory matches expected state
for filename, data in expected_new_dir.iteritems():
with open(os.path.join(config_provider.get_config_dir_path(), filename)) as f:
new_data = f.read()
assert new_data == data
# Now check that the old dir matches the original state
saved = config_provider.get_old_config_dir()
for filename, data in files_to_write.iteritems():
with open(os.path.join(saved, filename)) as f:
new_data = f.read()
assert new_data == data
config_provider.temp_dir.cleanup()

View file

@ -21,6 +21,9 @@ QE_CONTAINER_NAME = 'quay-enterprise-app'
# message is any string describing the state.
DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message'])
class K8sApiException(Exception):
pass
def _deployment_rollout_status_message(deployment, deployment_name):
"""
@ -225,11 +228,24 @@ class KubernetesAccessorSingleton(object):
}
}, api_prefix='apis/extensions/v1beta1', content_type='application/strategic-merge-patch+json'))
def _assert_success(self, response):
if response.status_code != 200:
def rollback_deployment(self, deployment_name):
deployment_rollback_url = 'namespaces/%s/deployments/%s/rollback' % (
self.kube_config.qe_namespace, deployment_name
)
self._assert_success(self._execute_k8s_api('POST', deployment_rollback_url, {
'name': deployment_name,
'rollbackTo': {
# revision=0 makes the deployment rollout to the previous revision
'revision': 0
}
}, api_prefix='apis/extensions/v1beta1'), 201)
def _assert_success(self, response, expected_code=200):
if response.status_code != expected_code:
logger.error('Kubernetes API call failed with response: %s => %s', response.status_code,
response.text)
raise Exception('Kubernetes API call failed: %s' % response.text)
raise K8sApiException('Kubernetes API call failed: %s' % response.text)
def _update_secret_file(self, relative_file_path, value=None):
if '/' in relative_file_path:

View file

@ -24,7 +24,9 @@ rules:
- "apps"
resources:
- deployments
- deployments/rollback
verbs:
- create
- get
- list
- patch

View file

@ -58,4 +58,4 @@
is-kubernetes="$ctrl.kubeNamespace !== false"
choose-deploy="$ctrl.chooseDeploy()">
</download-tarball-modal>
<kube-deploy-modal ng-if="$ctrl.state === 'deploy'"></kube-deploy-modal>
<kube-deploy-modal ng-if="$ctrl.state === 'deploy'" loaded-config="$ctrl.loadedConfig"></kube-deploy-modal>

View file

@ -47,6 +47,7 @@ export class ConfigSetupAppComponent {
this.apiService.scKubePopulateConfig()
.then(() => {
this.state = 'setup';
this.loadedConfig = true;
})
.catch(err => {
this.apiService.errorDisplay(

View file

@ -0,0 +1,5 @@
<div class="co-m-inline-loader co-an-fade-in-out">
<div class="co-m-loader-dot__one"></div>
<div class="co-m-loader-dot__two"></div>
<div class="co-m-loader-dot__three"></div>
</div>

View file

@ -1,9 +1,22 @@
const templateUrl = require('./cor-loader.html');
const loaderUrl = require('./cor-loader.html');
const inlineUrl = require('./cor-loader-inline.html');
angular.module('quay-config')
.directive('corLoader', function() {
var directiveDefinitionObject = {
templateUrl,
templateUrl: loaderUrl,
replace: true,
restrict: 'C',
scope: {
},
controller: function($rootScope, $scope, $element) {
}
};
return directiveDefinitionObject;
})
.directive('corLoaderInline', function() {
var directiveDefinitionObject = {
templateUrl: inlineUrl,
replace: true,
restrict: 'C',
scope: {

View file

@ -2,7 +2,7 @@
<div class="co-dialog modal fade initial-setup-modal in" id="setupModal" style="display: block;">
<div class="modal-backdrop fade in" style="height: 1000px;"></div>
<div class="modal-dialog fade in">
<div class="modal-content">
<div class="modal-content kube-deploy-modal">
<!-- Header -->
<div class="modal-header">
<span class="cor-step-bar">
@ -37,11 +37,13 @@
</div>
<div ng-if="$ctrl.state === 'cyclingDeployments'">
<div class="cor-loader"></div>
Cycling deployments...
<li class="kube-deploy-modal__list-item" ng-repeat="deployment in $ctrl.deploymentsStatus">
<i class="fa ci-k8s-logo"></i>
<code>{{deployment.name}}</code>: {{deployment.message || 'Waiting for deployment information...'}}
</li>
<span class="kube-deploy-modal__list-header">Cycling deployments...</span>
<ul class="kube-deploy-modal__list">
<li class="kube-deploy-modal__list-item" ng-repeat="deployment in $ctrl.deploymentsStatus">
<i class="fa ci-k8s-logo"></i>
<code>{{deployment.name}}</code>: {{deployment.message || 'Waiting for deployment information...'}}
</li>
</ul>
</div>
</div>
</div>
@ -50,12 +52,20 @@
<br>Note: The web interface of the Quay app may take a few minutes to come up.
</div>
<div ng-if="$ctrl.state === 'error'" class="modal-footer alert alert-danger">
{{ $ctrl.errorMessage }}
<div ng-if="$ctrl.offerRollback">
// todo
{{ $ctrl.errorMessage }}
<div ng-if="$ctrl.rollingBackStatus !== 'none'" class="rollback">
<button ng-if="$ctrl.rollingBackStatus === 'offer'" class="btn btn-default" ng-click="$ctrl.rollbackDeployments()">
<i class="fas fa-history" style="margin-right: 10px;"></i>
Rollback deployments
</button>
<div ng-if="$ctrl.rollingBackStatus === 'rolling'" class="cor-loader-inline"></div>
</div>
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div>
<div ng-if="$ctrl.state === 'rolledBackWarning'" class="modal-footer co-alert co-alert-warning">
Successfully rolled back changes. Please try deploying again with your configuration later.
</div>
</div>
</div><!-- /.modal-content -->
</div><!-- /.modal-dialog -->
</div>
</div>

View file

@ -1,5 +1,5 @@
import {Component, Inject, OnDestroy } from 'ng-metadata/core';
import {AngularPollChannel, PollHandle} from "../../services/services.types";
import { Input, Component, Inject, OnDestroy } from 'ng-metadata/core';
import { AngularPollChannel, PollHandle } from "../../services/services.types";
const templateUrl = require('./kube-deploy-modal.component.html');
const styleUrl = require('./kube-deploy-modal.css');
@ -24,22 +24,25 @@ const DEPLOYMENT_POLL_SLEEPTIME = 5000; /* 5 seconds */
styleUrls: [ styleUrl ],
})
export class KubeDeployModalComponent implements OnDestroy {
@Input('<') public loadedConfig;
private state
: 'loadingDeployments'
| 'readyToDeploy'
| 'deployingConfiguration'
| 'cyclingDeployments'
| 'deployed'
| 'error';
| 'error'
| 'rolledBackWarning' = 'loadingDeployments';
private errorMessage: string;
private offerRollback: boolean;
private deploymentsStatus: DeploymentStatus[] = [];
private deploymentsCycled: number = 0;
private onDestroyListeners: Function[] = [];
private rollingBackStatus
: 'none'
| 'offer'
| 'rolling' = 'none';
constructor(@Inject('ApiService') private ApiService, @Inject('AngularPollChannel') private AngularPollChannel: AngularPollChannel) {
this.state = 'loadingDeployments';
ApiService.scGetNumDeployments().then(resp => {
this.deploymentsStatus = resp.items.map(dep => ({ name: dep.metadata.name, numPods: dep.spec.replicas }));
this.state = 'readyToDeploy';
@ -80,7 +83,7 @@ export class KubeDeployModalComponent implements OnDestroy {
watchDeployments(): void {
this.deploymentsStatus.forEach(deployment => {
const pollChannel = this.AngularPollChannel.create( {
const pollChannel = this.AngularPollChannel.create({
// Have to mock the scope object for the poll channel since we're calling into angular1 code
// We register the onDestroy function to be called later when this object is destroyed
'$on': (_, onDestruction) => { this.onDestroyListeners.push(onDestruction) }
@ -113,14 +116,38 @@ export class KubeDeployModalComponent implements OnDestroy {
continue_callback(false);
deployment.message = deploymentRollout.message;
this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`;
this.offerRollback = true;
// Only offer rollback if we loaded/populated a config. (Can't rollback an initial setup)
if (this.loadedConfig) {
this.rollingBackStatus = 'offer';
this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`;
}
}
}).catch(err => {
continue_callback(false);
this.state = 'error';
this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
this.offerRollback = true;
this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}\
Would you like to rollback the deployment to its previous state?`;
// Only offer rollback if we loaded/populated a config. (Can't rollback an initial setup)
if (this.loadedConfig) {
this.rollingBackStatus = 'offer';
this.errorMessage = `Could not get deployment information for: ${deployment}`;
}
});
}
}
rollbackDeployments(): void {
this.rollingBackStatus = 'rolling';
const deploymentNames: string[] = this.deploymentsStatus.map(dep => dep.name);
this.ApiService.scRollbackDeployments({ deploymentNames }).then(() => {
this.state = 'rolledBackWarning';
this.rollingBackStatus = 'none';
}).catch(err => {
this.rollingBackStatus = 'none';
this.state = 'error';
this.errorMessage = `Could not cycle the deployments back to their previous states. Please contact support: ${err.toString()}`;
})
}
}

View file

@ -29,3 +29,18 @@
margin-top: 20px;
margin-bottom: 10px;
}
.kube-deploy-modal .rollback {
margin-left: auto;
}
.kube-deploy-modal .co-alert.co-alert-warning {
padding: 25px;
display: flex;
margin-bottom: 0;
}
.kube-deploy-modal .co-alert.co-alert-warning:before {
position: static;
padding-right: 15px;
}

View file

@ -62,6 +62,10 @@
position: static;
}
.co-alert.co-alert-danger:after {
/* Ignore the exclamation mark, it also messes with spacing elements */
content: none;
}
/* Fixes the transition to font awesome 5 */
.quay-config-app .co-alert.co-alert-warning::before {