vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Extract out deployment rollout and add tests

Browse source
This commit is contained in:
Sam Chow 2018-08-28 13:44:58 -04:00
parent 128cf0a28d
commit a6ffe49cba
5 changed files with 140 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
config_app/config_endpoints/api/kubeconfig.py
View file

@ -42,7 +42,11 @@ class QEDeploymentRolloutStatus(ApiResource):
@kubernetes_only
@nickname('scGetDeploymentRolloutStatus')
def get(self, deployment):
return KubernetesAccessorSingleton.get_instance().get_deployment_rollout_status(deployment).to_dict()
deployment_rollout_status = KubernetesAccessorSingleton.get_instance().get_deployment_rollout_status(deployment)
return {
'status': deployment_rollout_status.status,
'message': deployment_rollout_status.message,
}
@resource('/v1/superuser/config/kubernetes')

133
config_app/config_util/k8saccessor.py
View file

@ -5,6 +5,7 @@ import datetime
import os
from requests import Request, Session
from collections import namedtuple
from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
from config_app.config_util.k8sconfig import KubernetesConfig
@ -12,22 +13,74 @@ from config_app.config_util.k8sconfig import KubernetesConfig
logger = logging.getLogger(__name__)
QE_DEPLOYMENT_LABEL = 'quay-enterprise-component'
QE_CONTAINER_NAME = 'quay-enterprise-app'
class _DeploymentRolloutStatus:
"""
Class containing response of the deployment rollout status method.
status is one of: 'failed' | 'progressing' | 'available'
message is any string describing the state.
"""
def __init__(self, status, message):
self.status = status
self.message = message
def to_dict(self):
return {
'status': self.status,
'message': self.message
}
# Tuple containing response of the deployment rollout status method.
# status is one of: 'failed' | 'progressing' | 'available'
# message is any string describing the state.
DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message'])
def _deployment_rollout_status_message(deployment, deployment_name):
"""
Gets the friendly human readable message of the current state of the deployment rollout
:param deployment: python dict matching: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#deployment-v1-apps
:param deployment_name: string
:return: DeploymentRolloutStatus
"""
# Logic for rollout status pulled from the `kubectl rollout status` command:
# https://github.com/kubernetes/kubernetes/blob/d9ba19c751709c8608e09a0537eea98973f3a796/pkg/kubectl/rollout_status.go#L62
if deployment['metadata']['generation'] <= deployment['status']['observedGeneration']:
for cond in deployment['status']['conditions']:
if cond['type'] == 'Progressing' and cond['reason'] == 'ProgressDeadlineExceeded':
return DeploymentRolloutStatus(
status='failed',
message="Deployment %s's rollout failed. Please try again later." % deployment_name
)
desired_replicas = deployment['spec']['replicas']
current_replicas = deployment['status'].get('replicas', 0)
if current_replicas == 0:
return DeploymentRolloutStatus(
status='available',
message='Deployment %s updated (no replicas, so nothing to roll out)' % deployment_name
)
# Some fields are optional in the spec, so if they're omitted, replace with defaults that won't indicate a wrong status
available_replicas = deployment['status'].get('availableReplicas', 0)
updated_replicas = deployment['status'].get('updatedReplicas', 0)
if updated_replicas < desired_replicas:
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for rollout to finish: %d out of %d new replicas have been updated...' % (
updated_replicas, desired_replicas)
)
if current_replicas > updated_replicas:
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for rollout to finish: %d old replicas are pending termination...' % (
current_replicas - updated_replicas)
)
if available_replicas < updated_replicas:
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for rollout to finish: %d of %d updated replicas are available...' % (
available_replicas, updated_replicas)
)
return DeploymentRolloutStatus(
status='available',
message='Deployment %s successfully rolled out.' % deployment_name
)
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for deployment spec to be updated...'
)
class KubernetesAccessorSingleton(object):
@ -123,55 +176,11 @@ class KubernetesAccessorSingleton(object):
response = self._execute_k8s_api('GET', deployment_selector_url, api_prefix='apis/apps/v1')
if response.status_code != 200:
return _DeploymentRolloutStatus('failed', 'Could not get deployment. Please check that the deployment exists')
return DeploymentRolloutStatus('failed', 'Could not get deployment. Please check that the deployment exists')
deployment = json.loads(response.text)
# Logic for rollout status pulled from the `kubectl rollout status` command:
# https://github.com/kubernetes/kubernetes/blob/d9ba19c751709c8608e09a0537eea98973f3a796/pkg/kubectl/rollout_status.go#L62
if deployment['metadata']['generation'] <= deployment['status']['observedGeneration']:
for cond in deployment['status']['conditions']:
if cond['type'] == 'Progressing' and cond['reason'] == 'ProgressDeadlineExceeded':
return _DeploymentRolloutStatus(
'failed',
'Deployment %s\'s rollout failed. Please try again later.' % deployment_name
)
desired_replicas = deployment['spec']['replicas']
current_replicas = deployment['status'].get('replicas', 0)
if current_replicas == 0:
return _DeploymentRolloutStatus(
'available',
'Deployment %s updated (no replicas, so nothing to roll out)'
)
# Some fields are optional in the spec, so if they're omitted, replace with defaults that won't indicate a wrong status
available_replicas = deployment['status'].get('availableReplicas', 0)
updated_replicas = deployment['status'].get('updatedReplicas', 0)
if updated_replicas < desired_replicas:
return _DeploymentRolloutStatus(
'progressing',
'Waiting for rollout to finish: %d out of %d new replicas have been updated...' % (updated_replicas, desired_replicas)
)
if current_replicas > updated_replicas:
return _DeploymentRolloutStatus(
'progressing',
'Waiting for rollout to finish: %d old replicas are pending termination...' % (current_replicas - updated_replicas)
)
if available_replicas < updated_replicas:
return _DeploymentRolloutStatus(
'progressing',
'Waiting for rollout to finish: %d of %d updated replicas are available...' % (available_replicas, updated_replicas)
)
return _DeploymentRolloutStatus(
'available',
'Deployment %s successfully rolled out.' % deployment_name
)
return _DeploymentRolloutStatus(
'progressing',
'Waiting for deployment spec to be updated...'
)
return _deployment_rollout_status_message(deployment, deployment_name)
def get_qe_deployments(self):
""""
@ -205,7 +214,7 @@ class KubernetesAccessorSingleton(object):
'containers': [{
# Note: this name MUST match the deployment template's pod template
# (e.g. <template>.spec.template.spec.containers[0] == 'quay-enterprise-app')
'name': 'quay-enterprise-app',
'name': QE_CONTAINER_NAME,
'env': [{
'name': 'RESTART_TIME',
'value': str(datetime.datetime.now())

50
config_app/config_util/test/test_k8saccessor.py
View file

@ -2,10 +2,58 @@ import pytest
from httmock import urlmatch, HTTMock, response
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton, _deployment_rollout_status_message
from config_app.config_util.k8sconfig import KubernetesConfig
@pytest.mark.parametrize('deployment_object, expected_status, expected_message', [
({'metadata': {'generation': 1},
'status': {'observedGeneration': 0, 'conditions': []},
'spec': {'replicas': 0}},
'progressing',
'Waiting for deployment spec to be updated...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [{'type': 'Progressing', 'reason': 'ProgressDeadlineExceeded'}]},
'spec': {'replicas': 0}},
'failed',
"Deployment my-deployment's rollout failed. Please try again later."),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': []},
'spec': {'replicas': 0}},
'available',
'Deployment my-deployment updated (no replicas, so nothing to roll out)'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1},
'spec': {'replicas': 2}},
'progressing',
'Waiting for rollout to finish: 0 out of 2 new replicas have been updated...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 1},
'spec': {'replicas': 2}},
'progressing',
'Waiting for rollout to finish: 1 out of 2 new replicas have been updated...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 2, 'updatedReplicas': 1},
'spec': {'replicas': 1}},
'progressing',
'Waiting for rollout to finish: 1 old replicas are pending termination...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 2, 'availableReplicas': 0},
'spec': {'replicas': 0}},
'progressing',
'Waiting for rollout to finish: 0 of 2 updated replicas are available...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 2, 'availableReplicas': 2},
'spec': {'replicas': 0}},
'available',
'Deployment my-deployment successfully rolled out.'),
])
def test_deployment_rollout_status_message(deployment_object, expected_status, expected_message):
deployment_status = _deployment_rollout_status_message(deployment_object, 'my-deployment')
assert deployment_status.status == expected_status
assert deployment_status.message == expected_message
@pytest.mark.parametrize('kube_config, expected_api, expected_query', [
({'api_host': 'www.customhost.com'},
'/apis/extensions/v1beta1/namespaces/quay-enterprise/deployments', 'labelSelector=quay-enterprise-component%3Dapp'),

12
config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts
View file

@ -1,4 +1,4 @@
import {Component, EventEmitter, Inject, OnDestroy } from 'ng-metadata/core';
import {Component, Inject, OnDestroy } from 'ng-metadata/core';
import {AngularPollChannel, PollHandle} from "../../services/services.types";
const templateUrl = require('./kube-deploy-modal.component.html');
const styleUrl = require('./kube-deploy-modal.css');
@ -16,6 +16,8 @@ type DeploymentStatus = {
pollHandler?: PollHandle,
}
const DEPLOYMENT_POLL_SLEEPTIME = 5000; /* 5 seconds */
@Component({
selector: 'kube-deploy-modal',
templateUrl,
@ -78,12 +80,13 @@ export class KubeDeployModalComponent implements OnDestroy {
watchDeployments(): void {
this.deploymentsStatus.forEach(deployment => {
this.AngularPollChannel.create( {
const pollChannel = this.AngularPollChannel.create( {
// Have to mock the scope object for the poll channel since we're calling into angular1 code
// We register the onDestroy function to be called later when this object is destroyed
'$on': (_, onDestruction) => { this.onDestroyListeners.push(onDestruction) }
}, this.getDeploymentStatus(deployment), 5000 /* 5 seconds */)
.start();
}, this.getDeploymentStatus(deployment), DEPLOYMENT_POLL_SLEEPTIME);
pollChannel.start();
});
}
@ -116,6 +119,7 @@ export class KubeDeployModalComponent implements OnDestroy {
continue_callback(false);
this.state = 'error';
this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
this.offerRollback = true;
});
}
}

11
config_app/js/services/services.types.ts
View file

@ -1,10 +1,13 @@
export interface AngularPollChannel {
create(scope: { '$on': Function },
requester: (boolean) => void, // function that receives a callback to continue or halt polling
opt_sleeptime?: number
): PollHandle,
create: PollConstructor
}
type PollConstructor = (scope: MockAngularScope, requester: ShouldContinueCallback, opt_sleeptime?: number) => PollHandle;
type MockAngularScope = {
'$on': Function
};
type ShouldContinueCallback = (boolean) => void;
export interface PollHandle {
start(opt_skipFirstCall?: boolean): void,
stop(): void,