Add Rollout status to kube config tool

config_app/config_endpoints/api/kubeconfig.py

@@ -37,6 +37,13 @@ class SuperUserKubernetesDeployment(ApiResource):
     deployment_names = request.get_json()['deploymentNames']
     return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names)
 
+@resource('/v1/kubernetes/deployment/<deployment>/status')
+class QEDeploymentRolloutStatus(ApiResource):
+  @kubernetes_only
+  @nickname('scGetDeploymentRolloutStatus')
+  def get(self, deployment):
+    return KubernetesAccessorSingleton.get_instance().get_deployment_rollout_status(deployment)
+
 
 @resource('/v1/superuser/config/kubernetes')
 class SuperUserKubernetesConfiguration(ApiResource):

config_app/config_util/k8saccessor.py

@@ -96,6 +96,65 @@ class KubernetesAccessorSingleton(object):
 
     self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
 
+  def get_deployment_rollout_status(self, deployment_name):
+    """"
+    Returns the status of a rollout of a given deployment in the form:
+    {
+      'status': 'failed' | 'progressing' | 'available'
+      'message': <string>
+    }
+    """
+    deployment_selector_url = 'namespaces/%s/deployments/%s' % (
+      self.kube_config.qe_namespace, deployment_name
+    )
+
+    response = self._execute_k8s_api('GET', deployment_selector_url, api_prefix='apis/apps/v1')
+    if response.status_code != 200:
+      return None
+    deployment = json.loads(response.text)
+
+    # Logic for rollout status pulled from the `kubectl rollout status` command:
+    # https://github.com/kubernetes/kubernetes/blob/d9ba19c751709c8608e09a0537eea98973f3a796/pkg/kubectl/rollout_status.go#L62
+    if deployment['metadata']['generation'] <= deployment['status']['observedGeneration']:
+      for cond in deployment['status']['conditions']:
+        if cond['type'] == 'Progressing' and cond['reason'] == 'ProgressDeadlineExceeded':
+          return {
+            'status': 'failed',
+            'message': 'Deployment %s\'s rollout failed. Please try again later.' % deployment_name
+          }
+
+      desired_replicas = deployment['spec']['replicas']
+      current_replicas = deployment['status']['replicas']
+      # Some fields are optional in the spec, so if they're omitted, replace with defaults that won't indicate a wrong status
+      available_replicas = deployment['status'].get('availableReplicas', 0)
+      updated_replicas = deployment['status'].get('updatedReplicas', 0)
+
+      if updated_replicas < desired_replicas:
+        return {
+          'status': 'progressing',
+          'message': 'Waiting for rollout to finish: %d out of %d new replicas have been updated...' % (updated_replicas, desired_replicas)
+        }
+      if current_replicas > updated_replicas:
+        return {
+          'status': 'progressing',
+          'message': 'Waiting for rollout to finish: %d old replicas are pending termination...' % (current_replicas - updated_replicas)
+        }
+      if available_replicas < updated_replicas:
+        return {
+          'status': 'progressing',
+          'message': 'Waiting for rollout to finish: %d of %d updated replicas are available...' % (available_replicas, updated_replicas)
+        }
+
+      return {
+        'status': 'available',
+        'message': 'Deployment %s successfully rolled out.' % deployment_name
+      }
+
+    return {
+      'status': 'progressing',
+      'message': 'Waiting for deployment spec to be updated...'
+    }
+
   def get_qe_deployments(self):
     """"
     Returns all deployments matching the label selector provided in the KubeConfig
@@ -126,6 +185,8 @@ class KubernetesAccessorSingleton(object):
           'template': {
             'spec': {
               'containers': [{
+                # Note: this name MUST match the deployment template's pod template
+                # (e.g. <template>.spec.template.spec.containers[0] == 'quay-enterprise-app')
                 'name': 'quay-enterprise-app', 'env': [{
                   'name': 'RESTART_TIME',
                   'value': str(datetime.datetime.now())

config_app/docs/k8s_templates/config-tool-servicetoken-role.yml

@@ -25,5 +25,6 @@ rules:
   resources:
   - deployments
   verbs:
+  - get
   - list
   - patch

config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html

@@ -38,6 +38,10 @@
                         <div ng-if="$ctrl.state === 'cyclingDeployments'">
                             <div class="cor-loader"></div>
                             Cycling deployments...
+                            <li class="kube-deploy-modal__list-item" ng-repeat="deployment in $ctrl.deploymentsStatus">
+                                <i class="fa ci-k8s-logo"></i>
+                                <code>{{deployment.name}}</code>: {{deployment.message || 'Waiting for deployment information...'}}
+                            </li>
                         </div>
                         <div ng-if="$ctrl.state === 'deployed'">
                             Configuration successfully deployed!

config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts

@@ -1,7 +1,13 @@
-import {Component, Inject} from 'ng-metadata/core';
+import {Component, EventEmitter, Inject} from 'ng-metadata/core';
 const templateUrl = require('./kube-deploy-modal.component.html');
 const styleUrl = require('./kube-deploy-modal.css');
 
+// The response from the API about deployment rollout status
+type DeploymentRollout = {
+    status: 'available' | 'progressing' | 'failed',
+    message: string
+};
+
 @Component({
     selector: 'kube-deploy-modal',
     templateUrl,
@@ -18,7 +24,8 @@ export class KubeDeployModalComponent {
 
     private errorMessage: string;
 
-    private deploymentsStatus: { name: string, numPods: number }[] = [];
+    private deploymentsStatus: { name: string, numPods: number, message?: string }[] = [];
+    private deploymentsCycled: number = 0;
 
     constructor(@Inject('ApiService') private ApiService) {
         this.state = 'loadingDeployments';
@@ -37,17 +44,52 @@ export class KubeDeployModalComponent {
 
     deployConfiguration(): void {
         this.ApiService.scDeployConfiguration().then(() => {
+            this.state = 'deployingConfiguration';
             const deploymentNames: string[] = this.deploymentsStatus.map(dep => dep.name);
 
             this.ApiService.scCycleQEDeployments({ deploymentNames }).then(() => {
-                this.state = 'deployed'
+                this.state = 'cyclingDeployments';
+                this.watchDeployments();
             }).catch(err => {
                 this.state = 'error';
-                this.errorMessage = `Could cycle the deployments with the new configuration. Error: ${err.toString()}`;
+                this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
             })
         }).catch(err => {
             this.state = 'error';
             this.errorMessage = `Could not deploy the configuration. Error: ${err.toString()}`;
         })
     }
+
+    watchDeployments(): void {
+        this.deploymentsStatus.forEach(deployment => {
+            // Query each deployment every 500ms, and stop polling once it's either available or failed
+            const id: number = window.setInterval(() => {
+                const params = {
+                    'deployment': deployment.name
+                };
+
+                this.ApiService.scGetDeploymentRolloutStatus(null, params).then((deploymentRollout: DeploymentRollout) => {
+                    if (deploymentRollout.status === 'available') {
+                        window.clearInterval(id);
+
+                        this.deploymentsCycled++;
+                        if (this.deploymentsCycled === this.deploymentsStatus.length) {
+                            this.state = 'deployed';
+                        }
+                    } else if (deploymentRollout.status === 'progressing') {
+                        deployment.message = deploymentRollout.message;
+                    } else { // deployment rollout failed
+                        window.clearInterval(id);
+
+                        deployment.message = deploymentRollout.message;
+                    }
+                }).catch(err => {
+                    window.clearInterval(id);
+                    this.state = 'error';
+                    this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
+                });
+            }, 500);
+        });
+
+    }
 }