Merge pull request #3218 from quay/project/pod-status

Add Rollout status to kube config tool

config_app/config_endpoints/api/kubeconfig.py

@@ -37,6 +37,17 @@ class SuperUserKubernetesDeployment(ApiResource):
     deployment_names = request.get_json()['deploymentNames']
     return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names)
 
+@resource('/v1/kubernetes/deployment/<deployment>/status')
+class QEDeploymentRolloutStatus(ApiResource):
+  @kubernetes_only
+  @nickname('scGetDeploymentRolloutStatus')
+  def get(self, deployment):
+    deployment_rollout_status = KubernetesAccessorSingleton.get_instance().get_deployment_rollout_status(deployment)
+    return {
+      'status': deployment_rollout_status.status,
+      'message': deployment_rollout_status.message,
+    }
+
 
 @resource('/v1/superuser/config/kubernetes')
 class SuperUserKubernetesConfiguration(ApiResource):

config_app/config_util/k8saccessor.py

@@ -5,6 +5,7 @@ import datetime
 import os
 
 from requests import Request, Session
+from collections import namedtuple
 from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
 
 from config_app.config_util.k8sconfig import KubernetesConfig
@@ -12,6 +13,74 @@ from config_app.config_util.k8sconfig import KubernetesConfig
 logger = logging.getLogger(__name__)
 
 QE_DEPLOYMENT_LABEL = 'quay-enterprise-component'
+QE_CONTAINER_NAME = 'quay-enterprise-app'
+
+
+# Tuple containing response of the deployment rollout status method.
+# status is one of: 'failed' | 'progressing' | 'available'
+# message is any string describing the state.
+DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message'])
+
+
+def _deployment_rollout_status_message(deployment, deployment_name):
+  """
+  Gets the friendly human readable message of the current state of the deployment rollout
+  :param deployment: python dict matching: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#deployment-v1-apps
+  :param deployment_name: string
+  :return: DeploymentRolloutStatus
+  """
+  # Logic for rollout status pulled from the `kubectl rollout status` command:
+  # https://github.com/kubernetes/kubernetes/blob/d9ba19c751709c8608e09a0537eea98973f3a796/pkg/kubectl/rollout_status.go#L62
+  if deployment['metadata']['generation'] <= deployment['status']['observedGeneration']:
+    for cond in deployment['status']['conditions']:
+      if cond['type'] == 'Progressing' and cond['reason'] == 'ProgressDeadlineExceeded':
+        return DeploymentRolloutStatus(
+          status='failed',
+          message="Deployment %s's rollout failed. Please try again later." % deployment_name
+        )
+
+    desired_replicas = deployment['spec']['replicas']
+    current_replicas = deployment['status'].get('replicas', 0)
+    if current_replicas == 0:
+      return DeploymentRolloutStatus(
+        status='available',
+        message='Deployment %s updated (no replicas, so nothing to roll out)' % deployment_name
+      )
+
+    # Some fields are optional in the spec, so if they're omitted, replace with defaults that won't indicate a wrong status
+    available_replicas = deployment['status'].get('availableReplicas', 0)
+    updated_replicas = deployment['status'].get('updatedReplicas', 0)
+
+    if updated_replicas < desired_replicas:
+      return DeploymentRolloutStatus(
+        status='progressing',
+        message='Waiting for rollout to finish: %d out of %d new replicas have been updated...' % (
+        updated_replicas, desired_replicas)
+      )
+
+    if current_replicas > updated_replicas:
+      return DeploymentRolloutStatus(
+        status='progressing',
+        message='Waiting for rollout to finish: %d old replicas are pending termination...' % (
+              current_replicas - updated_replicas)
+      )
+
+    if available_replicas < updated_replicas:
+      return DeploymentRolloutStatus(
+        status='progressing',
+        message='Waiting for rollout to finish: %d of %d updated replicas are available...' % (
+        available_replicas, updated_replicas)
+      )
+
+    return DeploymentRolloutStatus(
+      status='available',
+      message='Deployment %s successfully rolled out.' % deployment_name
+    )
+
+  return DeploymentRolloutStatus(
+    status='progressing',
+    message='Waiting for deployment spec to be updated...'
+  )
 
 
 class KubernetesAccessorSingleton(object):
@@ -96,6 +165,23 @@ class KubernetesAccessorSingleton(object):
 
     self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
 
+  def get_deployment_rollout_status(self, deployment_name):
+    """"
+    Returns the status of a rollout of a given deployment
+    :return _DeploymentRolloutStatus
+    """
+    deployment_selector_url = 'namespaces/%s/deployments/%s' % (
+      self.kube_config.qe_namespace, deployment_name
+    )
+
+    response = self._execute_k8s_api('GET', deployment_selector_url, api_prefix='apis/apps/v1')
+    if response.status_code != 200:
+      return DeploymentRolloutStatus('failed', 'Could not get deployment. Please check that the deployment exists')
+
+    deployment = json.loads(response.text)
+
+    return _deployment_rollout_status_message(deployment, deployment_name)
+
   def get_qe_deployments(self):
     """"
     Returns all deployments matching the label selector provided in the KubeConfig
@@ -126,10 +212,13 @@ class KubernetesAccessorSingleton(object):
           'template': {
             'spec': {
               'containers': [{
-                'name': 'quay-enterprise-app', 'env': [{
+                # Note: this name MUST match the deployment template's pod template
+                # (e.g. <template>.spec.template.spec.containers[0] == 'quay-enterprise-app')
+                'name': QE_CONTAINER_NAME,
+                'env': [{
                   'name': 'RESTART_TIME',
                   'value': str(datetime.datetime.now())
-                }]
+                }],
               }]
             }
           }

config_app/config_util/test/test_k8saccessor.py

@@ -2,10 +2,58 @@ import pytest
 
 from httmock import urlmatch, HTTMock, response
 
-from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
+from config_app.config_util.k8saccessor import KubernetesAccessorSingleton, _deployment_rollout_status_message
 from config_app.config_util.k8sconfig import KubernetesConfig
 
 
+@pytest.mark.parametrize('deployment_object, expected_status, expected_message', [
+  ({'metadata': {'generation': 1},
+    'status': {'observedGeneration': 0, 'conditions': []},
+    'spec': {'replicas': 0}},
+   'progressing',
+   'Waiting for deployment spec to be updated...'),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': [{'type': 'Progressing', 'reason': 'ProgressDeadlineExceeded'}]},
+    'spec': {'replicas': 0}},
+   'failed',
+   "Deployment my-deployment's rollout failed. Please try again later."),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': []},
+    'spec': {'replicas': 0}},
+   'available',
+   'Deployment my-deployment updated (no replicas, so nothing to roll out)'),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1},
+    'spec': {'replicas': 2}},
+   'progressing',
+   'Waiting for rollout to finish: 0 out of 2 new replicas have been updated...'),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 1},
+    'spec': {'replicas': 2}},
+   'progressing',
+   'Waiting for rollout to finish: 1 out of 2 new replicas have been updated...'),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 2, 'updatedReplicas': 1},
+    'spec': {'replicas': 1}},
+   'progressing',
+   'Waiting for rollout to finish: 1 old replicas are pending termination...'),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 2, 'availableReplicas': 0},
+    'spec': {'replicas': 0}},
+   'progressing',
+   'Waiting for rollout to finish: 0 of 2 updated replicas are available...'),
+  ({'metadata': {'generation': 0},
+    'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 2, 'availableReplicas': 2},
+    'spec': {'replicas': 0}},
+   'available',
+   'Deployment my-deployment successfully rolled out.'),
+])
+def test_deployment_rollout_status_message(deployment_object, expected_status, expected_message):
+  deployment_status = _deployment_rollout_status_message(deployment_object, 'my-deployment')
+  assert deployment_status.status == expected_status
+  assert deployment_status.message == expected_message
+
+
 @pytest.mark.parametrize('kube_config, expected_api, expected_query', [
   ({'api_host': 'www.customhost.com'},
    '/apis/extensions/v1beta1/namespaces/quay-enterprise/deployments', 'labelSelector=quay-enterprise-component%3Dapp'),

config_app/docs/k8s_templates/config-tool-servicetoken-role.yml

@@ -25,5 +25,6 @@ rules:
   resources:
   - deployments
   verbs:
+  - get
   - list
   - patch

config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.html

@@ -38,13 +38,21 @@
                         <div ng-if="$ctrl.state === 'cyclingDeployments'">
                             <div class="cor-loader"></div>
                             Cycling deployments...
-                        </div>
-                        <div ng-if="$ctrl.state === 'deployed'">
-                            Configuration successfully deployed!
+                            <li class="kube-deploy-modal__list-item" ng-repeat="deployment in $ctrl.deploymentsStatus">
+                                <i class="fa ci-k8s-logo"></i>
+                                <code>{{deployment.name}}</code>: {{deployment.message || 'Waiting for deployment information...'}}
+                            </li>
                         </div>
                     </div>
-                    <div ng-if="$ctrl.state === 'error'">
+                </div>
+                <div ng-if="$ctrl.state === 'deployed'" class="modal-footer co-alert co-alert-success">
+                    Configuration successfully rolled out and deployed!
+                    <br>Note: The web interface of the Quay app may take a few minutes to come up.
+                </div>
+                <div ng-if="$ctrl.state === 'error'" class="modal-footer alert alert-danger">
                         {{ $ctrl.errorMessage }}
+                    <div ng-if="$ctrl.offerRollback">
+                        // todo
                     </div>
                 </div>
             </div><!-- /.modal-content -->

config_app/js/components/kube-deploy-modal/kube-deploy-modal.component.ts

@@ -1,13 +1,29 @@
-import {Component, Inject} from 'ng-metadata/core';
+import {Component, Inject, OnDestroy } from 'ng-metadata/core';
+import {AngularPollChannel, PollHandle} from "../../services/services.types";
 const templateUrl = require('./kube-deploy-modal.component.html');
 const styleUrl = require('./kube-deploy-modal.css');
 
+// The response from the API about deployment rollout status
+type DeploymentRollout = {
+    status: 'available' | 'progressing' | 'failed',
+    message: string
+};
+
+type DeploymentStatus = {
+    name: string,
+    numPods: number,
+    message?: string,
+    pollHandler?: PollHandle,
+}
+
+const DEPLOYMENT_POLL_SLEEPTIME = 5000; /* 5 seconds */
+
 @Component({
     selector: 'kube-deploy-modal',
     templateUrl,
     styleUrls: [ styleUrl ],
 })
-export class KubeDeployModalComponent {
+export class KubeDeployModalComponent implements OnDestroy {
     private state
         : 'loadingDeployments'
         | 'readyToDeploy'
@@ -15,12 +31,13 @@ export class KubeDeployModalComponent {
         | 'cyclingDeployments'
         | 'deployed'
         | 'error';
-
     private errorMessage: string;
+    private offerRollback: boolean;
+    private deploymentsStatus: DeploymentStatus[] = [];
+    private deploymentsCycled: number = 0;
+    private onDestroyListeners: Function[] = [];
 
-    private deploymentsStatus: { name: string, numPods: number }[] = [];
-
-    constructor(@Inject('ApiService') private ApiService) {
+    constructor(@Inject('ApiService') private ApiService, @Inject('AngularPollChannel') private AngularPollChannel: AngularPollChannel) {
         this.state = 'loadingDeployments';
 
         ApiService.scGetNumDeployments().then(resp => {
@@ -35,19 +52,75 @@ export class KubeDeployModalComponent {
         })
     }
 
+    // Call all listeners of the onDestroy
+    ngOnDestroy(): any {
+        this.onDestroyListeners.forEach(fn => {
+            fn()
+        });
+    }
+
+
     deployConfiguration(): void {
         this.ApiService.scDeployConfiguration().then(() => {
+            this.state = 'deployingConfiguration';
             const deploymentNames: string[] = this.deploymentsStatus.map(dep => dep.name);
 
             this.ApiService.scCycleQEDeployments({ deploymentNames }).then(() => {
-                this.state = 'deployed'
+                this.state = 'cyclingDeployments';
+                this.watchDeployments();
             }).catch(err => {
                 this.state = 'error';
-                this.errorMessage = `Could cycle the deployments with the new configuration. Error: ${err.toString()}`;
+                this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
             })
         }).catch(err => {
             this.state = 'error';
             this.errorMessage = `Could not deploy the configuration. Error: ${err.toString()}`;
         })
     }
+
+    watchDeployments(): void {
+        this.deploymentsStatus.forEach(deployment => {
+            const pollChannel = this.AngularPollChannel.create( {
+                // Have to mock the scope object for the poll channel since we're calling into angular1 code
+                // We register the onDestroy function to be called later when this object is destroyed
+                '$on': (_, onDestruction) => { this.onDestroyListeners.push(onDestruction) }
+            }, this.getDeploymentStatus(deployment), DEPLOYMENT_POLL_SLEEPTIME);
+
+            pollChannel.start();
+        });
+    }
+
+    // Query each deployment every 5s, and stop polling once it's either available or failed
+    getDeploymentStatus(deployment: DeploymentStatus): (boolean) => void {
+        return (continue_callback: (shouldContinue: boolean) => void) => {
+            const params = {
+                'deployment': deployment.name
+            };
+
+            this.ApiService.scGetDeploymentRolloutStatus(null, params).then((deploymentRollout: DeploymentRollout) => {
+                if (deploymentRollout.status === 'available') {
+                    continue_callback(false);
+
+                    this.deploymentsCycled++;
+                    if (this.deploymentsCycled === this.deploymentsStatus.length) {
+                        this.state = 'deployed';
+                    }
+                } else if (deploymentRollout.status === 'progressing') {
+                    continue_callback(true);
+                    deployment.message = deploymentRollout.message;
+                } else { // deployment rollout failed
+                    this.state = 'error';
+                    continue_callback(false);
+                    deployment.message = deploymentRollout.message;
+                    this.errorMessage = `Could not cycle deployments: ${deploymentRollout.message}`;
+                    this.offerRollback = true;
+                }
+            }).catch(err => {
+                continue_callback(false);
+                this.state = 'error';
+                this.errorMessage = `Could not cycle the deployments with the new configuration. Error: ${err.toString()}`;
+                this.offerRollback = true;
+            });
+        }
+    }
 }

config_app/js/services/angular-poll-channel.js

@@ -0,0 +1,107 @@
+/**
+ * Specialized class for conducting an HTTP poll, while properly preventing multiple calls.
+ */
+angular.module('quay-config').factory('AngularPollChannel',
+    ['ApiService', '$timeout', 'DocumentVisibilityService', 'CORE_EVENT', '$rootScope',
+    function(ApiService, $timeout, DocumentVisibilityService, CORE_EVENT, $rootScope) {
+  var _PollChannel = function(scope, requester, opt_sleeptime) {
+    this.scope_ = scope;
+    this.requester_ = requester;
+    this.sleeptime_ = opt_sleeptime || (60 * 1000 /* 60s */);
+    this.timer_ = null;
+
+    this.working = false;
+    this.polling = false;
+    this.skipping = false;
+
+    var that = this;
+
+    var visibilityHandler = $rootScope.$on(CORE_EVENT.DOC_VISIBILITY_CHANGE, function() {
+      // If the poll channel was skipping because the visibility was hidden, call it immediately.
+      if (that.skipping && !DocumentVisibilityService.isHidden()) {
+        that.call_();
+      }
+    });
+
+    scope.$on('$destroy', function() {
+      that.stop();
+      visibilityHandler();
+    });
+  };
+
+  _PollChannel.prototype.setSleepTime = function(sleepTime) {
+    this.sleeptime_ = sleepTime;
+    this.stop();
+    this.start(true);
+  };
+
+  _PollChannel.prototype.stop = function() {
+    if (this.timer_) {
+      $timeout.cancel(this.timer_);
+      this.timer_ = null;
+      this.polling = false;
+    }
+
+    this.skipping = false;
+    this.working = false;
+  };
+
+  _PollChannel.prototype.start = function(opt_skipFirstCall) {
+    // Make sure we invoke call outside the normal digest cycle, since
+    // we'll call $scope.$apply ourselves.
+    var that = this;
+    setTimeout(function() {
+      if (opt_skipFirstCall) {
+        that.setupTimer_();
+        return;
+      }
+
+      that.call_();
+    }, 0);
+  };
+
+  _PollChannel.prototype.call_ = function() {
+    if (this.working) { return; }
+
+    // If the document is currently hidden, skip the call.
+    if (DocumentVisibilityService.isHidden()) {
+      this.skipping = true;
+      this.setupTimer_();
+      return;
+    }
+
+    var that = this;
+    this.working = true;
+
+    $timeout(function() {
+      that.requester_(function(status) {
+        if (status) {
+          that.working = false;
+          that.skipping = false;
+          that.setupTimer_();
+        } else {
+          that.stop();
+        }
+      });
+    }, 0);
+  };
+
+  _PollChannel.prototype.setupTimer_ = function() {
+    if (this.timer_) { return; }
+
+    var that = this;
+    this.polling = true;
+    this.timer_ = $timeout(function() {
+      that.timer_ = null;
+      that.call_();
+    }, this.sleeptime_)
+  };
+
+  var service = {
+    'create': function(scope, requester, opt_sleeptime) {
+      return new _PollChannel(scope, requester, opt_sleeptime);
+    }
+  };
+
+  return service;
+}]);

config_app/js/services/document-visibility-service.js

@@ -0,0 +1,60 @@
+/**
+ * Helper service which fires off events when the document's visibility changes, as well as allowing
+ * other Angular code to query the state of the document's visibility directly.
+ */
+angular.module('quay-config').constant('CORE_EVENT', {
+  DOC_VISIBILITY_CHANGE: 'core.event.doc_visibility_change'
+});
+
+angular.module('quay-config').factory('DocumentVisibilityService', ['$rootScope', '$document', 'CORE_EVENT',
+    function($rootScope, $document, CORE_EVENT) {
+  var document = $document[0],
+  features,
+  detectedFeature;
+
+  function broadcastChangeEvent() {
+    $rootScope.$broadcast(CORE_EVENT.DOC_VISIBILITY_CHANGE,
+                          document[detectedFeature.propertyName]);
+  }
+
+  features = {
+    standard: {
+      eventName: 'visibilitychange',
+      propertyName: 'hidden'
+    },
+    moz: {
+      eventName: 'mozvisibilitychange',
+      propertyName: 'mozHidden'
+    },
+    ms: {
+      eventName: 'msvisibilitychange',
+      propertyName: 'msHidden'
+    },
+    webkit: {
+      eventName: 'webkitvisibilitychange',
+      propertyName: 'webkitHidden'
+    }
+  };
+
+  Object.keys(features).some(function(feature) {
+    if (document[features[feature].propertyName] !== undefined) {
+      detectedFeature = features[feature];
+      return true;
+    }
+  });
+
+  if (detectedFeature) {
+    $document.on(detectedFeature.eventName, broadcastChangeEvent);
+  }
+
+  return {
+    /**
+     * Is the window currently hidden or not.
+     */
+    isHidden: function() {
+      if (detectedFeature) {
+        return document[detectedFeature.propertyName];
+      }
+    }
+  };
+}]);

config_app/js/services/services.types.ts

@@ -0,0 +1,15 @@
+export interface AngularPollChannel {
+    create: PollConstructor
+}
+
+type PollConstructor = (scope: MockAngularScope, requester: ShouldContinueCallback, opt_sleeptime?: number) => PollHandle;
+type MockAngularScope = {
+    '$on': Function
+};
+type ShouldContinueCallback = (boolean) => void;
+
+export interface PollHandle {
+    start(opt_skipFirstCall?: boolean): void,
+    stop(): void,
+    setSleepTime(sleepTime: number): void,
+}

config_app/static/css/config-setup-app-component.css

@@ -28,6 +28,8 @@
     text-decoration: none;
 }
 
+/* Overrides for fixing old quay styles*/
+
 .quay-config-app .alert-danger {
     padding: 25px;
     display: flex;
@@ -43,7 +45,22 @@
     text-align: center;
 }
 
-/* Overrides for fixing old quay styles*/
+.quay-config-app .co-alert.co-alert-success {
+    padding: 25px;
+    display: flex;
+    margin-bottom: 0;
+    text-align: left;
+}
+
+.quay-config-app .co-alert.co-alert-success:before {
+    font-family: Font Awesome\ 5 Free;
+    font-weight: 900;
+    font-size: 30px;
+    padding-right: 15px;
+    color: green;
+    text-align: center;
+    position: static;
+}
 
 
 /* Fixes the transition to font awesome 5 */