diff --git a/config_app/config_endpoints/api/kubeconfig.py b/config_app/config_endpoints/api/kubeconfig.py index a2f9219c8..556e36504 100644 --- a/config_app/config_endpoints/api/kubeconfig.py +++ b/config_app/config_endpoints/api/kubeconfig.py @@ -37,6 +37,17 @@ class SuperUserKubernetesDeployment(ApiResource): deployment_names = request.get_json()['deploymentNames'] return KubernetesAccessorSingleton.get_instance().cycle_qe_deployments(deployment_names) +@resource('/v1/kubernetes/deployment//status') +class QEDeploymentRolloutStatus(ApiResource): + @kubernetes_only + @nickname('scGetDeploymentRolloutStatus') + def get(self, deployment): + deployment_rollout_status = KubernetesAccessorSingleton.get_instance().get_deployment_rollout_status(deployment) + return { + 'status': deployment_rollout_status.status, + 'message': deployment_rollout_status.message, + } + @resource('/v1/superuser/config/kubernetes') class SuperUserKubernetesConfiguration(ApiResource): diff --git a/config_app/config_util/k8saccessor.py b/config_app/config_util/k8saccessor.py index dd7f596e2..90597adfd 100644 --- a/config_app/config_util/k8saccessor.py +++ b/config_app/config_util/k8saccessor.py @@ -5,6 +5,7 @@ import datetime import os from requests import Request, Session +from collections import namedtuple from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX from config_app.config_util.k8sconfig import KubernetesConfig @@ -12,6 +13,74 @@ from config_app.config_util.k8sconfig import KubernetesConfig logger = logging.getLogger(__name__) QE_DEPLOYMENT_LABEL = 'quay-enterprise-component' +QE_CONTAINER_NAME = 'quay-enterprise-app' + + +# Tuple containing response of the deployment rollout status method. +# status is one of: 'failed' | 'progressing' | 'available' +# message is any string describing the state. +DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message']) + + +def _deployment_rollout_status_message(deployment, deployment_name): + """ + Gets the friendly human readable message of the current state of the deployment rollout + :param deployment: python dict matching: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#deployment-v1-apps + :param deployment_name: string + :return: DeploymentRolloutStatus + """ + # Logic for rollout status pulled from the `kubectl rollout status` command: + # https://github.com/kubernetes/kubernetes/blob/d9ba19c751709c8608e09a0537eea98973f3a796/pkg/kubectl/rollout_status.go#L62 + if deployment['metadata']['generation'] <= deployment['status']['observedGeneration']: + for cond in deployment['status']['conditions']: + if cond['type'] == 'Progressing' and cond['reason'] == 'ProgressDeadlineExceeded': + return DeploymentRolloutStatus( + status='failed', + message="Deployment %s's rollout failed. Please try again later." % deployment_name + ) + + desired_replicas = deployment['spec']['replicas'] + current_replicas = deployment['status'].get('replicas', 0) + if current_replicas == 0: + return DeploymentRolloutStatus( + status='available', + message='Deployment %s updated (no replicas, so nothing to roll out)' % deployment_name + ) + + # Some fields are optional in the spec, so if they're omitted, replace with defaults that won't indicate a wrong status + available_replicas = deployment['status'].get('availableReplicas', 0) + updated_replicas = deployment['status'].get('updatedReplicas', 0) + + if updated_replicas < desired_replicas: + return DeploymentRolloutStatus( + status='progressing', + message='Waiting for rollout to finish: %d out of %d new replicas have been updated...' % ( + updated_replicas, desired_replicas) + ) + + if current_replicas > updated_replicas: + return DeploymentRolloutStatus( + status='progressing', + message='Waiting for rollout to finish: %d old replicas are pending termination...' % ( + current_replicas - updated_replicas) + ) + + if available_replicas < updated_replicas: + return DeploymentRolloutStatus( + status='progressing', + message='Waiting for rollout to finish: %d of %d updated replicas are available...' % ( + available_replicas, updated_replicas) + ) + + return DeploymentRolloutStatus( + status='available', + message='Deployment %s successfully rolled out.' % deployment_name + ) + + return DeploymentRolloutStatus( + status='progressing', + message='Waiting for deployment spec to be updated...' + ) class KubernetesAccessorSingleton(object): @@ -96,6 +165,23 @@ class KubernetesAccessorSingleton(object): self._assert_success(self._execute_k8s_api('PUT', secret_url, secret)) + def get_deployment_rollout_status(self, deployment_name): + """" + Returns the status of a rollout of a given deployment + :return _DeploymentRolloutStatus + """ + deployment_selector_url = 'namespaces/%s/deployments/%s' % ( + self.kube_config.qe_namespace, deployment_name + ) + + response = self._execute_k8s_api('GET', deployment_selector_url, api_prefix='apis/apps/v1') + if response.status_code != 200: + return DeploymentRolloutStatus('failed', 'Could not get deployment. Please check that the deployment exists') + + deployment = json.loads(response.text) + + return _deployment_rollout_status_message(deployment, deployment_name) + def get_qe_deployments(self): """" Returns all deployments matching the label selector provided in the KubeConfig @@ -126,10 +212,13 @@ class KubernetesAccessorSingleton(object): 'template': { 'spec': { 'containers': [{ - 'name': 'quay-enterprise-app', 'env': [{ + # Note: this name MUST match the deployment template's pod template + # (e.g.