vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3218 from quay/project/pod-status

Browse source 
Add Rollout status to kube config tool
This commit is contained in:
Sam Chow 2018-08-29 12:42:33 -04:00 • committed by GitHub
commit 2b59432414
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 445 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

93
config_app/config_util/k8saccessor.py
View file

@ -5,6 +5,7 @@ import datetime
import os
from requests import Request, Session
from collections import namedtuple
from util.config.validator import EXTRA_CA_DIRECTORY, EXTRA_CA_DIRECTORY_PREFIX
from config_app.config_util.k8sconfig import KubernetesConfig
@ -12,6 +13,74 @@ from config_app.config_util.k8sconfig import KubernetesConfig
logger = logging.getLogger(__name__)
QE_DEPLOYMENT_LABEL = 'quay-enterprise-component'
QE_CONTAINER_NAME = 'quay-enterprise-app'
# Tuple containing response of the deployment rollout status method.
# status is one of: 'failed' | 'progressing' | 'available'
# message is any string describing the state.
DeploymentRolloutStatus = namedtuple('DeploymentRolloutStatus', ['status', 'message'])
def _deployment_rollout_status_message(deployment, deployment_name):
"""
Gets the friendly human readable message of the current state of the deployment rollout
:param deployment: python dict matching: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#deployment-v1-apps
:param deployment_name: string
:return: DeploymentRolloutStatus
"""
# Logic for rollout status pulled from the `kubectl rollout status` command:
# https://github.com/kubernetes/kubernetes/blob/d9ba19c751709c8608e09a0537eea98973f3a796/pkg/kubectl/rollout_status.go#L62
if deployment['metadata']['generation'] <= deployment['status']['observedGeneration']:
for cond in deployment['status']['conditions']:
if cond['type'] == 'Progressing' and cond['reason'] == 'ProgressDeadlineExceeded':
return DeploymentRolloutStatus(
status='failed',
message="Deployment %s's rollout failed. Please try again later." % deployment_name
)
desired_replicas = deployment['spec']['replicas']
current_replicas = deployment['status'].get('replicas', 0)
if current_replicas == 0:
return DeploymentRolloutStatus(
status='available',
message='Deployment %s updated (no replicas, so nothing to roll out)' % deployment_name
)
# Some fields are optional in the spec, so if they're omitted, replace with defaults that won't indicate a wrong status
available_replicas = deployment['status'].get('availableReplicas', 0)
updated_replicas = deployment['status'].get('updatedReplicas', 0)
if updated_replicas < desired_replicas:
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for rollout to finish: %d out of %d new replicas have been updated...' % (
updated_replicas, desired_replicas)
)
if current_replicas > updated_replicas:
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for rollout to finish: %d old replicas are pending termination...' % (
current_replicas - updated_replicas)
)
if available_replicas < updated_replicas:
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for rollout to finish: %d of %d updated replicas are available...' % (
available_replicas, updated_replicas)
)
return DeploymentRolloutStatus(
status='available',
message='Deployment %s successfully rolled out.' % deployment_name
)
return DeploymentRolloutStatus(
status='progressing',
message='Waiting for deployment spec to be updated...'
)
class KubernetesAccessorSingleton(object):
@ -96,6 +165,23 @@ class KubernetesAccessorSingleton(object):
self._assert_success(self._execute_k8s_api('PUT', secret_url, secret))
def get_deployment_rollout_status(self, deployment_name):
""""
Returns the status of a rollout of a given deployment
:return _DeploymentRolloutStatus
"""
deployment_selector_url = 'namespaces/%s/deployments/%s' % (
self.kube_config.qe_namespace, deployment_name
)
response = self._execute_k8s_api('GET', deployment_selector_url, api_prefix='apis/apps/v1')
if response.status_code != 200:
return DeploymentRolloutStatus('failed', 'Could not get deployment. Please check that the deployment exists')
deployment = json.loads(response.text)
return _deployment_rollout_status_message(deployment, deployment_name)
def get_qe_deployments(self):
""""
Returns all deployments matching the label selector provided in the KubeConfig
@ -126,10 +212,13 @@ class KubernetesAccessorSingleton(object):
'template': {
'spec': {
'containers': [{
'name': 'quay-enterprise-app', 'env': [{
# Note: this name MUST match the deployment template's pod template
# (e.g. <template>.spec.template.spec.containers[0] == 'quay-enterprise-app')
'name': QE_CONTAINER_NAME,
'env': [{
'name': 'RESTART_TIME',
'value': str(datetime.datetime.now())
}]
}],
}]
}
}

50
config_app/config_util/test/test_k8saccessor.py
View file

@ -2,10 +2,58 @@ import pytest
from httmock import urlmatch, HTTMock, response
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton
from config_app.config_util.k8saccessor import KubernetesAccessorSingleton, _deployment_rollout_status_message
from config_app.config_util.k8sconfig import KubernetesConfig
@pytest.mark.parametrize('deployment_object, expected_status, expected_message', [
({'metadata': {'generation': 1},
'status': {'observedGeneration': 0, 'conditions': []},
'spec': {'replicas': 0}},
'progressing',
'Waiting for deployment spec to be updated...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [{'type': 'Progressing', 'reason': 'ProgressDeadlineExceeded'}]},
'spec': {'replicas': 0}},
'failed',
"Deployment my-deployment's rollout failed. Please try again later."),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': []},
'spec': {'replicas': 0}},
'available',
'Deployment my-deployment updated (no replicas, so nothing to roll out)'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1},
'spec': {'replicas': 2}},
'progressing',
'Waiting for rollout to finish: 0 out of 2 new replicas have been updated...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 1},
'spec': {'replicas': 2}},
'progressing',
'Waiting for rollout to finish: 1 out of 2 new replicas have been updated...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 2, 'updatedReplicas': 1},
'spec': {'replicas': 1}},
'progressing',
'Waiting for rollout to finish: 1 old replicas are pending termination...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 2, 'availableReplicas': 0},
'spec': {'replicas': 0}},
'progressing',
'Waiting for rollout to finish: 0 of 2 updated replicas are available...'),
({'metadata': {'generation': 0},
'status': {'observedGeneration': 0, 'conditions': [], 'replicas': 1, 'updatedReplicas': 2, 'availableReplicas': 2},
'spec': {'replicas': 0}},
'available',
'Deployment my-deployment successfully rolled out.'),
])
def test_deployment_rollout_status_message(deployment_object, expected_status, expected_message):
deployment_status = _deployment_rollout_status_message(deployment_object, 'my-deployment')
assert deployment_status.status == expected_status
assert deployment_status.message == expected_message
@pytest.mark.parametrize('kube_config, expected_api, expected_query', [
({'api_host': 'www.customhost.com'},
'/apis/extensions/v1beta1/namespaces/quay-enterprise/deployments', 'labelSelector=quay-enterprise-component%3Dapp'),