vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add instance health checks for all gunicorn workers

Browse source 
Fixes https://jira.coreos.com/browse/QS-121
This commit is contained in:
Joseph Schorr 2018-01-12 16:07:10 -05:00
parent 18f1ccf80b
commit e91b83e1be
3 changed files with 36 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

45
health/services.py
View file

@ -5,27 +5,30 @@ from health.models_pre_oci import pre_oci_model as model
logger = logging.getLogger(__name__)
def _check_registry_gunicorn(app):
""" Returns the status of the registry gunicorn workers. """
# Compute the URL for checking the registry endpoint. We append a port if and only if the
# hostname contains one.
client = app.config['HTTPCLIENT']
hostname_parts = app.config['SERVER_HOSTNAME'].split(':')
port = ''
if len(hostname_parts) == 2:
port = ':' + hostname_parts[1]
def _check_gunicorn(endpoint):
def fn(app):
""" Returns the status of the gunicorn workers. """
# Compute the URL for checking the endpoint. We append a port if and only if the
# hostname contains one.
client = app.config['HTTPCLIENT']
hostname_parts = app.config['SERVER_HOSTNAME'].split(':')
port = ''
if len(hostname_parts) == 2:
port = ':' + hostname_parts[1]
scheme = app.config['PREFERRED_URL_SCHEME']
if app.config.get('EXTERNAL_TLS_TERMINATION', False):
scheme = 'http'
scheme = app.config['PREFERRED_URL_SCHEME']
if app.config.get('EXTERNAL_TLS_TERMINATION', False):
scheme = 'http'
registry_url = '%s://localhost%s/v1/_internal_ping' % (scheme, port)
try:
status_code = client.get(registry_url, verify=False, timeout=2).status_code
return (status_code == 200, 'Got non-200 response for registry: %s' % status_code)
except Exception as ex:
logger.exception('Exception when checking registry health: %s', registry_url)
return (False, 'Exception when checking registry health: %s' % registry_url)
registry_url = '%s://localhost%s/%s' % (scheme, port, endpoint)
try:
status_code = client.get(registry_url, verify=False, timeout=2).status_code
return (status_code == 200, 'Got non-200 response for worker: %s' % status_code)
except Exception as ex:
logger.exception('Exception when checking worker health: %s', registry_url)
return (False, 'Exception when checking worker health: %s' % registry_url)
return fn
def _check_database(app):
@ -53,7 +56,9 @@ def _check_auth(app):
_SERVICES = {
'registry_gunicorn': _check_registry_gunicorn,
'registry_gunicorn': _check_gunicorn('v1/_internal_ping'),
'web_gunicorn': _check_gunicorn('_internal_ping'),
'verbs_gunicorn': _check_gunicorn('c1/_internal_ping'),
'database': _check_database,
'redis': _check_redis,
'storage': _check_storage,