import logging from app import build_logs, storage, authentication, instance_keys from health.models_pre_oci import pre_oci_model as model logger = logging.getLogger(__name__) def _check_gunicorn(endpoint): def fn(app): """ Returns the status of the gunicorn workers. """ # Compute the URL for checking the endpoint. We append a port if and only if the # hostname contains one. client = app.config['HTTPCLIENT'] hostname_parts = app.config['SERVER_HOSTNAME'].split(':') port = '' if hostname_parts[0] == 'localhost': if len(hostname_parts) == 2: port = ':' + hostname_parts[1] scheme = app.config['PREFERRED_URL_SCHEME'] if app.config.get('EXTERNAL_TLS_TERMINATION', False): scheme = 'http' registry_url = '%s://localhost%s/%s' % (scheme, port, endpoint) try: status_code = client.get(registry_url, verify=False, timeout=2).status_code okay = status_code == 200 message = 'Got non-200 response for worker: %s' % status_code if not okay else None return (okay, message) except Exception as ex: logger.exception('Exception when checking worker health: %s', registry_url) return (False, 'Exception when checking worker health: %s' % registry_url) return fn def _check_database(app): """ Returns the status of the database, as accessed from this instance. """ return model.check_health(app.config) def _check_redis(app): """ Returns the status of Redis, as accessed from this instance. """ return build_logs.check_health() def _check_storage(app): """ Returns the status of storage, as accessed from this instance. """ try: storage.validate(storage.preferred_locations, app.config['HTTPCLIENT']) return (True, None) except Exception as ex: logger.exception('Storage check failed with exception %s', ex) return (False, 'Storage check failed with exception %s' % ex.message) def _check_auth(app): """ Returns the status of the auth engine, as accessed from this instance. """ return authentication.ping() def _check_service_key(app): """ Returns the status of the service key for this instance. If the key has disappeared or has expired, then will return False. """ if not app.config.get('SETUP_COMPLETE', False): return (True, 'Stack not fully setup; skipping check') try: kid = instance_keys.local_key_id except IOError as ex: # Key has not been created yet. return (True, 'Stack not fully setup; skipping check') try: key_is_valid = bool(instance_keys.get_service_key_public_key(kid)) message = 'Could not find valid instance service key %s' % kid if not key_is_valid else None return (key_is_valid, message) except Exception as ex: logger.exception('Got exception when trying to retrieve the instance key') # NOTE: We return *True* here if there was an exception when retrieving the key, as it means # the database is down, which will be handled by the database health check. return (True, 'Failed to get instance key due to a database issue; skipping check') _SERVICES = { 'registry_gunicorn': _check_gunicorn('v1/_internal_ping'), 'web_gunicorn': _check_gunicorn('_internal_ping'), 'verbs_gunicorn': _check_gunicorn('c1/_internal_ping'), 'database': _check_database, 'redis': _check_redis, 'storage': _check_storage, 'auth': _check_auth, 'service_key': _check_service_key, } def check_all_services(app, skip): """ Returns a dictionary containing the status of all the services defined. """ status = {} for name in _SERVICES: if name in skip: continue status[name] = _SERVICES[name](app) return status