Add storage validation to the status endpoint

Fixes #1659

health/healthcheck.py

@@ -76,6 +76,10 @@ class HealthCheck(object):
 
 
 class LocalHealthCheck(HealthCheck):
+  def __init__(self, app, config_provider, instance_keys):
+    super(LocalHealthCheck, self).__init__(app, config_provider, instance_keys,
+                                           ['redis', 'storage'])
+
   @classmethod
   def check_names(cls):
     return ['LocalHealthCheck']
@@ -84,7 +88,9 @@ class LocalHealthCheck(HealthCheck):
 class RDSAwareHealthCheck(HealthCheck):
   def __init__(self, app, config_provider, instance_keys, access_key, secret_key,
                db_instance='quay', region='us-east-1'):
-    super(RDSAwareHealthCheck, self).__init__(app, config_provider, instance_keys, ['redis'])
+    super(RDSAwareHealthCheck, self).__init__(app, config_provider, instance_keys,
+                                              ['redis', 'storage'])
+
     self.access_key = access_key
     self.secret_key = secret_key
     self.db_instance = db_instance

health/services.py

@@ -1,6 +1,6 @@
 import logging
 from data.model import health
-from app import build_logs
+from app import build_logs, storage
 
 
 logger = logging.getLogger(__name__)
@@ -32,11 +32,20 @@ def _check_redis(app):
   """ Returns the status of Redis, as accessed from this instance. """
   return build_logs.check_health()
 
+def _check_storage(app):
+  """ Returns the status of storage, as accessed from this instance. """
+  try:
+    storage.validate(storage.preferred_locations, app.config['HTTPCLIENT'])
+    return True
+  except:
+    return False
+
 
 _SERVICES = {
   'registry_gunicorn': _check_registry_gunicorn,
   'database': _check_database,
-  'redis': _check_redis
+  'redis': _check_redis,
+  'storage': _check_storage,
 }
 
 def check_all_services(app, skip):

storage/distributedstorage.py

@@ -46,6 +46,7 @@ class DistributedStorage(StoragePaths):
   stream_write = _location_aware(BaseStorage.stream_write)
   exists = _location_aware(BaseStorage.exists)
   remove = _location_aware(BaseStorage.remove)
+  validate = _location_aware(BaseStorage.validate)
   get_checksum = _location_aware(BaseStorage.get_checksum)
   get_supports_resumable_downloads = _location_aware(BaseStorage.get_supports_resumable_downloads)