Merge pull request #1660 from coreos-inc/storage-status

Add storage validation to the status endpoint

health/healthcheck.py

@@ -76,6 +76,10 @@ class HealthCheck(object):
 
 
 class LocalHealthCheck(HealthCheck):
+  def __init__(self, app, config_provider, instance_keys):
+    super(LocalHealthCheck, self).__init__(app, config_provider, instance_keys,
+                                           ['redis', 'storage'])
+
   @classmethod
   def check_names(cls):
     return ['LocalHealthCheck']
@@ -84,7 +88,9 @@ class LocalHealthCheck(HealthCheck):
 class RDSAwareHealthCheck(HealthCheck):
   def __init__(self, app, config_provider, instance_keys, access_key, secret_key,
                db_instance='quay', region='us-east-1'):
-    super(RDSAwareHealthCheck, self).__init__(app, config_provider, instance_keys, ['redis'])
+    super(RDSAwareHealthCheck, self).__init__(app, config_provider, instance_keys,
+                                              ['redis', 'storage'])
+
     self.access_key = access_key
     self.secret_key = secret_key
     self.db_instance = db_instance

health/services.py

@@ -1,6 +1,6 @@
 import logging
 from data.model import health
-from app import build_logs
+from app import build_logs, storage
 
 
 logger = logging.getLogger(__name__)
@@ -32,11 +32,20 @@ def _check_redis(app):
   """ Returns the status of Redis, as accessed from this instance. """
   return build_logs.check_health()
 
+def _check_storage(app):
+  """ Returns the status of storage, as accessed from this instance. """
+  try:
+    storage.validate(storage.preferred_locations, app.config['HTTPCLIENT'])
+    return True
+  except:
+    return False
+
 
 _SERVICES = {
   'registry_gunicorn': _check_registry_gunicorn,
   'database': _check_database,
-  'redis': _check_redis
+  'redis': _check_redis,
+  'storage': _check_storage,
 }
 
 def check_all_services(app, skip):

storage/basestorage.py

@@ -42,9 +42,12 @@ class BaseStorage(StoragePaths):
     pass
 
   def validate(self, client):
-    """ Called to perform any custom storage system validation. The client is an HTTP
+    """ Called to perform storage system validation. The client is an HTTP
         client to use for any external calls. """
-    pass
+    # Put a temporary file to make sure the normal storage paths work.
+    self.put_content('_verify', 'testing 123')
+    if not self.exists('_verify'):
+      raise Exception('Could not find verification file')
 
   def get_direct_download_url(self, path, expires_in=60, requires_cors=False, head=False):
     return None

storage/distributedstorage.py

@@ -46,6 +46,7 @@ class DistributedStorage(StoragePaths):
   stream_write = _location_aware(BaseStorage.stream_write)
   exists = _location_aware(BaseStorage.exists)
   remove = _location_aware(BaseStorage.remove)
+  validate = _location_aware(BaseStorage.validate)
   get_checksum = _location_aware(BaseStorage.get_checksum)
   get_supports_resumable_downloads = _location_aware(BaseStorage.get_supports_resumable_downloads)
 

storage/local.py

@@ -115,6 +115,8 @@ class LocalStorage(BaseStorageV2):
     os.remove(content_path)
 
   def validate(self, client):
+    super(LocalStorage, self).validate()
+
     # Load the set of disk mounts.
     try:
       mounts = psutil.disk_partitions(all=True)

storage/swift.py

@@ -158,23 +158,17 @@ class SwiftStorage(BaseStorage):
     return surl.format(scheme=scheme, host=hostname, full_path=full_path, sig=sig, expires=expires)
 
   def validate(self, client):
-    if self._temp_url_key:
-      # Add a file to test direct download.
-      self.put_content('dd_path', 'testing 3456')
+    super(SwiftStorage, self).validate()
 
+    if self._temp_url_key:
       # Generate a direct download URL.
-      dd_url = self.get_direct_download_url('dd_path')
+      dd_url = self.get_direct_download_url('_verify')
 
       if not dd_url:
-        self.remove('dd_path')
         raise Exception('Could not validate direct download URL; the token may be invalid.')
 
       # Try to retrieve the direct download URL.
       response = client.get(dd_url, timeout=2)
-
-      # Remove the test file.
-      self.remove('dd_path')
-
       if response.status_code != 200:
         logger.debug('Direct download failure: %s => %s with body %s', dd_url,
                      response.status_code, response.text)

util/config/validator.py

@@ -107,13 +107,9 @@ def _validate_registry_storage(config, _):
       if replication_enabled and storage_type == 'LocalStorage':
         raise Exception('Locally mounted directory not supported with storage replication')
 
-      # Run custom validation on the driver.
+      # Run validation on the driver.
       driver.validate(app.config['HTTPCLIENT'])
 
-      # Put and remove a temporary file to make sure the normal storage paths work.
-      driver.put_content('_verify', 'testing 123')
-      driver.remove('_verify')
-
       # Run setup on the driver if the read/write succeeded.
       driver.setup()
     except Exception as ex: