Merge pull request #2757 from coreos-inc/joseph.schorr/QUAY-606/logarchive-georep

Add support for QE customers to enable log rotation

static/directives/config/config-setup-tool.html

@@ -351,6 +351,47 @@
       </div>
     </div>
 
+    <!-- Action log archiving -->
+    <div class="co-panel">
+      <div class="co-panel-heading">
+        <i class="fa fa-archive"></i> Action Log Rotation and Archiving
+      </div>
+      <div class="co-panel-body">
+        <div class="description">
+          <p>
+            All actions performed in <span class="registry-name"></span> are automatically logged. These logs are stored in a database table, which can become quite large.
+            Enabling log rotation and archiving will move all logs older than 30 days into storage.
+          </p>
+        </div>
+        <div class="config-bool-field" binding="config.FEATURE_ACTION_LOG_ROTATION">
+          Enable Action Log Rotation
+        </div>
+
+        <table class="config-table" ng-if="config.FEATURE_ACTION_LOG_ROTATION">
+          <tr>
+            <td>Storage location:</td>
+            <td>
+              <select class="form-control" ng-model="config.ACTION_LOG_ARCHIVE_LOCATION">
+                <option ng-repeat="sc in storageConfig" value="{{ sc['location'] }}">{{ sc['location'] }}</option>
+              </select>
+              <div class="help-text">
+                The storage location in which to place archived action logs. Logs will only be archived to this single location.
+              </div>
+            </td>
+          </tr>
+          <tr>
+            <td>Storage path:</td>
+            <td>
+              <span class="config-string-field" binding="config.ACTION_LOG_ARCHIVE_PATH"
+                    placeholder="Path under storage to place archived logs"></span>
+              <div class="help-text">
+                The path under the configured storage engine in which to place the archived logs in JSON form.
+              </div>
+            </td>
+          </tr>
+        </table>
+    </div>
+
     <!-- Security Scanner -->
     <div class="co-panel">
       <div class="co-panel-heading">

static/js/core-config-setup.js

@@ -80,6 +80,10 @@ angular.module("core-config-setup", ['angularFileUpload'])
           {'id': 'oidc-login', 'title': 'OIDC Login(s)', 'condition': function(config) {
             return $scope.getOIDCProviders(config).length > 0;
           }},
+
+          {'id': 'actionlogarchiving', 'title': 'Action Log Rotation', 'condition': function(config) {
+            return config.FEATURE_ACTION_LOG_ROTATION;
+          }},
         ];
 
         $scope.STORAGE_CONFIG_FIELDS = {

util/config/validator.py

@@ -21,6 +21,7 @@ from util.config.validators.validate_github import GitHubLoginValidator, GitHubT
 from util.config.validators.validate_oidc import OIDCLoginValidator
 from util.config.validators.validate_timemachine import TimeMachineValidator
 from util.config.validators.validate_access import AccessSettingsValidator
+from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
 
 logger = logging.getLogger(__name__)
 
@@ -57,6 +58,7 @@ VALIDATORS = {
   OIDCLoginValidator.name: OIDCLoginValidator.validate,
   TimeMachineValidator.name: TimeMachineValidator.validate,
   AccessSettingsValidator.name: AccessSettingsValidator.validate,
+  ActionLogArchivingValidator.name: ActionLogArchivingValidator.validate,
 }
 
 def validate_service_for_config(service, config, password=None):

util/config/validators/test/test_validate_actionlog_archiving.py

@@ -0,0 +1,51 @@
+import pytest
+
+from util.config.validators import ConfigValidationException
+from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('unvalidated_config', [
+  ({}),
+  ({'ACTION_LOG_ARCHIVE_PATH': 'foo'}),
+  ({'ACTION_LOG_ARCHIVE_LOCATION': ''}),
+])
+def test_skip_validate_actionlog(unvalidated_config, app):
+  validator = ActionLogArchivingValidator()
+  validator.validate(unvalidated_config, None, None)
+
+
+@pytest.mark.parametrize('config, expected_error', [
+  ({'FEATURE_ACTION_LOG_ROTATION': True}, 'Missing action log archive path'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': ''}, 'Missing action log archive path'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'foo'}, 'Missing action log archive storage location'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'foo',
+    'ACTION_LOG_ARCHIVE_LOCATION': ''}, 'Missing action log archive storage location'),
+  ({'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'foo',
+    'ACTION_LOG_ARCHIVE_LOCATION': 'invalid'},
+    'Action log archive storage location `invalid` not found in storage config'),
+])
+def test_invalid_config(config, expected_error, app):
+  validator = ActionLogArchivingValidator()
+
+  with pytest.raises(ConfigValidationException) as ipe:
+    validator.validate(config, None, None)
+
+  assert ipe.value.message == expected_error
+
+def test_valid_config(app):
+  config = {
+    'FEATURE_ACTION_LOG_ROTATION': True,
+    'ACTION_LOG_ARCHIVE_PATH': 'somepath',
+    'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation',
+    'DISTRIBUTED_STORAGE_CONFIG': {
+      'somelocation': {},
+    },
+  }
+
+  validator = ActionLogArchivingValidator()
+  validator.validate(config, None, None)

util/config/validators/validate_actionlog_archiving.py

@@ -0,0 +1,22 @@
+from util.config.validators import BaseValidator, ConfigValidationException
+
+class ActionLogArchivingValidator(BaseValidator):
+  name = "actionlogarchiving"
+
+  @classmethod
+  def validate(cls, config, user, user_password):
+    """ Validates the action log archiving configuration. """
+    if not config.get('FEATURE_ACTION_LOG_ROTATION', False):
+      return
+
+    if not config.get('ACTION_LOG_ARCHIVE_PATH'):
+      raise ConfigValidationException('Missing action log archive path')
+
+    if not config.get('ACTION_LOG_ARCHIVE_LOCATION'):
+      raise ConfigValidationException('Missing action log archive storage location')
+
+    location = config['ACTION_LOG_ARCHIVE_LOCATION']
+    storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG') or {}
+    if location not in storage_config:
+      msg = 'Action log archive storage location `%s` not found in storage config' % location
+      raise ConfigValidationException(msg)

workers/logrotateworker.py

@@ -53,7 +53,13 @@ class LogRotateWorker(Worker):
         return
 
   def _perform_archiving(self, cutoff_id):
-    log_archive = DelegateUserfiles(app, storage, SAVE_LOCATION, SAVE_PATH)
+    save_location = SAVE_LOCATION
+    if not save_location:
+      # Pick the *same* save location for all instances. This is a fallback if
+      # a location was not configured.
+      save_location = storage.locations[0]
+
+    log_archive = DelegateUserfiles(app, storage, save_location, SAVE_PATH)
 
     with UseThenDisconnect(app.config):
       start_id = get_stale_logs_start_id()