From fa21e42ffb1dfbf5228829b45d7b54089f289a02 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 10 Jul 2017 12:37:44 +0300 Subject: [PATCH 1/4] Add default location for action log archiving Prevents the logs from being written to the preferred storage, which means they will jump around --- workers/logrotateworker.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/workers/logrotateworker.py b/workers/logrotateworker.py index 54ff0f25d..06e93d23d 100644 --- a/workers/logrotateworker.py +++ b/workers/logrotateworker.py @@ -53,7 +53,13 @@ class LogRotateWorker(Worker): return def _perform_archiving(self, cutoff_id): - log_archive = DelegateUserfiles(app, storage, SAVE_LOCATION, SAVE_PATH) + save_location = SAVE_LOCATION + if not save_location: + # Pick the *same* save location for all instances. This is a fallback if + # a location was not configured. + save_location = storage.locations[0] + + log_archive = DelegateUserfiles(app, storage, save_location, SAVE_PATH) with UseThenDisconnect(app.config): start_id = get_stale_logs_start_id() From 176c26e3f793506c648664ac0b143d300b9e9a8a Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 10 Jul 2017 13:09:33 +0300 Subject: [PATCH 2/4] Add config validation for action log archiving --- util/config/validator.py | 2 + .../test/test_validate_actionlog_archiving.py | 51 +++++++++++++++++++ .../validators/validate_actionlog_achiving.py | 22 ++++++++ 3 files changed, 75 insertions(+) create mode 100644 util/config/validators/test/test_validate_actionlog_archiving.py create mode 100644 util/config/validators/validate_actionlog_achiving.py diff --git a/util/config/validator.py b/util/config/validator.py index 4190caa34..1c66ac5fa 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -21,6 +21,7 @@ from util.config.validators.validate_github import GitHubLoginValidator, GitHubT from util.config.validators.validate_oidc import OIDCLoginValidator from util.config.validators.validate_timemachine import TimeMachineValidator from util.config.validators.validate_access import AccessSettingsValidator +from util.config.validators.validate_actionlog_achiving import ActionLogAchivingValidator logger = logging.getLogger(__name__) @@ -57,6 +58,7 @@ VALIDATORS = { OIDCLoginValidator.name: OIDCLoginValidator.validate, TimeMachineValidator.name: TimeMachineValidator.validate, AccessSettingsValidator.name: AccessSettingsValidator.validate, + ActionLogAchivingValidator.name: ActionLogAchivingValidator.validate, } def validate_service_for_config(service, config, password=None): diff --git a/util/config/validators/test/test_validate_actionlog_archiving.py b/util/config/validators/test/test_validate_actionlog_archiving.py new file mode 100644 index 000000000..915de58d1 --- /dev/null +++ b/util/config/validators/test/test_validate_actionlog_archiving.py @@ -0,0 +1,51 @@ +import pytest + +from util.config.validators import ConfigValidationException +from util.config.validators.validate_actionlog_achiving import ActionLogAchivingValidator + +from test.fixtures import * + +@pytest.mark.parametrize('unvalidated_config', [ + ({}), + ({'ACTION_LOG_ARCHIVE_PATH': 'foo'}), + ({'ACTION_LOG_ARCHIVE_LOCATION': ''}), +]) +def test_skip_validate_actionlog(unvalidated_config, app): + validator = ActionLogAchivingValidator() + validator.validate(unvalidated_config, None, None) + + +@pytest.mark.parametrize('config, expected_error', [ + ({'FEATURE_ACTION_LOG_ROTATION': True}, 'Missing action log archive path'), + ({'FEATURE_ACTION_LOG_ROTATION': True, + 'ACTION_LOG_ARCHIVE_PATH': ''}, 'Missing action log archive path'), + ({'FEATURE_ACTION_LOG_ROTATION': True, + 'ACTION_LOG_ARCHIVE_PATH': 'foo'}, 'Missing action log archive storage location'), + ({'FEATURE_ACTION_LOG_ROTATION': True, + 'ACTION_LOG_ARCHIVE_PATH': 'foo', + 'ACTION_LOG_ARCHIVE_LOCATION': ''}, 'Missing action log archive storage location'), + ({'FEATURE_ACTION_LOG_ROTATION': True, + 'ACTION_LOG_ARCHIVE_PATH': 'foo', + 'ACTION_LOG_ARCHIVE_LOCATION': 'invalid'}, + 'Action log archive storage location `invalid` not found in storage config'), +]) +def test_invalid_config(config, expected_error, app): + validator = ActionLogAchivingValidator() + + with pytest.raises(ConfigValidationException) as ipe: + validator.validate(config, None, None) + + assert ipe.value.message == expected_error + +def test_valid_config(app): + config = { + 'FEATURE_ACTION_LOG_ROTATION': True, + 'ACTION_LOG_ARCHIVE_PATH': 'somepath', + 'ACTION_LOG_ARCHIVE_LOCATION': 'somelocation', + 'DISTRIBUTED_STORAGE_CONFIG': { + 'somelocation': {}, + }, + } + + validator = ActionLogAchivingValidator() + validator.validate(config, None, None) diff --git a/util/config/validators/validate_actionlog_achiving.py b/util/config/validators/validate_actionlog_achiving.py new file mode 100644 index 000000000..f6d1d5eba --- /dev/null +++ b/util/config/validators/validate_actionlog_achiving.py @@ -0,0 +1,22 @@ +from util.config.validators import BaseValidator, ConfigValidationException + +class ActionLogAchivingValidator(BaseValidator): + name = "actionlogachiving" + + @classmethod + def validate(cls, config, user, user_password): + """ Validates the action log archiving configuration. """ + if not config.get('FEATURE_ACTION_LOG_ROTATION', False): + return + + if not config.get('ACTION_LOG_ARCHIVE_PATH'): + raise ConfigValidationException('Missing action log archive path') + + if not config.get('ACTION_LOG_ARCHIVE_LOCATION'): + raise ConfigValidationException('Missing action log archive storage location') + + location = config['ACTION_LOG_ARCHIVE_LOCATION'] + storage_config = config.get('DISTRIBUTED_STORAGE_CONFIG') or {} + if location not in storage_config: + msg = 'Action log archive storage location `%s` not found in storage config' % location + raise ConfigValidationException(msg) From 661c0e643265068270fd3da255cfb28aa5ea6516 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 10 Jul 2017 13:22:29 +0300 Subject: [PATCH 3/4] Add superuser configuration for action log rotation --- .../directives/config/config-setup-tool.html | 41 +++++++++++++++++++ static/js/core-config-setup.js | 4 ++ 2 files changed, 45 insertions(+) diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 3e78cc92e..f40b2b40f 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -351,6 +351,47 @@ + +
+
+ Action Log Rotation and Archiving +
+
+
+

+ All actions performed in are automatically logged. These logs are stored in a database table, which can become quite large. + Enabling log rotation and archiving will move all logs older than 30 days into storage. +

+
+
+ Enable Action Log Rotation +
+ + + + + + + + + + +
Storage location: + +
+ The storage location in which to place archived action logs. Logs will only be archived to this single location. +
+
Storage path: + +
+ The path under the configured storage engine in which to place the archived logs in JSON form. +
+
+
+
diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 763330ff6..d99ef4e7b 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -80,6 +80,10 @@ angular.module("core-config-setup", ['angularFileUpload']) {'id': 'oidc-login', 'title': 'OIDC Login(s)', 'condition': function(config) { return $scope.getOIDCProviders(config).length > 0; }}, + + {'id': 'actionlogachiving', 'title': 'Action Log Rotation', 'condition': function(config) { + return config.FEATURE_ACTION_LOG_ROTATION; + }}, ]; $scope.STORAGE_CONFIG_FIELDS = { From a13235c0321228c2576071e9de56f84f71d76976 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Mon, 10 Jul 2017 18:35:51 +0300 Subject: [PATCH 4/4] Fix typo --- static/js/core-config-setup.js | 2 +- util/config/validator.py | 4 ++-- .../validators/test/test_validate_actionlog_archiving.py | 8 ++++---- ...ionlog_achiving.py => validate_actionlog_archiving.py} | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) rename util/config/validators/{validate_actionlog_achiving.py => validate_actionlog_archiving.py} (91%) diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index d99ef4e7b..bc208ef5a 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -81,7 +81,7 @@ angular.module("core-config-setup", ['angularFileUpload']) return $scope.getOIDCProviders(config).length > 0; }}, - {'id': 'actionlogachiving', 'title': 'Action Log Rotation', 'condition': function(config) { + {'id': 'actionlogarchiving', 'title': 'Action Log Rotation', 'condition': function(config) { return config.FEATURE_ACTION_LOG_ROTATION; }}, ]; diff --git a/util/config/validator.py b/util/config/validator.py index 1c66ac5fa..dda3bd666 100644 --- a/util/config/validator.py +++ b/util/config/validator.py @@ -21,7 +21,7 @@ from util.config.validators.validate_github import GitHubLoginValidator, GitHubT from util.config.validators.validate_oidc import OIDCLoginValidator from util.config.validators.validate_timemachine import TimeMachineValidator from util.config.validators.validate_access import AccessSettingsValidator -from util.config.validators.validate_actionlog_achiving import ActionLogAchivingValidator +from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator logger = logging.getLogger(__name__) @@ -58,7 +58,7 @@ VALIDATORS = { OIDCLoginValidator.name: OIDCLoginValidator.validate, TimeMachineValidator.name: TimeMachineValidator.validate, AccessSettingsValidator.name: AccessSettingsValidator.validate, - ActionLogAchivingValidator.name: ActionLogAchivingValidator.validate, + ActionLogArchivingValidator.name: ActionLogArchivingValidator.validate, } def validate_service_for_config(service, config, password=None): diff --git a/util/config/validators/test/test_validate_actionlog_archiving.py b/util/config/validators/test/test_validate_actionlog_archiving.py index 915de58d1..c14555441 100644 --- a/util/config/validators/test/test_validate_actionlog_archiving.py +++ b/util/config/validators/test/test_validate_actionlog_archiving.py @@ -1,7 +1,7 @@ import pytest from util.config.validators import ConfigValidationException -from util.config.validators.validate_actionlog_achiving import ActionLogAchivingValidator +from util.config.validators.validate_actionlog_archiving import ActionLogArchivingValidator from test.fixtures import * @@ -11,7 +11,7 @@ from test.fixtures import * ({'ACTION_LOG_ARCHIVE_LOCATION': ''}), ]) def test_skip_validate_actionlog(unvalidated_config, app): - validator = ActionLogAchivingValidator() + validator = ActionLogArchivingValidator() validator.validate(unvalidated_config, None, None) @@ -30,7 +30,7 @@ def test_skip_validate_actionlog(unvalidated_config, app): 'Action log archive storage location `invalid` not found in storage config'), ]) def test_invalid_config(config, expected_error, app): - validator = ActionLogAchivingValidator() + validator = ActionLogArchivingValidator() with pytest.raises(ConfigValidationException) as ipe: validator.validate(config, None, None) @@ -47,5 +47,5 @@ def test_valid_config(app): }, } - validator = ActionLogAchivingValidator() + validator = ActionLogArchivingValidator() validator.validate(config, None, None) diff --git a/util/config/validators/validate_actionlog_achiving.py b/util/config/validators/validate_actionlog_archiving.py similarity index 91% rename from util/config/validators/validate_actionlog_achiving.py rename to util/config/validators/validate_actionlog_archiving.py index f6d1d5eba..e8fb79a50 100644 --- a/util/config/validators/validate_actionlog_achiving.py +++ b/util/config/validators/validate_actionlog_archiving.py @@ -1,7 +1,7 @@ from util.config.validators import BaseValidator, ConfigValidationException -class ActionLogAchivingValidator(BaseValidator): - name = "actionlogachiving" +class ActionLogArchivingValidator(BaseValidator): + name = "actionlogarchiving" @classmethod def validate(cls, config, user, user_password):