Implement a worker for batch exporting of usage logs

This will allow customers to request their usage logs for a repository or an entire namespace, and we can export the logs in a manner that doesn't absolutely destroy the database, with every step along the way timed.
This commit is contained in:
Joseph Schorr 2018-11-27 18:28:32 +02:00
parent b8d2e1be9c
commit 8a212728a3
18 changed files with 768 additions and 15 deletions

View file

@ -113,4 +113,29 @@
float: none !important;
text-align: right;
margin-bottom: 20px;
}
.logs-view-element .toggle-icon {
vertical-align: middle;
}
.logs-view-element .download-btn {
margin-left: 6px;
vertical-align: middle;
}
.logs-view-element .download-btn i.fa {
margin-right: 4px;
}
.logs-view-element .help-text {
font-size: 12px;
color: #888;
padding-top: 0px;
padding: 10px;
}
.logs-view-element code.date {
font-size: 70%;
color: #2581c7;
}

View file

@ -23,6 +23,8 @@
<span class="hidden-xs right">
<i class="fa fa-bar-chart-o toggle-icon" ng-class="chartVisible ? 'active' : ''"
ng-click="toggleChart()" data-title="Toggle Chart" bs-tooltip="tooltip.title"></i>
<button class="btn btn-default download-btn" ng-click="showExportLogs()"
ng-if="user || organization || repository"><i class="fa fa-download"></i>Export Logs</button>
</span>
</div>
@ -98,4 +100,27 @@
<div class="cor-loader" ng-show="loading"></div>
</div>
</div>
<!-- Modal Dialog -->
<div class="cor-confirm-dialog"
dialog-context="exportLogsInfo"
dialog-action="exportLogs(exportLogsInfo, callback)"
dialog-title="Export Usage Logs"
dialog-action-title="Start Logs Export"
dialog-form="context.exportform">
<form name="context.exportform">
<div style="margin-bottom: 14px;">
Enter an e-mail address or callback URL (must start with <code>http://</code> or <code>https://</code>)
at which to receive the exported logs once they have been fully processed:
</div>
<input class="form-control" type="text" ng-model="exportLogsInfo.urlOrEmail"
placeholder="E-mail address or callback URL"
ng-pattern="'(http(s)?:.+)|.+@.+'">
<div class="help-text">
Note: The export process can take <strong>up to an hour</strong> to process if there are many logs. As well,
only a <strong>single</strong> export process can run at a time for each namespace. Additional export requests will be
queued.
</div>
</form>
</div>
</div>

View file

@ -28,6 +28,7 @@ angular.module('quay').directive('logsView', function () {
$scope.chartLoading = true;
$scope.options = {};
$scope.context = {};
var datetime = new Date();
$scope.options.logStartDate = new Date(datetime.getUTCFullYear(), datetime.getUTCMonth(), datetime.getUTCDate() - 7);
@ -479,6 +480,33 @@ angular.module('quay').directive('logsView', function () {
return StringBuilderService.buildString(logDescriptions[log.kind] || log.kind, log.metadata);
};
$scope.showExportLogs = function() {
$scope.exportLogsInfo = {};
};
$scope.exportLogs = function(exportLogsInfo, callback) {
if (!exportLogsInfo.urlOrEmail) {
callback(false);
return;
}
var exportURL = getUrl('exportlogs').toString();
var runExport = Restangular.one(exportURL);
var urlOrEmail = exportLogsInfo.urlOrEmail;
var data = {};
if (urlOrEmail.indexOf('http://') == 0 || urlOrEmail.indexOf('https://') == 0) {
data['callback_url'] = urlOrEmail;
} else {
data['callback_email'] = urlOrEmail;
}
runExport.customPOST(data).then(function(resp) {
bootbox.alert('Usage logs export queued with ID `' + resp['export_id'] + '`')
callback(true);
}, ApiService.errorDisplay('Could not start logs export', callback));
};
$scope.$watch('organization', update);
$scope.$watch('user', update);
$scope.$watch('repository', update);