Add ability to download system logs

config.py

@@ -187,3 +187,11 @@ class DefaultConfig(object):
 
   # For enterprise:
   MAXIMUM_REPOSITORY_USAGE = 20
+
+  # System logs.
+  SYSTEM_LOGS_PATH = "/var/log/"
+  SYSTEM_SERVICE_LOGS_PATH = "/var/log/%s/current"
+  SYSTEM_SERVICES_PATH = "conf/init/"
+
+  # Services that should not be shown in the logs view.
+  SYSTEM_SERVICE_BLACKLIST = ['tutumdocker', 'dockerfilebuild']

endpoints/api/superuser.py

@@ -23,12 +23,14 @@ import features
 
 logger = logging.getLogger(__name__)
 
-LOGS_PATH = "/var/log/%s/current"
-SERVICES_PATH = "conf/init/"
-
 def get_immediate_subdirectories(directory):
   return [name for name in os.listdir(directory) if os.path.isdir(os.path.join(directory, name))]
 
+def get_services():
+  services = set(get_immediate_subdirectories(app.config['SYSTEM_SERVICES_PATH']))
+  services = services - set(app.config['SYSTEM_SERVICE_BLACKLIST'])
+  return services
+
 
 @resource('/v1/superuser/systemlogs/<service>')
 @internal_only
@@ -39,12 +41,11 @@ class SuperUserGetLogsForService(ApiResource):
   def get(self, service):
     """ Returns the logs for the specific service. """
     if SuperUserPermission().can():
-      services = get_immediate_subdirectories(SERVICES_PATH)
+      if not service in get_services():
-      if not service in services:
         abort(404)
 
       try:
-        with open(LOGS_PATH % service, 'r') as f:
+        with open(app.config['SYSTEM_SERVICE_LOGS_PATH'] % service, 'r') as f:
           logs = f.read()
       except Exception as ex:
         logger.exception('Cannot read logs')
@@ -67,7 +68,7 @@ class SuperUserSystemLogServices(ApiResource):
     """ List the system logs for the current system. """
     if SuperUserPermission().can():
       return {
-        'services': get_immediate_subdirectories(SERVICES_PATH)
+        'services': list(get_services())
       }
 
     abort(403)

endpoints/csrf.py

@@ -19,12 +19,7 @@ def generate_csrf_token():
 
   return session['_csrf_token']
 
-
+def verify_csrf():
-def csrf_protect(func):
-  @wraps(func)
-  def wrapper(*args, **kwargs):
-    oauth_token = get_validated_oauth_token()
-    if oauth_token is None and request.method != "GET" and request.method != "HEAD":
   token = session.get('_csrf_token', None)
   found_token = request.values.get('_csrf_token', None)
 
@@ -33,6 +28,13 @@ def csrf_protect(func):
     logger.error(msg, token, found_token)
     abort(403, message='CSRF token was invalid or missing.')
 
+def csrf_protect(func):
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    oauth_token = get_validated_oauth_token()
+    if oauth_token is None and request.method != "GET" and request.method != "HEAD":
+      verify_csrf()
+
     return func(*args, **kwargs)
   return wrapper
 

endpoints/web.py

@@ -12,15 +12,18 @@ from data import model
 from data.model.oauth import DatabaseAuthorizationProvider
 from app import app, billing as stripe, build_logs, avatar
 from auth.auth import require_session_login, process_oauth
-from auth.permissions import AdministerOrganizationPermission, ReadRepositoryPermission
+from auth.permissions import (AdministerOrganizationPermission, ReadRepositoryPermission,
+                              SuperUserPermission)
+
 from util.invoice import renderInvoiceToPdf
 from util.seo import render_snapshot
 from util.cache import no_cache
 from endpoints.common import common_login, render_page_template, route_show_if, param_required
-from endpoints.csrf import csrf_protect, generate_csrf_token
+from endpoints.csrf import csrf_protect, generate_csrf_token, verify_csrf
 from endpoints.registry import set_cache_headers
 from util.names import parse_repository_name
 from util.useremails import send_email_changed
+from util.systemlogs import build_logs_archive
 from auth import scopes
 
 import features
@@ -466,3 +469,21 @@ def exchange_code_for_token():
 
   provider = FlaskAuthorizationProvider()
   return provider.get_token(grant_type, client_id, client_secret, redirect_uri, code, scope=scope)
+
+
+@web.route('/systemlogsarchive', methods=['GET'])
+@process_oauth
+@route_show_if(features.SUPER_USERS)
+@no_cache
+def download_logs_archive():
+  # Note: We cannot use the decorator here because this is a GET method. That being said, this
+  # information is sensitive enough that we want the extra protection.
+  verify_csrf()
+
+  if SuperUserPermission().can():
+    archive_data = build_logs_archive(app)
+    return Response(archive_data,
+                    mimetype="application/octet-stream",
+                    headers={"Content-Disposition": "attachment;filename=erlogs.tar.gz"})
+
+  abort(403)

static/css/quay.css

@@ -4879,4 +4879,10 @@ i.slack-icon {
 
 .system-log-download-panel {
   padding: 20px;
+  text-align: center;
+  font-size: 18px;
+}
+
+.system-log-download-panel a {
+  margin-top: 20px;
 }

static/js/controllers.js

@@ -2826,6 +2826,7 @@ function SuperUserAdminCtrl($scope, $timeout, ApiService, Features, UserService,
   $scope.debugLogs = null;
   $scope.pollChannel = null;
   $scope.logsScrolled = false;
+  $scope.csrf_token = window.__token;
 
   $scope.viewSystemLogs = function(service) {
     if ($scope.pollChannel) {
@@ -2835,7 +2836,7 @@ function SuperUserAdminCtrl($scope, $timeout, ApiService, Features, UserService,
     $scope.debugService = service;
     $scope.debugLogs = null;
 
-    $scope.pollChannel = AngularPollChannel.create($scope, $scope.loadServiceLogs, 1 * 1000 /* 1s */);
+    $scope.pollChannel = AngularPollChannel.create($scope, $scope.loadServiceLogs, 2 * 1000 /* 2s */);
     $scope.pollChannel.start();
   };
 

static/js/core-ui.js

@@ -20,7 +20,6 @@ angular.module("core-ui", [])
           if (isAnimatedScrolling) { return; }
           var element = $element.find("#co-log-viewer")[0];
           isScrollBottom = element.scrollHeight - element.scrollTop === element.clientHeight;
-
           if (isScrollBottom) {
             $scope.hasNewLogs = false;
           }

static/partials/super-user.html

@@ -44,7 +44,13 @@
          </ul>
 
          <div class="system-log-download-panel" ng-if="!debugService">
-          Please choose a service above to view its logs.
+          Select a service above to view its local logs
+
+          <div>
+            <a class="btn btn-primary" href="/systemlogsarchive?_csrf_token={{ csrf_token }}" target="_blank">
+              <i class="fa fa-download fa-lg" style="margin-right: 4px;"></i> Download All Local Logs (.tar.gz)
+            </a>
+          </div>
          </div>
          <div class="cor-log-box" logs="debugLogs" ng-show="debugService"></div>
         </div>

util/systemlogs.py

@@ -0,0 +1,15 @@
+import tarfile
+import os
+import cStringIO
+
+def build_logs_archive(app):
+  """ Builds a .tar.gz with the contents of the system logs found for the given app and returns
+      the binary contents.
+  """
+  path = app.config['SYSTEM_LOGS_PATH']
+  buf = cStringIO.StringIO()
+
+  with tarfile.open(mode="w:gz", fileobj=buf) as tar:
+    tar.add(path, arcname=os.path.basename(path))
+
+  return buf.getvalue()