Merge pull request #1830 from coreos-inc/superuser-dashboard

Add prometheus stats to enable better dashboarding

endpoints/realtime.py

@@ -4,63 +4,14 @@ import json
 from flask import request, Blueprint, abort, Response
 from flask.ext.login import current_user
 from auth.auth import require_session_login
-from endpoints.common import route_show_if
-from app import app, userevents
-from auth.permissions import SuperUserPermission
+from app import userevents
 from data.userevent import CannotReadUserEventsException
 
-import features
-import psutil
-import time
-
 logger = logging.getLogger(__name__)
 
 realtime = Blueprint('realtime', __name__)
 
 
-@realtime.route("/ps")
-@route_show_if(features.SUPER_USERS)
-@require_session_login
-def ps():
-  if not SuperUserPermission().can():
-    abort(403)
-
-  def generator():
-    while True:
-      build_status = {}
-      try:
-        builder_data = app.config['HTTPCLIENT'].get('http://localhost:8686/status', timeout=1)
-        if builder_data.status_code == 200:
-          build_status = json.loads(builder_data.text)
-      except:
-        pass
-
-      try:
-        data = {
-          'count': {
-            'cpu': psutil.cpu_percent(interval=1, percpu=True),
-            'virtual_mem': psutil.virtual_memory(),
-            'swap_mem': psutil.swap_memory(),
-            'connections': len(psutil.net_connections()),
-            'processes': len(psutil.pids()),
-            'network': psutil.net_io_counters()
-          },
-          'build': build_status
-        }
-      except psutil.AccessDenied:
-        data = {}
-
-      json_string = json.dumps(data)
-      yield 'data: %s\n\n' % json_string
-      time.sleep(1)
-
-  try:
-    return Response(generator(), mimetype="text/event-stream")
-  except:
-    pass
-
-
-
 @realtime.route("/user/")
 @require_session_login
 def index():

endpoints/v1/index.py

@@ -7,7 +7,7 @@ from functools import wraps
 from flask import request, make_response, jsonify, session
 
 from data import model
-from app import authentication, userevents
+from app import authentication, userevents, metric_queue
 from auth.auth import process_auth, generate_signed_token
 from auth.auth_context import get_authenticated_user, get_validated_token, get_validated_oauth_token
 from auth.permissions import (ModifyRepositoryPermission, UserAdminPermission,
@@ -247,6 +247,7 @@ def update_images(namespace_name, repo_name):
 
     track_and_log('push_repo', repo)
     spawn_notification(repo, 'repo_push', event_data)
+    metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v1'])
     return make_response('Updated', 204)
 
   abort(403)
@@ -273,6 +274,7 @@ def get_repository_images(namespace_name, repo_name):
     resp.mimetype = 'application/json'
 
     track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
+    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v1'])
     return resp
 
   abort(403)

endpoints/v2/manifest.py

@@ -14,7 +14,7 @@ from jwkest.jws import SIGNER_ALGS, keyrep
 
 import features
 
-from app import docker_v2_signing_key, app
+from app import docker_v2_signing_key, app, metric_queue
 from auth.registry_jwt_auth import process_registry_jwt_auth
 from endpoints.common import parse_repository_name
 from endpoints.decorators import anon_protect
@@ -261,6 +261,7 @@ def fetch_manifest_by_tagname(namespace_name, repo_name, manifest_ref):
   repo = model.repository.get_repository(namespace_name, repo_name)
   if repo is not None:
     track_and_log('pull_repo', repo, analytics_name='pull_repo_100x', analytics_sample=0.01)
+    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
 
   response = make_response(manifest.json_data, 200)
   response.headers['Content-Type'] = MANIFEST_CONTENT_TYPE
@@ -283,6 +284,7 @@ def fetch_manifest_by_digest(namespace_name, repo_name, manifest_ref):
   repo = model.repository.get_repository(namespace_name, repo_name)
   if repo is not None:
     track_and_log('pull_repo', repo)
+    metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
 
   response = make_response(manifest.json_data, 200)
   response.headers['Content-Type'] = MANIFEST_CONTENT_TYPE
@@ -487,6 +489,7 @@ def _write_manifest(namespace_name, repo_name, manifest):
 
   track_and_log('push_repo', repo, tag=tag_name)
   spawn_notification(repo, 'repo_push', event_data)
+  metric_queue.repository_push.Inc(labelvalues=[namespace_name, repo_name, 'v2'])
 
   response = make_response('OK', 202)
   response.headers['Docker-Content-Digest'] = manifest_digest

endpoints/verbs.py

@@ -249,10 +249,12 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
   # Check for torrent. If found, we return a torrent for the repo verb image (if the derived
   # image already exists).
   if request.accept_mimetypes.best == 'application/x-bittorrent':
+    metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb + '+torrent'])
     return _torrent_repo_verb(repo_image, tag, verb, **kwargs)
 
   # Log the action.
   track_and_log('repo_verb', repo_image.repository, tag=tag, verb=verb, **kwargs)
+  metric_queue.repository_pull.Inc(labelvalues=[namespace, repository, verb])
 
   # Lookup/create the derived image storage for the verb and repo image.
   derived = model.image.find_or_create_derived_storage(repo_image, verb,
@@ -402,4 +404,5 @@ def get_tag_torrent(namespace_name, repo_name, digest):
   except model.BlobDoesNotExist:
     abort(404)
 
+  metric_queue.repository_pull.Inc(labelvalues=[namespace_name, repo_name, 'torrent'])
   return _torrent_for_storage(blob, public_repo)