diff --git a/buildman/component/basecomponent.py b/buildman/component/basecomponent.py index 47781dff5..bd4032776 100644 --- a/buildman/component/basecomponent.py +++ b/buildman/component/basecomponent.py @@ -8,3 +8,6 @@ class BaseComponent(ApplicationSession): self.parent_manager = None self.build_logs = None self.user_files = None + + def kind(self): + raise NotImplementedError \ No newline at end of file diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index d518d3453..f31bf8d34 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -49,6 +49,9 @@ class BuildComponent(BaseComponent): BaseComponent.__init__(self, config, **kwargs) + def kind(self): + return 'builder' + def onConnect(self): self.join(self.builder_realm) diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py index 6583284a8..b49ddd0f3 100644 --- a/buildman/manager/enterprise.py +++ b/buildman/manager/enterprise.py @@ -13,6 +13,9 @@ logger = logging.getLogger(__name__) class DynamicRegistrationComponent(BaseComponent): """ Component session that handles dynamic registration of the builder components. """ + def kind(self): + return 'registration' + def onConnect(self): self.join(REGISTRATION_REALM) @@ -69,6 +72,7 @@ class EnterpriseManager(BaseManager): def build_component_disposed(self, build_component, timed_out): self.build_components.remove(build_component) + self.unregister_component(build_component) def num_workers(self): return len(self.build_components) diff --git a/buildman/server.py b/buildman/server.py index e6d254536..002ccea07 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -1,5 +1,6 @@ import logging import trollius +import json from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory from autobahn.asyncio.websocket import WampWebSocketServerFactory @@ -63,7 +64,20 @@ class BuilderServer(object): @controller_app.route('/status') def status(): - return server._current_status + (running_count, available_count) = server._queue.get_metrics() + + workers = [component for component in server._current_components + if component.kind() == 'builder'] + + data = { + 'status': server._current_status, + 'running_local': server._job_count, + 'running_total': running_count, + 'workers': len(workers), + 'job_total': available_count + } + + return json.dumps(data) self._controller_app = controller_app diff --git a/data/queue.py b/data/queue.py index 5c720eed2..0e93a273f 100644 --- a/data/queue.py +++ b/data/queue.py @@ -41,10 +41,7 @@ class WorkQueue(object): def _name_match_query(self): return '%s%%' % self._canonical_name([self._queue_name] + self._canonical_name_match_list) - def update_metrics(self): - if self._reporter is None: - return - + def get_metrics(self): with self._transaction_factory(db): now = datetime.utcnow() name_match_query = self._name_match_query() @@ -52,9 +49,16 @@ class WorkQueue(object): running_query = self._running_jobs(now, name_match_query) running_count = running_query.distinct().count() - avialable_query = self._available_jobs(now, name_match_query, running_query) - available_count = avialable_query.select(QueueItem.queue_name).distinct().count() + available_query = self._available_jobs(now, name_match_query, running_query) + available_count = available_query.select(QueueItem.queue_name).distinct().count() + return (running_count, available_count) + + def update_metrics(self): + if self._reporter is None: + return + + (running_count, available_count) = self.get_metrics() self._reporter(self._currently_processing, running_count, running_count + available_count) def put(self, canonical_name_list, message, available_after=0, retries_remaining=5): diff --git a/endpoints/realtime.py b/endpoints/realtime.py index cfa3ec7ad..ac2a6c483 100644 --- a/endpoints/realtime.py +++ b/endpoints/realtime.py @@ -5,7 +5,7 @@ from flask import request, Blueprint, abort, Response from flask.ext.login import current_user from auth.auth import require_session_login from endpoints.common import route_show_if -from app import userevents +from app import app, userevents from auth.permissions import SuperUserPermission import features @@ -26,6 +26,11 @@ def ps(): def generator(): while True: + builder_data = app.config['HTTPCLIENT'].get('http://localhost:8686/status', timeout=1) + build_status = {} + if builder_data.status_code == 200: + build_status = json.loads(builder_data.text) + data = { 'count': { 'cpu': psutil.cpu_percent(interval=1, percpu=True), @@ -34,7 +39,8 @@ def ps(): 'connections': len(psutil.net_connections()), 'processes': len(psutil.pids()), 'network': psutil.net_io_counters() - } + }, + 'build': build_status } json_string = json.dumps(data) yield 'data: %s\n\n' % json_string diff --git a/static/directives/ps-usage-graph.html b/static/directives/ps-usage-graph.html index ddba677d0..9407519f0 100644 --- a/static/directives/ps-usage-graph.html +++ b/static/directives/ps-usage-graph.html @@ -1,4 +1,17 @@
+ Build Workers: +
+ +
+ CPU:
diff --git a/static/directives/realtime-area-chart.html b/static/directives/realtime-area-chart.html new file mode 100644 index 000000000..553bd30b1 --- /dev/null +++ b/static/directives/realtime-area-chart.html @@ -0,0 +1,3 @@ +
+
+
\ No newline at end of file diff --git a/static/js/app.js b/static/js/app.js index 7c5fd8d1e..0791b2f40 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -6522,6 +6522,81 @@ quayApp.directive('locationView', function () { }); +quayApp.directive('realtimeAreaChart', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/realtime-area-chart.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'data': '=data', + 'labels': '=labels', + 'colors': '=colors', + 'counter': '=counter' + }, + controller: function($scope, $element) { + var graph = null; + var series = []; + var palette = new Rickshaw.Color.Palette( { scheme: 'spectrum14' } ); + var colors = $scope.colors || []; + + var setupGraph = function() { + for (var i = 0; i < $scope.labels.length; ++i) { + series.push({ + name: $scope.labels[i], + color: i >= colors.length ? palette.color(): $scope.colors[i], + stroke: 'rgba(0,0,0,0.15)', + data: [] + }); + } + + graph = new Rickshaw.Graph( { + element: $element.find('.chart')[0], + renderer: 'area', + stroke: true, + series: series, + min: 0 + }); + }; + + var refresh = function(data) { + if (!data || $scope.counter < 0) { return; } + if (!graph) { + setupGraph(); + } + + for (var i = 0; i < $scope.data.length; ++i) { + series[i].data.push( + {'x': $scope.counter, 'y': $scope.data[i] } + ); + } + + hoverDetail = new Rickshaw.Graph.HoverDetail({ + graph: graph, + xFormatter: function(x) { + return x.toString(); + } + }); + + graph.renderer.unstack = true; + graph.render(); + }; + + $scope.$watch('counter', function() { + refresh($scope.data_raw); + }); + + $scope.$watch('data', function(data) { + $scope.data_raw = data; + refresh($scope.data_raw); + }); + } + }; + return directiveDefinitionObject; +}); + + quayApp.directive('realtimeLineChart', function () { var directiveDefinitionObject = { priority: 0,