diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py index c56830a1c..0ce69e508 100644 --- a/buildman/manager/enterprise.py +++ b/buildman/manager/enterprise.py @@ -25,6 +25,9 @@ class DynamicRegistrationComponent(BaseComponent): logger.debug('Registering new build component+worker with realm %s', realm) return realm + def kind(self): + return 'registration' + class EnterpriseManager(BaseManager): """ Build manager implementation for the Enterprise Registry. """ @@ -82,5 +85,7 @@ class EnterpriseManager(BaseManager): if build_component in self.ready_components: self.ready_components.remove(build_component) + self.unregister_component(build_component) + def num_workers(self): return len(self.all_components) diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py index cfb52f8ad..473e75fb3 100644 --- a/buildman/manager/ephemeral.py +++ b/buildman/manager/ephemeral.py @@ -271,8 +271,6 @@ class EphemeralBuilderManager(BaseManager): def build_component_disposed(self, build_component, timed_out): logger.debug('Calling build_component_disposed.') - - # TODO make it so that I don't have to unregister the component if it timed out self.unregister_component(build_component) @coroutine diff --git a/buildman/server.py b/buildman/server.py index 28db129ad..f6ba9b4bc 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -70,7 +70,8 @@ class BuilderServer(object): @controller_app.route('/status') def status(): - (running_count, available_count) = server._queue.get_metrics() + metrics = server._queue.get_metrics(require_transaction=False) + (running_count, available_not_running_count, available_count) = metrics workers = [component for component in server._current_components if component.kind() == 'builder'] @@ -80,7 +81,7 @@ class BuilderServer(object): 'running_local': server._job_count, 'running_total': running_count, 'workers': len(workers), - 'job_total': available_count + 'job_total': available_count + running_count } return json.dumps(data) diff --git a/data/queue.py b/data/queue.py index 40a94c6e9..c1fb871ad 100644 --- a/data/queue.py +++ b/data/queue.py @@ -6,6 +6,12 @@ from util.morecollections import AttrDict MINIMUM_EXTENSION = timedelta(seconds=20) +class NoopWith: + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + pass class WorkQueue(object): def __init__(self, queue_name, transaction_factory, @@ -49,21 +55,32 @@ class WorkQueue(object): def _item_by_id_for_update(self, queue_id): return db_for_update(QueueItem.select().where(QueueItem.id == queue_id)).get() - def update_metrics(self): - if self._reporter is None: - return - - with self._transaction_factory(db): + def get_metrics(self, require_transaction=True): + guard = self._transaction_factory(db) if require_transaction else NoopWith() + with guard: now = datetime.utcnow() name_match_query = self._name_match_query() running_query = self._running_jobs(now, name_match_query) running_count = running_query.distinct().count() - available_query = self._available_jobs_not_running(now, name_match_query, running_query) + available_query = self._available_jobs(now, name_match_query) available_count = available_query.select(QueueItem.queue_name).distinct().count() - self._reporter(self._currently_processing, running_count, running_count + available_count) + available_not_running_query = self._available_jobs_not_running(now, name_match_query, + running_query) + available_not_running_count = (available_not_running_query.select(QueueItem.queue_name) + .distinct().count()) + + return (running_count, available_not_running_count, available_count) + + def update_metrics(self): + if self._reporter is None: + return + + (running_count, available_not_running_count, available_count) = self.get_metrics() + self._reporter(self._currently_processing, running_count, + running_count + available_not_running_count) def put(self, canonical_name_list, message, available_after=0, retries_remaining=5): """ diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index ee48fdf19..10741d9f3 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -50,10 +50,6 @@ class SuperUserRegistryStatus(ApiResource): @verify_not_prod def get(self): """ Returns the status of the registry. """ - if app.config.get('DEBUGGING', False): - return { - 'status': 'ready' - } # If there is no conf/stack volume, then report that status. if not CONFIG_PROVIDER.volume_exists(): diff --git a/endpoints/realtime.py b/endpoints/realtime.py index 212ca6eee..483a4c76c 100644 --- a/endpoints/realtime.py +++ b/endpoints/realtime.py @@ -47,9 +47,12 @@ def ps(): } json_string = json.dumps(data) yield 'data: %s\n\n' % json_string - time.sleep(0.25) + time.sleep(1) - return Response(generator(), mimetype="text/event-stream") + try: + return Response(generator(), mimetype="text/event-stream") + except: + pass diff --git a/external_libraries.py b/external_libraries.py index 3fa48c44a..3ab6bfd4a 100644 --- a/external_libraries.py +++ b/external_libraries.py @@ -6,7 +6,7 @@ LOCAL_DIRECTORY = 'static/ldn/' EXTERNAL_JS = [ 'code.jquery.com/jquery.js', - 'netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js', + 'netdna.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js', 'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular.min.js', 'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-route.min.js', 'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-sanitize.min.js', @@ -19,7 +19,7 @@ EXTERNAL_JS = [ EXTERNAL_CSS = [ 'netdna.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.css', - 'netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.no-icons.min.css', + 'netdna.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css', 'fonts.googleapis.com/css?family=Source+Sans+Pro:400,700', ] diff --git a/static/css/core-ui.css b/static/css/core-ui.css index d19e23dbc..51a6d57c9 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -705,5 +705,11 @@ } .realtime-area-chart, .realtime-line-chart { - display: inline-block; + margin: 10px; + text-align: center; +} + +.rickshaw_graph { + overflow: hidden; + padding-bottom: 40px; } \ No newline at end of file diff --git a/static/css/quay.css b/static/css/quay.css index e09fce346..2e1c11136 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -4972,3 +4972,9 @@ i.slack-icon { left: 16px; font-size: 28px; } + +.chart-col h4, .chart-col h5 { + display: block; + text-align: center; +} + diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index 6b40f1fd5..7863ab1a4 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -1,5 +1,5 @@
-
+
@@ -289,7 +289,7 @@

Authentication for the registry can be handled by either the registry itself or LDAP. - External authentication providers (such as Github) can be used on top of this choice. + External authentication providers (such as GitHub) can be used on top of this choice.

@@ -339,20 +339,20 @@
- +
- Github (Enterprise) Authentication + GitHub (Enterprise) Authentication

- If enabled, users can use Github or Github Enterprise to authenticate to the registry. + If enabled, users can use GitHub or GitHub Enterprise to authenticate to the registry.

- Note: A registered Github (Enterprise) OAuth application is required. + Note: A registered GitHub (Enterprise) OAuth application is required. View instructions on how to - + Create an OAuth Application in GitHub

@@ -360,21 +360,21 @@
- +
- + - + @@ -402,7 +402,7 @@
Github:GitHub:
Github Endpoint:GitHub Endpoint:
- The Github Enterprise endpoint. Must start with http:// or https://. + The GitHub Enterprise endpoint. Must start with http:// or https://.
-
+
@@ -471,20 +471,20 @@
- +
- Github (Enterprise) Build Triggers + GitHub (Enterprise) Build Triggers

- If enabled, users can setup Github or Github Enterprise triggers to invoke Registry builds. + If enabled, users can setup GitHub or GitHub Enterprise triggers to invoke Registry builds.

- Note: A registered Github (Enterprise) OAuth application (separate from Github Authentication) is required. + Note: A registered GitHub (Enterprise) OAuth application (separate from GitHub Authentication) is required. View instructions on how to - + Create an OAuth Application in GitHub

@@ -492,21 +492,21 @@
- +
- + - + @@ -534,7 +534,7 @@
Github:GitHub:
Github Endpoint:GitHub Endpoint:
- The Github Enterprise endpoint. Must start with http:// or https://. + The GitHub Enterprise endpoint. Must start with http:// or https://.
-
+
diff --git a/static/directives/ps-usage-graph.html b/static/directives/ps-usage-graph.html index 88d4a2ba4..90c67c22b 100644 --- a/static/directives/ps-usage-graph.html +++ b/static/directives/ps-usage-graph.html @@ -1,39 +1,75 @@
-
- Build Workers: -
+ +
+
+ Cannot load build system status. Please restart your container. +
+
+
+

Build Queue

+
+ Running Jobs: {{ data.build.running_total }} | Total Jobs: {{ data.build.job_total }} +
+
+
-
+
+

Local Build Workers

+
+ Local Workers: {{ data.build.workers }} | Working: {{ data.build.running_local }} +
+
+
+
- CPU: -
+ +
+

CPU Usage %

+
+
- Process Count: -
+
+

Process Count

+
+
- Virtual Memory: -
+
+

Virtual Memory %

+
+
- Swap Memory: -
+
+

Swap Memory

+
+
- Network Connections: -
+
+

Network Connections

+
+
- Network Usage: -
+
+

Network Usage (Bytes)

+
+
\ No newline at end of file diff --git a/static/directives/realtime-area-chart.html b/static/directives/realtime-area-chart.html index 553bd30b1..d42d46784 100644 --- a/static/directives/realtime-area-chart.html +++ b/static/directives/realtime-area-chart.html @@ -1,3 +1,6 @@
-
+
+
+
+
\ No newline at end of file diff --git a/static/directives/realtime-line-chart.html b/static/directives/realtime-line-chart.html index a99ba56c4..74e8f748c 100644 --- a/static/directives/realtime-line-chart.html +++ b/static/directives/realtime-line-chart.html @@ -1,3 +1,6 @@
-
+
+
+
+
\ No newline at end of file diff --git a/static/js/app.js b/static/js/app.js index dd8a2e5eb..6cc07552f 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -6602,178 +6602,6 @@ quayApp.directive('locationView', function () { }); -quayApp.directive('realtimeAreaChart', function () { - var directiveDefinitionObject = { - priority: 0, - templateUrl: '/static/directives/realtime-area-chart.html', - replace: false, - transclude: false, - restrict: 'C', - scope: { - 'data': '=data', - 'labels': '=labels', - 'colors': '=colors', - 'counter': '=counter' - }, - controller: function($scope, $element) { - var graph = null; - var series = []; - var palette = new Rickshaw.Color.Palette( { scheme: 'spectrum14' } ); - var colors = $scope.colors || []; - - var setupGraph = function() { - for (var i = 0; i < $scope.labels.length; ++i) { - series.push({ - name: $scope.labels[i], - color: i >= colors.length ? palette.color(): $scope.colors[i], - stroke: 'rgba(0,0,0,0.15)', - data: [] - }); - } - - graph = new Rickshaw.Graph( { - element: $element.find('.chart')[0], - renderer: 'area', - stroke: true, - series: series, - min: 0 - }); - }; - - var refresh = function(data) { - if (!data || $scope.counter < 0) { return; } - if (!graph) { - setupGraph(); - } - - for (var i = 0; i < $scope.data.length; ++i) { - series[i].data.push( - {'x': $scope.counter, 'y': $scope.data[i] } - ); - } - - hoverDetail = new Rickshaw.Graph.HoverDetail({ - graph: graph, - xFormatter: function(x) { - return x.toString(); - } - }); - - graph.renderer.unstack = true; - graph.render(); - }; - - $scope.$watch('counter', function() { - refresh($scope.data_raw); - }); - - $scope.$watch('data', function(data) { - $scope.data_raw = data; - refresh($scope.data_raw); - }); - } - }; - return directiveDefinitionObject; -}); - - -quayApp.directive('realtimeLineChart', function () { - var directiveDefinitionObject = { - priority: 0, - templateUrl: '/static/directives/realtime-line-chart.html', - replace: false, - transclude: false, - restrict: 'C', - scope: { - 'data': '=data', - 'labels': '=labels', - 'counter': '=counter', - 'labelTemplate': '@labelTemplate' - }, - controller: function($scope, $element) { - var graph = null; - var hoverDetail = null; - var series = []; - var counter = 0; - var palette = new Rickshaw.Color.Palette( { scheme: 'spectrum14' } ); - - var setupGraph = function() { - graph = new Rickshaw.Graph({ - element: $element.find('.chart')[0], - renderer: 'line', - series: series, - min: 'auto', - padding: { - 'top': 0.1, - 'left': 0.01, - 'right': 0.01, - 'bottom': 0.1 - } - }); - - hoverDetail = new Rickshaw.Graph.HoverDetail({ - graph: graph, - xFormatter: function(x) { - return x.toString(); - } - }); - }; - - var refresh = function(data) { - if (!data) { return; } - if (!graph) { - setupGraph(); - } - - if (typeof data == 'number') { - data = [data]; - } - - if ($scope.labels) { - data = data.slice(0, $scope.labels.length); - } - - if (series.length == 0){ - for (var i = 0; i < data.length; ++i) { - var title = $scope.labels ? $scope.labels[i] : $scope.labelTemplate.replace('{x}', i + 1); - series.push({ - 'color': palette.color(), - 'data': [], - 'name': title - }) - } - } - - counter++; - - for (var i = 0; i < data.length; ++i) { - var arr = series[i].data; - arr.push({ - 'x': counter, - 'y': data[i] - }) - - if (arr.length > 10) { - series[i].data = arr.slice(arr.length - 10, arr.length); - } - } - - graph.render(); - }; - - $scope.$watch('counter', function(counter) { - refresh($scope.data_raw); - }); - - $scope.$watch('data', function(data) { - $scope.data_raw = data; - }); - } - }; - return directiveDefinitionObject; -}); - - quayApp.directive('psUsageGraph', function () { var directiveDefinitionObject = { priority: 0, @@ -6782,18 +6610,40 @@ quayApp.directive('psUsageGraph', function () { transclude: false, restrict: 'C', scope: { + 'isEnabled': '=isEnabled' }, controller: function($scope, $element) { $scope.counter = -1; $scope.data = null; - var source = new EventSource('/realtime/ps'); - source.onmessage = function(e) { - $scope.$apply(function() { - $scope.counter++; - $scope.data = JSON.parse(e.data); - }); + var source = null; + + var connect = function() { + if (source) { return; } + source = new EventSource('/realtime/ps'); + source.onmessage = function(e) { + $scope.$apply(function() { + $scope.counter++; + $scope.data = JSON.parse(e.data); + }); + }; }; + + var disconnect = function() { + if (!source) { return; } + source.close(); + source = null; + }; + + $scope.$watch('isEnabled', function(value) { + if (value) { + connect(); + } else { + disconnect(); + } + }); + + $scope.$on("$destroy", disconnect); } }; return directiveDefinitionObject; diff --git a/static/js/controllers/superuser.js b/static/js/controllers/superuser.js index ddaee7d5c..f867cd43b 100644 --- a/static/js/controllers/superuser.js +++ b/static/js/controllers/superuser.js @@ -17,6 +17,11 @@ function SuperUserAdminCtrl($scope, $timeout, ApiService, Features, UserService, $scope.pollChannel = null; $scope.logsScrolled = false; $scope.csrf_token = encodeURIComponent(window.__token); + $scope.dashboardActive = false; + + $scope.setDashboardActive = function(active) { + $scope.dashboardActive = active; + }; $scope.configurationSaved = function() { $scope.requiresRestart = true; diff --git a/static/js/core-config-setup.js b/static/js/core-config-setup.js index 0b22da9a2..569169201 100644 --- a/static/js/core-config-setup.js +++ b/static/js/core-config-setup.js @@ -307,10 +307,10 @@ angular.module("core-config-setup", ['angularFileUpload']) if (!value) { return; } ApiService.scGetConfig().then(function(resp) { - $scope.config = resp['config']; + $scope.config = resp['config'] || {}; initializeMappedLogic($scope.config); $scope.mapped['$hasChanges'] = false; - }); + }, ApiService.errorDisplay('Could not load config')); }); } }; diff --git a/static/js/core-ui.js b/static/js/core-ui.js index fc1ea029c..ed5e982e5 100644 --- a/static/js/core-ui.js +++ b/static/js/core-ui.js @@ -270,9 +270,18 @@ angular.module("core-ui", []) 'tabActive': '@tabActive', 'tabTitle': '@tabTitle', 'tabTarget': '@tabTarget', - 'tabInit': '&tabInit' + 'tabInit': '&tabInit', + 'tabShown': '&tabShown', + 'tabHidden': '&tabHidden' }, controller: function($rootScope, $scope, $element) { + $element.find('a[data-toggle="tab"]').on('hidden.bs.tab', function (e) { + $scope.tabHidden({}); + }); + + $element.find('a[data-toggle="tab"]').on('shown.bs.tab', function (e) { + $scope.tabShown({}); + }); } }; return directiveDefinitionObject; @@ -297,6 +306,236 @@ angular.module("core-ui", []) return directiveDefinitionObject; }) + .directive('realtimeAreaChart', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/realtime-area-chart.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'data': '=data', + 'labels': '=labels', + 'colors': '=colors', + 'counter': '=counter' + }, + controller: function($scope, $element) { + var graph = null; + var series = []; + var palette = new Rickshaw.Color.Palette( { scheme: 'spectrum14' } ); + var colors = $scope.colors || []; + + var setupGraph = function() { + for (var i = 0; i < $scope.labels.length; ++i) { + series.push({ + name: $scope.labels[i], + color: i >= colors.length ? palette.color(): $scope.colors[i], + stroke: 'rgba(0,0,0,0.15)', + data: [] + }); + } + + var options = { + element: $element.find('.chart')[0], + renderer: 'area', + stroke: true, + series: series, + min: 0, + padding: { + 'top': 0.3, + 'left': 0, + 'right': 0, + 'bottom': 0.3 + } + }; + + if ($scope.minimum != null) { + options['min'] = $scope.minimum == 'auto' ? 'auto' : $scope.minimum * 1; + } else { + options['min'] = 0; + } + + if ($scope.maximum != null) { + options['max'] = $scope.maximum == 'auto' ? 'auto' : $scope.maximum * 1; + } + + graph = new Rickshaw.Graph(options); + + xaxes = new Rickshaw.Graph.Axis.Time({ + graph: graph, + timeFixture: new Rickshaw.Fixtures.Time.Local() + }); + + yaxes = new Rickshaw.Graph.Axis.Y({ + graph: graph, + tickFormat: Rickshaw.Fixtures.Number.formatKMBT + }); + + hoverDetail = new Rickshaw.Graph.HoverDetail({ + graph: graph, + xFormatter: function(x) { + return new Date(x * 1000).toString(); + } + }); + }; + + var refresh = function(data) { + if (!data || $scope.counter < 0) { return; } + if (!graph) { + setupGraph(); + } + + var timecode = new Date().getTime() / 1000; + for (var i = 0; i < $scope.data.length; ++i) { + var arr = series[i].data; + arr.push( + {'x': timecode, 'y': $scope.data[i] } + ); + + if (arr.length > 10) { + series[i].data = arr.slice(arr.length - 10, arr.length); + } + } + + graph.renderer.unstack = true; + graph.update(); + }; + + $scope.$watch('counter', function() { + refresh($scope.data_raw); + }); + + $scope.$watch('data', function(data) { + $scope.data_raw = data; + refresh($scope.data_raw); + }); + } + }; + return directiveDefinitionObject; + }) + + + .directive('realtimeLineChart', function () { + var directiveDefinitionObject = { + priority: 0, + templateUrl: '/static/directives/realtime-line-chart.html', + replace: false, + transclude: false, + restrict: 'C', + scope: { + 'data': '=data', + 'labels': '=labels', + 'counter': '=counter', + 'labelTemplate': '@labelTemplate', + 'minimum': '@minimum', + 'maximum': '@maximum' + }, + controller: function($scope, $element) { + var graph = null; + var xaxes = null; + var yaxes = null; + var hoverDetail = null; + var series = []; + var counter = 0; + var palette = new Rickshaw.Color.Palette( { scheme: 'spectrum14' } ); + + var setupGraph = function() { + var options = { + element: $element.find('.chart')[0], + renderer: 'line', + series: series, + padding: { + 'top': 0.3, + 'left': 0, + 'right': 0, + 'bottom': 0.3 + } + }; + + if ($scope.minimum != null) { + options['min'] = $scope.minimum == 'auto' ? 'auto' : $scope.minimum * 1; + } else { + options['min'] = 0; + } + + if ($scope.maximum != null) { + options['max'] = $scope.maximum == 'auto' ? 'auto' : $scope.maximum * 1; + } + + graph = new Rickshaw.Graph(options); + xaxes = new Rickshaw.Graph.Axis.Time({ + graph: graph, + timeFixture: new Rickshaw.Fixtures.Time.Local() + }); + + yaxes = new Rickshaw.Graph.Axis.Y({ + graph: graph, + tickFormat: Rickshaw.Fixtures.Number.formatKMBT + }); + + hoverDetail = new Rickshaw.Graph.HoverDetail({ + graph: graph, + xFormatter: function(x) { + return new Date(x * 1000).toString(); + } + }); + }; + + var refresh = function(data) { + if (data == null) { return; } + if (!graph) { + setupGraph(); + } + + if (typeof data == 'number') { + data = [data]; + } + + if ($scope.labels) { + data = data.slice(0, $scope.labels.length); + } + + if (series.length == 0){ + for (var i = 0; i < data.length; ++i) { + var title = $scope.labels ? $scope.labels[i] : $scope.labelTemplate.replace('{x}', i + 1); + series.push({ + 'color': palette.color(), + 'data': [], + 'name': title + }) + } + } + + counter++; + var timecode = new Date().getTime() / 1000; + + for (var i = 0; i < data.length; ++i) { + var arr = series[i].data; + arr.push({ + 'x': timecode, + 'y': data[i] + }) + + if (arr.length > 10) { + series[i].data = arr.slice(arr.length - 10, arr.length); + } + } + + graph.update(); + }; + + $scope.$watch('counter', function(counter) { + refresh($scope.data_raw); + }); + + $scope.$watch('data', function(data) { + $scope.data_raw = data; + }); + } + }; + return directiveDefinitionObject; + }) + .directive('corStepBar', function() { var directiveDefinitionObject = { priority: 4, diff --git a/static/partials/super-user.html b/static/partials/super-user.html index 2e72bfe85..6d150634d 100644 --- a/static/partials/super-user.html +++ b/static/partials/super-user.html @@ -20,7 +20,8 @@ tab-target="#users" tab-init="loadUsers()"> - + @@ -47,7 +48,7 @@
-
+