diff --git a/buildman/component/basecomponent.py b/buildman/component/basecomponent.py index 47781dff5..bd4032776 100644 --- a/buildman/component/basecomponent.py +++ b/buildman/component/basecomponent.py @@ -8,3 +8,6 @@ class BaseComponent(ApplicationSession): self.parent_manager = None self.build_logs = None self.user_files = None + + def kind(self): + raise NotImplementedError \ No newline at end of file diff --git a/buildman/component/buildcomponent.py b/buildman/component/buildcomponent.py index e20ee822d..647161190 100644 --- a/buildman/component/buildcomponent.py +++ b/buildman/component/buildcomponent.py @@ -49,6 +49,9 @@ class BuildComponent(BaseComponent): BaseComponent.__init__(self, config, **kwargs) + def kind(self): + return 'builder' + def onConnect(self): self.join(self.builder_realm) diff --git a/buildman/manager/enterprise.py b/buildman/manager/enterprise.py index c56830a1c..0ce69e508 100644 --- a/buildman/manager/enterprise.py +++ b/buildman/manager/enterprise.py @@ -25,6 +25,9 @@ class DynamicRegistrationComponent(BaseComponent): logger.debug('Registering new build component+worker with realm %s', realm) return realm + def kind(self): + return 'registration' + class EnterpriseManager(BaseManager): """ Build manager implementation for the Enterprise Registry. """ @@ -82,5 +85,7 @@ class EnterpriseManager(BaseManager): if build_component in self.ready_components: self.ready_components.remove(build_component) + self.unregister_component(build_component) + def num_workers(self): return len(self.all_components) diff --git a/buildman/manager/ephemeral.py b/buildman/manager/ephemeral.py index cfb52f8ad..473e75fb3 100644 --- a/buildman/manager/ephemeral.py +++ b/buildman/manager/ephemeral.py @@ -271,8 +271,6 @@ class EphemeralBuilderManager(BaseManager): def build_component_disposed(self, build_component, timed_out): logger.debug('Calling build_component_disposed.') - - # TODO make it so that I don't have to unregister the component if it timed out self.unregister_component(build_component) @coroutine diff --git a/buildman/server.py b/buildman/server.py index ebbc558bb..f6ba9b4bc 100644 --- a/buildman/server.py +++ b/buildman/server.py @@ -1,5 +1,6 @@ import logging import trollius +import json from autobahn.asyncio.wamp import RouterFactory, RouterSessionFactory from autobahn.asyncio.websocket import WampWebSocketServerFactory @@ -69,7 +70,21 @@ class BuilderServer(object): @controller_app.route('/status') def status(): - return server._current_status + metrics = server._queue.get_metrics(require_transaction=False) + (running_count, available_not_running_count, available_count) = metrics + + workers = [component for component in server._current_components + if component.kind() == 'builder'] + + data = { + 'status': server._current_status, + 'running_local': server._job_count, + 'running_total': running_count, + 'workers': len(workers), + 'job_total': available_count + running_count + } + + return json.dumps(data) self._controller_app = controller_app diff --git a/data/model/legacy.py b/data/model/legacy.py index ab80aa1a1..fe675767e 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -170,8 +170,7 @@ def _create_user(username, email): pass try: - new_user = User.create(username=username, email=email) - return new_user + return User.create(username=username, email=email) except Exception as ex: raise DataModelException(ex.message) diff --git a/data/queue.py b/data/queue.py index 40a94c6e9..c1fb871ad 100644 --- a/data/queue.py +++ b/data/queue.py @@ -6,6 +6,12 @@ from util.morecollections import AttrDict MINIMUM_EXTENSION = timedelta(seconds=20) +class NoopWith: + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + pass class WorkQueue(object): def __init__(self, queue_name, transaction_factory, @@ -49,21 +55,32 @@ class WorkQueue(object): def _item_by_id_for_update(self, queue_id): return db_for_update(QueueItem.select().where(QueueItem.id == queue_id)).get() - def update_metrics(self): - if self._reporter is None: - return - - with self._transaction_factory(db): + def get_metrics(self, require_transaction=True): + guard = self._transaction_factory(db) if require_transaction else NoopWith() + with guard: now = datetime.utcnow() name_match_query = self._name_match_query() running_query = self._running_jobs(now, name_match_query) running_count = running_query.distinct().count() - available_query = self._available_jobs_not_running(now, name_match_query, running_query) + available_query = self._available_jobs(now, name_match_query) available_count = available_query.select(QueueItem.queue_name).distinct().count() - self._reporter(self._currently_processing, running_count, running_count + available_count) + available_not_running_query = self._available_jobs_not_running(now, name_match_query, + running_query) + available_not_running_count = (available_not_running_query.select(QueueItem.queue_name) + .distinct().count()) + + return (running_count, available_not_running_count, available_count) + + def update_metrics(self): + if self._reporter is None: + return + + (running_count, available_not_running_count, available_count) = self.get_metrics() + self._reporter(self._currently_processing, running_count, + running_count + available_not_running_count) def put(self, canonical_name_list, message, available_after=0, retries_remaining=5): """ diff --git a/endpoints/api/__init__.py b/endpoints/api/__init__.py index 377834002..0252851a1 100644 --- a/endpoints/api/__init__.py +++ b/endpoints/api/__init__.py @@ -334,7 +334,11 @@ def validate_json_request(schema_name): def wrapped(self, *args, **kwargs): schema = self.schemas[schema_name] try: - validate(request.get_json(), schema) + json_data = request.get_json() + if json_data is None: + raise InvalidRequest('Missing JSON body') + + validate(json_data, schema) return func(self, *args, **kwargs) except ValidationError as ex: raise InvalidRequest(ex.message) diff --git a/endpoints/api/suconfig.py b/endpoints/api/suconfig.py index daaba41ce..10741d9f3 100644 --- a/endpoints/api/suconfig.py +++ b/endpoints/api/suconfig.py @@ -50,6 +50,7 @@ class SuperUserRegistryStatus(ApiResource): @verify_not_prod def get(self): """ Returns the status of the registry. """ + # If there is no conf/stack volume, then report that status. if not CONFIG_PROVIDER.volume_exists(): return { diff --git a/endpoints/realtime.py b/endpoints/realtime.py index 9f1d5c44f..44b806ce1 100644 --- a/endpoints/realtime.py +++ b/endpoints/realtime.py @@ -4,13 +4,62 @@ import json from flask import request, Blueprint, abort, Response from flask.ext.login import current_user from auth.auth import require_session_login -from app import userevents +from endpoints.common import route_show_if +from app import app, userevents +from auth.permissions import SuperUserPermission + +import features +import psutil +import time logger = logging.getLogger(__name__) realtime = Blueprint('realtime', __name__) +@realtime.route("/ps") +@route_show_if(features.SUPER_USERS) +@require_session_login +def ps(): + if not SuperUserPermission().can(): + abort(403) + + def generator(): + while True: + build_status = {} + try: + builder_data = app.config['HTTPCLIENT'].get('http://localhost:8686/status', timeout=1) + if builder_data.status_code == 200: + build_status = json.loads(builder_data.text) + except: + pass + + try: + data = { + 'count': { + 'cpu': psutil.cpu_percent(interval=1, percpu=True), + 'virtual_mem': psutil.virtual_memory(), + 'swap_mem': psutil.swap_memory(), + 'connections': len(psutil.net_connections()), + 'processes': len(psutil.pids()), + 'network': psutil.net_io_counters() + }, + 'build': build_status + } + except psutil.AccessDenied: + data = {} + + json_string = json.dumps(data) + yield 'data: %s\n\n' % json_string + time.sleep(1) + + try: + return Response(generator(), mimetype="text/event-stream") + except: + pass + + + @realtime.route("/user/") @require_session_login def index(): diff --git a/external_libraries.py b/external_libraries.py index 3fa48c44a..3ab6bfd4a 100644 --- a/external_libraries.py +++ b/external_libraries.py @@ -6,7 +6,7 @@ LOCAL_DIRECTORY = 'static/ldn/' EXTERNAL_JS = [ 'code.jquery.com/jquery.js', - 'netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js', + 'netdna.bootstrapcdn.com/bootstrap/3.3.2/js/bootstrap.min.js', 'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular.min.js', 'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-route.min.js', 'ajax.googleapis.com/ajax/libs/angularjs/1.2.9/angular-sanitize.min.js', @@ -19,7 +19,7 @@ EXTERNAL_JS = [ EXTERNAL_CSS = [ 'netdna.bootstrapcdn.com/font-awesome/4.2.0/css/font-awesome.css', - 'netdna.bootstrapcdn.com/bootstrap/3.0.0/css/bootstrap.no-icons.min.css', + 'netdna.bootstrapcdn.com/bootstrap/3.3.2/css/bootstrap.min.css', 'fonts.googleapis.com/css?family=Source+Sans+Pro:400,700', ] diff --git a/requirements-nover.txt b/requirements-nover.txt index 9b8707870..b81936ec7 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -47,3 +47,4 @@ pyOpenSSL pygpgme cachetools mock +psutil \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4e51c6245..ee41fcc56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ mixpanel-py==3.2.1 mock==1.0.1 paramiko==1.15.2 peewee==2.4.7 +psutil==2.2.1 psycopg2==2.5.4 py-bcrypt==0.4 pycrypto==2.6.1 diff --git a/static/css/core-ui.css b/static/css/core-ui.css index a89f07f39..2012129c1 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -702,4 +702,19 @@ .co-alert .co-step-bar { float: right; margin-top: 6px; +} + +.realtime-area-chart, .realtime-line-chart { + margin: 10px; + text-align: center; +} + +.rickshaw_graph { + overflow: hidden; + padding-bottom: 40px; +} + +.cor-container { + padding-left: 15px; + padding-right: 15px; } \ No newline at end of file diff --git a/static/css/quay.css b/static/css/quay.css index e09fce346..4fc72cc49 100644 --- a/static/css/quay.css +++ b/static/css/quay.css @@ -3,6 +3,10 @@ margin: 0; } +.btn { + outline: none !important; +} + @media (max-width: 410px) { .olrk-normal { display: none; @@ -159,7 +163,7 @@ nav.navbar-default .navbar-nav>li>a.active { .notification-view-element .circle { position: absolute; - top: 14px; + top: 15px; left: 0px; width: 12px; @@ -179,13 +183,13 @@ nav.navbar-default .navbar-nav>li>a.active { margin-bottom: 4px; } -.notification-view-element .container { +.notification-view-element .notification-content { padding: 10px; border-radius: 6px; margin-left: 16px; } -.notification-view-element .container:hover { +.notification-view-element .notification-content:hover { background: rgba(66, 139, 202, 0.1); } @@ -1140,59 +1144,59 @@ i.toggle-icon:hover { } .visible-sm-inline { - display: none; + display: none !important; } .visible-md-inline { - display: none; + display: none !important; } .hidden-sm-inline { - display: inline; + display: inline !important; } .hidden-xs-inline { - display: inline; + display: inline !important; } @media (min-width: 991px) { .visible-md-inline { - display: inline; + display: inline !important; } } @media (max-width: 991px) and (min-width: 768px) { .visible-sm-inline { - display: inline; + display: inline !important; } .hidden-sm-inline { - display: none; + display: none !important; } } @media (max-width: 700px) { .hidden-xs-inline { - display: none; + display: none !important; } } .visible-xl { - display: none; + display: none !important; } .visible-xl-inline { - display: none; + display: none !important; } @media (min-width: 1200px) { .visible-xl { - display: block; + display: block !important; } .visible-xl-inline { - display: inline-block; + display: inline-block !important; } } @@ -1413,6 +1417,10 @@ i.toggle-icon:hover { background: transparent; } +.jumbotron p { + font-size: 100%; +} + .jumbotron .disclaimer-link { font-size: .3em; vertical-align: 23px; @@ -1483,6 +1491,8 @@ i.toggle-icon:hover { .landing-content { z-index: 2; + padding-left: 20px; + padding-right: 20px; } .landing .call-to-action i.fa { @@ -3762,7 +3772,7 @@ p.editable:hover i { text-align: center; position: relative; color: white; - left: -42px; + left: -38px; top: -9px; font-weight: bold; font-size: .4em; @@ -3772,23 +3782,6 @@ p.editable:hover i { margin-bottom: 40px; } -.landing .social-alternate { - color: #777; - font-size: 2em; - margin-left: 43px; - line-height: 1em; -} - -.landing .social-alternate .inner-text { - text-align: center; - position: relative; - color: white; - left: -43px; - top: -9px; - font-weight: bold; - font-size: .4em; -} - .contact-options { margin-top: 60px; } @@ -4972,3 +4965,9 @@ i.slack-icon { left: 16px; font-size: 28px; } + +.chart-col h4, .chart-col h5 { + display: block; + text-align: center; +} + diff --git a/static/directives/application-manager.html b/static/directives/application-manager.html index 824fd36f7..ba7fb90ae 100644 --- a/static/directives/application-manager.html +++ b/static/directives/application-manager.html @@ -1,7 +1,7 @@