From 265520d07117a4abe897dd93a2dc40a25380a571 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 11 Jul 2017 13:52:15 +0300 Subject: [PATCH 1/4] Move globalpromstats worker into its own package --- conf/init/service/globalpromstats/run | 2 +- workers/globalpromstats/__init__.py | 0 workers/{ => globalpromstats}/globalpromstats.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 workers/globalpromstats/__init__.py rename workers/{ => globalpromstats}/globalpromstats.py (100%) diff --git a/conf/init/service/globalpromstats/run b/conf/init/service/globalpromstats/run index cd31b67ea..a8f5627cd 100755 --- a/conf/init/service/globalpromstats/run +++ b/conf/init/service/globalpromstats/run @@ -4,6 +4,6 @@ echo 'Starting global prometheus stats worker' QUAYPATH=${QUAYPATH:-"."} cd ${QUAYDIR:-"/"} -PYTHONPATH=$QUAYPATH venv/bin/python -m workers.globalpromstats +PYTHONPATH=$QUAYPATH venv/bin/python -m workers.globalpromstats.globalpromstats echo 'Global prometheus stats exited' diff --git a/workers/globalpromstats/__init__.py b/workers/globalpromstats/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/workers/globalpromstats.py b/workers/globalpromstats/globalpromstats.py similarity index 100% rename from workers/globalpromstats.py rename to workers/globalpromstats/globalpromstats.py From 13922fd19491f5a00c9d72c05d40c4561dc3672b Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 11 Jul 2017 13:52:35 +0300 Subject: [PATCH 2/4] Remove unused imports --- workers/globalpromstats/globalpromstats.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/workers/globalpromstats/globalpromstats.py b/workers/globalpromstats/globalpromstats.py index 5092a9b19..dd4cabf74 100644 --- a/workers/globalpromstats/globalpromstats.py +++ b/workers/globalpromstats/globalpromstats.py @@ -1,11 +1,9 @@ import logging import time -import features from app import app, metric_queue from data.database import UseThenDisconnect from data import model -from data.model.image import total_image_count from util.locking import GlobalLock, LockNotAcquiredException from util.log import logfile_path from workers.worker import Worker From 0629a13da2c47ba884c11b7942020dd130a62eb4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 11 Jul 2017 13:57:20 +0300 Subject: [PATCH 3/4] Add very basic test for stats worker --- .../globalpromstats/test/test_globalpromstats.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 workers/globalpromstats/test/test_globalpromstats.py diff --git a/workers/globalpromstats/test/test_globalpromstats.py b/workers/globalpromstats/test/test_globalpromstats.py new file mode 100644 index 000000000..3256f251f --- /dev/null +++ b/workers/globalpromstats/test/test_globalpromstats.py @@ -0,0 +1,15 @@ +from mock import patch, Mock + +from workers.globalpromstats.globalpromstats import GlobalPrometheusStatsWorker + +from test.fixtures import * + +def test_reportstats(initialized_db): + mock = Mock() + with patch('workers.globalpromstats.globalpromstats.metric_queue', mock): + worker = GlobalPrometheusStatsWorker() + worker._report_stats() + + mock.repository_count.Set.assert_called_once() + mock.org_count.Set.assert_called_once() + mock.robot_count.Set.assert_called_once() From 8e179cb865cd70b3610c524a3cd5a239001d58bf Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 11 Jul 2017 14:01:07 +0300 Subject: [PATCH 4/4] Switch globalpromstats worker to use a data interface --- workers/globalpromstats/globalpromstats.py | 10 ++++---- workers/globalpromstats/models_interface.py | 27 +++++++++++++++++++++ workers/globalpromstats/models_pre_oci.py | 18 ++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 workers/globalpromstats/models_interface.py create mode 100644 workers/globalpromstats/models_pre_oci.py diff --git a/workers/globalpromstats/globalpromstats.py b/workers/globalpromstats/globalpromstats.py index dd4cabf74..e0ea4676c 100644 --- a/workers/globalpromstats/globalpromstats.py +++ b/workers/globalpromstats/globalpromstats.py @@ -3,7 +3,7 @@ import time from app import app, metric_queue from data.database import UseThenDisconnect -from data import model +from workers.globalpromstats.models_pre_oci import pre_oci_model as model from util.locking import GlobalLock, LockNotAcquiredException from util.log import logfile_path from workers.worker import Worker @@ -34,12 +34,12 @@ class GlobalPrometheusStatsWorker(Worker): logger.debug('Reporting global stats') with UseThenDisconnect(app.config): # Repository count. - metric_queue.repository_count.Set(model.repository.get_repository_count()) + metric_queue.repository_count.Set(model.get_repository_count()) # User counts. - metric_queue.user_count.Set(model.user.get_active_user_count()) - metric_queue.org_count.Set(model.organization.get_active_org_count()) - metric_queue.robot_count.Set(model.user.get_robot_count()) + metric_queue.user_count.Set(model.get_active_user_count()) + metric_queue.org_count.Set(model.get_active_org_count()) + metric_queue.robot_count.Set(model.get_robot_count()) def main(): diff --git a/workers/globalpromstats/models_interface.py b/workers/globalpromstats/models_interface.py new file mode 100644 index 000000000..0abaa4dfb --- /dev/null +++ b/workers/globalpromstats/models_interface.py @@ -0,0 +1,27 @@ +from abc import ABCMeta, abstractmethod +from six import add_metaclass + +@add_metaclass(ABCMeta) +class GlobalPromStatsWorkerDataInterface(object): + """ + Interface that represents all data store interactions required by the global prom stats worker. + """ + @abstractmethod + def get_repository_count(self): + """ Returns the number of repositories in the database. """ + pass + + @abstractmethod + def get_active_user_count(self): + """ Returns the number of active users in the database. """ + pass + + @abstractmethod + def get_active_org_count(self): + """ Returns the number of active organizations in the database. """ + pass + + @abstractmethod + def get_robot_count(self): + """ Returns the number of robots in the database. """ + pass diff --git a/workers/globalpromstats/models_pre_oci.py b/workers/globalpromstats/models_pre_oci.py new file mode 100644 index 000000000..cef40291a --- /dev/null +++ b/workers/globalpromstats/models_pre_oci.py @@ -0,0 +1,18 @@ +from data import model + +from workers.globalpromstats.models_interface import GlobalPromStatsWorkerDataInterface + +class PreOCIModel(GlobalPromStatsWorkerDataInterface): + def get_repository_count(self): + return model.repository.get_repository_count() + + def get_active_user_count(self): + return model.user.get_active_user_count() + + def get_active_org_count(self): + return model.organization.get_active_org_count() + + def get_robot_count(self): + return model.user.get_robot_count() + +pre_oci_model = PreOCIModel()