From 3b496e27590b4b0e113043d274e6616c278fd840 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 12 Jul 2017 15:57:02 +0300 Subject: [PATCH 1/4] Move serverkeyworker into its own package --- .../service/{service_key_worker => servicekeyworker}/log/run | 0 conf/init/service/{service_key_worker => servicekeyworker}/run | 2 +- workers/servicekeyworker/__init__.py | 0 .../servicekeyworker.py} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename conf/init/service/{service_key_worker => servicekeyworker}/log/run (100%) rename conf/init/service/{service_key_worker => servicekeyworker}/run (59%) create mode 100644 workers/servicekeyworker/__init__.py rename workers/{service_key_worker.py => servicekeyworker/servicekeyworker.py} (100%) diff --git a/conf/init/service/service_key_worker/log/run b/conf/init/service/servicekeyworker/log/run similarity index 100% rename from conf/init/service/service_key_worker/log/run rename to conf/init/service/servicekeyworker/log/run diff --git a/conf/init/service/service_key_worker/run b/conf/init/service/servicekeyworker/run similarity index 59% rename from conf/init/service/service_key_worker/run rename to conf/init/service/servicekeyworker/run index 470913439..f1b9635e5 100755 --- a/conf/init/service/service_key_worker/run +++ b/conf/init/service/servicekeyworker/run @@ -4,6 +4,6 @@ echo 'Starting service key worker' QUAYPATH=${QUAYPATH:-"."} cd ${QUAYDIR:-"/"} -PYTHONPATH=$QUAYPATH venv/bin/python -m workers.service_key_worker 2>&1 +PYTHONPATH=$QUAYPATH venv/bin/python -m workers.servicekeyworker.servicekeyworker 2>&1 echo 'Service key worker exited' diff --git a/workers/servicekeyworker/__init__.py b/workers/servicekeyworker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/workers/service_key_worker.py b/workers/servicekeyworker/servicekeyworker.py similarity index 100% rename from workers/service_key_worker.py rename to workers/servicekeyworker/servicekeyworker.py From 0afc222214df4b8b24cfa564d05afb5e2179d5ce Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 12 Jul 2017 16:19:30 +0300 Subject: [PATCH 2/4] Add basic unit test for the servicekeyworker --- workers/servicekeyworker/servicekeyworker.py | 6 ++--- .../test/test_servicekeyworker.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 workers/servicekeyworker/test/test_servicekeyworker.py diff --git a/workers/servicekeyworker/servicekeyworker.py b/workers/servicekeyworker/servicekeyworker.py index adbcf5465..8539699a0 100644 --- a/workers/servicekeyworker/servicekeyworker.py +++ b/workers/servicekeyworker/servicekeyworker.py @@ -10,12 +10,12 @@ logger = logging.getLogger(__name__) class ServiceKeyWorker(Worker): def __init__(self): super(ServiceKeyWorker, self).__init__() - self.add_operation(self._refresh_service_keys, + self.add_operation(self._refresh_service_key, app.config.get('INSTANCE_SERVICE_KEY_REFRESH', 60)*60) - def _refresh_service_keys(self): + def _refresh_service_key(self): """ - Refreshes active service keys so they don't get garbage collected. + Refreshes the instance's active service key so it doesn't get garbage collected. """ expiration = timedelta(minutes=instance_keys.service_key_expiration) logger.debug('Starting refresh of automatic service keys') diff --git a/workers/servicekeyworker/test/test_servicekeyworker.py b/workers/servicekeyworker/test/test_servicekeyworker.py new file mode 100644 index 000000000..40ddcf139 --- /dev/null +++ b/workers/servicekeyworker/test/test_servicekeyworker.py @@ -0,0 +1,23 @@ +from datetime import datetime, timedelta +from mock import patch + +from data import model +from workers.servicekeyworker.servicekeyworker import ServiceKeyWorker +from util.morecollections import AttrDict + +from test.fixtures import * + +def test_refresh_service_key(initialized_db): + # Create a service key for testing. + test_key = model.service_keys.create_service_key('test', 'somekid', 'quay', '', {}, + datetime.now() + timedelta(minutes=10)) + + instance_keys = AttrDict(dict(local_key_id=test_key.kid, service_key_expiration=10)) + + with patch('workers.servicekeyworker.servicekeyworker.instance_keys', instance_keys): + worker = ServiceKeyWorker() + worker._refresh_service_key() + + # Ensure the key's expiration was changed. + updated_key = model.service_keys.get_service_key(test_key.kid, approved_only=False) + assert updated_key.expiration_date > test_key.expiration_date From 932db23a5ca00206945908398140f586baecb9ce Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 12 Jul 2017 16:37:32 +0300 Subject: [PATCH 3/4] Change servicekeyworker to use a data interface --- workers/servicekeyworker/models_interface.py | 27 +++++++++++++++++++ workers/servicekeyworker/models_pre_oci.py | 19 +++++++++++++ workers/servicekeyworker/servicekeyworker.py | 4 +-- .../test/test_servicekeyworker.py | 11 ++++---- 4 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 workers/servicekeyworker/models_interface.py create mode 100644 workers/servicekeyworker/models_pre_oci.py diff --git a/workers/servicekeyworker/models_interface.py b/workers/servicekeyworker/models_interface.py new file mode 100644 index 000000000..6b9292b91 --- /dev/null +++ b/workers/servicekeyworker/models_interface.py @@ -0,0 +1,27 @@ +from abc import ABCMeta, abstractmethod +from six import add_metaclass + +@add_metaclass(ABCMeta) +class ServiceKeyWorkerDataInterface(object): + """ + Interface that represents all data store interactions required by the service key worker. + """ + + @abstractmethod + def set_key_expiration(self, key_id, expiration_date): + """ Sets the expiration date of the service key with the given key ID to that given. """ + pass + + @abstractmethod + def create_service_key_for_testing(self, expiration): + """ Creates a service key for testing with the given expiration. Returns the KID for + key. + """ + pass + + @abstractmethod + def get_service_key_expiration(self, key_id): + """ Returns the expiration date for the key with the given ID. If the key doesn't exist or + does not have an expiration, returns None. + """ + pass diff --git a/workers/servicekeyworker/models_pre_oci.py b/workers/servicekeyworker/models_pre_oci.py new file mode 100644 index 000000000..906572397 --- /dev/null +++ b/workers/servicekeyworker/models_pre_oci.py @@ -0,0 +1,19 @@ +from data import model +from workers.servicekeyworker.models_interface import ServiceKeyWorkerDataInterface + +class PreOCIModel(ServiceKeyWorkerDataInterface): + def set_key_expiration(self, kid, expiration_date): + model.service_keys.set_key_expiration(kid, expiration_date) + + def create_service_key_for_testing(self, expiration): + key = model.service_keys.create_service_key('test', 'somekid', 'quay', '', {}, expiration) + return key.kid + + def get_service_key_expiration(self, kid): + try: + key = model.service_keys.get_service_key(kid, approved_only=False) + return key.expiration_date + except model.ServiceKeyDoesNotExist: + return None + +pre_oci_model = PreOCIModel() diff --git a/workers/servicekeyworker/servicekeyworker.py b/workers/servicekeyworker/servicekeyworker.py index 8539699a0..90ddd0e33 100644 --- a/workers/servicekeyworker/servicekeyworker.py +++ b/workers/servicekeyworker/servicekeyworker.py @@ -2,7 +2,7 @@ import logging from datetime import datetime, timedelta from app import app, instance_keys -from data.model.service_keys import set_key_expiration +from workers.servicekeyworker.models_pre_oci import pre_oci_model as model from workers.worker import Worker logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ class ServiceKeyWorker(Worker): """ expiration = timedelta(minutes=instance_keys.service_key_expiration) logger.debug('Starting refresh of automatic service keys') - set_key_expiration(instance_keys.local_key_id, datetime.now() + expiration) + model.set_key_expiration(instance_keys.local_key_id, datetime.now() + expiration) logger.debug('Finished refresh of automatic service keys') if __name__ == "__main__": diff --git a/workers/servicekeyworker/test/test_servicekeyworker.py b/workers/servicekeyworker/test/test_servicekeyworker.py index 40ddcf139..7db38c62b 100644 --- a/workers/servicekeyworker/test/test_servicekeyworker.py +++ b/workers/servicekeyworker/test/test_servicekeyworker.py @@ -6,18 +6,17 @@ from workers.servicekeyworker.servicekeyworker import ServiceKeyWorker from util.morecollections import AttrDict from test.fixtures import * +from workers.servicekeyworker.models_pre_oci import pre_oci_model as model def test_refresh_service_key(initialized_db): # Create a service key for testing. - test_key = model.service_keys.create_service_key('test', 'somekid', 'quay', '', {}, - datetime.now() + timedelta(minutes=10)) - - instance_keys = AttrDict(dict(local_key_id=test_key.kid, service_key_expiration=10)) + original_expiration = datetime.now() + timedelta(minutes=10) + test_key_kid = model.create_service_key_for_testing(original_expiration) + instance_keys = AttrDict(dict(local_key_id=test_key_kid, service_key_expiration=10)) with patch('workers.servicekeyworker.servicekeyworker.instance_keys', instance_keys): worker = ServiceKeyWorker() worker._refresh_service_key() # Ensure the key's expiration was changed. - updated_key = model.service_keys.get_service_key(test_key.kid, approved_only=False) - assert updated_key.expiration_date > test_key.expiration_date + assert model.get_service_key_expiration(test_key_kid) > original_expiration From 4ed73d247b04ffb64ac0437b43f6ad5d02708da5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 12 Jul 2017 16:37:51 +0300 Subject: [PATCH 4/4] yapf --- workers/servicekeyworker/models_interface.py | 1 + workers/servicekeyworker/models_pre_oci.py | 6 ++++-- workers/servicekeyworker/servicekeyworker.py | 4 +++- workers/servicekeyworker/test/test_servicekeyworker.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/workers/servicekeyworker/models_interface.py b/workers/servicekeyworker/models_interface.py index 6b9292b91..909ac2367 100644 --- a/workers/servicekeyworker/models_interface.py +++ b/workers/servicekeyworker/models_interface.py @@ -1,6 +1,7 @@ from abc import ABCMeta, abstractmethod from six import add_metaclass + @add_metaclass(ABCMeta) class ServiceKeyWorkerDataInterface(object): """ diff --git a/workers/servicekeyworker/models_pre_oci.py b/workers/servicekeyworker/models_pre_oci.py index 906572397..5a82d3127 100644 --- a/workers/servicekeyworker/models_pre_oci.py +++ b/workers/servicekeyworker/models_pre_oci.py @@ -1,13 +1,14 @@ from data import model from workers.servicekeyworker.models_interface import ServiceKeyWorkerDataInterface + class PreOCIModel(ServiceKeyWorkerDataInterface): def set_key_expiration(self, kid, expiration_date): model.service_keys.set_key_expiration(kid, expiration_date) def create_service_key_for_testing(self, expiration): - key = model.service_keys.create_service_key('test', 'somekid', 'quay', '', {}, expiration) - return key.kid + key = model.service_keys.create_service_key('test', 'somekid', 'quay', '', {}, expiration) + return key.kid def get_service_key_expiration(self, kid): try: @@ -16,4 +17,5 @@ class PreOCIModel(ServiceKeyWorkerDataInterface): except model.ServiceKeyDoesNotExist: return None + pre_oci_model = PreOCIModel() diff --git a/workers/servicekeyworker/servicekeyworker.py b/workers/servicekeyworker/servicekeyworker.py index 90ddd0e33..23ad26024 100644 --- a/workers/servicekeyworker/servicekeyworker.py +++ b/workers/servicekeyworker/servicekeyworker.py @@ -7,11 +7,12 @@ from workers.worker import Worker logger = logging.getLogger(__name__) + class ServiceKeyWorker(Worker): def __init__(self): super(ServiceKeyWorker, self).__init__() self.add_operation(self._refresh_service_key, - app.config.get('INSTANCE_SERVICE_KEY_REFRESH', 60)*60) + app.config.get('INSTANCE_SERVICE_KEY_REFRESH', 60) * 60) def _refresh_service_key(self): """ @@ -22,6 +23,7 @@ class ServiceKeyWorker(Worker): model.set_key_expiration(instance_keys.local_key_id, datetime.now() + expiration) logger.debug('Finished refresh of automatic service keys') + if __name__ == "__main__": worker = ServiceKeyWorker() worker.start() diff --git a/workers/servicekeyworker/test/test_servicekeyworker.py b/workers/servicekeyworker/test/test_servicekeyworker.py index 7db38c62b..2cf6103d3 100644 --- a/workers/servicekeyworker/test/test_servicekeyworker.py +++ b/workers/servicekeyworker/test/test_servicekeyworker.py @@ -13,7 +13,7 @@ def test_refresh_service_key(initialized_db): original_expiration = datetime.now() + timedelta(minutes=10) test_key_kid = model.create_service_key_for_testing(original_expiration) - instance_keys = AttrDict(dict(local_key_id=test_key_kid, service_key_expiration=10)) + instance_keys = AttrDict(dict(local_key_id=test_key_kid, service_key_expiration=30)) with patch('workers.servicekeyworker.servicekeyworker.instance_keys', instance_keys): worker = ServiceKeyWorker() worker._refresh_service_key()