From 38f1752a2d209c057067a9b29447e982d97cf200 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 28 Jun 2017 15:04:10 +0300 Subject: [PATCH 1/4] Move gcworker into its own package --- conf/init/service/gcworker/run | 2 +- workers/gcworker/__init__.py | 0 workers/{ => gcworker}/gcworker.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 workers/gcworker/__init__.py rename workers/{ => gcworker}/gcworker.py (100%) diff --git a/conf/init/service/gcworker/run b/conf/init/service/gcworker/run index 6a843d4b8..50b974b6a 100755 --- a/conf/init/service/gcworker/run +++ b/conf/init/service/gcworker/run @@ -3,6 +3,6 @@ echo 'Starting GC worker' cd / -venv/bin/python -m workers.gcworker 2>&1 +venv/bin/python -m workers.gcworker.gcworker 2>&1 echo 'Repository GC exited' \ No newline at end of file diff --git a/workers/gcworker/__init__.py b/workers/gcworker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/workers/gcworker.py b/workers/gcworker/gcworker.py similarity index 100% rename from workers/gcworker.py rename to workers/gcworker/gcworker.py From 420a5e5a3a07e266a1e6de64460d0def770b67a5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 28 Jun 2017 15:13:11 +0300 Subject: [PATCH 2/4] Change GC worker to use data interface --- workers/gcworker/gcworker.py | 19 ++++--------------- workers/gcworker/models_interface.py | 13 +++++++++++++ workers/gcworker/models_pre_oci.py | 21 +++++++++++++++++++++ workers/gcworker/test/test_gcworker.py | 7 +++++++ 4 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 workers/gcworker/models_interface.py create mode 100644 workers/gcworker/models_pre_oci.py create mode 100644 workers/gcworker/test/test_gcworker.py diff --git a/workers/gcworker/gcworker.py b/workers/gcworker/gcworker.py index e43bd2262..d53a4ac3b 100644 --- a/workers/gcworker/gcworker.py +++ b/workers/gcworker/gcworker.py @@ -1,13 +1,8 @@ -import logging - from app import app -from data.model.repository import (find_repository_with_garbage, garbage_collect_repo, - get_random_gc_policy) - +from data.database import UseThenDisconnect +from workers.gcworker.models_pre_oci import pre_oci_model as model from workers.worker import Worker -logger = logging.getLogger(__name__) - class GarbageCollectionWorker(Worker): def __init__(self): super(GarbageCollectionWorker, self).__init__() @@ -16,14 +11,8 @@ class GarbageCollectionWorker(Worker): def _garbage_collection_repos(self): """ Performs garbage collection on repositories. """ - repository = find_repository_with_garbage(get_random_gc_policy()) - if repository is None: - logger.debug('No repository with garbage found') - return - - logger.debug('Starting GC of repository #%s (%s)', repository.id, repository.name) - garbage_collect_repo(repository) - logger.debug('Finished GC of repository #%s (%s)', repository.id, repository.name) + with UseThenDisconnect(app.config): + model.perform_garbage_collection() if __name__ == "__main__": worker = GarbageCollectionWorker() diff --git a/workers/gcworker/models_interface.py b/workers/gcworker/models_interface.py new file mode 100644 index 000000000..073447653 --- /dev/null +++ b/workers/gcworker/models_interface.py @@ -0,0 +1,13 @@ +from abc import ABCMeta, abstractmethod + +from six import add_metaclass + +@add_metaclass(ABCMeta) +class GCWorkerDataInterface(object): + """ + Interface that represents all data store interactions required by the GC worker. + """ + @abstractmethod + def perform_garbage_collection(self): + """ Performs garbage collection on a single repository, if any can be found with garbage. """ + pass diff --git a/workers/gcworker/models_pre_oci.py b/workers/gcworker/models_pre_oci.py new file mode 100644 index 000000000..ec6e3ea99 --- /dev/null +++ b/workers/gcworker/models_pre_oci.py @@ -0,0 +1,21 @@ +import logging + +from data.model.repository import (find_repository_with_garbage, garbage_collect_repo, + get_random_gc_policy) + +from workers.gcworker.models_interface import GCWorkerDataInterface + +logger = logging.getLogger(__name__) + +class PreOCIModel(GCWorkerDataInterface): + def perform_garbage_collection(self): + repository = find_repository_with_garbage(get_random_gc_policy()) + if repository is None: + logger.debug('No repository with garbage found') + return + + logger.debug('Starting GC of repository #%s (%s)', repository.id, repository.name) + garbage_collect_repo(repository) + logger.debug('Finished GC of repository #%s (%s)', repository.id, repository.name) + +pre_oci_model = PreOCIModel() diff --git a/workers/gcworker/test/test_gcworker.py b/workers/gcworker/test/test_gcworker.py new file mode 100644 index 000000000..45f77ea82 --- /dev/null +++ b/workers/gcworker/test/test_gcworker.py @@ -0,0 +1,7 @@ +from workers.gcworker.gcworker import GarbageCollectionWorker + +from test.fixtures import * + +def test_gc(initialized_db): + worker = GarbageCollectionWorker() + worker._garbage_collection_repos() From 76c9339453b00e6772e6890edfcf3d4482c1bc51 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 29 Jun 2017 09:37:32 +0300 Subject: [PATCH 3/4] Rename GC worker package to `gc` --- conf/init/service/gcworker/run | 2 +- workers/{gcworker => gc}/__init__.py | 0 workers/{gcworker => gc}/gcworker.py | 2 +- workers/{gcworker => gc}/models_interface.py | 0 workers/{gcworker => gc}/models_pre_oci.py | 2 +- workers/{gcworker => gc}/test/test_gcworker.py | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename workers/{gcworker => gc}/__init__.py (100%) rename workers/{gcworker => gc}/gcworker.py (89%) rename workers/{gcworker => gc}/models_interface.py (100%) rename workers/{gcworker => gc}/models_pre_oci.py (91%) rename workers/{gcworker => gc}/test/test_gcworker.py (68%) diff --git a/conf/init/service/gcworker/run b/conf/init/service/gcworker/run index 50b974b6a..d31f814a4 100755 --- a/conf/init/service/gcworker/run +++ b/conf/init/service/gcworker/run @@ -3,6 +3,6 @@ echo 'Starting GC worker' cd / -venv/bin/python -m workers.gcworker.gcworker 2>&1 +venv/bin/python -m workers.gc.gcworker 2>&1 echo 'Repository GC exited' \ No newline at end of file diff --git a/workers/gcworker/__init__.py b/workers/gc/__init__.py similarity index 100% rename from workers/gcworker/__init__.py rename to workers/gc/__init__.py diff --git a/workers/gcworker/gcworker.py b/workers/gc/gcworker.py similarity index 89% rename from workers/gcworker/gcworker.py rename to workers/gc/gcworker.py index d53a4ac3b..5cf434eba 100644 --- a/workers/gcworker/gcworker.py +++ b/workers/gc/gcworker.py @@ -1,6 +1,6 @@ from app import app from data.database import UseThenDisconnect -from workers.gcworker.models_pre_oci import pre_oci_model as model +from workers.gc.models_pre_oci import pre_oci_model as model from workers.worker import Worker class GarbageCollectionWorker(Worker): diff --git a/workers/gcworker/models_interface.py b/workers/gc/models_interface.py similarity index 100% rename from workers/gcworker/models_interface.py rename to workers/gc/models_interface.py diff --git a/workers/gcworker/models_pre_oci.py b/workers/gc/models_pre_oci.py similarity index 91% rename from workers/gcworker/models_pre_oci.py rename to workers/gc/models_pre_oci.py index ec6e3ea99..bb1fda906 100644 --- a/workers/gcworker/models_pre_oci.py +++ b/workers/gc/models_pre_oci.py @@ -3,7 +3,7 @@ import logging from data.model.repository import (find_repository_with_garbage, garbage_collect_repo, get_random_gc_policy) -from workers.gcworker.models_interface import GCWorkerDataInterface +from workers.gc.models_interface import GCWorkerDataInterface logger = logging.getLogger(__name__) diff --git a/workers/gcworker/test/test_gcworker.py b/workers/gc/test/test_gcworker.py similarity index 68% rename from workers/gcworker/test/test_gcworker.py rename to workers/gc/test/test_gcworker.py index 45f77ea82..c301fc4b9 100644 --- a/workers/gcworker/test/test_gcworker.py +++ b/workers/gc/test/test_gcworker.py @@ -1,4 +1,4 @@ -from workers.gcworker.gcworker import GarbageCollectionWorker +from workers.gc.gcworker import GarbageCollectionWorker from test.fixtures import * From 138881dab84e3505dd267583ae5cebd23d88e865 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 29 Jun 2017 09:40:39 +0300 Subject: [PATCH 4/4] yapf format --- workers/gc/gcworker.py | 2 ++ workers/gc/models_interface.py | 2 ++ workers/gc/models_pre_oci.py | 6 ++++-- workers/gc/test/test_gcworker.py | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/workers/gc/gcworker.py b/workers/gc/gcworker.py index 5cf434eba..0fc769a13 100644 --- a/workers/gc/gcworker.py +++ b/workers/gc/gcworker.py @@ -3,6 +3,7 @@ from data.database import UseThenDisconnect from workers.gc.models_pre_oci import pre_oci_model as model from workers.worker import Worker + class GarbageCollectionWorker(Worker): def __init__(self): super(GarbageCollectionWorker, self).__init__() @@ -14,6 +15,7 @@ class GarbageCollectionWorker(Worker): with UseThenDisconnect(app.config): model.perform_garbage_collection() + if __name__ == "__main__": worker = GarbageCollectionWorker() worker.start() diff --git a/workers/gc/models_interface.py b/workers/gc/models_interface.py index 073447653..561d1ae0d 100644 --- a/workers/gc/models_interface.py +++ b/workers/gc/models_interface.py @@ -2,11 +2,13 @@ from abc import ABCMeta, abstractmethod from six import add_metaclass + @add_metaclass(ABCMeta) class GCWorkerDataInterface(object): """ Interface that represents all data store interactions required by the GC worker. """ + @abstractmethod def perform_garbage_collection(self): """ Performs garbage collection on a single repository, if any can be found with garbage. """ diff --git a/workers/gc/models_pre_oci.py b/workers/gc/models_pre_oci.py index bb1fda906..e672007e0 100644 --- a/workers/gc/models_pre_oci.py +++ b/workers/gc/models_pre_oci.py @@ -1,12 +1,13 @@ import logging -from data.model.repository import (find_repository_with_garbage, garbage_collect_repo, - get_random_gc_policy) +from data.model.repository import ( + find_repository_with_garbage, garbage_collect_repo, get_random_gc_policy) from workers.gc.models_interface import GCWorkerDataInterface logger = logging.getLogger(__name__) + class PreOCIModel(GCWorkerDataInterface): def perform_garbage_collection(self): repository = find_repository_with_garbage(get_random_gc_policy()) @@ -18,4 +19,5 @@ class PreOCIModel(GCWorkerDataInterface): garbage_collect_repo(repository) logger.debug('Finished GC of repository #%s (%s)', repository.id, repository.name) + pre_oci_model = PreOCIModel() diff --git a/workers/gc/test/test_gcworker.py b/workers/gc/test/test_gcworker.py index c301fc4b9..a153cf460 100644 --- a/workers/gc/test/test_gcworker.py +++ b/workers/gc/test/test_gcworker.py @@ -2,6 +2,7 @@ from workers.gc.gcworker import GarbageCollectionWorker from test.fixtures import * + def test_gc(initialized_db): worker = GarbageCollectionWorker() worker._garbage_collection_repos()