diff --git a/README.md b/README.md index 02d4cbb2c..7391cf8e8 100644 --- a/README.md +++ b/README.md @@ -21,4 +21,13 @@ running: ``` sudo nginx -c `pwd`/nginx.conf STACK=prod gunicorn -D --workers 4 -b unix:/tmp/gunicorn.sock --worker-class eventlet -t 500 application:application -``` \ No newline at end of file +``` + +set up the snapshot script: +(instructions in the seo-snapshots directory)[seo-snapshots/README.md] + +start the workers: + +``` +python -m workers.diffsworker -D +``` diff --git a/data/queue.py b/data/queue.py index fb3421466..9ebf9c659 100644 --- a/data/queue.py +++ b/data/queue.py @@ -7,7 +7,7 @@ class WorkQueue(object): def __init__(self, queue_name): self.queue_name = queue_name - def put(message, available_after=0): + def put(self, message, available_after=0): """ Put an item, if it shouldn't be processed for some number of seconds, specify that amount as available_after. @@ -24,7 +24,7 @@ class WorkQueue(object): QueueItem.create(**params) - def get(processing_time=300): + def get(self, processing_time=300): """ Get an available item and mark it as unavailable for the default of five minutes. @@ -35,9 +35,9 @@ class WorkQueue(object): # TODO the query and the update should be atomic, but for now we only # have one worker. - avaial = QueueItem.select().where(QueueItem.queue_name = self.queue_name, - QueueItem.available_after <= now, - available_or_expired) + avail = QueueItem.select().where(QueueItem.queue_name == self.queue_name, + QueueItem.available_after <= now, + available_or_expired) found = list(avail.limit(1).order_by(QueueItem.available_after)) @@ -51,5 +51,5 @@ class WorkQueue(object): return None - def complete(completed_item): + def complete(self, completed_item): item.delete_instance() diff --git a/workers/image_diffs.py b/workers/diffsworker.py similarity index 86% rename from workers/image_diffs.py rename to workers/diffsworker.py index 93bcd1b45..024aa0d89 100644 --- a/workers/image_diffs.py +++ b/workers/diffsworker.py @@ -1,5 +1,7 @@ import logging import json +import daemon +import time from apscheduler.scheduler import Scheduler @@ -15,7 +17,7 @@ image_diff_queue = WorkQueue('imagediff') def process_work_items(): logger.debug('Getting work item from queue.') - item = imagediff.get() + item = image_diff_queue.get() if item: logger.debug('Queue gave us some work: %s' % item.body) @@ -26,7 +28,6 @@ def process_work_items(): else: logger.debug('No work today.') - FORMAT = '%(asctime)-15s - %(levelname)s - %(pathname)s - %(funcName)s - %(message)s' logging.basicConfig(format=FORMAT, level=logging.DEBUG) @@ -34,3 +35,6 @@ sched = Scheduler() sched.start() sched.add_interval_job(process_work_items, seconds=10) + +while True: + time.sleep(60 * 60 * 24) # sleep one day, basically forever \ No newline at end of file