Switch to a unified worker system

- Handles logging
- Handles reporting to Sentry
- Removes old code around serving a web endpoint (unused now)
This commit is contained in:
Joseph Schorr 2015-07-28 17:25:12 -04:00
parent dbd9a32c85
commit ac0cca2d90
7 changed files with 264 additions and 268 deletions

View file

@ -1,54 +1,51 @@
import logging
from apscheduler.schedulers.blocking import BlockingScheduler
from app import app
from data.database import (Repository, LogEntry, RepositoryActionCount, db_random_func, fn,
UseThenDisconnect)
from datetime import date, datetime, timedelta
from workers.worker import Worker
POLL_PERIOD_SECONDS = 30
POLL_PERIOD_SECONDS = 10
logger = logging.getLogger(__name__)
sched = BlockingScheduler()
@sched.scheduled_job(trigger='interval', seconds=10)
def count_repository_actions():
""" Counts actions for a random repository for the previous day. """
class RepositoryActionCountWorker(Worker):
def __init__(self):
super(RepositoryActionCountWorker, self).__init__()
self.add_operation(self._count_repository_actions, POLL_PERIOD_SECONDS)
with UseThenDisconnect(app.config):
try:
# Get a random repository to count.
today = date.today()
yesterday = today - timedelta(days=1)
has_yesterday_actions = (RepositoryActionCount.select(RepositoryActionCount.repository)
.where(RepositoryActionCount.date == yesterday))
def _count_repository_actions(self):
""" Counts actions for a random repository for the previous day. """
to_count = (Repository.select()
.where(~(Repository.id << (has_yesterday_actions)))
.order_by(db_random_func()).get())
logger.debug('Counting: %s', to_count.id)
actions = (LogEntry.select()
.where(LogEntry.repository == to_count,
LogEntry.datetime >= yesterday,
LogEntry.datetime < today)
.count())
# Create the row.
with UseThenDisconnect(app.config):
try:
RepositoryActionCount.create(repository=to_count, date=yesterday, count=actions)
except:
logger.exception('Exception when writing count')
# Get a random repository to count.
today = date.today()
yesterday = today - timedelta(days=1)
has_yesterday_actions = (RepositoryActionCount.select(RepositoryActionCount.repository)
.where(RepositoryActionCount.date == yesterday))
return True
to_count = (Repository.select()
.where(~(Repository.id << (has_yesterday_actions)))
.order_by(db_random_func()).get())
except Repository.DoesNotExist:
logger.debug('No further repositories to count')
return False
logger.debug('Counting: %s', to_count.id)
actions = (LogEntry.select()
.where(LogEntry.repository == to_count,
LogEntry.datetime >= yesterday,
LogEntry.datetime < today)
.count())
# Create the row.
try:
RepositoryActionCount.create(repository=to_count, date=yesterday, count=actions)
except:
logger.exception('Exception when writing count')
except Repository.DoesNotExist:
logger.debug('No further repositories to count')
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
sched.start()
worker = RepositoryActionCountWorker()
worker.start()