vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Unify the database connection lifecycle across all workers

Browse source
This commit is contained in:
Jake Moshenko 2015-12-04 15:51:53 -05:00
parent 38cb63d195
commit 2f626f2691
7 changed files with 111 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

27
workers/buildlogsarchiver.py
View file

@ -1,13 +1,12 @@
import logging
from peewee import fn
from tempfile import SpooledTemporaryFile
from gzip import GzipFile
from data import model
from data.archivedlogs import JSON_MIMETYPE
from data.database import RepositoryBuild, db_random_func
from app import build_logs, log_archive
from data.database import CloseForLongOperation
from app import build_logs, log_archive, app
from util.streamingjsonencoder import StreamingJSONEncoder
from workers.worker import Worker
@ -39,19 +38,21 @@ class ArchiveBuildLogsWorker(Worker):
'logs': entries,
}
with SpooledTemporaryFile(MEMORY_TEMPFILE_SIZE) as tempfile:
with GzipFile('testarchive', fileobj=tempfile) as zipstream:
for chunk in StreamingJSONEncoder().iterencode(to_encode):
zipstream.write(chunk)
with CloseForLongOperation(app.config):
with SpooledTemporaryFile(MEMORY_TEMPFILE_SIZE) as tempfile:
with GzipFile('testarchive', fileobj=tempfile) as zipstream:
for chunk in StreamingJSONEncoder().iterencode(to_encode):
zipstream.write(chunk)
tempfile.seek(0)
log_archive.store_file(tempfile, JSON_MIMETYPE, content_encoding='gzip',
file_id=to_archive.uuid)
tempfile.seek(0)
log_archive.store_file(tempfile, JSON_MIMETYPE, content_encoding='gzip',
file_id=to_archive.uuid)
to_archive.logs_archived = True
to_archive.save()
to_update = model.build.get_repository_build(to_archive.uuid)
to_update.logs_archived = True
to_update.save()
build_logs.expire_log_entries(to_archive.uuid)
build_logs.expire_log_entries(to_update.uuid)
if __name__ == "__main__":