Have the squashing system write the data (via a double queue system and multiprocessing) to both the client and the stack's storage. On subsequent calls, if the synthetic image exists, it will be returned directly instead of being recomputed

This commit is contained in:
Joseph Schorr 2014-09-16 22:43:19 -04:00
parent 5cca609c55
commit 1cfb6fc353
3 changed files with 146 additions and 11 deletions

37
util/queuefile.py Normal file
View file

@ -0,0 +1,37 @@
from multiprocessing import Queue
import os
class QueueFile(object):
""" Class which implements a file-like interface and reads from a blocking
multiprocessing queue.
"""
def __init__(self, queue, name=None):
self._queue = queue
self._closed = False
self._done = False
self._buffer = ''
self._total_size = 0
self._name = name
def read(self, size=8192):
if self._closed or self._done:
return None
while len(self._buffer) < size:
result = self._queue.get(block=True)
if result is None:
self._done = True
break
self._buffer += result
self._total_size += len(result)
buf = self._buffer[0:size]
self._buffer = self._buffer[size:]
return buf
def flush(self):
pass
def close(self):
self._closed = True