""" Defines utility methods for working with gzip streams. """ import zlib # Window size for decompressing GZIP streams. # This results in ZLIB automatically detecting the GZIP headers. # http://stackoverflow.com/questions/3122145/zlib-error-error-3-while-decompressing-incorrect-header-check/22310760#22310760 ZLIB_GZIP_WINDOW = zlib.MAX_WBITS | 32 CHUNK_SIZE = 5 * 1024 * 1024 class SizeInfo(object): def __init__(self): self.size = 0 def calculate_size_handler(): """ Returns an object and a SocketReader handler. The handler will gunzip the data it receives, adding the size found to the object. """ size_info = SizeInfo() decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW) def fn(buf): # Note: We set a maximum CHUNK_SIZE to prevent the decompress from taking too much # memory. As a result, we have to loop until the unconsumed tail is empty. current_data = buf while len(current_data) > 0: size_info.size += len(decompressor.decompress(current_data, CHUNK_SIZE)) current_data = decompressor.unconsumed_tail return size_info, fn