From c07dec4d39d3595ff0c63bf088d3e8c31d82d9f9 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 25 Aug 2015 13:49:21 -0400 Subject: [PATCH] File reader fixes for verbs - Fix local file reader to always read in chunks - Have gzip stream raise an exception if the full data is requested --- storage/local.py | 8 +++++--- util/registry/gzipwrap.py | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/storage/local.py b/storage/local.py index 6e18004e0..9495aed9c 100644 --- a/storage/local.py +++ b/storage/local.py @@ -61,9 +61,11 @@ class LocalStorage(BaseStorageV2): num_bytes < 0 copy until the stream ends. """ bytes_copied = 0 - bytes_remaining = num_bytes - while bytes_remaining > 0 or num_bytes < 0: - size_to_read = min(bytes_remaining, self.buffer_size) + while bytes_copied < num_bytes or num_bytes == -1: + size_to_read = min(num_bytes - bytes_copied, self.buffer_size) + if size_to_read < 0: + size_to_read = self.buffer_size + try: buf = in_fp.read(size_to_read) if not buf: diff --git a/util/registry/gzipwrap.py b/util/registry/gzipwrap.py index 604fa343d..685e5bb13 100644 --- a/util/registry/gzipwrap.py +++ b/util/registry/gzipwrap.py @@ -11,6 +11,9 @@ class GzipWrap(object): self.is_done = False def read(self, size=-1): + if size is None or size < 0: + raise Exception('Call to GzipWrap with unbound size will result in poor performance') + # If the buffer already has enough bytes, then simply pop them off of # the beginning and return them. if len(self.buffer) >= size or self.is_done: