File reader fixes for verbs

- Fix local file reader to always read in chunks
- Have gzip stream raise an exception if the full data is requested
This commit is contained in:
Joseph Schorr 2015-08-25 13:49:21 -04:00
parent 0c7839203e
commit c07dec4d39
2 changed files with 8 additions and 3 deletions

View file

@ -61,9 +61,11 @@ class LocalStorage(BaseStorageV2):
num_bytes < 0 copy until the stream ends.
"""
bytes_copied = 0
bytes_remaining = num_bytes
while bytes_remaining > 0 or num_bytes < 0:
size_to_read = min(bytes_remaining, self.buffer_size)
while bytes_copied < num_bytes or num_bytes == -1:
size_to_read = min(num_bytes - bytes_copied, self.buffer_size)
if size_to_read < 0:
size_to_read = self.buffer_size
try:
buf = in_fp.read(size_to_read)
if not buf:

View file

@ -11,6 +11,9 @@ class GzipWrap(object):
self.is_done = False
def read(self, size=-1):
if size is None or size < 0:
raise Exception('Call to GzipWrap with unbound size will result in poor performance')
# If the buffer already has enough bytes, then simply pop them off of
# the beginning and return them.
if len(self.buffer) >= size or self.is_done: