File reader fixes for verbs
- Fix local file reader to always read in chunks - Have gzip stream raise an exception if the full data is requested
This commit is contained in:
parent
0c7839203e
commit
c07dec4d39
2 changed files with 8 additions and 3 deletions
|
@ -61,9 +61,11 @@ class LocalStorage(BaseStorageV2):
|
|||
num_bytes < 0 copy until the stream ends.
|
||||
"""
|
||||
bytes_copied = 0
|
||||
bytes_remaining = num_bytes
|
||||
while bytes_remaining > 0 or num_bytes < 0:
|
||||
size_to_read = min(bytes_remaining, self.buffer_size)
|
||||
while bytes_copied < num_bytes or num_bytes == -1:
|
||||
size_to_read = min(num_bytes - bytes_copied, self.buffer_size)
|
||||
if size_to_read < 0:
|
||||
size_to_read = self.buffer_size
|
||||
|
||||
try:
|
||||
buf = in_fp.read(size_to_read)
|
||||
if not buf:
|
||||
|
|
|
@ -11,6 +11,9 @@ class GzipWrap(object):
|
|||
self.is_done = False
|
||||
|
||||
def read(self, size=-1):
|
||||
if size is None or size < 0:
|
||||
raise Exception('Call to GzipWrap with unbound size will result in poor performance')
|
||||
|
||||
# If the buffer already has enough bytes, then simply pop them off of
|
||||
# the beginning and return them.
|
||||
if len(self.buffer) >= size or self.is_done:
|
||||
|
|
Reference in a new issue