diff --git a/storage/s3.py b/storage/s3.py index 26e73d127..e4f1671cd 100644 --- a/storage/s3.py +++ b/storage/s3.py @@ -1,6 +1,6 @@ - import cStringIO as StringIO import os +import logging import boto.s3.connection import boto.s3.key @@ -8,6 +8,30 @@ import boto.s3.key from . import Storage +logger = logging.getLogger(__name__) + + +class StreamReadKeyAsFile(object): + def __init__(self, key): + self._key = key + self._finished = False + + def __enter__(self): + return self + + def __exit__(self, type, value, tb): + self._key.close(fast=True) + + def read(self, amt=None): + if self._finished: + return None + + resp = self._key.read(amt) + if not resp: + self._finished = True + return resp + + class S3Storage(Storage): def __init__(self, storage_path, s3_access_key, s3_secret_key, s3_bucket): @@ -65,7 +89,7 @@ class S3Storage(Storage): key = boto.s3.key.Key(self._s3_bucket, path) if not key.exists(): raise IOError('No such key: \'{0}\''.format(path)) - return key + return StreamReadKeyAsFile(key) def stream_write(self, path, fp): # Minimum size of upload part size on S3 is 5MB