vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Change the backfill script to use a spooled temp file

Browse source
This commit is contained in:
Joseph Schorr 2014-09-27 14:55:24 -04:00
parent 21d8c41320
commit 0a93b39c54
1 changed files with 11 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
tools/uncompressedsize.py
View file

@ -6,7 +6,7 @@ from data.database import ImageStorage
from app import app, storage as store from app import app, storage as store
from data.database import db from data.database import db
from gzip import GzipFile from gzip import GzipFile
from StringIO import StringIO from tempfile import SpooledTemporaryFile
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -67,22 +67,19 @@ def backfill_sizes_from_data():
with_locations = model.get_storage_by_uuid(uuid) with_locations = model.get_storage_by_uuid(uuid)
layer_size = -2 layer_size = -2
# Read the layer from backing storage. # Read the layer from backing storage and calculate the uncompressed size.
logger.debug('Reading entry: %s (%s bytes)', uuid, with_locations.image_size)
try: try:
layer_data = store.get_content(with_locations.locations, store.image_layer_path(uuid)) logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size)
except Exception as ex: CHUNK_SIZE = 512 * 1024
logger.debug('Could not read entry: %s. Reason: %s', uuid, ex) with SpooledTemporaryFile(CHUNK_SIZE) as tarball:
continue layer_data = store.get_content(with_locations.locations, store.image_layer_path(uuid))
tarball.write(layer_data)
tarball.seek(0)
# Calculate the uncompressed size. with GzipFile(fileobj=tarball, mode='rb') as gzip_file:
try: gzip_file.read()
layer_stream = StringIO(layer_data) layer_size = gzip_file.size
with GzipFile(fileobj=layer_stream, mode='rb') as gzip_file:
gzip_file.read()
layer_size = gzip_file.size
layer_stream.close()
except Exception as ex: except Exception as ex:
logger.debug('Could not gunzip entry: %s. Reason: %s', uuid, ex) logger.debug('Could not gunzip entry: %s. Reason: %s', uuid, ex)
continue continue