vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Better error handling in the uncompressedsize script.

Browse source
This commit is contained in:
Jake Moshenko 2014-10-07 10:22:02 -04:00
parent 4ad592e7ce
commit 55460cdcba
1 changed files with 32 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
tools/uncompressedsize.py
View file

@ -30,17 +30,18 @@ def backfill_sizes_from_data():
for record in batch_ids: for record in batch_ids:
uuid = record.uuid uuid = record.uuid
with_locations = model.get_storage_by_uuid(uuid) try:
if with_locations.uncompressed_size is not None: with_locs = model.get_storage_by_uuid(uuid)
if with_locs.uncompressed_size is not None:
logger.debug('Somebody else already filled this in for us: %s', uuid) logger.debug('Somebody else already filled this in for us: %s', uuid)
continue continue
# Read the layer from backing storage and calculate the uncompressed size. # Read the layer from backing storage and calculate the uncompressed size.
logger.debug('Loading data: %s (%s bytes)', uuid, with_locations.image_size) logger.debug('Loading data: %s (%s bytes)', uuid, with_locs.image_size)
decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW) decompressor = zlib.decompressobj(ZLIB_GZIP_WINDOW)
uncompressed_size = 0 uncompressed_size = 0
with store.stream_read_file(with_locations.locations, store.image_layer_path(uuid)) as stream: with store.stream_read_file(with_locs.locations, store.image_layer_path(uuid)) as stream:
while True: while True:
current_data = stream.read(CHUNK_SIZE) current_data = stream.read(CHUNK_SIZE)
if len(current_data) == 0: if len(current_data) == 0:
@ -52,19 +53,21 @@ def backfill_sizes_from_data():
# make sure the image storage still exists and has not changed. # make sure the image storage still exists and has not changed.
logger.debug('Writing entry: %s. Size: %s', uuid, uncompressed_size) logger.debug('Writing entry: %s. Size: %s', uuid, uncompressed_size)
with app.config['DB_TRANSACTION_FACTORY'](db): with app.config['DB_TRANSACTION_FACTORY'](db):
try:
current_record = model.get_storage_by_uuid(uuid) current_record = model.get_storage_by_uuid(uuid)
except model.InvalidImageException:
logger.warning('Storage with uuid no longer exists: %s', uuid)
continue
if not current_record.uploading and current_record.uncompressed_size == None: if not current_record.uploading and current_record.uncompressed_size == None:
current_record.uncompressed_size = uncompressed_size current_record.uncompressed_size = uncompressed_size
current_record.save() current_record.save()
else:
logger.debug('Somebody else already filled this in for us, after we did the work: %s',
uuid)
except model.InvalidImageException:
logger.warning('Storage with uuid no longer exists: %s', uuid)
if __name__ == "__main__": if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
logging.getLogger('boto').setLevel(logging.CRITICAL) logging.getLogger('boto').setLevel(logging.CRITICAL)
logging.getLogger('peewee').setLevel(logging.CRITICAL)
backfill_sizes_from_data() backfill_sizes_from_data()