import logging from data.database import ImageStorage, Image, db from app import app LOGGER = logging.getLogger(__name__) def backfill_aggregate_sizes(): """ Generates aggregate sizes for any image storage entries without them """ LOGGER.setLevel(logging.DEBUG) LOGGER.debug('Aggregate sizes backfill: Began execution') while True: batch_storage_ids = list(ImageStorage .select(ImageStorage.id) .where(ImageStorage.aggregate_size >> None) .limit(10)) if len(batch_storage_ids) == 0: # There are no storages left to backfill. We're done! LOGGER.debug('Aggregate sizes backfill: Backfill completed') return LOGGER.debug('Aggregate sizes backfill: Found %s records to update', len(batch_storage_ids)) for image_storage_id in batch_storage_ids: LOGGER.debug('Updating image storage: %s', image_storage_id.id) with app.config['DB_TRANSACTION_FACTORY'](db): try: storage = ImageStorage.select().where(ImageStorage.id == image_storage_id.id).get() image = Image.select().where(Image.storage == storage).get() image_ids = image.ancestors.split('/')[1:-1] aggregate_size = storage.image_size for image_id in image_ids: current_image = Image.select().where(Image.id == image_id).join(ImageStorage) aggregate_size += image.storage.image_size storage.aggregate_size = aggregate_size storage.save() except ImageStorage.DoesNotExist: pass except Image.DoesNotExist: pass if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) logging.getLogger('peewee').setLevel(logging.CRITICAL) backfill_aggregate_sizes()