import logging from data.database import ImageStorage, Image, db, db_for_update from app import app logger = logging.getLogger(__name__) def backfill_aggregate_sizes(): """ Generates aggregate sizes for any image storage entries without them """ logger.debug('Aggregate sizes backfill: Began execution') while True: batch_image_ids = list(Image .select(Image.id) .where(Image.aggregate_size >> None) .limit(100)) if len(batch_image_ids) == 0: # There are no storages left to backfill. We're done! logger.debug('Aggregate sizes backfill: Backfill completed') return logger.debug('Aggregate sizes backfill: Found %s records to update', len(batch_image_ids)) for image_id in batch_image_ids: logger.debug('Updating image : %s', image_id.id) with app.config['DB_TRANSACTION_FACTORY'](db): try: image = (Image .select(Image, ImageStorage) .join(ImageStorage) .where(Image.id == image_id) .get()) aggregate_size = image.storage.image_size image_ids = image.ancestors.split('/')[1:-1] for image_id in image_ids: to_add = db_for_update(Image .select(Image, ImageStorage) .join(ImageStorage) .where(Image.id == image_id)).get() aggregate_size += to_add.storage.image_size image.aggregate_size = aggregate_size image.save() except Image.DoesNotExist: pass if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) logging.getLogger('peewee').setLevel(logging.CRITICAL) backfill_aggregate_sizes()