vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Fixes for backfill_aggregate_size script.

Browse source
This commit is contained in:
Jake Moshenko 2015-09-15 19:08:31 -04:00
parent 8baacd2741
commit a887125c3f
1 changed files with 28 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
util/migrate/backfill_aggregate_sizes.py
View file

@ -1,44 +1,50 @@
import logging import logging
from data.database import ImageStorage, Image, db from data.database import ImageStorage, Image, db, db_for_update
from app import app from app import app
LOGGER = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
def backfill_aggregate_sizes(): def backfill_aggregate_sizes():
""" Generates aggregate sizes for any image storage entries without them """ """ Generates aggregate sizes for any image storage entries without them """
LOGGER.setLevel(logging.DEBUG) logger.debug('Aggregate sizes backfill: Began execution')
LOGGER.debug('Aggregate sizes backfill: Began execution')
while True: while True:
batch_storage_ids = list(ImageStorage batch_image_ids = list(Image
.select(ImageStorage.id) .select(Image.id)
.where(ImageStorage.aggregate_size >> None) .where(Image.aggregate_size >> None)
.limit(10)) .limit(100))
if len(batch_storage_ids) == 0: if len(batch_image_ids) == 0:
# There are no storages left to backfill. We're done! # There are no storages left to backfill. We're done!
LOGGER.debug('Aggregate sizes backfill: Backfill completed') logger.debug('Aggregate sizes backfill: Backfill completed')
return return
LOGGER.debug('Aggregate sizes backfill: Found %s records to update', len(batch_storage_ids)) logger.debug('Aggregate sizes backfill: Found %s records to update', len(batch_image_ids))
for image_storage_id in batch_storage_ids: for image_id in batch_image_ids:
LOGGER.debug('Updating image storage: %s', image_storage_id.id) logger.debug('Updating image : %s', image_id.id)
with app.config['DB_TRANSACTION_FACTORY'](db): with app.config['DB_TRANSACTION_FACTORY'](db):
try: try:
storage = ImageStorage.select().where(ImageStorage.id == image_storage_id.id).get() image = (Image
image = Image.select().where(Image.storage == storage).get() .select(Image, ImageStorage)
.join(ImageStorage)
.where(Image.id == image_id)
.get())
aggregate_size = image.storage.image_size
image_ids = image.ancestors.split('/')[1:-1] image_ids = image.ancestors.split('/')[1:-1]
aggregate_size = storage.image_size
for image_id in image_ids: for image_id in image_ids:
current_image = Image.select().where(Image.id == image_id).join(ImageStorage) to_add = db_for_update(Image
aggregate_size += image.storage.image_size .select(Image, ImageStorage)
.join(ImageStorage)
.where(Image.id == image_id)).get()
aggregate_size += to_add.storage.image_size
storage.aggregate_size = aggregate_size image.aggregate_size = aggregate_size
storage.save() image.save()
except ImageStorage.DoesNotExist:
pass
except Image.DoesNotExist: except Image.DoesNotExist:
pass pass