vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Revert "Merge pull request #491 from jakedt/migratebackp2"

Browse source 
This reverts commit 7ad2522dbe, reversing
changes made to a0b191ffa1.
This commit is contained in:
Silas Sewell 2015-09-28 16:09:22 -04:00
parent 7ad2522dbe
commit 9000169b53
11 changed files with 66 additions and 232 deletions
Download patch file Download diff file Expand all files Collapse all files

50
util/migrate/backfill_aggregate_sizes.py
View file

@ -1,50 +1,44 @@
import logging
from data.database import ImageStorage, Image, db, db_for_update
from data.database import ImageStorage, Image, db
from app import app
logger = logging.getLogger(__name__)
LOGGER = logging.getLogger(__name__)
def backfill_aggregate_sizes():
""" Generates aggregate sizes for any image storage entries without them """
logger.debug('Aggregate sizes backfill: Began execution')
LOGGER.setLevel(logging.DEBUG)
LOGGER.debug('Aggregate sizes backfill: Began execution')
while True:
batch_image_ids = list(Image
.select(Image.id)
.where(Image.aggregate_size >> None)
.limit(100))
batch_storage_ids = list(ImageStorage
.select(ImageStorage.id)
.where(ImageStorage.aggregate_size >> None)
.limit(10))
if len(batch_image_ids) == 0:
if len(batch_storage_ids) == 0:
# There are no storages left to backfill. We're done!
logger.debug('Aggregate sizes backfill: Backfill completed')
LOGGER.debug('Aggregate sizes backfill: Backfill completed')
return
logger.debug('Aggregate sizes backfill: Found %s records to update', len(batch_image_ids))
for image_id in batch_image_ids:
logger.debug('Updating image : %s', image_id.id)
LOGGER.debug('Aggregate sizes backfill: Found %s records to update', len(batch_storage_ids))
for image_storage_id in batch_storage_ids:
LOGGER.debug('Updating image storage: %s', image_storage_id.id)
with app.config['DB_TRANSACTION_FACTORY'](db):
try:
image = (Image
.select(Image, ImageStorage)
.join(ImageStorage)
.where(Image.id == image_id)
.get())
aggregate_size = image.storage.image_size
storage = ImageStorage.select().where(ImageStorage.id == image_storage_id.id).get()
image = Image.select().where(Image.storage == storage).get()
image_ids = image.ancestors.split('/')[1:-1]
aggregate_size = storage.image_size
for image_id in image_ids:
to_add = db_for_update(Image
.select(Image, ImageStorage)
.join(ImageStorage)
.where(Image.id == image_id)).get()
aggregate_size += to_add.storage.image_size
current_image = Image.select().where(Image.id == image_id).join(ImageStorage)
aggregate_size += image.storage.image_size
image.aggregate_size = aggregate_size
image.save()
storage.aggregate_size = aggregate_size
storage.save()
except ImageStorage.DoesNotExist:
pass
except Image.DoesNotExist:
pass