vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Re-enable parent id backfill, use new backfill style

Browse source
This commit is contained in:
Jake Moshenko 2015-11-06 15:45:39 -05:00
parent 9036ca2f2f
commit fd3f88f489
2 changed files with 24 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
data/migrations/versions/22af01f81722_backfill_parent_id_and_v1_checksums.py
View file

@ -11,8 +11,10 @@ revision = '22af01f81722'
down_revision = '2827d36939e4' down_revision = '2827d36939e4'
from util.migrate.backfill_v1_checksums import backfill_checksums from util.migrate.backfill_v1_checksums import backfill_checksums
from util.migrate.backfill_parent_id import backfill_parent_id
def upgrade(tables): def upgrade(tables):
backfill_parent_id()
backfill_checksums() backfill_checksums()
def downgrade(tables): def downgrade(tables):

52
util/migrate/backfill_parent_id.py
View file

@ -1,46 +1,38 @@
import logging import logging
from data.database import Image, ImageStorage, db
from data.database import Image, ImageStorage, db, db_for_update
from app import app from app import app
from util.migrate import yield_random_entries
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def backfill_parent_id(): def backfill_parent_id():
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
logger.debug('backfill_parent_id: Starting') logger.debug('backfill_parent_id: Starting')
logger.debug('backfill_parent_id: This can be a LONG RUNNING OPERATION. Please wait!') logger.debug('backfill_parent_id: This can be a LONG RUNNING OPERATION. Please wait!')
# Check for any images without parent def fetch_batch():
has_images = bool(list(Image return (Image
.select(Image.id) .select(Image.id, Image.ancestors)
.join(ImageStorage) .join(ImageStorage)
.where(Image.parent >> None, Image.ancestors != '/', ImageStorage.uploading == False) .where(Image.parent >> None, Image.ancestors != '/',
.limit(1))) ImageStorage.uploading == False))
if not has_images: for to_backfill in yield_random_entries(fetch_batch, 10000, 0.3):
logger.debug('backfill_parent_id: No migration needed') with app.config['DB_TRANSACTION_FACTORY'](db):
return try:
image = db_for_update(Image
.select()
.where(Image.id == to_backfill.id)).get()
image.parent = to_backfill.ancestors.split('/')[-2]
image.save()
except Image.DoesNotExist:
pass
while True: logger.debug('backfill_parent_id: Completed')
# Load the record from the DB.
batch_images_ids = list(Image
.select(Image.id)
.join(ImageStorage)
.where(Image.parent >> None, Image.ancestors != '/', ImageStorage.uploading == False)
.limit(100))
if len(batch_images_ids) == 0:
logger.debug('backfill_parent_id: Completed')
return
for image_id in batch_images_ids:
with app.config['DB_TRANSACTION_FACTORY'](db):
try:
image = Image.select(Image.id, Image.ancestors).where(Image.id == image_id).get()
image.parent = image.ancestors.split('/')[-2]
image.save()
except Image.DoesNotExist:
pass
if __name__ == "__main__": if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)