quay/tools/uncompressedsize.py

from data import model
from data.database import ImageStorage
from app import app, storage as store
from data.database import db

import logging

logger = logging.getLogger(__name__)

def backfill_sizes():
  query = (ImageStorage
           .select()
           .where(ImageStorage.uncompressed_size == None, ImageStorage.uploading == False)
           .limit(100))

  total = 0
  missing = 0
  batch_processed = 1

  while batch_processed > 0:
    batch_processed = 0
    with app.config['DB_TRANSACTION_FACTORY'](db):
      for image_storage in query.clone():
        total += 1
        batch_processed += 1

        if (total - 1) % 100 == 0:
          logger.debug('Storing entry: %s', total)

        # Lookup the JSON for the image.
        uuid = image_storage.uuid
        with_locations = model.get_storage_by_uuid(uuid)

        json_data = store.get_content(with_locations.locations, store.image_json_path(uuid))
        size = json_data.get('Size', json_data.get('size', -1))

        if size == -1:
          missing += 1

          logger.debug('Missing entry %s (%s/%s)', uuid, missing, total)

        image_storage.uncompressed_size = size
        image_storage.save()


if __name__ == "__main__":
  logging.basicConfig(level=logging.DEBUG)
  logging.getLogger('boto').setLevel(logging.CRITICAL)

  backfill_sizes()