58 lines
1.5 KiB
Python
58 lines
1.5 KiB
Python
import json
|
|
import logging
|
|
|
|
from data import model
|
|
from data.database import ImageStorage
|
|
from app import app, storage as store
|
|
from data.database import db
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def backfill_sizes():
|
|
query = (ImageStorage
|
|
.select()
|
|
.where(ImageStorage.uncompressed_size == None, ImageStorage.uploading == False)
|
|
.limit(100))
|
|
|
|
total = 0
|
|
missing = 0
|
|
batch_processed = 1
|
|
|
|
while batch_processed > 0:
|
|
batch_processed = 0
|
|
with app.config['DB_TRANSACTION_FACTORY'](db):
|
|
for image_storage in query.clone():
|
|
total += 1
|
|
batch_processed += 1
|
|
|
|
if (total - 1) % 100 == 0:
|
|
logger.debug('Storing entry: %s', total)
|
|
|
|
# Lookup the JSON for the image.
|
|
uuid = image_storage.uuid
|
|
with_locations = model.get_storage_by_uuid(uuid)
|
|
|
|
try:
|
|
json_string = store.get_content(with_locations.locations, store.image_json_path(uuid))
|
|
json_data = json.loads(json_string)
|
|
size = json_data.get('Size', json_data.get('size', -1))
|
|
except IOError:
|
|
logger.debug('Image storage with no json %s', uuid)
|
|
size = -1
|
|
|
|
if size == -1:
|
|
missing += 1
|
|
|
|
logger.debug('Missing entry %s (%s/%s)', uuid, missing, total)
|
|
|
|
image_storage.uncompressed_size = size
|
|
image_storage.save()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.DEBUG)
|
|
logging.getLogger('boto').setLevel(logging.CRITICAL)
|
|
|
|
backfill_sizes()
|