77 lines
2.4 KiB
Python
77 lines
2.4 KiB
Python
|
"""
|
||
|
|
Usage (from the root in the container): venv/bin/python -m util.verifyplacements
|
||
|
|
|
||
|
|
This script verifies that if a blob is listed as being in a specific storage location, the file
|
||
|
|
actually exists there. If the file is not found in that storage location, the placement entry in
|
||
|
|
the database is removed.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import features
|
||
|
|
|
||
|
|
from peewee import fn
|
||
|
|
|
||
|
|
from app import storage
|
||
|
|
from data import model
|
||
|
|
from data.database import ImageStorage, ImageStoragePlacement, ImageStorageLocation
|
||
|
|
from util.migrate.allocator import yield_random_entries
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
LOCATION_MAP = {}
|
||
|
|
|
||
|
|
|
||
|
|
def _get_location_row(location):
|
||
|
|
if location in LOCATION_MAP:
|
||
|
|
return LOCATION_MAP[location]
|
||
|
|
|
||
|
|
location_row = ImageStorageLocation.get(name=location)
|
||
|
|
LOCATION_MAP[location] = location_row
|
||
|
|
return location_row
|
||
|
|
|
||
|
|
|
||
|
|
def verify_placements():
|
||
|
|
encountered = set()
|
||
|
|
|
||
|
|
iterator = yield_random_entries(
|
||
|
|
lambda: ImageStorage.select().where(ImageStorage.uploading == False),
|
||
|
|
ImageStorage.id,
|
||
|
|
1000,
|
||
|
|
ImageStorage.select(fn.Max(ImageStorage.id)).scalar(),
|
||
|
|
1,
|
||
|
|
)
|
||
|
|
|
||
|
|
for storage_row, abt, _ in iterator:
|
||
|
|
if storage_row.id in encountered:
|
||
|
|
continue
|
||
|
|
|
||
|
|
encountered.add(storage_row.id)
|
||
|
|
|
||
|
|
logger.info('Checking placements for storage `%s`', storage_row.uuid)
|
||
|
|
try:
|
||
|
|
with_locations = model.storage.get_storage_by_uuid(storage_row.uuid)
|
||
|
|
except model.InvalidImageException:
|
||
|
|
logger.exception('Could not find storage `%s`', storage_row.uuid)
|
||
|
|
continue
|
||
|
|
|
||
|
|
storage_path = model.storage.get_layer_path(storage_row)
|
||
|
|
locations_to_check = set(with_locations.locations)
|
||
|
|
if locations_to_check:
|
||
|
|
logger.info('Checking locations `%s` for storage `%s`', locations_to_check, storage_row.uuid)
|
||
|
|
for location in locations_to_check:
|
||
|
|
logger.info('Checking location `%s` for storage `%s`', location, storage_row.uuid)
|
||
|
|
if not storage.exists([location], storage_path):
|
||
|
|
location_row = _get_location_row(location)
|
||
|
|
logger.info('Location `%s` is missing for storage `%s`; removing', location,
|
||
|
|
storage_row.uuid)
|
||
|
|
(ImageStoragePlacement
|
||
|
|
.delete()
|
||
|
|
.where(ImageStoragePlacement.storage == storage_row,
|
||
|
|
ImageStoragePlacement.location == location_row)
|
||
|
|
.execute())
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
logging.basicConfig(level=logging.INFO)
|
||
|
|
verify_placements()
|