54 lines
1.5 KiB
Python
54 lines
1.5 KiB
Python
|
import logging
|
||
|
|
|
||
|
|
from peewee import fn
|
||
|
|
|
||
|
|
from app import app
|
||
|
|
from util.migrate.allocator import yield_random_entries
|
||
|
|
from endpoints.v2.manifest import SignedManifest
|
||
|
|
from data.database import TagManifest
|
||
|
|
from data import model
|
||
|
|
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
PRINT_EVERY = 10
|
||
|
|
BATCH_SIZE = 100
|
||
|
|
|
||
|
|
|
||
|
|
def batch_query():
|
||
|
|
return TagManifest.select()
|
||
|
|
|
||
|
|
|
||
|
|
def remove_stale_manifests():
|
||
|
|
max_manifest_id = TagManifest.select(fn.Max(TagManifest.id)).scalar()
|
||
|
|
problematic = 0
|
||
|
|
checked = 0
|
||
|
|
for found, _ in yield_random_entries(batch_query, TagManifest.id, BATCH_SIZE, max_manifest_id):
|
||
|
|
checked += 1
|
||
|
|
parsed = SignedManifest(found.json_data, validate=False)
|
||
|
|
logger.debug('Auditing manifest with id: %s for %s/%s', found.digest, parsed.namespace,
|
||
|
|
parsed.repo_name)
|
||
|
|
|
||
|
|
try:
|
||
|
|
for layer_mdata in parsed.layers:
|
||
|
|
digest = layer_mdata.digest
|
||
|
|
found = model.blob.get_repo_blob_by_digest(parsed.namespace, parsed.repo_name, digest)
|
||
|
|
|
||
|
|
except model.BlobDoesNotExist:
|
||
|
|
logger.warning('Manifest missing layer: %s, deleting', digest)
|
||
|
|
found.delete_instance()
|
||
|
|
problematic += 1
|
||
|
|
|
||
|
|
if checked % PRINT_EVERY == 0:
|
||
|
|
logger.info('Removed %s/%s manifests', problematic, checked)
|
||
|
|
|
||
|
|
logger.info('Final Summary: %s/%s manifests removed', problematic, checked)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
logging.basicConfig(level=logging.INFO)
|
||
|
|
logging.getLogger('endpoints.v2.manifest').setLevel(logging.WARNING)
|
||
|
|
logging.getLogger('peewee').setLevel(logging.WARNING)
|
||
|
|
remove_stale_manifests()
|