vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Update the GC code to do everything with subqueries, making each GC run a bounded finite number of queries with a fixed length.

Browse source
This commit is contained in:
Jake Moshenko 2014-10-17 14:33:04 -04:00
parent 380eb49e58
commit d8149295ab
1 changed files with 34 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
data/model/legacy.py
View file

@ -1390,50 +1390,52 @@ def garbage_collect_repository(namespace_name, repository_name):
to_remove = set(all_images.keys()).difference(referenced_anscestors)
logger.info('Cleaning up unreferenced images: %s', to_remove)
uuids_to_check_for_gc = {all_images[id_to_remove].storage.uuid for id_to_remove in to_remove}
Image.delete().where(Image.id << list(to_remove)).execute()
logger.debug('Checking image storages for being orphaned: %s', uuids_to_check_for_gc)
# We are going to make the concious decision to not delete image storage inside the transaction
# This may end up producing garbage in s3, trading off for higher availability in the database
def placements_query_to_paths_set(placements_query):
return {(placement.location.name, config.store.image_path(placement.storage.uuid))
for placement in placements_query}
def remove_storages(placements_query):
storages_to_remove = {placement.storage.uuid: placement.storage
for placement in placements_query}
for storage in storages_to_remove.values():
# Deletes all placements as well, but leaves derived storages
storage.delete_instance(recursive=True)
def orphaned_storage_query(select_base_query):
return (select_base_query
.switch(ImageStorage)
.join(Image, JOIN_LEFT_OUTER)
.switch(ImageStorage)
.join(DerivedImageStorage, JOIN_LEFT_OUTER, on=(ImageStorage.id ==
DerivedImageStorage.derivative))
.group_by(ImageStorage)
.having((fn.Count(Image.id) == 0) & (fn.Count(DerivedImageStorage.id) == 0)))
paths_to_remove = set()
with config.app_config['DB_TRANSACTION_FACTORY'](db):
placements_to_remove = (ImageStoragePlacement
.select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
.join(ImageStorage)
.join(Image, JOIN_LEFT_OUTER)
.switch(ImageStoragePlacement)
.join(ImageStorageLocation)
.group_by(ImageStorage)
.where(ImageStorage.uuid << list(uuids_to_check_for_gc))
.having(fn.Count(Image.id) == 0))
paths_to_remove.update(placements_query_to_paths_set(placements_to_remove))
remove_storages(placements_to_remove)
# Remove the dervived image storages with sources of orphaned storages
DerivedImageStorage.delete().where(DerivedImageStorage.source <<
orphaned_storage_query(ImageStorage.select())).execute()
with config.app_config['DB_TRANSACTION_FACTORY'](db):
derived_to_remove = (ImageStoragePlacement
.select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)
.join(ImageStorage)
.join(DerivedImageStorage, on=(ImageStorage.id ==
DerivedImageStorage.derivative))
.switch(ImageStoragePlacement)
.join(ImageStorageLocation)
.where(DerivedImageStorage.source >> None))
paths_to_remove.update(placements_query_to_paths_set(derived_to_remove))
remove_storages(derived_to_remove)
# Track all of the data that should be removed from blob storage
placements_to_remove = orphaned_storage_query(ImageStoragePlacement
.select(ImageStoragePlacement,
ImageStorage,
ImageStorageLocation)
.join(ImageStorageLocation)
.switch(ImageStoragePlacement)
.join(ImageStorage))
paths_to_remove.update(placements_query_to_paths_set(placements_to_remove.clone()))
# Remove the placements for orphaned storages
placements_subquery = placements_to_remove.clone().select(ImageStoragePlacement.id)
(ImageStoragePlacement
.delete()
.where(ImageStoragePlacement.id << placements_subquery)
.execute())
# Remove the all orphaned storages
(ImageStorage
.delete()
.where(ImageStorage.id << orphaned_storage_query(ImageStorage.select(ImageStorage.id)))
.execute())
# Delete the actual blob storage
for location_name, image_path in paths_to_remove: