vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Revert "Add GC of layers in Clair"

Browse source 
This reverts 49872838ab
This commit is contained in:
Evan Cordell 2016-12-13 18:13:08 -05:00
parent dd5f7cbe6c
commit 5686c80af1
5 changed files with 23 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

44
data/model/repository.py
View file

@ -6,8 +6,8 @@ from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
from cachetools import ttl_cache
from data.model import (DataModelException, tag, db_transaction, storage, permission,
_basequery, config)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User, ImageStorage,
_basequery)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, User,
Visibility, RepositoryPermission, RepositoryActionCount,
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
@ -173,26 +173,29 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
referenced_candidates = (direct_referenced | ancestor_referenced)
# We desire three pieces of information from the database from the following
# We desire two pieces of information from the database from the following
# query: all of the image ids which are associated with this repository,
# the storages which are associated with those images and their docker IDs.
# and the storages which are associated with those images. In order to
# fetch just this information, and bypass all of the peewee model parsing
# code, which is overkill for just two fields, we use a tuple query, and
# feed that directly to the dictionary tuple constructor which takes an
# iterable of tuples containing [(k, v), (k, v), ...]
unreferenced_candidates = (Image
.select(Image.id, Image.storage, Image.docker_image_id, ImageStorage)
.join(ImageStorage)
.select(Image.id, Image.storage)
.where(Image.id << candidates_orphans,
~(Image.id << referenced_candidates)))
~(Image.id << referenced_candidates))
.tuples())
tuples = [(image.id, image.storage_id) for image in unreferenced_candidates]
unreferenced_images_to_storage_ids = dict(tuples)
image_ids_to_remove = unreferenced_images_to_storage_ids.keys()
unreferecend_images_to_storages = dict(unreferenced_candidates)
to_remove = unreferecend_images_to_storages.keys()
if len(image_ids_to_remove) > 0:
logger.info('Cleaning up unreferenced images: %s', image_ids_to_remove)
storage_id_whitelist = set(unreferenced_images_to_storage_ids.values())
if len(to_remove) > 0:
logger.info('Cleaning up unreferenced images: %s', to_remove)
storage_id_whitelist = set(unreferecend_images_to_storages.values())
# Lookup any derived images for the images to remove.
derived = DerivedStorageForImage.select().where(
DerivedStorageForImage.source_image << image_ids_to_remove)
DerivedStorageForImage.source_image << to_remove)
has_derived = False
for derived_image in derived:
@ -204,22 +207,21 @@ def garbage_collect_repo(repo, extra_candidate_set=None):
try:
(DerivedStorageForImage
.delete()
.where(DerivedStorageForImage.source_image << image_ids_to_remove)
.where(DerivedStorageForImage.source_image << to_remove)
.execute())
except IntegrityError:
logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove)
logger.info('Could not GC derived images %s; will try again soon', to_remove)
return False
try:
Image.delete().where(Image.id << image_ids_to_remove).execute()
Image.delete().where(Image.id << to_remove).execute()
except IntegrityError:
logger.info('Could not GC images %s; will try again soon', image_ids_to_remove)
logger.info('Could not GC images %s; will try again soon', to_remove)
return False
if len(image_ids_to_remove) > 0:
logger.info('Garbage collecting storage for images: %s', image_ids_to_remove)
if len(to_remove) > 0:
logger.info('Garbage collecting storage for images: %s', to_remove)
storage.garbage_collect_storage(storage_id_whitelist)
config.secscan_api.delete_layers(unreferenced_candidates)
return True