Add new Manifest, ManifestLabel, ManifestLegacyImage and ManifestBlob tables and start writing and GCing to/from them

This change also starts passing in the manifest interface, rather than the raw data, to the model for writing.

Note that this change does *not* backfill the existing rows in to the new tables; that will occur in a followup PR. The new columns in `tagmanifest` and `tagmanifestlabel` will be used to track the backfill, as it will occur in a worker.
This commit is contained in:
Joseph Schorr 2018-07-31 15:43:09 -04:00
parent 36c7482385
commit a46660a06f
13 changed files with 476 additions and 120 deletions

View file

@ -10,9 +10,10 @@ from data.model import (
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import (
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind, ApprTag)
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail,
DerivedStorageForImage, Label, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind, ApprTag,
ManifestLegacyImage, Manifest)
from data.text import prefix_search
from util.itertoolrecipes import take
@ -275,6 +276,13 @@ def garbage_collect_repo(repo, extra_candidate_set=None, is_purge=False):
logger.info('Could not GC derived images %s; will try again soon', image_ids_to_remove)
return False
# Delete any legacy references to the images.
(ManifestLegacyImage
.delete()
.where(ManifestLegacyImage.image << image_ids_to_remove)
.execute())
# Delete the images themselves.
try:
Image.delete().where(Image.id << image_ids_to_remove).execute()
except IntegrityError: