vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Phase 3 of Appr migration

Browse source 
Deletes the old models and their code
This commit is contained in:
Joseph Schorr 2018-07-16 17:12:04 -04:00
parent 223077ef53
commit 487edf0ba1
9 changed files with 218 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

8
data/model/repository.py
View file

@ -9,11 +9,11 @@ from cachetools import ttl_cache
from data.model import (
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import (
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag,
from data.database import (
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility,
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind)
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind, ApprTag)
from data.text import prefix_search
from util.itertoolrecipes import take
@ -90,8 +90,6 @@ def purge_repository(namespace_name, repository_name):
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute()
Tag.delete().where(Tag.repository == repo).execute()
ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute()
ApprTag.delete().where(ApprTag.repository == repo).execute()

16
data/model/storage.py
View file

@ -8,8 +8,8 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
DataModelException, _basequery)
from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignature,
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
ApprBlob, ensure_under_transaction)
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, ApprBlob,
ensure_under_transaction)
logger = logging.getLogger(__name__)
@ -105,21 +105,15 @@ def garbage_collect_storage(storage_id_whitelist):
logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
is_referenced_checksums)
# Check the ApprBlob tables as well.
query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
blob_referenced_checksums = set([blob.digest for blob in query])
if blob_referenced_checksums:
logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
blob_referenced_checksums)
# Check the ApprBlob table as well.
query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums))
appr_blob_referenced_checksums = set([blob.digest for blob in query])
if appr_blob_referenced_checksums:
logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced',
appr_blob_referenced_checksums)
unreferenced_checksums = (content_checksums - blob_referenced_checksums -
appr_blob_referenced_checksums - is_referenced_checksums)
unreferenced_checksums = (content_checksums - appr_blob_referenced_checksums -
is_referenced_checksums)
# Return all placements for all image storages found not at a CAS path or with a content
# checksum that is referenced.

6
data/model/test/test_gc.py
View file

@ -10,7 +10,7 @@ from playhouse.test_utils import assert_query_count
from data import model, database
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
Blob, ApprBlob)
ApprBlob)
from test.fixtures import *
@ -187,9 +187,6 @@ def assert_gc_integrity(expect_storage_removed=True):
if storage_row.cas_path:
storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
for blob_row in Blob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
@ -592,7 +589,6 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d
media_type = database.MediaType.get(name='text/plain')
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
database.Blob.create(digest=digest, size=0, media_type=media_type)
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
location = database.ImageStorageLocation.get(name=preferred)