Phase 1 of migrating APPR-specific tables to tables with the Appr prefix
Fixes https://jira.coreos.com/browse/QUAY-950
This commit is contained in:
Joseph Schorr
parent
6622f27c93
commit
113bb96f29
28 changed files with 699 additions and 176 deletions
|
|
@ -3,7 +3,6 @@ import logging
|
|||
from peewee import IntegrityError
|
||||
|
||||
from data.model import db_transaction
|
||||
from data.database import Blob, BlobPlacementLocation, BlobPlacement
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -13,17 +12,20 @@ def _ensure_sha256_header(digest):
|
|||
return 'sha256:' + digest
|
||||
|
||||
|
||||
def get_blob(digest):
|
||||
def get_blob(digest, models_ref):
|
||||
""" Find a blob by its digest. """
|
||||
Blob = models_ref.Blob
|
||||
return Blob.select().where(Blob.digest == _ensure_sha256_header(digest)).get()
|
||||
|
||||
|
||||
def get_or_create_blob(digest, size, media_type_name, locations):
|
||||
def get_or_create_blob(digest, size, media_type_name, locations, models_ref):
|
||||
""" Try to find a blob by its digest or create it. """
|
||||
Blob = models_ref.Blob
|
||||
BlobPlacement = models_ref.BlobPlacement
|
||||
|
||||
# Get or create the blog entry for the digest.
|
||||
try:
|
||||
blob = get_blob(digest)
|
||||
blob = get_blob(digest, models_ref)
|
||||
logger.debug('Retrieved blob with digest %s', digest)
|
||||
except Blob.DoesNotExist:
|
||||
blob = Blob.create(digest=_ensure_sha256_header(digest),
|
||||
|
|
@ -38,13 +40,16 @@ def get_or_create_blob(digest, size, media_type_name, locations):
|
|||
BlobPlacement.create(blob=blob, location=location_id)
|
||||
except IntegrityError:
|
||||
logger.debug('Location %s already existing for blob %s', location_name, blob.id)
|
||||
pass
|
||||
|
||||
return blob
|
||||
|
||||
|
||||
def get_blob_locations(digest):
|
||||
def get_blob_locations(digest, models_ref):
|
||||
""" Find all locations names for a blob. """
|
||||
Blob = models_ref.Blob
|
||||
BlobPlacement = models_ref.BlobPlacement
|
||||
BlobPlacementLocation = models_ref.BlobPlacementLocation
|
||||
|
||||
return [x.name for x in
|
||||
BlobPlacementLocation
|
||||
.select()
|
||||
|
|
@ -53,7 +58,9 @@ def get_blob_locations(digest):
|
|||
.where(Blob.digest == _ensure_sha256_header(digest))]
|
||||
|
||||
|
||||
def ensure_blob_locations(*names):
|
||||
def ensure_blob_locations(models_ref, *names):
|
||||
BlobPlacementLocation = models_ref.BlobPlacementLocation
|
||||
|
||||
with db_transaction():
|
||||
locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
from data.database import Tag, Channel
|
||||
from data.appr_model import tag as tag_model
|
||||
|
||||
|
||||
def get_channel_releases(repo, channel):
|
||||
def get_channel_releases(repo, channel, models_ref):
|
||||
""" Return all previously linked tags.
|
||||
This works based upon Tag lifetimes.
|
||||
"""
|
||||
Channel = models_ref.Channel
|
||||
Tag = models_ref.Tag
|
||||
|
||||
tag_kind_id = Channel.tag_kind.get_id('channel')
|
||||
channel_name = channel.name
|
||||
return (Tag
|
||||
|
|
@ -17,40 +19,46 @@ def get_channel_releases(repo, channel):
|
|||
.order_by(Tag.lifetime_end))
|
||||
|
||||
|
||||
def get_channel(repo, channel_name):
|
||||
def get_channel(repo, channel_name, models_ref):
|
||||
""" Find a Channel by name. """
|
||||
channel = tag_model.get_tag(repo, channel_name, "channel")
|
||||
channel = tag_model.get_tag(repo, channel_name, models_ref, "channel")
|
||||
return channel
|
||||
|
||||
|
||||
def get_tag_channels(repo, tag_name, active=True):
|
||||
def get_tag_channels(repo, tag_name, models_ref, active=True):
|
||||
""" Find the Channels associated with a Tag. """
|
||||
tag = tag_model.get_tag(repo, tag_name, "release")
|
||||
Tag = models_ref.Tag
|
||||
|
||||
tag = tag_model.get_tag(repo, tag_name, models_ref, "release")
|
||||
query = tag.tag_parents
|
||||
|
||||
if active:
|
||||
query = tag_model.tag_alive_oci(query)
|
||||
query = tag_model.tag_is_alive(query, Tag)
|
||||
|
||||
return query
|
||||
|
||||
|
||||
def delete_channel(repo, channel_name):
|
||||
def delete_channel(repo, channel_name, models_ref):
|
||||
""" Delete a channel by name. """
|
||||
return tag_model.delete_tag(repo, channel_name, "channel")
|
||||
return tag_model.delete_tag(repo, channel_name, models_ref, "channel")
|
||||
|
||||
|
||||
def create_or_update_channel(repo, channel_name, tag_name):
|
||||
def create_or_update_channel(repo, channel_name, tag_name, models_ref):
|
||||
""" Creates or updates a channel to include a particular tag. """
|
||||
tag = tag_model.get_tag(repo, tag_name, 'release')
|
||||
return tag_model.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel")
|
||||
tag = tag_model.get_tag(repo, tag_name, models_ref, 'release')
|
||||
return tag_model.create_or_update_tag(repo, channel_name, models_ref, linked_tag=tag,
|
||||
tag_kind="channel")
|
||||
|
||||
|
||||
def get_repo_channels(repo):
|
||||
def get_repo_channels(repo, models_ref):
|
||||
""" Creates or updates a channel to include a particular tag. """
|
||||
Channel = models_ref.Channel
|
||||
Tag = models_ref.Tag
|
||||
|
||||
tag_kind_id = Channel.tag_kind.get_id('channel')
|
||||
query = (Channel
|
||||
.select(Channel, Tag)
|
||||
.join(Tag, on=(Tag.id == Channel.linked_tag))
|
||||
.where(Channel.repository == repo,
|
||||
Channel.tag_kind == tag_kind_id))
|
||||
return tag_model.tag_alive_oci(query, cls=Channel)
|
||||
return tag_model.tag_is_alive(query, Channel)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import json
|
|||
|
||||
from cnr.models.package_base import get_media_type
|
||||
|
||||
from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
|
||||
from data.database import db_transaction, MediaType
|
||||
from data.appr_model import tag as tag_model
|
||||
|
||||
|
||||
|
|
@ -21,20 +21,23 @@ def _digest(manifestjson):
|
|||
return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest())
|
||||
|
||||
|
||||
def get_manifest_query(digest, media_type):
|
||||
def get_manifest_query(digest, media_type, models_ref):
|
||||
Manifest = models_ref.Manifest
|
||||
return Manifest.select().where(Manifest.digest == _ensure_sha256_header(digest),
|
||||
Manifest.media_type == Manifest.media_type.get_id(media_type))
|
||||
|
||||
|
||||
def get_manifest_with_blob(digest, media_type):
|
||||
query = get_manifest_query(digest, media_type)
|
||||
def get_manifest_with_blob(digest, media_type, models_ref):
|
||||
Blob = models_ref.Blob
|
||||
query = get_manifest_query(digest, media_type, models_ref)
|
||||
return query.join(Blob).get()
|
||||
|
||||
|
||||
def get_or_create_manifest(manifest_json, media_type_name):
|
||||
def get_or_create_manifest(manifest_json, media_type_name, models_ref):
|
||||
Manifest = models_ref.Manifest
|
||||
digest = _digest(manifest_json)
|
||||
try:
|
||||
manifest = get_manifest_query(digest, media_type_name).get()
|
||||
manifest = get_manifest_query(digest, media_type_name, models_ref).get()
|
||||
except Manifest.DoesNotExist:
|
||||
with db_transaction():
|
||||
manifest = Manifest.create(digest=digest,
|
||||
|
|
@ -42,16 +45,19 @@ def get_or_create_manifest(manifest_json, media_type_name):
|
|||
media_type=Manifest.media_type.get_id(media_type_name))
|
||||
return manifest
|
||||
|
||||
def get_manifest_types(repo, release=None):
|
||||
def get_manifest_types(repo, models_ref, release=None):
|
||||
""" Returns an array of MediaTypes.name for a repo, can filter by tag """
|
||||
query = tag_model.tag_alive_oci(Tag
|
||||
Tag = models_ref.Tag
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
|
||||
query = tag_model.tag_is_alive(Tag
|
||||
.select(MediaType.name)
|
||||
.join(ManifestListManifest,
|
||||
on=(ManifestListManifest.manifest_list == Tag.manifest_list))
|
||||
.join(MediaType,
|
||||
on=(ManifestListManifest.media_type == MediaType.id))
|
||||
.where(Tag.repository == repo,
|
||||
Tag.tag_kind == Tag.tag_kind.get_id('release')))
|
||||
Tag.tag_kind == Tag.tag_kind.get_id('release')), Tag)
|
||||
if release:
|
||||
query = query.where(Tag.name == release)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import logging
|
|||
import hashlib
|
||||
import json
|
||||
|
||||
from data.database import ManifestList, ManifestListManifest, db_transaction
|
||||
from data.database import db_transaction
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -18,16 +18,19 @@ def _digest(manifestjson):
|
|||
return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest())
|
||||
|
||||
|
||||
def get_manifest_list(digest):
|
||||
def get_manifest_list(digest, models_ref):
|
||||
ManifestList = models_ref.ManifestList
|
||||
return ManifestList.select().where(ManifestList.digest == _ensure_sha256_header(digest)).get()
|
||||
|
||||
|
||||
def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version):
|
||||
def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version, models_ref):
|
||||
ManifestList = models_ref.ManifestList
|
||||
|
||||
digest = _digest(manifest_list_json)
|
||||
media_type_id = ManifestList.media_type.get_id(media_type_name)
|
||||
|
||||
try:
|
||||
return get_manifest_list(digest)
|
||||
return get_manifest_list(digest, models_ref)
|
||||
except ManifestList.DoesNotExist:
|
||||
with db_transaction():
|
||||
manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json,
|
||||
|
|
@ -35,7 +38,7 @@ def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_vers
|
|||
return manifestlist
|
||||
|
||||
|
||||
def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
|
||||
def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json, models_ref):
|
||||
""" From a manifestlist, manifests, and the manifest list blob,
|
||||
create if doesn't exist the manfiestlistmanifest for each manifest """
|
||||
for pos in xrange(len(manifest_ids)):
|
||||
|
|
@ -43,10 +46,13 @@ def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
|
|||
manifest_json = manifest_list_json[pos]
|
||||
get_or_create_manifestlistmanifest(manifest=manifest_id,
|
||||
manifestlist=manifestlist,
|
||||
media_type_name=manifest_json['mediaType'])
|
||||
media_type_name=manifest_json['mediaType'],
|
||||
models_ref=models_ref)
|
||||
|
||||
|
||||
def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name):
|
||||
def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name, models_ref):
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
|
||||
media_type_id = ManifestListManifest.media_type.get_id(media_type_name)
|
||||
try:
|
||||
ml = (ManifestListManifest
|
||||
|
|
|
|||
21
data/appr_model/models.py
Normal file
21
data/appr_model/models.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
from collections import namedtuple
|
||||
|
||||
from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
|
||||
ManifestListManifest, Manifest, BlobPlacement, Channel)
|
||||
from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
|
||||
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
|
||||
ApprBlobPlacement, ApprChannel)
|
||||
|
||||
ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation', 'ManifestList',
|
||||
'ManifestBlob', 'Blob', 'ManifestListManifest', 'Manifest',
|
||||
'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name',
|
||||
'tag_set_prefetch_name'])
|
||||
|
||||
OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
|
||||
ManifestListManifest, Manifest, BlobPlacement, Channel,
|
||||
'manifestlistmanifest_set', 'tag_set_prefetch')
|
||||
|
||||
NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
|
||||
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
|
||||
ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set',
|
||||
'apprtag_set_prefetch')
|
||||
|
|
@ -3,12 +3,14 @@ from peewee import prefetch
|
|||
|
||||
|
||||
from data import model
|
||||
from data.database import Repository, Namespace, Tag, ManifestListManifest
|
||||
from data.database import Repository, Namespace
|
||||
from data.appr_model import tag as tag_model
|
||||
|
||||
|
||||
def list_packages_query(namespace=None, media_type=None, search_query=None, username=None):
|
||||
def list_packages_query(models_ref, namespace=None, media_type=None, search_query=None,
|
||||
username=None):
|
||||
""" List and filter repository by search query. """
|
||||
Tag = models_ref.Tag
|
||||
fields = [model.repository.SEARCH_FIELDS.name.name]
|
||||
|
||||
if search_query is not None:
|
||||
|
|
@ -40,9 +42,9 @@ def list_packages_query(namespace=None, media_type=None, search_query=None, user
|
|||
.order_by(Tag.lifetime_start))
|
||||
|
||||
if media_type:
|
||||
tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type)
|
||||
tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type, models_ref)
|
||||
|
||||
tag_query = tag_model.tag_alive_oci(tag_query)
|
||||
tag_query = tag_model.tag_is_alive(tag_query, Tag)
|
||||
query = prefetch(repo_query, tag_query)
|
||||
|
||||
return query
|
||||
|
|
|
|||
|
|
@ -3,8 +3,7 @@ import bisect
|
|||
from cnr.exception import PackageAlreadyExists
|
||||
from cnr.models.package_base import manifest_media_type
|
||||
|
||||
from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
|
||||
ManifestListManifest, Blob, ManifestBlob)
|
||||
from data.database import db_transaction, get_epoch_timestamp
|
||||
from data.appr_model import (blob as blob_model, manifest as manifest_model,
|
||||
manifest_list as manifest_list_model,
|
||||
tag as tag_model)
|
||||
|
|
@ -20,11 +19,17 @@ def _ensure_sha256_header(digest):
|
|||
return 'sha256:' + digest
|
||||
|
||||
|
||||
def get_app_release(repo, tag_name, media_type):
|
||||
def get_app_release(repo, tag_name, media_type, models_ref):
|
||||
""" Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """
|
||||
tag = tag_model.get_tag(repo, tag_name, tag_kind='release')
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
Manifest = models_ref.Manifest
|
||||
Blob = models_ref.Blob
|
||||
ManifestBlob = models_ref.ManifestBlob
|
||||
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
|
||||
|
||||
tag = tag_model.get_tag(repo, tag_name, models_ref, tag_kind='release')
|
||||
media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
|
||||
manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set
|
||||
manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name)
|
||||
.join(Manifest)
|
||||
.where(ManifestListManifest.media_type == media_type_id).get())
|
||||
manifest = manifestlistmanifest.manifest
|
||||
|
|
@ -32,25 +37,26 @@ def get_app_release(repo, tag_name, media_type):
|
|||
return (tag, manifest, blob)
|
||||
|
||||
|
||||
def delete_app_release(repo, tag_name, media_type):
|
||||
def delete_app_release(repo, tag_name, media_type, models_ref):
|
||||
""" Terminate a Tag/media-type couple
|
||||
It find the corresponding tag/manifest and remove from the manifestlistmanifest the manifest
|
||||
1. it terminates the current tag (in all-cases)
|
||||
2. if the new manifestlist is not empty, it creates a new tag for it
|
||||
"""
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
|
||||
|
||||
media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
|
||||
|
||||
with db_transaction():
|
||||
tag = tag_model.get_tag(repo, tag_name)
|
||||
tag = tag_model.get_tag(repo, tag_name, models_ref)
|
||||
manifest_list = tag.manifest_list
|
||||
list_json = manifest_list.manifest_list_json
|
||||
mlm_query = (ManifestListManifest
|
||||
.select()
|
||||
.where(ManifestListManifest.manifest_list == tag.manifest_list))
|
||||
list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
|
||||
manifestlistmanifest = (tag
|
||||
.manifest_list
|
||||
.manifestlistmanifest_set
|
||||
manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name)
|
||||
.where(ManifestListManifest.media_type == media_type_id).get())
|
||||
index = list_manifest_ids.index(manifestlistmanifest.manifest_id)
|
||||
list_manifest_ids.pop(index)
|
||||
|
|
@ -61,36 +67,42 @@ def delete_app_release(repo, tag_name, media_type):
|
|||
tag.save()
|
||||
else:
|
||||
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
|
||||
SCHEMA_VERSION)
|
||||
SCHEMA_VERSION, models_ref)
|
||||
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids,
|
||||
list_json)
|
||||
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
|
||||
list_json, models_ref)
|
||||
tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist,
|
||||
tag_kind="release")
|
||||
return tag
|
||||
|
||||
|
||||
def create_app_release(repo, tag_name, manifest_data, digest, force=False):
|
||||
def create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False):
|
||||
""" Create a new application release, it includes creating a new Tag, ManifestList,
|
||||
ManifestListManifests, Manifest, ManifestBlob.
|
||||
|
||||
To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id.
|
||||
To find the insert point in the ManifestList it uses bisect on the manifest-ids list.
|
||||
"""
|
||||
ManifestList = models_ref.ManifestList
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
Blob = models_ref.Blob
|
||||
ManifestBlob = models_ref.ManifestBlob
|
||||
|
||||
with db_transaction():
|
||||
# Create/get the package manifest
|
||||
manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType'])
|
||||
manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType'],
|
||||
models_ref)
|
||||
# get the tag
|
||||
tag = tag_model.get_or_initialize_tag(repo, tag_name)
|
||||
tag = tag_model.get_or_initialize_tag(repo, tag_name, models_ref)
|
||||
|
||||
if tag.manifest_list is None:
|
||||
tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE),
|
||||
schema_version=SCHEMA_VERSION,
|
||||
manifest_list_json=[])
|
||||
manifest_list_json=[], )
|
||||
|
||||
elif tag_model.tag_media_type_exists(tag, manifest.media_type):
|
||||
elif tag_model.tag_media_type_exists(tag, manifest.media_type, models_ref):
|
||||
if force:
|
||||
delete_app_release(repo, tag_name, manifest.media_type.name)
|
||||
return create_app_release(repo, tag_name, manifest_data, digest, force=False)
|
||||
delete_app_release(repo, tag_name, manifest.media_type.name, models_ref)
|
||||
return create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False)
|
||||
else:
|
||||
raise PackageAlreadyExists("package exists already")
|
||||
|
||||
|
|
@ -103,10 +115,11 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
|
|||
list_json.insert(insert_point, manifest.manifest_json)
|
||||
list_manifest_ids.insert(insert_point, manifest.id)
|
||||
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
|
||||
SCHEMA_VERSION)
|
||||
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json)
|
||||
SCHEMA_VERSION, models_ref)
|
||||
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json,
|
||||
models_ref)
|
||||
|
||||
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
|
||||
tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist,
|
||||
tag_kind="release")
|
||||
blob_digest = digest
|
||||
|
||||
|
|
@ -117,21 +130,23 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
|
|||
.where(ManifestBlob.manifest == manifest,
|
||||
Blob.digest == _ensure_sha256_header(blob_digest)).get())
|
||||
except ManifestBlob.DoesNotExist:
|
||||
blob = blob_model.get_blob(blob_digest)
|
||||
blob = blob_model.get_blob(blob_digest, models_ref)
|
||||
ManifestBlob.create(manifest=manifest, blob=blob)
|
||||
return tag
|
||||
|
||||
def get_release_objs(repo, media_type=None):
|
||||
def get_release_objs(repo, models_ref, media_type=None):
|
||||
""" Returns an array of Tag for a repo, with optional filtering by media_type. """
|
||||
Tag = models_ref.Tag
|
||||
|
||||
release_query = (Tag
|
||||
.select()
|
||||
.where(Tag.repository == repo,
|
||||
Tag.tag_kind == Tag.tag_kind.get_id("release")))
|
||||
if media_type:
|
||||
release_query = tag_model.filter_tags_by_media_type(release_query, media_type)
|
||||
release_query = tag_model.filter_tags_by_media_type(release_query, media_type, models_ref)
|
||||
|
||||
return tag_model.tag_alive_oci(release_query)
|
||||
return tag_model.tag_is_alive(release_query, Tag)
|
||||
|
||||
def get_releases(repo, media_type=None):
|
||||
def get_releases(repo, model_refs, media_type=None):
|
||||
""" Returns an array of Tag.name for a repo, can filter by media_type. """
|
||||
return [t.name for t in get_release_objs(repo, media_type)]
|
||||
return [t.name for t in get_release_objs(repo, model_refs, media_type)]
|
||||
|
|
|
|||
|
|
@ -4,32 +4,37 @@ from cnr.models.package_base import manifest_media_type
|
|||
from peewee import IntegrityError
|
||||
|
||||
from data.model import (db_transaction, TagAlreadyCreatedException)
|
||||
from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update
|
||||
from data.database import get_epoch_timestamp_ms, db_for_update
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def tag_alive_oci(query, now_ts=None, cls=Tag):
|
||||
def tag_is_alive(query, cls, now_ts=None):
|
||||
return query.where((cls.lifetime_end >> None) |
|
||||
(cls.lifetime_end > now_ts))
|
||||
|
||||
|
||||
def tag_media_type_exists(tag, media_type):
|
||||
return (tag.manifest_list.manifestlistmanifest_set
|
||||
def tag_media_type_exists(tag, media_type, models_ref):
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
|
||||
return (getattr(tag.manifest_list, manifestlistmanifest_set_name)
|
||||
.where(ManifestListManifest.media_type == media_type).count() > 0)
|
||||
|
||||
|
||||
def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"):
|
||||
def create_or_update_tag(repo, tag_name, models_ref, manifest_list=None, linked_tag=None,
|
||||
tag_kind="release"):
|
||||
Tag = models_ref.Tag
|
||||
|
||||
now_ts = get_epoch_timestamp_ms()
|
||||
tag_kind_id = Tag.tag_kind.get_id(tag_kind)
|
||||
with db_transaction():
|
||||
try:
|
||||
tag = db_for_update(tag_alive_oci(Tag
|
||||
tag = db_for_update(tag_is_alive(Tag
|
||||
.select()
|
||||
.where(Tag.repository == repo,
|
||||
Tag.name == tag_name,
|
||||
Tag.tag_kind == tag_kind_id), now_ts)).get()
|
||||
Tag.tag_kind == tag_kind_id), Tag, now_ts)).get()
|
||||
if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag:
|
||||
return tag
|
||||
tag.lifetime_end = now_ts
|
||||
|
|
@ -46,40 +51,47 @@ def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, ta
|
|||
raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name))
|
||||
|
||||
|
||||
def get_or_initialize_tag(repo, tag_name, tag_kind="release"):
|
||||
def get_or_initialize_tag(repo, tag_name, models_ref, tag_kind="release"):
|
||||
Tag = models_ref.Tag
|
||||
|
||||
try:
|
||||
return tag_alive_oci(Tag.select().where(Tag.repository == repo, Tag.name == tag_name)).get()
|
||||
return tag_is_alive(Tag.select().where(Tag.repository == repo, Tag.name == tag_name), Tag).get()
|
||||
except Tag.DoesNotExist:
|
||||
return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind))
|
||||
|
||||
|
||||
def get_tag(repo, tag_name, tag_kind="release"):
|
||||
return tag_alive_oci(Tag.select()
|
||||
def get_tag(repo, tag_name, models_ref, tag_kind="release"):
|
||||
Tag = models_ref.Tag
|
||||
return tag_is_alive(Tag.select()
|
||||
.where(Tag.repository == repo,
|
||||
Tag.name == tag_name,
|
||||
Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get()
|
||||
Tag.tag_kind == Tag.tag_kind.get_id(tag_kind)), Tag).get()
|
||||
|
||||
|
||||
def delete_tag(repo, tag_name, tag_kind="release"):
|
||||
def delete_tag(repo, tag_name, models_ref, tag_kind="release"):
|
||||
Tag = models_ref.Tag
|
||||
tag_kind_id = Tag.tag_kind.get_id(tag_kind)
|
||||
tag = tag_alive_oci(Tag.select()
|
||||
tag = tag_is_alive(Tag.select()
|
||||
.where(Tag.repository == repo,
|
||||
Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get()
|
||||
Tag.name == tag_name, Tag.tag_kind == tag_kind_id), Tag).get()
|
||||
tag.lifetime_end = get_epoch_timestamp_ms()
|
||||
tag.save()
|
||||
return tag
|
||||
|
||||
|
||||
def tag_exists(repo, tag_name, tag_kind="release"):
|
||||
def tag_exists(repo, tag_name, models_ref, tag_kind="release"):
|
||||
Tag = models_ref.Tag
|
||||
try:
|
||||
get_tag(repo, tag_name, tag_kind)
|
||||
get_tag(repo, tag_name, models_ref, tag_kind)
|
||||
return True
|
||||
except Tag.DoesNotExist:
|
||||
return False
|
||||
|
||||
|
||||
def filter_tags_by_media_type(tag_query, media_type):
|
||||
def filter_tags_by_media_type(tag_query, media_type, models_ref):
|
||||
""" Return only available tag for a media_type. """
|
||||
ManifestListManifest = models_ref.ManifestListManifest
|
||||
Tag = models_ref.Tag
|
||||
media_type = manifest_media_type(media_type)
|
||||
t = (tag_query
|
||||
.join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list))
|
||||
|
|
|
|||
151
data/database.py
151
data/database.py
|
|
@ -470,8 +470,8 @@ class User(BaseModel):
|
|||
RepositoryTag, PermissionPrototype, DerivedStorageForImage,
|
||||
TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
|
||||
RepositoryNotification, OAuthAuthorizationCode,
|
||||
RepositoryActionCount, TagManifestLabel, Tag,
|
||||
TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes
|
||||
RepositoryActionCount, TagManifestLabel,
|
||||
TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes
|
||||
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
|
||||
|
||||
|
||||
|
|
@ -619,7 +619,7 @@ class Repository(BaseModel):
|
|||
# are cleaned up directly
|
||||
skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
|
||||
Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
|
||||
RepositorySearchScore} | cnr_classes
|
||||
RepositorySearchScore} | cnr_classes | appr_classes
|
||||
|
||||
delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
|
||||
|
||||
|
|
@ -1187,7 +1187,6 @@ class ServiceKey(BaseModel):
|
|||
|
||||
class MediaType(BaseModel):
|
||||
""" MediaType is an enumeration of the possible formats of various objects in the data model.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
|
@ -1201,7 +1200,6 @@ class Messages(BaseModel):
|
|||
|
||||
class LabelSourceType(BaseModel):
|
||||
""" LabelSourceType is an enumeration of the possible sources for a label.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
mutable = BooleanField(default=False)
|
||||
|
|
@ -1210,7 +1208,6 @@ class LabelSourceType(BaseModel):
|
|||
class Label(BaseModel):
|
||||
""" Label represents user-facing metadata associated with another entry in the database (e.g. a
|
||||
Manifest).
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
"""
|
||||
uuid = CharField(default=uuid_generator, index=True, unique=True)
|
||||
key = CharField(index=True)
|
||||
|
|
@ -1221,7 +1218,6 @@ class Label(BaseModel):
|
|||
|
||||
class TagManifestLabel(BaseModel):
|
||||
""" Mapping from a tag manifest to a label.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
"""
|
||||
repository = ForeignKeyField(Repository, index=True)
|
||||
annotated = ForeignKeyField(TagManifest, index=True)
|
||||
|
|
@ -1237,8 +1233,17 @@ class TagManifestLabel(BaseModel):
|
|||
|
||||
class Blob(BaseModel):
|
||||
""" Blob represents a content-addressable object stored outside of the database.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This is deprecated in favor of ApprBlob.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
media_type = EnumField(MediaType)
|
||||
size = BigIntegerField()
|
||||
uncompressed_size = BigIntegerField(null=True)
|
||||
|
||||
|
||||
|
||||
class ApprBlob(BaseModel):
|
||||
""" ApprBlob represents a content-addressable object stored outside of the database.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
media_type = EnumField(MediaType)
|
||||
|
|
@ -1248,16 +1253,20 @@ class Blob(BaseModel):
|
|||
|
||||
class BlobPlacementLocation(BaseModel):
|
||||
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This is deprecated in favor of ApprBlobPlacementLocation.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class ApprBlobPlacementLocation(BaseModel):
|
||||
""" ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class BlobPlacement(BaseModel):
|
||||
""" BlobPlacement represents the location of a Blob.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This is deprecated in favor of ApprBlobPlacement.
|
||||
"""
|
||||
blob = ForeignKeyField(Blob)
|
||||
location = EnumField(BlobPlacementLocation)
|
||||
|
|
@ -1270,10 +1279,31 @@ class BlobPlacement(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ApprBlobPlacement(BaseModel):
|
||||
""" ApprBlobPlacement represents the location of a Blob.
|
||||
"""
|
||||
blob = ForeignKeyField(ApprBlob)
|
||||
location = EnumField(ApprBlobPlacementLocation)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('blob', 'location'), True),
|
||||
)
|
||||
|
||||
|
||||
class Manifest(BaseModel):
|
||||
""" Manifest represents the metadata and collection of blobs that comprise a container image.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This is deprecated in favor of ApprManifest.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
media_type = EnumField(MediaType)
|
||||
manifest_json = JSONField()
|
||||
|
||||
|
||||
class ApprManifest(BaseModel):
|
||||
""" ApprManifest represents the metadata and collection of blobs that comprise an Appr image.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
media_type = EnumField(MediaType)
|
||||
|
|
@ -1282,8 +1312,7 @@ class Manifest(BaseModel):
|
|||
|
||||
class ManifestBlob(BaseModel):
|
||||
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This is deprecated in favor of ApprManifestBlob.
|
||||
"""
|
||||
manifest = ForeignKeyField(Manifest, index=True)
|
||||
blob = ForeignKeyField(Blob, index=True)
|
||||
|
|
@ -1296,10 +1325,32 @@ class ManifestBlob(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ApprManifestBlob(BaseModel):
|
||||
""" ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs.
|
||||
"""
|
||||
manifest = ForeignKeyField(ApprManifest, index=True)
|
||||
blob = ForeignKeyField(ApprBlob, index=True)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('manifest', 'blob'), True),
|
||||
)
|
||||
|
||||
|
||||
class ManifestList(BaseModel):
|
||||
""" ManifestList represents all of the various manifests that compose a Tag.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This is deprecated in favor of ApprManifestList.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
manifest_list_json = JSONField()
|
||||
schema_version = CharField()
|
||||
media_type = EnumField(MediaType)
|
||||
|
||||
|
||||
class ApprManifestList(BaseModel):
|
||||
""" ApprManifestList represents all of the various Appr manifests that compose an ApprTag.
|
||||
"""
|
||||
digest = CharField(index=True, unique=True)
|
||||
manifest_list_json = JSONField()
|
||||
|
|
@ -1309,16 +1360,20 @@ class ManifestList(BaseModel):
|
|||
|
||||
class TagKind(BaseModel):
|
||||
""" TagKind is a enumtable to reference tag kinds.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This model is deprecated in favor of ApprTagKind.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class ApprTagKind(BaseModel):
|
||||
""" ApprTagKind is a enumtable to reference tag kinds.
|
||||
"""
|
||||
name = CharField(index=True, unique=True)
|
||||
|
||||
|
||||
class Tag(BaseModel):
|
||||
""" Tag represents a user-facing alias for referencing a ManifestList.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This model is deprecated in favor of ApprTag.
|
||||
"""
|
||||
name = CharField()
|
||||
repository = ForeignKeyField(Repository)
|
||||
|
|
@ -1342,13 +1397,37 @@ class Tag(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ApprTag(BaseModel):
|
||||
""" ApprTag represents a user-facing alias for referencing an ApprManifestList.
|
||||
"""
|
||||
name = CharField()
|
||||
repository = ForeignKeyField(Repository)
|
||||
manifest_list = ForeignKeyField(ApprManifestList, null=True)
|
||||
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
|
||||
lifetime_end = BigIntegerField(null=True, index=True)
|
||||
hidden = BooleanField(default=False)
|
||||
reverted = BooleanField(default=False)
|
||||
protected = BooleanField(default=False)
|
||||
tag_kind = EnumField(ApprTagKind)
|
||||
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('repository', 'name'), False),
|
||||
(('repository', 'name', 'hidden'), False),
|
||||
# This unique index prevents deadlocks when concurrently moving and deleting tags
|
||||
(('repository', 'name', 'lifetime_end'), True),
|
||||
)
|
||||
|
||||
Channel = Tag.alias()
|
||||
ApprChannel = ApprTag.alias()
|
||||
|
||||
|
||||
class ManifestListManifest(BaseModel):
|
||||
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
|
||||
This model is a part of the new OCI/CNR model set.
|
||||
CNR
|
||||
This model is deprecated in favor of ApprManifestListManifest.
|
||||
"""
|
||||
manifest_list = ForeignKeyField(ManifestList, index=True)
|
||||
manifest = ForeignKeyField(Manifest, index=True)
|
||||
|
|
@ -1366,6 +1445,25 @@ class ManifestListManifest(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ApprManifestListManifest(BaseModel):
|
||||
""" ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and
|
||||
ApprManifests.
|
||||
"""
|
||||
manifest_list = ForeignKeyField(ApprManifestList, index=True)
|
||||
manifest = ForeignKeyField(ApprManifest, index=True)
|
||||
operating_system = CharField(null=True)
|
||||
architecture = CharField(null=True)
|
||||
platform_json = JSONField(null=True)
|
||||
media_type = EnumField(MediaType)
|
||||
|
||||
class Meta:
|
||||
database = db
|
||||
read_slaves = (read_slave,)
|
||||
indexes = (
|
||||
(('manifest_list', 'media_type'), False),
|
||||
)
|
||||
|
||||
|
||||
class AppSpecificAuthToken(BaseModel):
|
||||
""" AppSpecificAuthToken represents a token generated by a user for use with an external
|
||||
application where putting the user's credentials, even encrypted, is deemed too risky.
|
||||
|
|
@ -1388,5 +1486,8 @@ class AppSpecificAuthToken(BaseModel):
|
|||
|
||||
cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
|
||||
ManifestListManifest, Manifest, BlobPlacement])
|
||||
appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
|
||||
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
|
||||
ApprBlobPlacement])
|
||||
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
|
||||
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ up_mysql() {
|
|||
sleep 25
|
||||
|
||||
# Add the database to mysql.
|
||||
docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
|
||||
docker run --rm --link mysql:mysql mysql:5.7 sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
|
||||
}
|
||||
|
||||
down_mysql() {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import sqlalchemy as sa
|
|||
from sqlalchemy.dialects import mysql
|
||||
from util.migrate import UTF8LongText, UTF8CharField
|
||||
|
||||
def upgrade(tables):
|
||||
def upgrade(tables, tester):
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('derivedimage')
|
||||
op.drop_table('manifestlabel')
|
||||
|
|
@ -28,7 +28,7 @@ def upgrade(tables):
|
|||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade(tables):
|
||||
def downgrade(tables, tester):
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
'manifestlayer',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
"""Add new Appr-specific tables
|
||||
|
||||
Revision ID: 610320e9dacf
|
||||
Revises: 5cbbfc95bac7
|
||||
Create Date: 2018-05-24 16:46:13.514562
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '610320e9dacf'
|
||||
down_revision = '5cbbfc95bac7'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
from util.migrate.table_ops import copy_table_contents
|
||||
|
||||
def upgrade(tables, tester):
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('apprblobplacementlocation',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacementlocation'))
|
||||
)
|
||||
op.create_index('apprblobplacementlocation_name', 'apprblobplacementlocation', ['name'], unique=True)
|
||||
op.create_table('apprtagkind',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtagkind'))
|
||||
)
|
||||
op.create_index('apprtagkind_name', 'apprtagkind', ['name'], unique=True)
|
||||
op.create_table('apprblob',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('digest', sa.String(length=255), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.Column('size', sa.BigInteger(), nullable=False),
|
||||
sa.Column('uncompressed_size', sa.BigInteger(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprblob_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblob'))
|
||||
)
|
||||
op.create_index('apprblob_digest', 'apprblob', ['digest'], unique=True)
|
||||
op.create_index('apprblob_media_type_id', 'apprblob', ['media_type_id'], unique=False)
|
||||
op.create_table('apprmanifest',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('digest', sa.String(length=255), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_json', sa.Text(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifest_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifest'))
|
||||
)
|
||||
op.create_index('apprmanifest_digest', 'apprmanifest', ['digest'], unique=True)
|
||||
op.create_index('apprmanifest_media_type_id', 'apprmanifest', ['media_type_id'], unique=False)
|
||||
op.create_table('apprmanifestlist',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('digest', sa.String(length=255), nullable=False),
|
||||
sa.Column('manifest_list_json', sa.Text(), nullable=False),
|
||||
sa.Column('schema_version', sa.String(length=255), nullable=False),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlist_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlist'))
|
||||
)
|
||||
op.create_index('apprmanifestlist_digest', 'apprmanifestlist', ['digest'], unique=True)
|
||||
op.create_index('apprmanifestlist_media_type_id', 'apprmanifestlist', ['media_type_id'], unique=False)
|
||||
op.create_table('apprblobplacement',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('blob_id', sa.Integer(), nullable=False),
|
||||
sa.Column('location_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprblobplacement_blob_id_apprblob')),
|
||||
sa.ForeignKeyConstraint(['location_id'], ['apprblobplacementlocation.id'], name=op.f('fk_apprblobplacement_location_id_apprblobplacementlocation')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacement'))
|
||||
)
|
||||
op.create_index('apprblobplacement_blob_id', 'apprblobplacement', ['blob_id'], unique=False)
|
||||
op.create_index('apprblobplacement_blob_id_location_id', 'apprblobplacement', ['blob_id', 'location_id'], unique=True)
|
||||
op.create_index('apprblobplacement_location_id', 'apprblobplacement', ['location_id'], unique=False)
|
||||
op.create_table('apprmanifestblob',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_id', sa.Integer(), nullable=False),
|
||||
sa.Column('blob_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprmanifestblob_blob_id_apprblob')),
|
||||
sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestblob_manifest_id_apprmanifest')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestblob'))
|
||||
)
|
||||
op.create_index('apprmanifestblob_blob_id', 'apprmanifestblob', ['blob_id'], unique=False)
|
||||
op.create_index('apprmanifestblob_manifest_id', 'apprmanifestblob', ['manifest_id'], unique=False)
|
||||
op.create_index('apprmanifestblob_manifest_id_blob_id', 'apprmanifestblob', ['manifest_id', 'blob_id'], unique=True)
|
||||
op.create_table('apprmanifestlistmanifest',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_list_id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_id', sa.Integer(), nullable=False),
|
||||
sa.Column('operating_system', sa.String(length=255), nullable=True),
|
||||
sa.Column('architecture', sa.String(length=255), nullable=True),
|
||||
sa.Column('platform_json', sa.Text(), nullable=True),
|
||||
sa.Column('media_type_id', sa.Integer(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_id_apprmanifest')),
|
||||
sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_list_id_apprmanifestlist')),
|
||||
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlistmanifest_media_type_id_mediatype')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlistmanifest'))
|
||||
)
|
||||
op.create_index('apprmanifestlistmanifest_manifest_id', 'apprmanifestlistmanifest', ['manifest_id'], unique=False)
|
||||
op.create_index('apprmanifestlistmanifest_manifest_list_id', 'apprmanifestlistmanifest', ['manifest_list_id'], unique=False)
|
||||
op.create_index('apprmanifestlistmanifest_manifest_list_id_media_type_id', 'apprmanifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False)
|
||||
op.create_index('apprmanifestlistmanifest_manifest_list_id_operating_system_arch', 'apprmanifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False)
|
||||
op.create_index('apprmanifestlistmanifest_media_type_id', 'apprmanifestlistmanifest', ['media_type_id'], unique=False)
|
||||
op.create_table('apprtag',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('repository_id', sa.Integer(), nullable=False),
|
||||
sa.Column('manifest_list_id', sa.Integer(), nullable=True),
|
||||
sa.Column('lifetime_start', sa.BigInteger(), nullable=False),
|
||||
sa.Column('lifetime_end', sa.BigInteger(), nullable=True),
|
||||
sa.Column('hidden', sa.Boolean(), nullable=False),
|
||||
sa.Column('reverted', sa.Boolean(), nullable=False),
|
||||
sa.Column('protected', sa.Boolean(), nullable=False),
|
||||
sa.Column('tag_kind_id', sa.Integer(), nullable=False),
|
||||
sa.Column('linked_tag_id', sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['linked_tag_id'], ['apprtag.id'], name=op.f('fk_apprtag_linked_tag_id_apprtag')),
|
||||
sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprtag_manifest_list_id_apprmanifestlist')),
|
||||
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_apprtag_repository_id_repository')),
|
||||
sa.ForeignKeyConstraint(['tag_kind_id'], ['apprtagkind.id'], name=op.f('fk_apprtag_tag_kind_id_apprtagkind')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtag'))
|
||||
)
|
||||
op.create_index('apprtag_lifetime_end', 'apprtag', ['lifetime_end'], unique=False)
|
||||
op.create_index('apprtag_linked_tag_id', 'apprtag', ['linked_tag_id'], unique=False)
|
||||
op.create_index('apprtag_manifest_list_id', 'apprtag', ['manifest_list_id'], unique=False)
|
||||
op.create_index('apprtag_repository_id', 'apprtag', ['repository_id'], unique=False)
|
||||
op.create_index('apprtag_repository_id_name', 'apprtag', ['repository_id', 'name'], unique=False)
|
||||
op.create_index('apprtag_repository_id_name_hidden', 'apprtag', ['repository_id', 'name', 'hidden'], unique=False)
|
||||
op.create_index('apprtag_repository_id_name_lifetime_end', 'apprtag', ['repository_id', 'name', 'lifetime_end'], unique=True)
|
||||
op.create_index('apprtag_tag_kind_id', 'apprtag', ['tag_kind_id'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
conn = op.get_bind()
|
||||
copy_table_contents('blobplacementlocation', 'apprblobplacementlocation', conn)
|
||||
copy_table_contents('tagkind', 'apprtagkind', conn)
|
||||
|
||||
# ### population of test data ### #
|
||||
|
||||
tester.populate_table('apprmanifest', [
|
||||
('digest', tester.TestDataType.String),
|
||||
('media_type_id', tester.TestDataType.Foreign('mediatype')),
|
||||
('manifest_json', tester.TestDataType.JSON),
|
||||
])
|
||||
|
||||
tester.populate_table('apprmanifestlist', [
|
||||
('digest', tester.TestDataType.String),
|
||||
('manifest_list_json', tester.TestDataType.JSON),
|
||||
('schema_version', tester.TestDataType.String),
|
||||
('media_type_id', tester.TestDataType.Foreign('mediatype')),
|
||||
])
|
||||
|
||||
tester.populate_table('apprmanifestlistmanifest', [
|
||||
('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')),
|
||||
('manifest_id', tester.TestDataType.Foreign('apprmanifest')),
|
||||
('operating_system', tester.TestDataType.String),
|
||||
('architecture', tester.TestDataType.String),
|
||||
('platform_json', tester.TestDataType.JSON),
|
||||
('media_type_id', tester.TestDataType.Foreign('mediatype')),
|
||||
])
|
||||
|
||||
tester.populate_table('apprblob', [
|
||||
('digest', tester.TestDataType.String),
|
||||
('media_type_id', tester.TestDataType.Foreign('mediatype')),
|
||||
('size', tester.TestDataType.BigInteger),
|
||||
('uncompressed_size', tester.TestDataType.BigInteger),
|
||||
])
|
||||
|
||||
tester.populate_table('apprmanifestblob', [
|
||||
('manifest_id', tester.TestDataType.Foreign('apprmanifest')),
|
||||
('blob_id', tester.TestDataType.Foreign('apprblob')),
|
||||
])
|
||||
|
||||
tester.populate_table('apprtag', [
|
||||
('name', tester.TestDataType.String),
|
||||
('repository_id', tester.TestDataType.Foreign('repository')),
|
||||
('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')),
|
||||
('lifetime_start', tester.TestDataType.Integer),
|
||||
('hidden', tester.TestDataType.Boolean),
|
||||
('reverted', tester.TestDataType.Boolean),
|
||||
('protected', tester.TestDataType.Boolean),
|
||||
('tag_kind_id', tester.TestDataType.Foreign('apprtagkind')),
|
||||
])
|
||||
|
||||
tester.populate_table('apprblobplacement', [
|
||||
('blob_id', tester.TestDataType.Foreign('apprmanifestblob')),
|
||||
('location_id', tester.TestDataType.Foreign('apprblobplacementlocation')),
|
||||
])
|
||||
|
||||
# ### end population of test data ### #
|
||||
|
||||
|
||||
|
||||
def downgrade(tables, tester):
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table('apprtag')
|
||||
op.drop_table('apprmanifestlistmanifest')
|
||||
op.drop_table('apprmanifestblob')
|
||||
op.drop_table('apprblobplacement')
|
||||
op.drop_table('apprmanifestlist')
|
||||
op.drop_table('apprmanifest')
|
||||
op.drop_table('apprblob')
|
||||
op.drop_table('apprtagkind')
|
||||
op.drop_table('apprblobplacementlocation')
|
||||
# ### end Alembic commands ###
|
||||
|
|
@ -10,7 +10,7 @@ from cachetools import ttl_cache
|
|||
from data.model import (
|
||||
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
|
||||
from data.database import (
|
||||
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag,
|
||||
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag,
|
||||
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
|
||||
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
|
||||
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind)
|
||||
|
|
@ -87,11 +87,13 @@ def purge_repository(namespace_name, repository_name):
|
|||
except Repository.DoesNotExist:
|
||||
return False
|
||||
|
||||
# Delete the repository of all OCI-referenced entries.
|
||||
# Delete the repository of all Appr-referenced entries.
|
||||
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
|
||||
# and MySQL is... particular... about such relationships when deleting.
|
||||
Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute()
|
||||
Tag.delete().where(Tag.repository == repo).execute()
|
||||
ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute()
|
||||
ApprTag.delete().where(ApprTag.repository == repo).execute()
|
||||
|
||||
# Delete all tags to allow gc to reclaim storage
|
||||
previously_referenced = tag.purge_all_tags(repo)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ OPTION_TRANSLATIONS = {
|
|||
'null': 'nullable',
|
||||
}
|
||||
|
||||
|
||||
def gen_sqlalchemy_metadata(peewee_model_list):
|
||||
metadata = MetaData(naming_convention={
|
||||
"ix": 'ix_%(column_0_label)s',
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
|
|||
from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
|
||||
ImageStorageTransformation, ImageStorageSignature,
|
||||
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
|
||||
ensure_under_transaction)
|
||||
ApprBlob, ensure_under_transaction)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -105,15 +105,21 @@ def garbage_collect_storage(storage_id_whitelist):
|
|||
logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
|
||||
is_referenced_checksums)
|
||||
|
||||
# Check the new Blob table as well.
|
||||
# Check the ApprBlob tables as well.
|
||||
query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
|
||||
blob_referenced_checksums = set([blob.digest for blob in query])
|
||||
if blob_referenced_checksums:
|
||||
logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
|
||||
blob_referenced_checksums)
|
||||
|
||||
query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums))
|
||||
appr_blob_referenced_checksums = set([blob.digest for blob in query])
|
||||
if appr_blob_referenced_checksums:
|
||||
logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced',
|
||||
appr_blob_referenced_checksums)
|
||||
|
||||
unreferenced_checksums = (content_checksums - blob_referenced_checksums -
|
||||
is_referenced_checksums)
|
||||
appr_blob_referenced_checksums - is_referenced_checksums)
|
||||
|
||||
# Return all placements for all image storages found not at a CAS path or with a content
|
||||
# checksum that is referenced.
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ from contextlib import contextmanager
|
|||
from playhouse.test_utils import assert_query_count
|
||||
|
||||
from data import model, database
|
||||
from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, Blob
|
||||
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
|
||||
Blob, ApprBlob)
|
||||
from test.fixtures import *
|
||||
|
||||
|
||||
|
|
@ -187,7 +188,10 @@ def assert_gc_integrity(expect_storage_removed=True):
|
|||
storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
|
||||
|
||||
for blob_row in Blob.select():
|
||||
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
|
||||
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
|
||||
|
||||
for blob_row in ApprBlob.select():
|
||||
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
|
||||
|
||||
|
||||
def test_has_garbage(default_tag_policy, initialized_db):
|
||||
|
|
@ -589,6 +593,7 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d
|
|||
|
||||
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
|
||||
database.Blob.create(digest=digest, size=0, media_type=media_type)
|
||||
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
|
||||
|
||||
location = database.ImageStorageLocation.get(name=preferred)
|
||||
database.ImageStoragePlacement.create(location=location, storage=is1)
|
||||
|
|
|
|||
Reference in a new issue