vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Phase 1 of migrating APPR-specific tables to tables with the Appr prefix

Browse source 
Fixes https://jira.coreos.com/browse/QUAY-950
This commit is contained in:
Joseph Schorr 2018-05-24 17:54:51 -04:00
parent 6622f27c93
commit 113bb96f29
28 changed files with 699 additions and 176 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
config.py
View file

@ -264,6 +264,9 @@ class DefaultConfig(ImmutableConfig):
# Feature Flag: Whether to enable support for App repositories. # Feature Flag: Whether to enable support for App repositories.
FEATURE_APP_REGISTRY = False FEATURE_APP_REGISTRY = False
# Feature Flag: Whether app registry is in a read-only mode.
FEATURE_READONLY_APP_REGISTRY = False
# Feature Flag: If set to true, the _catalog endpoint returns public repositories. Otherwise, # Feature Flag: If set to true, the _catalog endpoint returns public repositories. Otherwise,
# only private repositories can be returned. # only private repositories can be returned.
FEATURE_PUBLIC_CATALOG = False FEATURE_PUBLIC_CATALOG = False

21
data/appr_model/blob.py
View file

@ -3,7 +3,6 @@ import logging
from peewee import IntegrityError from peewee import IntegrityError
from data.model import db_transaction from data.model import db_transaction
from data.database import Blob, BlobPlacementLocation, BlobPlacement
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -13,17 +12,20 @@ def _ensure_sha256_header(digest):
return 'sha256:' + digest return 'sha256:' + digest
def get_blob(digest): def get_blob(digest, models_ref):
""" Find a blob by its digest. """ """ Find a blob by its digest. """
Blob = models_ref.Blob
return Blob.select().where(Blob.digest == _ensure_sha256_header(digest)).get() return Blob.select().where(Blob.digest == _ensure_sha256_header(digest)).get()
def get_or_create_blob(digest, size, media_type_name, locations): def get_or_create_blob(digest, size, media_type_name, locations, models_ref):
""" Try to find a blob by its digest or create it. """ """ Try to find a blob by its digest or create it. """
Blob = models_ref.Blob
BlobPlacement = models_ref.BlobPlacement
# Get or create the blog entry for the digest. # Get or create the blog entry for the digest.
try: try:
blob = get_blob(digest) blob = get_blob(digest, models_ref)
logger.debug('Retrieved blob with digest %s', digest) logger.debug('Retrieved blob with digest %s', digest)
except Blob.DoesNotExist: except Blob.DoesNotExist:
blob = Blob.create(digest=_ensure_sha256_header(digest), blob = Blob.create(digest=_ensure_sha256_header(digest),
@ -38,13 +40,16 @@ def get_or_create_blob(digest, size, media_type_name, locations):
BlobPlacement.create(blob=blob, location=location_id) BlobPlacement.create(blob=blob, location=location_id)
except IntegrityError: except IntegrityError:
logger.debug('Location %s already existing for blob %s', location_name, blob.id) logger.debug('Location %s already existing for blob %s', location_name, blob.id)
pass
return blob return blob
def get_blob_locations(digest): def get_blob_locations(digest, models_ref):
""" Find all locations names for a blob. """ """ Find all locations names for a blob. """
Blob = models_ref.Blob
BlobPlacement = models_ref.BlobPlacement
BlobPlacementLocation = models_ref.BlobPlacementLocation
return [x.name for x in return [x.name for x in
BlobPlacementLocation BlobPlacementLocation
.select() .select()
@ -53,7 +58,9 @@ def get_blob_locations(digest):
.where(Blob.digest == _ensure_sha256_header(digest))] .where(Blob.digest == _ensure_sha256_header(digest))]
def ensure_blob_locations(*names): def ensure_blob_locations(models_ref, *names):
BlobPlacementLocation = models_ref.BlobPlacementLocation
with db_transaction(): with db_transaction():
locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names) locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names)

36
data/appr_model/channel.py
View file

@ -1,11 +1,13 @@
from data.database import Tag, Channel
from data.appr_model import tag as tag_model from data.appr_model import tag as tag_model
def get_channel_releases(repo, channel): def get_channel_releases(repo, channel, models_ref):
""" Return all previously linked tags. """ Return all previously linked tags.
This works based upon Tag lifetimes. This works based upon Tag lifetimes.
""" """
Channel = models_ref.Channel
Tag = models_ref.Tag
tag_kind_id = Channel.tag_kind.get_id('channel') tag_kind_id = Channel.tag_kind.get_id('channel')
channel_name = channel.name channel_name = channel.name
return (Tag return (Tag
@ -17,40 +19,46 @@ def get_channel_releases(repo, channel):
.order_by(Tag.lifetime_end)) .order_by(Tag.lifetime_end))
def get_channel(repo, channel_name): def get_channel(repo, channel_name, models_ref):
""" Find a Channel by name. """ """ Find a Channel by name. """
channel = tag_model.get_tag(repo, channel_name, "channel") channel = tag_model.get_tag(repo, channel_name, models_ref, "channel")
return channel return channel
def get_tag_channels(repo, tag_name, active=True): def get_tag_channels(repo, tag_name, models_ref, active=True):
""" Find the Channels associated with a Tag. """ """ Find the Channels associated with a Tag. """
tag = tag_model.get_tag(repo, tag_name, "release") Tag = models_ref.Tag
tag = tag_model.get_tag(repo, tag_name, models_ref, "release")
query = tag.tag_parents query = tag.tag_parents
if active: if active:
query = tag_model.tag_alive_oci(query) query = tag_model.tag_is_alive(query, Tag)
return query return query
def delete_channel(repo, channel_name): def delete_channel(repo, channel_name, models_ref):
""" Delete a channel by name. """ """ Delete a channel by name. """
return tag_model.delete_tag(repo, channel_name, "channel") return tag_model.delete_tag(repo, channel_name, models_ref, "channel")
def create_or_update_channel(repo, channel_name, tag_name): def create_or_update_channel(repo, channel_name, tag_name, models_ref):
""" Creates or updates a channel to include a particular tag. """ """ Creates or updates a channel to include a particular tag. """
tag = tag_model.get_tag(repo, tag_name, 'release') tag = tag_model.get_tag(repo, tag_name, models_ref, 'release')
return tag_model.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel") return tag_model.create_or_update_tag(repo, channel_name, models_ref, linked_tag=tag,
tag_kind="channel")
def get_repo_channels(repo): def get_repo_channels(repo, models_ref):
""" Creates or updates a channel to include a particular tag. """ """ Creates or updates a channel to include a particular tag. """
Channel = models_ref.Channel
Tag = models_ref.Tag
tag_kind_id = Channel.tag_kind.get_id('channel') tag_kind_id = Channel.tag_kind.get_id('channel')
query = (Channel query = (Channel
.select(Channel, Tag) .select(Channel, Tag)
.join(Tag, on=(Tag.id == Channel.linked_tag)) .join(Tag, on=(Tag.id == Channel.linked_tag))
.where(Channel.repository == repo, .where(Channel.repository == repo,
Channel.tag_kind == tag_kind_id)) Channel.tag_kind == tag_kind_id))
return tag_model.tag_alive_oci(query, cls=Channel) return tag_model.tag_is_alive(query, Channel)

24
data/appr_model/manifest.py
View file

@ -4,7 +4,7 @@ import json
from cnr.models.package_base import get_media_type from cnr.models.package_base import get_media_type
from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag from data.database import db_transaction, MediaType
from data.appr_model import tag as tag_model from data.appr_model import tag as tag_model
@ -21,20 +21,23 @@ def _digest(manifestjson):
return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()) return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest())
def get_manifest_query(digest, media_type): def get_manifest_query(digest, media_type, models_ref):
Manifest = models_ref.Manifest
return Manifest.select().where(Manifest.digest == _ensure_sha256_header(digest), return Manifest.select().where(Manifest.digest == _ensure_sha256_header(digest),
Manifest.media_type == Manifest.media_type.get_id(media_type)) Manifest.media_type == Manifest.media_type.get_id(media_type))
def get_manifest_with_blob(digest, media_type): def get_manifest_with_blob(digest, media_type, models_ref):
query = get_manifest_query(digest, media_type) Blob = models_ref.Blob
query = get_manifest_query(digest, media_type, models_ref)
return query.join(Blob).get() return query.join(Blob).get()
def get_or_create_manifest(manifest_json, media_type_name): def get_or_create_manifest(manifest_json, media_type_name, models_ref):
Manifest = models_ref.Manifest
digest = _digest(manifest_json) digest = _digest(manifest_json)
try: try:
manifest = get_manifest_query(digest, media_type_name).get() manifest = get_manifest_query(digest, media_type_name, models_ref).get()
except Manifest.DoesNotExist: except Manifest.DoesNotExist:
with db_transaction(): with db_transaction():
manifest = Manifest.create(digest=digest, manifest = Manifest.create(digest=digest,
@ -42,16 +45,19 @@ def get_or_create_manifest(manifest_json, media_type_name):
media_type=Manifest.media_type.get_id(media_type_name)) media_type=Manifest.media_type.get_id(media_type_name))
return manifest return manifest
def get_manifest_types(repo, release=None): def get_manifest_types(repo, models_ref, release=None):
""" Returns an array of MediaTypes.name for a repo, can filter by tag """ """ Returns an array of MediaTypes.name for a repo, can filter by tag """
query = tag_model.tag_alive_oci(Tag Tag = models_ref.Tag
ManifestListManifest = models_ref.ManifestListManifest
query = tag_model.tag_is_alive(Tag
.select(MediaType.name) .select(MediaType.name)
.join(ManifestListManifest, .join(ManifestListManifest,
on=(ManifestListManifest.manifest_list == Tag.manifest_list)) on=(ManifestListManifest.manifest_list == Tag.manifest_list))
.join(MediaType, .join(MediaType,
on=(ManifestListManifest.media_type == MediaType.id)) on=(ManifestListManifest.media_type == MediaType.id))
.where(Tag.repository == repo, .where(Tag.repository == repo,
Tag.tag_kind == Tag.tag_kind.get_id('release'))) Tag.tag_kind == Tag.tag_kind.get_id('release')), Tag)
if release: if release:
query = query.where(Tag.name == release) query = query.where(Tag.name == release)

20
data/appr_model/manifest_list.py
View file

@ -2,7 +2,7 @@ import logging
import hashlib import hashlib
import json import json
from data.database import ManifestList, ManifestListManifest, db_transaction from data.database import db_transaction
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -18,16 +18,19 @@ def _digest(manifestjson):
return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()) return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest())
def get_manifest_list(digest): def get_manifest_list(digest, models_ref):
ManifestList = models_ref.ManifestList
return ManifestList.select().where(ManifestList.digest == _ensure_sha256_header(digest)).get() return ManifestList.select().where(ManifestList.digest == _ensure_sha256_header(digest)).get()
def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version): def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version, models_ref):
ManifestList = models_ref.ManifestList
digest = _digest(manifest_list_json) digest = _digest(manifest_list_json)
media_type_id = ManifestList.media_type.get_id(media_type_name) media_type_id = ManifestList.media_type.get_id(media_type_name)
try: try:
return get_manifest_list(digest) return get_manifest_list(digest, models_ref)
except ManifestList.DoesNotExist: except ManifestList.DoesNotExist:
with db_transaction(): with db_transaction():
manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json, manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json,
@ -35,7 +38,7 @@ def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_vers
return manifestlist return manifestlist
def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json): def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json, models_ref):
""" From a manifestlist, manifests, and the manifest list blob, """ From a manifestlist, manifests, and the manifest list blob,
create if doesn't exist the manfiestlistmanifest for each manifest """ create if doesn't exist the manfiestlistmanifest for each manifest """
for pos in xrange(len(manifest_ids)): for pos in xrange(len(manifest_ids)):
@ -43,10 +46,13 @@ def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
manifest_json = manifest_list_json[pos] manifest_json = manifest_list_json[pos]
get_or_create_manifestlistmanifest(manifest=manifest_id, get_or_create_manifestlistmanifest(manifest=manifest_id,
manifestlist=manifestlist, manifestlist=manifestlist,
media_type_name=manifest_json['mediaType']) media_type_name=manifest_json['mediaType'],
models_ref=models_ref)
def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name): def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name, models_ref):
ManifestListManifest = models_ref.ManifestListManifest
media_type_id = ManifestListManifest.media_type.get_id(media_type_name) media_type_id = ManifestListManifest.media_type.get_id(media_type_name)
try: try:
ml = (ManifestListManifest ml = (ManifestListManifest

21
data/appr_model/models.py Normal file
View file

@ -0,0 +1,21 @@
from collections import namedtuple
from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement, Channel)
from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement, ApprChannel)
ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation', 'ManifestList',
'ManifestBlob', 'Blob', 'ManifestListManifest', 'Manifest',
'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name',
'tag_set_prefetch_name'])
OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement, Channel,
'manifestlistmanifest_set', 'tag_set_prefetch')
NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set',
'apprtag_set_prefetch')

10
data/appr_model/package.py
View file

@ -3,12 +3,14 @@ from peewee import prefetch
from data import model from data import model
from data.database import Repository, Namespace, Tag, ManifestListManifest from data.database import Repository, Namespace
from data.appr_model import tag as tag_model from data.appr_model import tag as tag_model
def list_packages_query(namespace=None, media_type=None, search_query=None, username=None): def list_packages_query(models_ref, namespace=None, media_type=None, search_query=None,
username=None):
""" List and filter repository by search query. """ """ List and filter repository by search query. """
Tag = models_ref.Tag
fields = [model.repository.SEARCH_FIELDS.name.name] fields = [model.repository.SEARCH_FIELDS.name.name]
if search_query is not None: if search_query is not None:
@ -40,9 +42,9 @@ def list_packages_query(namespace=None, media_type=None, search_query=None, user
.order_by(Tag.lifetime_start)) .order_by(Tag.lifetime_start))
if media_type: if media_type:
tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type) tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type, models_ref)
tag_query = tag_model.tag_alive_oci(tag_query) tag_query = tag_model.tag_is_alive(tag_query, Tag)
query = prefetch(repo_query, tag_query) query = prefetch(repo_query, tag_query)
return query return query

73
data/appr_model/release.py
View file

@ -3,8 +3,7 @@ import bisect
from cnr.exception import PackageAlreadyExists from cnr.exception import PackageAlreadyExists
from cnr.models.package_base import manifest_media_type from cnr.models.package_base import manifest_media_type
from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag, from data.database import db_transaction, get_epoch_timestamp
ManifestListManifest, Blob, ManifestBlob)
from data.appr_model import (blob as blob_model, manifest as manifest_model, from data.appr_model import (blob as blob_model, manifest as manifest_model,
manifest_list as manifest_list_model, manifest_list as manifest_list_model,
tag as tag_model) tag as tag_model)
@ -20,11 +19,17 @@ def _ensure_sha256_header(digest):
return 'sha256:' + digest return 'sha256:' + digest
def get_app_release(repo, tag_name, media_type): def get_app_release(repo, tag_name, media_type, models_ref):
""" Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """ """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """
tag = tag_model.get_tag(repo, tag_name, tag_kind='release') ManifestListManifest = models_ref.ManifestListManifest
Manifest = models_ref.Manifest
Blob = models_ref.Blob
ManifestBlob = models_ref.ManifestBlob
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
tag = tag_model.get_tag(repo, tag_name, models_ref, tag_kind='release')
media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name)
.join(Manifest) .join(Manifest)
.where(ManifestListManifest.media_type == media_type_id).get()) .where(ManifestListManifest.media_type == media_type_id).get())
manifest = manifestlistmanifest.manifest manifest = manifestlistmanifest.manifest
@ -32,25 +37,26 @@ def get_app_release(repo, tag_name, media_type):
return (tag, manifest, blob) return (tag, manifest, blob)
def delete_app_release(repo, tag_name, media_type): def delete_app_release(repo, tag_name, media_type, models_ref):
""" Terminate a Tag/media-type couple """ Terminate a Tag/media-type couple
It find the corresponding tag/manifest and remove from the manifestlistmanifest the manifest It find the corresponding tag/manifest and remove from the manifestlistmanifest the manifest
1. it terminates the current tag (in all-cases) 1. it terminates the current tag (in all-cases)
2. if the new manifestlist is not empty, it creates a new tag for it 2. if the new manifestlist is not empty, it creates a new tag for it
""" """
ManifestListManifest = models_ref.ManifestListManifest
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
with db_transaction(): with db_transaction():
tag = tag_model.get_tag(repo, tag_name) tag = tag_model.get_tag(repo, tag_name, models_ref)
manifest_list = tag.manifest_list manifest_list = tag.manifest_list
list_json = manifest_list.manifest_list_json list_json = manifest_list.manifest_list_json
mlm_query = (ManifestListManifest mlm_query = (ManifestListManifest
.select() .select()
.where(ManifestListManifest.manifest_list == tag.manifest_list)) .where(ManifestListManifest.manifest_list == tag.manifest_list))
list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query]) list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
manifestlistmanifest = (tag manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name)
.manifest_list
.manifestlistmanifest_set
.where(ManifestListManifest.media_type == media_type_id).get()) .where(ManifestListManifest.media_type == media_type_id).get())
index = list_manifest_ids.index(manifestlistmanifest.manifest_id) index = list_manifest_ids.index(manifestlistmanifest.manifest_id)
list_manifest_ids.pop(index) list_manifest_ids.pop(index)
@ -61,36 +67,42 @@ def delete_app_release(repo, tag_name, media_type):
tag.save() tag.save()
else: else:
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
SCHEMA_VERSION) SCHEMA_VERSION, models_ref)
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids,
list_json) list_json, models_ref)
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist,
tag_kind="release") tag_kind="release")
return tag return tag
def create_app_release(repo, tag_name, manifest_data, digest, force=False): def create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False):
""" Create a new application release, it includes creating a new Tag, ManifestList, """ Create a new application release, it includes creating a new Tag, ManifestList,
ManifestListManifests, Manifest, ManifestBlob. ManifestListManifests, Manifest, ManifestBlob.
To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id. To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id.
To find the insert point in the ManifestList it uses bisect on the manifest-ids list. To find the insert point in the ManifestList it uses bisect on the manifest-ids list.
""" """
ManifestList = models_ref.ManifestList
ManifestListManifest = models_ref.ManifestListManifest
Blob = models_ref.Blob
ManifestBlob = models_ref.ManifestBlob
with db_transaction(): with db_transaction():
# Create/get the package manifest # Create/get the package manifest
manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType']) manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType'],
models_ref)
# get the tag # get the tag
tag = tag_model.get_or_initialize_tag(repo, tag_name) tag = tag_model.get_or_initialize_tag(repo, tag_name, models_ref)
if tag.manifest_list is None: if tag.manifest_list is None:
tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE), tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE),
schema_version=SCHEMA_VERSION, schema_version=SCHEMA_VERSION,
manifest_list_json=[]) manifest_list_json=[], )
elif tag_model.tag_media_type_exists(tag, manifest.media_type): elif tag_model.tag_media_type_exists(tag, manifest.media_type, models_ref):
if force: if force:
delete_app_release(repo, tag_name, manifest.media_type.name) delete_app_release(repo, tag_name, manifest.media_type.name, models_ref)
return create_app_release(repo, tag_name, manifest_data, digest, force=False) return create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False)
else: else:
raise PackageAlreadyExists("package exists already") raise PackageAlreadyExists("package exists already")
@ -103,10 +115,11 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
list_json.insert(insert_point, manifest.manifest_json) list_json.insert(insert_point, manifest.manifest_json)
list_manifest_ids.insert(insert_point, manifest.id) list_manifest_ids.insert(insert_point, manifest.id)
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
SCHEMA_VERSION) SCHEMA_VERSION, models_ref)
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json,
models_ref)
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist,
tag_kind="release") tag_kind="release")
blob_digest = digest blob_digest = digest
@ -117,21 +130,23 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
.where(ManifestBlob.manifest == manifest, .where(ManifestBlob.manifest == manifest,
Blob.digest == _ensure_sha256_header(blob_digest)).get()) Blob.digest == _ensure_sha256_header(blob_digest)).get())
except ManifestBlob.DoesNotExist: except ManifestBlob.DoesNotExist:
blob = blob_model.get_blob(blob_digest) blob = blob_model.get_blob(blob_digest, models_ref)
ManifestBlob.create(manifest=manifest, blob=blob) ManifestBlob.create(manifest=manifest, blob=blob)
return tag return tag
def get_release_objs(repo, media_type=None): def get_release_objs(repo, models_ref, media_type=None):
""" Returns an array of Tag for a repo, with optional filtering by media_type. """ """ Returns an array of Tag for a repo, with optional filtering by media_type. """
Tag = models_ref.Tag
release_query = (Tag release_query = (Tag
.select() .select()
.where(Tag.repository == repo, .where(Tag.repository == repo,
Tag.tag_kind == Tag.tag_kind.get_id("release"))) Tag.tag_kind == Tag.tag_kind.get_id("release")))
if media_type: if media_type:
release_query = tag_model.filter_tags_by_media_type(release_query, media_type) release_query = tag_model.filter_tags_by_media_type(release_query, media_type, models_ref)
return tag_model.tag_alive_oci(release_query) return tag_model.tag_is_alive(release_query, Tag)
def get_releases(repo, media_type=None): def get_releases(repo, model_refs, media_type=None):
""" Returns an array of Tag.name for a repo, can filter by media_type. """ """ Returns an array of Tag.name for a repo, can filter by media_type. """
return [t.name for t in get_release_objs(repo, media_type)] return [t.name for t in get_release_objs(repo, model_refs, media_type)]

48
data/appr_model/tag.py
View file

@ -4,32 +4,37 @@ from cnr.models.package_base import manifest_media_type
from peewee import IntegrityError from peewee import IntegrityError
from data.model import (db_transaction, TagAlreadyCreatedException) from data.model import (db_transaction, TagAlreadyCreatedException)
from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update from data.database import get_epoch_timestamp_ms, db_for_update
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def tag_alive_oci(query, now_ts=None, cls=Tag): def tag_is_alive(query, cls, now_ts=None):
return query.where((cls.lifetime_end >> None) | return query.where((cls.lifetime_end >> None) |
(cls.lifetime_end > now_ts)) (cls.lifetime_end > now_ts))
def tag_media_type_exists(tag, media_type): def tag_media_type_exists(tag, media_type, models_ref):
return (tag.manifest_list.manifestlistmanifest_set ManifestListManifest = models_ref.ManifestListManifest
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
return (getattr(tag.manifest_list, manifestlistmanifest_set_name)
.where(ManifestListManifest.media_type == media_type).count() > 0) .where(ManifestListManifest.media_type == media_type).count() > 0)
def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"): def create_or_update_tag(repo, tag_name, models_ref, manifest_list=None, linked_tag=None,
tag_kind="release"):
Tag = models_ref.Tag
now_ts = get_epoch_timestamp_ms() now_ts = get_epoch_timestamp_ms()
tag_kind_id = Tag.tag_kind.get_id(tag_kind) tag_kind_id = Tag.tag_kind.get_id(tag_kind)
with db_transaction(): with db_transaction():
try: try:
tag = db_for_update(tag_alive_oci(Tag tag = db_for_update(tag_is_alive(Tag
.select() .select()
.where(Tag.repository == repo, .where(Tag.repository == repo,
Tag.name == tag_name, Tag.name == tag_name,
Tag.tag_kind == tag_kind_id), now_ts)).get() Tag.tag_kind == tag_kind_id), Tag, now_ts)).get()
if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag: if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag:
return tag return tag
tag.lifetime_end = now_ts tag.lifetime_end = now_ts
@ -46,40 +51,47 @@ def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, ta
raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name)) raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name))
def get_or_initialize_tag(repo, tag_name, tag_kind="release"): def get_or_initialize_tag(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
try: try:
return tag_alive_oci(Tag.select().where(Tag.repository == repo, Tag.name == tag_name)).get() return tag_is_alive(Tag.select().where(Tag.repository == repo, Tag.name == tag_name), Tag).get()
except Tag.DoesNotExist: except Tag.DoesNotExist:
return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind)) return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind))
def get_tag(repo, tag_name, tag_kind="release"): def get_tag(repo, tag_name, models_ref, tag_kind="release"):
return tag_alive_oci(Tag.select() Tag = models_ref.Tag
return tag_is_alive(Tag.select()
.where(Tag.repository == repo, .where(Tag.repository == repo,
Tag.name == tag_name, Tag.name == tag_name,
Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get() Tag.tag_kind == Tag.tag_kind.get_id(tag_kind)), Tag).get()
def delete_tag(repo, tag_name, tag_kind="release"): def delete_tag(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
tag_kind_id = Tag.tag_kind.get_id(tag_kind) tag_kind_id = Tag.tag_kind.get_id(tag_kind)
tag = tag_alive_oci(Tag.select() tag = tag_is_alive(Tag.select()
.where(Tag.repository == repo, .where(Tag.repository == repo,
Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get() Tag.name == tag_name, Tag.tag_kind == tag_kind_id), Tag).get()
tag.lifetime_end = get_epoch_timestamp_ms() tag.lifetime_end = get_epoch_timestamp_ms()
tag.save() tag.save()
return tag return tag
def tag_exists(repo, tag_name, tag_kind="release"): def tag_exists(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
try: try:
get_tag(repo, tag_name, tag_kind) get_tag(repo, tag_name, models_ref, tag_kind)
return True return True
except Tag.DoesNotExist: except Tag.DoesNotExist:
return False return False
def filter_tags_by_media_type(tag_query, media_type): def filter_tags_by_media_type(tag_query, media_type, models_ref):
""" Return only available tag for a media_type. """ """ Return only available tag for a media_type. """
ManifestListManifest = models_ref.ManifestListManifest
Tag = models_ref.Tag
media_type = manifest_media_type(media_type) media_type = manifest_media_type(media_type)
t = (tag_query t = (tag_query
.join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list)) .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list))

151
data/database.py
View file

@ -470,8 +470,8 @@ class User(BaseModel):
RepositoryTag, PermissionPrototype, DerivedStorageForImage, RepositoryTag, PermissionPrototype, DerivedStorageForImage,
TagManifest, AccessToken, OAuthAccessToken, BlobUpload, TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
RepositoryNotification, OAuthAuthorizationCode, RepositoryNotification, OAuthAuthorizationCode,
RepositoryActionCount, TagManifestLabel, Tag, RepositoryActionCount, TagManifestLabel,
TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes) delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
@ -619,7 +619,7 @@ class Repository(BaseModel):
# are cleaned up directly # are cleaned up directly
skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload, skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage, Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
RepositorySearchScore} | cnr_classes RepositorySearchScore} | cnr_classes | appr_classes
delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes) delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
@ -1187,7 +1187,6 @@ class ServiceKey(BaseModel):
class MediaType(BaseModel): class MediaType(BaseModel):
""" MediaType is an enumeration of the possible formats of various objects in the data model. """ MediaType is an enumeration of the possible formats of various objects in the data model.
This model is a part of the new OCI/CNR model set.
""" """
name = CharField(index=True, unique=True) name = CharField(index=True, unique=True)
@ -1201,7 +1200,6 @@ class Messages(BaseModel):
class LabelSourceType(BaseModel): class LabelSourceType(BaseModel):
""" LabelSourceType is an enumeration of the possible sources for a label. """ LabelSourceType is an enumeration of the possible sources for a label.
This model is a part of the new OCI/CNR model set.
""" """
name = CharField(index=True, unique=True) name = CharField(index=True, unique=True)
mutable = BooleanField(default=False) mutable = BooleanField(default=False)
@ -1210,7 +1208,6 @@ class LabelSourceType(BaseModel):
class Label(BaseModel): class Label(BaseModel):
""" Label represents user-facing metadata associated with another entry in the database (e.g. a """ Label represents user-facing metadata associated with another entry in the database (e.g. a
Manifest). Manifest).
This model is a part of the new OCI/CNR model set.
""" """
uuid = CharField(default=uuid_generator, index=True, unique=True) uuid = CharField(default=uuid_generator, index=True, unique=True)
key = CharField(index=True) key = CharField(index=True)
@ -1221,7 +1218,6 @@ class Label(BaseModel):
class TagManifestLabel(BaseModel): class TagManifestLabel(BaseModel):
""" Mapping from a tag manifest to a label. """ Mapping from a tag manifest to a label.
This model is a part of the new OCI/CNR model set.
""" """
repository = ForeignKeyField(Repository, index=True) repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(TagManifest, index=True) annotated = ForeignKeyField(TagManifest, index=True)
@ -1237,8 +1233,17 @@ class TagManifestLabel(BaseModel):
class Blob(BaseModel): class Blob(BaseModel):
""" Blob represents a content-addressable object stored outside of the database. """ Blob represents a content-addressable object stored outside of the database.
This model is a part of the new OCI/CNR model set. This is deprecated in favor of ApprBlob.
CNR """
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
size = BigIntegerField()
uncompressed_size = BigIntegerField(null=True)
class ApprBlob(BaseModel):
""" ApprBlob represents a content-addressable object stored outside of the database.
""" """
digest = CharField(index=True, unique=True) digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType) media_type = EnumField(MediaType)
@ -1248,16 +1253,20 @@ class Blob(BaseModel):
class BlobPlacementLocation(BaseModel): class BlobPlacementLocation(BaseModel):
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs. """ BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
This model is a part of the new OCI/CNR model set. This is deprecated in favor of ApprBlobPlacementLocation.
CNR """
name = CharField(index=True, unique=True)
class ApprBlobPlacementLocation(BaseModel):
""" ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs.
""" """
name = CharField(index=True, unique=True) name = CharField(index=True, unique=True)
class BlobPlacement(BaseModel): class BlobPlacement(BaseModel):
""" BlobPlacement represents the location of a Blob. """ BlobPlacement represents the location of a Blob.
This model is a part of the new OCI/CNR model set. This is deprecated in favor of ApprBlobPlacement.
CNR
""" """
blob = ForeignKeyField(Blob) blob = ForeignKeyField(Blob)
location = EnumField(BlobPlacementLocation) location = EnumField(BlobPlacementLocation)
@ -1270,10 +1279,31 @@ class BlobPlacement(BaseModel):
) )
class ApprBlobPlacement(BaseModel):
""" ApprBlobPlacement represents the location of a Blob.
"""
blob = ForeignKeyField(ApprBlob)
location = EnumField(ApprBlobPlacementLocation)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('blob', 'location'), True),
)
class Manifest(BaseModel): class Manifest(BaseModel):
""" Manifest represents the metadata and collection of blobs that comprise a container image. """ Manifest represents the metadata and collection of blobs that comprise a container image.
This model is a part of the new OCI/CNR model set. This is deprecated in favor of ApprManifest.
CNR """
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
manifest_json = JSONField()
class ApprManifest(BaseModel):
""" ApprManifest represents the metadata and collection of blobs that comprise an Appr image.
""" """
digest = CharField(index=True, unique=True) digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType) media_type = EnumField(MediaType)
@ -1282,8 +1312,7 @@ class Manifest(BaseModel):
class ManifestBlob(BaseModel): class ManifestBlob(BaseModel):
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs. """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
This model is a part of the new OCI/CNR model set. This is deprecated in favor of ApprManifestBlob.
CNR
""" """
manifest = ForeignKeyField(Manifest, index=True) manifest = ForeignKeyField(Manifest, index=True)
blob = ForeignKeyField(Blob, index=True) blob = ForeignKeyField(Blob, index=True)
@ -1296,10 +1325,32 @@ class ManifestBlob(BaseModel):
) )
class ApprManifestBlob(BaseModel):
""" ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs.
"""
manifest = ForeignKeyField(ApprManifest, index=True)
blob = ForeignKeyField(ApprBlob, index=True)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest', 'blob'), True),
)
class ManifestList(BaseModel): class ManifestList(BaseModel):
""" ManifestList represents all of the various manifests that compose a Tag. """ ManifestList represents all of the various manifests that compose a Tag.
This model is a part of the new OCI/CNR model set. This is deprecated in favor of ApprManifestList.
CNR """
digest = CharField(index=True, unique=True)
manifest_list_json = JSONField()
schema_version = CharField()
media_type = EnumField(MediaType)
class ApprManifestList(BaseModel):
""" ApprManifestList represents all of the various Appr manifests that compose an ApprTag.
""" """
digest = CharField(index=True, unique=True) digest = CharField(index=True, unique=True)
manifest_list_json = JSONField() manifest_list_json = JSONField()
@ -1309,16 +1360,20 @@ class ManifestList(BaseModel):
class TagKind(BaseModel): class TagKind(BaseModel):
""" TagKind is a enumtable to reference tag kinds. """ TagKind is a enumtable to reference tag kinds.
This model is a part of the new OCI/CNR model set. This model is deprecated in favor of ApprTagKind.
CNR """
name = CharField(index=True, unique=True)
class ApprTagKind(BaseModel):
""" ApprTagKind is a enumtable to reference tag kinds.
""" """
name = CharField(index=True, unique=True) name = CharField(index=True, unique=True)
class Tag(BaseModel): class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList. """ Tag represents a user-facing alias for referencing a ManifestList.
This model is a part of the new OCI/CNR model set. This model is deprecated in favor of ApprTag.
CNR
""" """
name = CharField() name = CharField()
repository = ForeignKeyField(Repository) repository = ForeignKeyField(Repository)
@ -1342,13 +1397,37 @@ class Tag(BaseModel):
) )
class ApprTag(BaseModel):
""" ApprTag represents a user-facing alias for referencing an ApprManifestList.
"""
name = CharField()
repository = ForeignKeyField(Repository)
manifest_list = ForeignKeyField(ApprManifestList, null=True)
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reverted = BooleanField(default=False)
protected = BooleanField(default=False)
tag_kind = EnumField(ApprTagKind)
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'name'), False),
(('repository', 'name', 'hidden'), False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end'), True),
)
Channel = Tag.alias() Channel = Tag.alias()
ApprChannel = ApprTag.alias()
class ManifestListManifest(BaseModel): class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
This model is a part of the new OCI/CNR model set. This model is deprecated in favor of ApprManifestListManifest.
CNR
""" """
manifest_list = ForeignKeyField(ManifestList, index=True) manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True) manifest = ForeignKeyField(Manifest, index=True)
@ -1366,6 +1445,25 @@ class ManifestListManifest(BaseModel):
) )
class ApprManifestListManifest(BaseModel):
""" ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and
ApprManifests.
"""
manifest_list = ForeignKeyField(ApprManifestList, index=True)
manifest = ForeignKeyField(ApprManifest, index=True)
operating_system = CharField(null=True)
architecture = CharField(null=True)
platform_json = JSONField(null=True)
media_type = EnumField(MediaType)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest_list', 'media_type'), False),
)
class AppSpecificAuthToken(BaseModel): class AppSpecificAuthToken(BaseModel):
""" AppSpecificAuthToken represents a token generated by a user for use with an external """ AppSpecificAuthToken represents a token generated by a user for use with an external
application where putting the user's credentials, even encrypted, is deemed too risky. application where putting the user's credentials, even encrypted, is deemed too risky.
@ -1388,5 +1486,8 @@ class AppSpecificAuthToken(BaseModel):
cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement]) ManifestListManifest, Manifest, BlobPlacement])
appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

2
data/migrations/migration.sh
View file

@ -14,7 +14,7 @@ up_mysql() {
sleep 25 sleep 25
# Add the database to mysql. # Add the database to mysql.
docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' docker run --rm --link mysql:mysql mysql:5.7 sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
} }
down_mysql() { down_mysql() {

4
data/migrations/versions/5cbbfc95bac7_remove_oci_tables_not_used_by_cnr_the_.py
View file

@ -15,7 +15,7 @@ import sqlalchemy as sa
from sqlalchemy.dialects import mysql from sqlalchemy.dialects import mysql
from util.migrate import UTF8LongText, UTF8CharField from util.migrate import UTF8LongText, UTF8CharField
def upgrade(tables): def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ### # ### commands auto generated by Alembic - please adjust! ###
op.drop_table('derivedimage') op.drop_table('derivedimage')
op.drop_table('manifestlabel') op.drop_table('manifestlabel')
@ -28,7 +28,7 @@ def upgrade(tables):
# ### end Alembic commands ### # ### end Alembic commands ###
def downgrade(tables): def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ### # ### commands auto generated by Alembic - please adjust! ###
op.create_table( op.create_table(
'manifestlayer', 'manifestlayer',

203
data/migrations/versions/610320e9dacf_add_new_appr_specific_tables.py Normal file
View file

@ -0,0 +1,203 @@
"""Add new Appr-specific tables
Revision ID: 610320e9dacf
Revises: 5cbbfc95bac7
Create Date: 2018-05-24 16:46:13.514562
"""
# revision identifiers, used by Alembic.
revision = '610320e9dacf'
down_revision = '5cbbfc95bac7'
from alembic import op
import sqlalchemy as sa
from util.migrate.table_ops import copy_table_contents
def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('apprblobplacementlocation',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacementlocation'))
)
op.create_index('apprblobplacementlocation_name', 'apprblobplacementlocation', ['name'], unique=True)
op.create_table('apprtagkind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtagkind'))
)
op.create_index('apprtagkind_name', 'apprtagkind', ['name'], unique=True)
op.create_table('apprblob',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('size', sa.BigInteger(), nullable=False),
sa.Column('uncompressed_size', sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprblob_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblob'))
)
op.create_index('apprblob_digest', 'apprblob', ['digest'], unique=True)
op.create_index('apprblob_media_type_id', 'apprblob', ['media_type_id'], unique=False)
op.create_table('apprmanifest',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('manifest_json', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifest_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifest'))
)
op.create_index('apprmanifest_digest', 'apprmanifest', ['digest'], unique=True)
op.create_index('apprmanifest_media_type_id', 'apprmanifest', ['media_type_id'], unique=False)
op.create_table('apprmanifestlist',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('manifest_list_json', sa.Text(), nullable=False),
sa.Column('schema_version', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlist_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlist'))
)
op.create_index('apprmanifestlist_digest', 'apprmanifestlist', ['digest'], unique=True)
op.create_index('apprmanifestlist_media_type_id', 'apprmanifestlist', ['media_type_id'], unique=False)
op.create_table('apprblobplacement',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprblobplacement_blob_id_apprblob')),
sa.ForeignKeyConstraint(['location_id'], ['apprblobplacementlocation.id'], name=op.f('fk_apprblobplacement_location_id_apprblobplacementlocation')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacement'))
)
op.create_index('apprblobplacement_blob_id', 'apprblobplacement', ['blob_id'], unique=False)
op.create_index('apprblobplacement_blob_id_location_id', 'apprblobplacement', ['blob_id', 'location_id'], unique=True)
op.create_index('apprblobplacement_location_id', 'apprblobplacement', ['location_id'], unique=False)
op.create_table('apprmanifestblob',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprmanifestblob_blob_id_apprblob')),
sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestblob_manifest_id_apprmanifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestblob'))
)
op.create_index('apprmanifestblob_blob_id', 'apprmanifestblob', ['blob_id'], unique=False)
op.create_index('apprmanifestblob_manifest_id', 'apprmanifestblob', ['manifest_id'], unique=False)
op.create_index('apprmanifestblob_manifest_id_blob_id', 'apprmanifestblob', ['manifest_id', 'blob_id'], unique=True)
op.create_table('apprmanifestlistmanifest',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_list_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('operating_system', sa.String(length=255), nullable=True),
sa.Column('architecture', sa.String(length=255), nullable=True),
sa.Column('platform_json', sa.Text(), nullable=True),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_id_apprmanifest')),
sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_list_id_apprmanifestlist')),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlistmanifest_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlistmanifest'))
)
op.create_index('apprmanifestlistmanifest_manifest_id', 'apprmanifestlistmanifest', ['manifest_id'], unique=False)
op.create_index('apprmanifestlistmanifest_manifest_list_id', 'apprmanifestlistmanifest', ['manifest_list_id'], unique=False)
op.create_index('apprmanifestlistmanifest_manifest_list_id_media_type_id', 'apprmanifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False)
op.create_index('apprmanifestlistmanifest_manifest_list_id_operating_system_arch', 'apprmanifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False)
op.create_index('apprmanifestlistmanifest_media_type_id', 'apprmanifestlistmanifest', ['media_type_id'], unique=False)
op.create_table('apprtag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('manifest_list_id', sa.Integer(), nullable=True),
sa.Column('lifetime_start', sa.BigInteger(), nullable=False),
sa.Column('lifetime_end', sa.BigInteger(), nullable=True),
sa.Column('hidden', sa.Boolean(), nullable=False),
sa.Column('reverted', sa.Boolean(), nullable=False),
sa.Column('protected', sa.Boolean(), nullable=False),
sa.Column('tag_kind_id', sa.Integer(), nullable=False),
sa.Column('linked_tag_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['linked_tag_id'], ['apprtag.id'], name=op.f('fk_apprtag_linked_tag_id_apprtag')),
sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprtag_manifest_list_id_apprmanifestlist')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_apprtag_repository_id_repository')),
sa.ForeignKeyConstraint(['tag_kind_id'], ['apprtagkind.id'], name=op.f('fk_apprtag_tag_kind_id_apprtagkind')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtag'))
)
op.create_index('apprtag_lifetime_end', 'apprtag', ['lifetime_end'], unique=False)
op.create_index('apprtag_linked_tag_id', 'apprtag', ['linked_tag_id'], unique=False)
op.create_index('apprtag_manifest_list_id', 'apprtag', ['manifest_list_id'], unique=False)
op.create_index('apprtag_repository_id', 'apprtag', ['repository_id'], unique=False)
op.create_index('apprtag_repository_id_name', 'apprtag', ['repository_id', 'name'], unique=False)
op.create_index('apprtag_repository_id_name_hidden', 'apprtag', ['repository_id', 'name', 'hidden'], unique=False)
op.create_index('apprtag_repository_id_name_lifetime_end', 'apprtag', ['repository_id', 'name', 'lifetime_end'], unique=True)
op.create_index('apprtag_tag_kind_id', 'apprtag', ['tag_kind_id'], unique=False)
# ### end Alembic commands ###
conn = op.get_bind()
copy_table_contents('blobplacementlocation', 'apprblobplacementlocation', conn)
copy_table_contents('tagkind', 'apprtagkind', conn)
# ### population of test data ### #
tester.populate_table('apprmanifest', [
('digest', tester.TestDataType.String),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
('manifest_json', tester.TestDataType.JSON),
])
tester.populate_table('apprmanifestlist', [
('digest', tester.TestDataType.String),
('manifest_list_json', tester.TestDataType.JSON),
('schema_version', tester.TestDataType.String),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
])
tester.populate_table('apprmanifestlistmanifest', [
('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')),
('manifest_id', tester.TestDataType.Foreign('apprmanifest')),
('operating_system', tester.TestDataType.String),
('architecture', tester.TestDataType.String),
('platform_json', tester.TestDataType.JSON),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
])
tester.populate_table('apprblob', [
('digest', tester.TestDataType.String),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
('size', tester.TestDataType.BigInteger),
('uncompressed_size', tester.TestDataType.BigInteger),
])
tester.populate_table('apprmanifestblob', [
('manifest_id', tester.TestDataType.Foreign('apprmanifest')),
('blob_id', tester.TestDataType.Foreign('apprblob')),
])
tester.populate_table('apprtag', [
('name', tester.TestDataType.String),
('repository_id', tester.TestDataType.Foreign('repository')),
('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')),
('lifetime_start', tester.TestDataType.Integer),
('hidden', tester.TestDataType.Boolean),
('reverted', tester.TestDataType.Boolean),
('protected', tester.TestDataType.Boolean),
('tag_kind_id', tester.TestDataType.Foreign('apprtagkind')),
])
tester.populate_table('apprblobplacement', [
('blob_id', tester.TestDataType.Foreign('apprmanifestblob')),
('location_id', tester.TestDataType.Foreign('apprblobplacementlocation')),
])
# ### end population of test data ### #
def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('apprtag')
op.drop_table('apprmanifestlistmanifest')
op.drop_table('apprmanifestblob')
op.drop_table('apprblobplacement')
op.drop_table('apprmanifestlist')
op.drop_table('apprmanifest')
op.drop_table('apprblob')
op.drop_table('apprtagkind')
op.drop_table('apprblobplacementlocation')
# ### end Alembic commands ###

6
data/model/repository.py
View file

@ -10,7 +10,7 @@ from cachetools import ttl_cache
from data.model import ( from data.model import (
config, DataModelException, tag, db_transaction, storage, permission, _basequery) config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import ( from data.database import (
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag,
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest, RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp, DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind) db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind)
@ -87,11 +87,13 @@ def purge_repository(namespace_name, repository_name):
except Repository.DoesNotExist: except Repository.DoesNotExist:
return False return False
# Delete the repository of all OCI-referenced entries. # Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags, # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting. # and MySQL is... particular... about such relationships when deleting.
Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute() Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute()
Tag.delete().where(Tag.repository == repo).execute() Tag.delete().where(Tag.repository == repo).execute()
ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute()
ApprTag.delete().where(ApprTag.repository == repo).execute()
# Delete all tags to allow gc to reclaim storage # Delete all tags to allow gc to reclaim storage
previously_referenced = tag.purge_all_tags(repo) previously_referenced = tag.purge_all_tags(repo)

1
data/model/sqlalchemybridge.py
View file

@ -15,7 +15,6 @@ OPTION_TRANSLATIONS = {
'null': 'nullable', 'null': 'nullable',
} }
def gen_sqlalchemy_metadata(peewee_model_list): def gen_sqlalchemy_metadata(peewee_model_list):
metadata = MetaData(naming_convention={ metadata = MetaData(naming_convention={
"ix": 'ix_%(column_0_label)s', "ix": 'ix_%(column_0_label)s',

12
data/model/storage.py
View file

@ -9,7 +9,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation, from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignature, ImageStorageTransformation, ImageStorageSignature,
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob, ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
ensure_under_transaction) ApprBlob, ensure_under_transaction)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -105,15 +105,21 @@ def garbage_collect_storage(storage_id_whitelist):
logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced', logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
is_referenced_checksums) is_referenced_checksums)
# Check the new Blob table as well. # Check the ApprBlob tables as well.
query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums)) query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
blob_referenced_checksums = set([blob.digest for blob in query]) blob_referenced_checksums = set([blob.digest for blob in query])
if blob_referenced_checksums: if blob_referenced_checksums:
logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced', logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
blob_referenced_checksums) blob_referenced_checksums)
query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums))
appr_blob_referenced_checksums = set([blob.digest for blob in query])
if appr_blob_referenced_checksums:
logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced',
appr_blob_referenced_checksums)
unreferenced_checksums = (content_checksums - blob_referenced_checksums - unreferenced_checksums = (content_checksums - blob_referenced_checksums -
is_referenced_checksums) appr_blob_referenced_checksums - is_referenced_checksums)
# Return all placements for all image storages found not at a CAS path or with a content # Return all placements for all image storages found not at a CAS path or with a content
# checksum that is referenced. # checksum that is referenced.

9
data/model/test/test_gc.py
View file

@ -9,7 +9,8 @@ from contextlib import contextmanager
from playhouse.test_utils import assert_query_count from playhouse.test_utils import assert_query_count
from data import model, database from data import model, database
from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, Blob from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
Blob, ApprBlob)
from test.fixtures import * from test.fixtures import *
@ -187,7 +188,10 @@ def assert_gc_integrity(expect_storage_removed=True):
storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum)) storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
for blob_row in Blob.select(): for blob_row in Blob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest)) storage.get_content({preferred}, storage.blob_path(blob_row.digest))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
def test_has_garbage(default_tag_policy, initialized_db): def test_has_garbage(default_tag_policy, initialized_db):
@ -589,6 +593,7 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False) is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
database.Blob.create(digest=digest, size=0, media_type=media_type) database.Blob.create(digest=digest, size=0, media_type=media_type)
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
location = database.ImageStorageLocation.get(name=preferred) location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1) database.ImageStoragePlacement.create(location=location, storage=is1)

8
endpoints/api/repository_models_pre_oci.py
View file

@ -3,7 +3,9 @@ from collections import defaultdict
from datetime import datetime, timedelta from datetime import datetime, timedelta
from auth.permissions import ReadRepositoryPermission from auth.permissions import ReadRepositoryPermission
from data import model, appr_model from data import model
from data.appr_model import channel as channel_model, release as release_model
from endpoints.appr.models_cnr import model as appr_model
from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \ from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \
ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count
@ -143,8 +145,8 @@ class PreOCIModel(RepositoryDataInterface):
False, False, False) False, False, False)
if base.kind_name == 'application': if base.kind_name == 'application':
channels = appr_model.channel.get_repo_channels(repo) channels = channel_model.get_repo_channels(repo, appr_model.models_ref)
releases = appr_model.release.get_release_objs(repo) releases = release_model.get_release_objs(repo, appr_model.models_ref)
releases_channels_map = defaultdict(list) releases_channels_map = defaultdict(list)
return ApplicationRepository( return ApplicationRepository(
base, [_create_channel(channel, releases_channels_map) for channel in channels], [ base, [_create_channel(channel, releases_channels_map) for channel in channels], [

27
endpoints/api/test/test_repository.py
View file

@ -2,7 +2,9 @@ import pytest
from mock import patch, ANY, MagicMock from mock import patch, ANY, MagicMock
from data import model from data import model, database
from data.appr_model import release, channel, blob
from endpoints.appr.models_cnr import model as appr_model
from endpoints.api.test.shared import conduct_api_call from endpoints.api.test.shared import conduct_api_call
from endpoints.api.repository import RepositoryTrust, Repository, RepositoryList from endpoints.api.repository import RepositoryTrust, Repository, RepositoryList
from endpoints.test.shared import client_with_identity from endpoints.test.shared import client_with_identity
@ -117,3 +119,26 @@ def test_create_repository(repo_name, expected_status, client):
if expected_status == 201: if expected_status == 201:
assert result['name'] == repo_name assert result['name'] == repo_name
assert model.repository.get_repository('devtable', repo_name).name == repo_name assert model.repository.get_repository('devtable', repo_name).name == repo_name
def test_get_app_repo(client, initialized_db):
with client_with_identity('devtable', client) as cl:
devtable = model.user.get_user('devtable')
repo = model.repository.create_repository('devtable', 'someappr', devtable,
repo_kind='application')
models_ref = appr_model.models_ref
blob.get_or_create_blob('sha256:somedigest', 0, 'application/vnd.cnr.blob.v0.tar+gzip',
['local_us'], models_ref)
release.create_app_release(repo, 'test',
dict(mediaType='application/vnd.cnr.package-manifest.helm.v0.json'),
'sha256:somedigest', models_ref, False)
channel.create_or_update_channel(repo, 'somechannel', 'test', models_ref)
params = {'repository': 'devtable/someappr'}
response = conduct_api_call(cl, Repository, 'GET', params).json
assert response['kind'] == 'application'
assert response['channels']
assert response['releases']

5
endpoints/api/test/test_search.py
View file

@ -2,6 +2,7 @@ import pytest
from playhouse.test_utils import assert_query_count from playhouse.test_utils import assert_query_count
from data import model, database
from endpoints.api.search import ConductRepositorySearch, ConductSearch from endpoints.api.search import ConductRepositorySearch, ConductSearch
from endpoints.api.test.shared import conduct_api_call from endpoints.api.test.shared import conduct_api_call
from endpoints.test.shared import client_with_identity from endpoints.test.shared import client_with_identity
@ -14,6 +15,10 @@ from test.fixtures import *
('repository'), ('repository'),
]) ])
def test_repository_search(query, client): def test_repository_search(query, client):
# Prime the caches.
database.Repository.kind.get_id('image')
database.Repository.kind.get_name(1)
with client_with_identity('devtable', client) as cl: with client_with_identity('devtable', client) as cl:
params = {'query': query} params = {'query': query}
with assert_query_count(7): with assert_query_count(7):

101
endpoints/appr/models_cnr.py
View file

@ -2,13 +2,15 @@ from datetime import datetime
import cnr.semver import cnr.semver
from cnr.exception import raise_package_not_found, raise_channel_not_found from cnr.exception import raise_package_not_found, raise_channel_not_found, CnrException
import features
import data.model import data.model
from app import storage, authentication from app import storage, authentication
from data import appr_model from data import appr_model
from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel from data.database import Repository, MediaType, db_transaction
from data.appr_model.models import OLD_MODELS, NEW_MODELS
from endpoints.appr.models_interface import ( from endpoints.appr.models_interface import (
ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface, ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface,
BlobDescriptor, ChannelView, ChannelReleasesView) BlobDescriptor, ChannelView, ChannelReleasesView)
@ -17,6 +19,12 @@ from util.morecollections import AttrDict
from util.names import parse_robot_username from util.names import parse_robot_username
class ReadOnlyException(CnrException):
status_code = 405
errorcode = "read-only"
def _strip_sha256_header(digest): def _strip_sha256_header(digest):
if digest.startswith('sha256:'): if digest.startswith('sha256:'):
return digest.split('sha256:')[1] return digest.split('sha256:')[1]
@ -48,6 +56,10 @@ def _application(package):
class CNRAppModel(AppRegistryDataInterface): class CNRAppModel(AppRegistryDataInterface):
def __init__(self, models_ref, is_readonly):
self.models_ref = models_ref
self.is_readonly = is_readonly
def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None, def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None,
analytics_sample=1, metadata=None): analytics_sample=1, metadata=None):
metadata = {} if metadata is None else metadata metadata = {} if metadata is None else metadata
@ -70,9 +82,10 @@ class CNRAppModel(AppRegistryDataInterface):
""" """
views = [] views = []
for repo in appr_model.package.list_packages_query(namespace, media_type, search, for repo in appr_model.package.list_packages_query(self.models_ref, namespace, media_type,
username=username): search, username=username):
releases = [t.name for t in repo.tag_set_prefetch] tag_set_prefetch = getattr(repo, self.models_ref.tag_set_prefetch_name)
releases = [t.name for t in tag_set_prefetch]
if not releases: if not releases:
continue continue
available_releases = [ available_releases = [
@ -81,7 +94,7 @@ class CNRAppModel(AppRegistryDataInterface):
if with_channels: if with_channels:
channels = [ channels = [
ChannelView(name=chan.name, current=chan.linked_tag.name) ChannelView(name=chan.name, current=chan.linked_tag.name)
for chan in appr_model.channel.get_repo_channels(repo)] for chan in appr_model.channel.get_repo_channels(repo, self.models_ref)]
app_name = _join_package_name(repo.namespace_user.username, repo.name) app_name = _join_package_name(repo.namespace_user.username, repo.name)
manifests = self.list_manifests(app_name, available_releases[0]) manifests = self.list_manifests(app_name, available_releases[0])
@ -93,8 +106,8 @@ class CNRAppModel(AppRegistryDataInterface):
channels=channels, channels=channels,
manifests=manifests, manifests=manifests,
releases=available_releases, releases=available_releases,
updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start), updated_at=_timestamp_to_iso(tag_set_prefetch[-1].lifetime_start),
created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start),) created_at=_timestamp_to_iso(tag_set_prefetch[0].lifetime_start),)
views.append(view) views.append(view)
return views return views
@ -108,6 +121,9 @@ class CNRAppModel(AppRegistryDataInterface):
def create_application(self, package_name, visibility, owner): def create_application(self, package_name, visibility, owner):
""" Create a new app repository, owner is the user who creates it """ """ Create a new app repository, owner is the user who creates it """
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
ns, name = _split_package_name(package_name) ns, name = _split_package_name(package_name)
data.model.repository.create_repository(ns, name, owner, visibility, 'application') data.model.repository.create_repository(ns, name, owner, visibility, 'application')
@ -137,7 +153,7 @@ class CNRAppModel(AppRegistryDataInterface):
Todo: Todo:
* Paginate * Paginate
""" """
return appr_model.release.get_releases(_application(package_name), media_type) return appr_model.release.get_releases(_application(package_name), self.models_ref, media_type)
def list_manifests(self, package_name, release=None): def list_manifests(self, package_name, release=None):
""" Returns the list of all manifests of an Application. """ Returns the list of all manifests of an Application.
@ -147,8 +163,8 @@ class CNRAppModel(AppRegistryDataInterface):
""" """
try: try:
repo = _application(package_name) repo = _application(package_name)
return list(appr_model.manifest.get_manifest_types(repo, release)) return list(appr_model.manifest.get_manifest_types(repo, self.models_ref, release))
except (Repository.DoesNotExist, Tag.DoesNotExist): except (Repository.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_package_not_found(package_name, release) raise_package_not_found(package_name, release)
def fetch_release(self, package_name, release, media_type): def fetch_release(self, package_name, release, media_type):
@ -157,7 +173,8 @@ class CNRAppModel(AppRegistryDataInterface):
""" """
repo = _application(package_name) repo = _application(package_name)
try: try:
tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type) tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type,
self.models_ref)
created_at = _timestamp_to_iso(tag.lifetime_start) created_at = _timestamp_to_iso(tag.lifetime_start)
blob_descriptor = BlobDescriptor(digest=_strip_sha256_header(blob.digest), blob_descriptor = BlobDescriptor(digest=_strip_sha256_header(blob.digest),
@ -169,17 +186,23 @@ class CNRAppModel(AppRegistryDataInterface):
app_release = ApplicationRelease(release=tag.name, created_at=created_at, name=package_name, app_release = ApplicationRelease(release=tag.name, created_at=created_at, name=package_name,
manifest=app_manifest) manifest=app_manifest)
return app_release return app_release
except (Tag.DoesNotExist, Manifest.DoesNotExist, Blob.DoesNotExist, Repository.DoesNotExist, except (self.models_ref.Tag.DoesNotExist,
self.models_ref.Manifest.DoesNotExist,
self.models_ref.Blob.DoesNotExist,
Repository.DoesNotExist,
MediaType.DoesNotExist): MediaType.DoesNotExist):
raise_package_not_found(package_name, release, media_type) raise_package_not_found(package_name, release, media_type)
def store_blob(self, cnrblob, content_media_type): def store_blob(self, cnrblob, content_media_type):
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
fp = cnrblob.packager.io_file fp = cnrblob.packager.io_file
path = cnrblob.upload_url(cnrblob.digest) path = cnrblob.upload_url(cnrblob.digest)
locations = storage.preferred_locations locations = storage.preferred_locations
storage.stream_write(locations, path, fp, 'application/x-gzip') storage.stream_write(locations, path, fp, 'application/x-gzip')
db_blob = appr_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type, db_blob = appr_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type,
locations) locations, self.models_ref)
return BlobDescriptor(mediaType=content_media_type, return BlobDescriptor(mediaType=content_media_type,
digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[]) digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[])
@ -187,49 +210,60 @@ class CNRAppModel(AppRegistryDataInterface):
""" Add an app-release to a repository """ Add an app-release to a repository
package is an instance of data.cnr.package.Package package is an instance of data.cnr.package.Package
""" """
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
manifest = package.manifest() manifest = package.manifest()
ns, name = package.namespace, package.name ns, name = package.namespace, package.name
repo = data.model.repository.get_or_create_repository(ns, name, user, visibility=visibility, repo = data.model.repository.get_or_create_repository(ns, name, user, visibility=visibility,
repo_kind='application') repo_kind='application')
tag_name = package.release tag_name = package.release
appr_model.release.create_app_release(repo, tag_name, appr_model.release.create_app_release(repo, tag_name, package.manifest(),
package.manifest(), manifest['content']['digest'], force) manifest['content']['digest'], self.models_ref, force)
def delete_release(self, package_name, release, media_type): def delete_release(self, package_name, release, media_type):
""" Remove/Delete an app-release from an app-repository. """ Remove/Delete an app-release from an app-repository.
It does not delete the entire app-repository, only a single release It does not delete the entire app-repository, only a single release
""" """
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
repo = _application(package_name) repo = _application(package_name)
try: try:
appr_model.release.delete_app_release(repo, release, media_type) appr_model.release.delete_app_release(repo, release, media_type, self.models_ref)
except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist): except (self.models_ref.Channel.DoesNotExist,
self.models_ref.Tag.DoesNotExist,
MediaType.DoesNotExist):
raise_package_not_found(package_name, release, media_type) raise_package_not_found(package_name, release, media_type)
def release_exists(self, package, release): def release_exists(self, package, release):
""" Return true if a release with that name already exist or """ Return true if a release with that name already exist or
have existed (include deleted ones) """ have existed (include deleted ones) """
# TODO: Figure out why this isn't implemented.
def channel_exists(self, package_name, channel_name): def channel_exists(self, package_name, channel_name):
""" Returns true if channel exists """ """ Returns true if channel exists """
repo = _application(package_name) repo = _application(package_name)
return appr_model.tag.tag_exists(repo, channel_name, "channel") return appr_model.tag.tag_exists(repo, channel_name, self.models_ref, "channel")
def delete_channel(self, package_name, channel_name): def delete_channel(self, package_name, channel_name):
""" Delete an AppChannel """ Delete an AppChannel
Note: Note:
It doesn't delete the AppReleases It doesn't delete the AppReleases
""" """
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
repo = _application(package_name) repo = _application(package_name)
try: try:
appr_model.channel.delete_channel(repo, channel_name) appr_model.channel.delete_channel(repo, channel_name, self.models_ref)
except (Channel.DoesNotExist, Tag.DoesNotExist): except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_channel_not_found(package_name, channel_name) raise_channel_not_found(package_name, channel_name)
def list_channels(self, package_name): def list_channels(self, package_name):
""" Returns all AppChannel for a package """ """ Returns all AppChannel for a package """
repo = _application(package_name) repo = _application(package_name)
channels = appr_model.channel.get_repo_channels(repo) channels = appr_model.channel.get_repo_channels(repo, self.models_ref)
return [ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in channels] return [ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in channels]
def fetch_channel(self, package_name, channel_name, with_releases=True): def fetch_channel(self, package_name, channel_name, with_releases=True):
@ -237,12 +271,12 @@ class CNRAppModel(AppRegistryDataInterface):
repo = _application(package_name) repo = _application(package_name)
try: try:
channel = appr_model.channel.get_channel(repo, channel_name) channel = appr_model.channel.get_channel(repo, channel_name, self.models_ref)
except (Channel.DoesNotExist, Tag.DoesNotExist): except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_channel_not_found(package_name, channel_name) raise_channel_not_found(package_name, channel_name)
if with_releases: if with_releases:
releases = appr_model.channel.get_channel_releases(repo, channel) releases = appr_model.channel.get_channel_releases(repo, channel, self.models_ref)
chanview = ChannelReleasesView( chanview = ChannelReleasesView(
current=channel.linked_tag.name, name=channel.name, current=channel.linked_tag.name, name=channel.name,
releases=[channel.linked_tag.name] + [c.name for c in releases]) releases=[channel.linked_tag.name] + [c.name for c in releases])
@ -254,9 +288,9 @@ class CNRAppModel(AppRegistryDataInterface):
def list_release_channels(self, package_name, release, active=True): def list_release_channels(self, package_name, release, active=True):
repo = _application(package_name) repo = _application(package_name)
try: try:
channels = appr_model.channel.get_tag_channels(repo, release, active=active) channels = appr_model.channel.get_tag_channels(repo, release, self.models_ref, active=active)
return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels] return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels]
except (Channel.DoesNotExist, Tag.DoesNotExist): except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_package_not_found(package_name, release) raise_package_not_found(package_name, release)
def update_channel(self, package_name, channel_name, release): def update_channel(self, package_name, channel_name, release):
@ -264,12 +298,17 @@ class CNRAppModel(AppRegistryDataInterface):
Returns: Returns:
A new AppChannel with the release A new AppChannel with the release
""" """
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
repo = _application(package_name) repo = _application(package_name)
channel = appr_model.channel.create_or_update_channel(repo, channel_name, release) channel = appr_model.channel.create_or_update_channel(repo, channel_name, release,
self.models_ref)
return ChannelView(current=channel.linked_tag.name, name=channel.name) return ChannelView(current=channel.linked_tag.name, name=channel.name)
def get_blob_locations(self, digest): def get_blob_locations(self, digest):
return appr_model.blob.get_blob_locations(digest) return appr_model.blob.get_blob_locations(digest, self.models_ref)
model = CNRAppModel() # Phase 1: Read from old tables, disallow writing.
model = CNRAppModel(OLD_MODELS, features.READONLY_APP_REGISTRY)

16
endpoints/appr/test/test_api.py
View file

@ -6,13 +6,13 @@ from cnr.tests.conftest import *
from cnr.tests.test_apiserver import BaseTestServer from cnr.tests.test_apiserver import BaseTestServer
from cnr.tests.test_models import CnrTestModels from cnr.tests.test_models import CnrTestModels
import data.appr_model.blob as oci_blob import data.appr_model.blob as appr_blob
from data.database import User from data.database import User
from data.model import organization, user from data.model import organization, user
from endpoints.appr import registry # Needed to register the endpoint from endpoints.appr import registry # Needed to register the endpoint
from endpoints.appr.cnr_backend import Channel, Package, QuayDB from endpoints.appr.cnr_backend import Channel, Package, QuayDB
from endpoints.appr.models_cnr import model as oci_app_model from endpoints.appr.models_cnr import model as appr_app_model
from test.fixtures import * from test.fixtures import *
@ -28,7 +28,7 @@ class ChannelTest(Channel):
@classmethod @classmethod
def dump_all(cls, package_class=None): def dump_all(cls, package_class=None):
result = [] result = []
for repo in oci_app_model.list_applications(with_channels=True): for repo in appr_app_model.list_applications(with_channels=True):
for chan in repo.channels: for chan in repo.channels:
result.append({'name': chan.name, 'current': chan.current, 'package': repo.name}) result.append({'name': chan.name, 'current': chan.current, 'package': repo.name})
return result return result
@ -51,17 +51,17 @@ class PackageTest(Package):
@classmethod @classmethod
def dump_all(cls, blob_cls): def dump_all(cls, blob_cls):
result = [] result = []
for repo in oci_app_model.list_applications(with_channels=True): for repo in appr_app_model.list_applications(with_channels=True):
package_name = repo.name package_name = repo.name
for release in repo.releases: for release in repo.releases:
for mtype in cls.manifests(package_name, release): for mtype in cls.manifests(package_name, release):
package = oci_app_model.fetch_release(package_name, release, mtype) package = appr_app_model.fetch_release(package_name, release, mtype)
blob = blob_cls.get(package_name, package.manifest.content.digest) blob = blob_cls.get(package_name, package.manifest.content.digest)
app_data = cls._apptuple_to_dict(package) app_data = cls._apptuple_to_dict(package)
app_data.pop('digest') app_data.pop('digest')
app_data['channels'] = [ app_data['channels'] = [
x.name x.name
for x in oci_app_model.list_release_channels(package_name, package.release, False) for x in appr_app_model.list_release_channels(package_name, package.release, False)
] ]
app_data['blob'] = blob.b64blob app_data['blob'] = blob.b64blob
result.append(app_data) result.append(app_data)
@ -141,11 +141,11 @@ class TestQuayModels(CnrTestModels):
assert p.release == "2.0.1" assert p.release == "2.0.1"
assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80" assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80"
blob = db_with_data1.Blob.get("titi/rocketchat", p.digest) blob = db_with_data1.Blob.get("titi/rocketchat", p.digest)
bdb = oci_blob.get_blob(p.digest) bdb = appr_blob.get_blob(p.digest, appr_app_model.models_ref)
newblob = db_with_data1.Blob("titi/app2", blob.b64blob) newblob = db_with_data1.Blob("titi/app2", blob.b64blob)
p2 = db_with_data1.Package("titi/app2", "1.0.0", "helm", newblob) p2 = db_with_data1.Package("titi/app2", "1.0.0", "helm", newblob)
p2.save() p2.save()
b2db = oci_blob.get_blob(p2.digest) b2db = appr_blob.get_blob(p2.digest, appr_app_model.models_ref)
assert b2db.id == bdb.id assert b2db.id == bdb.id
def test_force_push_different_blob(self, db_with_data1): def test_force_push_different_blob(self, db_with_data1):

30
endpoints/appr/test/test_registry.py
View file

@ -1,14 +1,18 @@
import base64 import base64
import json import json
from flask import url_for from mock import patch
import pytest import pytest
from flask import url_for
from data import model from data import model
from endpoints.appr.registry import appr_bp from endpoints.appr.registry import appr_bp
from test.fixtures import * from test.fixtures import *
@pytest.mark.parametrize('login_data, expected_code', [ @pytest.mark.parametrize('login_data, expected_code', [
({ ({
"username": "devtable", "username": "devtable",
@ -62,3 +66,27 @@ def test_invalid_release_name(release_name, app, client):
rv = client.open(url, method='POST', data=json.dumps(data), headers=headers) rv = client.open(url, method='POST', data=json.dumps(data), headers=headers)
assert rv.status_code == 422 assert rv.status_code == 422
@pytest.mark.parametrize('readonly, expected_status', [
(True, 405),
(False, 422),
])
def test_readonly(readonly, expected_status, app, client):
params = {
'namespace': 'devtable',
'package_name': 'someapprepo',
}
url = url_for('appr.push', **params)
auth = base64.b64encode('devtable:password')
headers = {'Content-Type': 'application/json', 'Authorization': 'Basic ' + auth}
data = {
'release': '1.0',
'media_type': 'application/vnd.cnr.manifest.v0+json',
'blob': 'H4sIAFQwWVoAA+3PMQrCQBAF0Bxlb+Bk143nETGIIEoSC29vMMFOu3TvNb/5DH/Ot8f02jWbiohDremT3ZKR90uuUlty7nKJNmqKtkQuTarbzlo8x+k4zFOu4+lyH4afvbnW93/urH98EwAAAAAAAAAAADb0BsdwExIAKAAA',
}
with patch('endpoints.appr.models_cnr.model.is_readonly', readonly):
rv = client.open(url, method='POST', data=json.dumps(data), headers=headers)
assert rv.status_code == expected_status

11
initdb.py
View file

@ -21,7 +21,7 @@ from data.database import (db, all_models, cnr_classes, Role, TeamRole, Visibili
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind, ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason, RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason,
DeletedNamespace) DeletedNamespace, appr_classes, ApprTagKind, ApprBlobPlacementLocation)
from data import model from data import model
from data.queue import WorkQueue from data.queue import WorkQueue
from app import app, storage as store, tf from app import app, storage as store, tf
@ -364,6 +364,9 @@ def initialize_database():
BlobPlacementLocation.create(name='local_eu') BlobPlacementLocation.create(name='local_eu')
BlobPlacementLocation.create(name='local_us') BlobPlacementLocation.create(name='local_us')
ApprBlobPlacementLocation.create(name='local_eu')
ApprBlobPlacementLocation.create(name='local_us')
ImageStorageTransformation.create(name='squash') ImageStorageTransformation.create(name='squash')
ImageStorageTransformation.create(name='aci') ImageStorageTransformation.create(name='aci')
@ -435,6 +438,10 @@ def initialize_database():
TagKind.create(name='release') TagKind.create(name='release')
TagKind.create(name='channel') TagKind.create(name='channel')
ApprTagKind.create(name='tag')
ApprTagKind.create(name='release')
ApprTagKind.create(name='channel')
DisableReason.create(name='user_toggled') DisableReason.create(name='user_toggled')
DisableReason.create(name='successive_build_failures') DisableReason.create(name='successive_build_failures')
DisableReason.create(name='successive_build_internal_errors') DisableReason.create(name='successive_build_internal_errors')
@ -906,7 +913,7 @@ def find_models_missing_data():
# whitelisted. # whitelisted.
models_missing_data = set() models_missing_data = set()
for one_model in all_models: for one_model in all_models:
if one_model in cnr_classes: if one_model in cnr_classes or one_model in appr_classes:
continue continue
try: try:

9
test/test_api_usage.py
View file

@ -27,6 +27,7 @@ from app import app, config_provider, all_queues, dockerfile_build_queue, notifi
from buildtrigger.basehandler import BuildTriggerHandler from buildtrigger.basehandler import BuildTriggerHandler
from initdb import setup_database_for_testing, finished_database_for_testing from initdb import setup_database_for_testing, finished_database_for_testing
from data import database, model, appr_model from data import database, model, appr_model
from data.appr_model.models import OLD_MODELS
from data.database import RepositoryActionCount, Repository as RepositoryTable from data.database import RepositoryActionCount, Repository as RepositoryTable
from test.helpers import assert_action_logged from test.helpers import assert_action_logged
from util.secscan.fake import fake_security_scanner from util.secscan.fake import fake_security_scanner
@ -2154,10 +2155,10 @@ class TestDeleteRepository(ApiTestCase):
repository = model.repository.get_repository(ADMIN_ACCESS_USER, 'complex') repository = model.repository.get_repository(ADMIN_ACCESS_USER, 'complex')
# Add some CNR tags and linked tags. # Add some CNR tags and linked tags.
base_tag = appr_model.tag.create_or_update_tag(repository, 'somebasetag') base_tag = appr_model.tag.create_or_update_tag(repository, 'somebasetag', OLD_MODELS)
base_tag2 = appr_model.tag.create_or_update_tag(repository, 'somebasetag2') base_tag2 = appr_model.tag.create_or_update_tag(repository, 'somebasetag2', OLD_MODELS)
appr_model.tag.create_or_update_tag(repository, 'somelinkedtag', linked_tag=base_tag) appr_model.tag.create_or_update_tag(repository, 'somelinkedtag', OLD_MODELS, linked_tag=base_tag)
appr_model.tag.create_or_update_tag(repository, 'somelinkedtag2', linked_tag=base_tag2) appr_model.tag.create_or_update_tag(repository, 'somelinkedtag2', OLD_MODELS, linked_tag=base_tag2)
# Create some access tokens. # Create some access tokens.
access_token = model.token.create_access_token(repository, 'read') access_token = model.token.create_access_token(repository, 'read')

6
util/config/database.py
View file

@ -1,4 +1,6 @@
from data import model, appr_model from data import model
from data.appr_model import blob
from data.appr_model.models import OLD_MODELS
def sync_database_with_config(config): def sync_database_with_config(config):
@ -7,4 +9,4 @@ def sync_database_with_config(config):
location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys() location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
if location_names: if location_names:
model.image.ensure_image_locations(*location_names) model.image.ensure_image_locations(*location_names)
appr_model.blob.ensure_blob_locations(*location_names) blob.ensure_blob_locations(OLD_MODELS, *location_names)

7
util/config/schema.py
View file

@ -881,6 +881,13 @@ CONFIG_SCHEMA = {
'x-example': False, 'x-example': False,
}, },
# Feature Flag: Read only app registry.
'FEATURE_READONLY_APP_REGISTRY': {
'type': 'boolean',
'description': 'Whether to App repositories are read-only. Defaults to False',
'x-example': True,
},
# Feature Flag: Public Reposiotires in _catalog Endpoint. # Feature Flag: Public Reposiotires in _catalog Endpoint.
'FEATURE_PUBLIC_CATALOG': { 'FEATURE_PUBLIC_CATALOG': {
'type': 'boolean', 'type': 'boolean',

11
util/migrate/table_ops.py Normal file
View file

@ -0,0 +1,11 @@
def copy_table_contents(source_table, destination_table, conn):
if conn.engine.name == 'postgresql':
conn.execute('INSERT INTO "%s" SELECT * FROM "%s"' % (destination_table, source_table))
result = list(conn.execute('Select Max(id) from "%s"' % destination_table))[0]
new_start_id = result[0] + 1
conn.execute('ALTER SEQUENCE "%s_id_seq" RESTART WITH %s' % (destination_table, new_start_id))
else:
conn.execute("INSERT INTO `%s` SELECT * FROM `%s` WHERE 1" % (destination_table, source_table))
result = list(conn.execute('Select Max(id) from `%s` WHERE 1' % destination_table))[0]
new_start_id = result[0] + 1
conn.execute("ALTER TABLE `%s` AUTO_INCREMENT = %s" % (destination_table, new_start_id))