From 113bb96f2972cdfe3e06e269a0ebe23a1964dea1 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 24 May 2018 17:54:51 -0400 Subject: [PATCH] Phase 1 of migrating APPR-specific tables to tables with the `Appr` prefix Fixes https://jira.coreos.com/browse/QUAY-950 --- config.py | 3 + data/appr_model/blob.py | 21 +- data/appr_model/channel.py | 36 ++-- data/appr_model/manifest.py | 24 ++- data/appr_model/manifest_list.py | 20 +- data/appr_model/models.py | 21 ++ data/appr_model/package.py | 10 +- data/appr_model/release.py | 73 ++++--- data/appr_model/tag.py | 48 +++-- data/database.py | 151 ++++++++++--- data/migrations/migration.sh | 2 +- ..._remove_oci_tables_not_used_by_cnr_the_.py | 4 +- ...0320e9dacf_add_new_appr_specific_tables.py | 203 ++++++++++++++++++ data/model/repository.py | 6 +- data/model/sqlalchemybridge.py | 1 - data/model/storage.py | 12 +- data/model/test/test_gc.py | 9 +- endpoints/api/repository_models_pre_oci.py | 8 +- endpoints/api/test/test_repository.py | 27 ++- endpoints/api/test/test_search.py | 5 + endpoints/appr/models_cnr.py | 101 ++++++--- endpoints/appr/test/test_api.py | 16 +- endpoints/appr/test/test_registry.py | 30 ++- initdb.py | 11 +- test/test_api_usage.py | 9 +- util/config/database.py | 6 +- util/config/schema.py | 7 + util/migrate/table_ops.py | 11 + 28 files changed, 699 insertions(+), 176 deletions(-) create mode 100644 data/appr_model/models.py create mode 100644 data/migrations/versions/610320e9dacf_add_new_appr_specific_tables.py create mode 100644 util/migrate/table_ops.py diff --git a/config.py b/config.py index 7e6a10747..75794515e 100644 --- a/config.py +++ b/config.py @@ -264,6 +264,9 @@ class DefaultConfig(ImmutableConfig): # Feature Flag: Whether to enable support for App repositories. FEATURE_APP_REGISTRY = False + # Feature Flag: Whether app registry is in a read-only mode. + FEATURE_READONLY_APP_REGISTRY = False + # Feature Flag: If set to true, the _catalog endpoint returns public repositories. Otherwise, # only private repositories can be returned. FEATURE_PUBLIC_CATALOG = False diff --git a/data/appr_model/blob.py b/data/appr_model/blob.py index 7cc5a65d7..d340a7491 100644 --- a/data/appr_model/blob.py +++ b/data/appr_model/blob.py @@ -3,7 +3,6 @@ import logging from peewee import IntegrityError from data.model import db_transaction -from data.database import Blob, BlobPlacementLocation, BlobPlacement logger = logging.getLogger(__name__) @@ -13,17 +12,20 @@ def _ensure_sha256_header(digest): return 'sha256:' + digest -def get_blob(digest): +def get_blob(digest, models_ref): """ Find a blob by its digest. """ + Blob = models_ref.Blob return Blob.select().where(Blob.digest == _ensure_sha256_header(digest)).get() -def get_or_create_blob(digest, size, media_type_name, locations): +def get_or_create_blob(digest, size, media_type_name, locations, models_ref): """ Try to find a blob by its digest or create it. """ + Blob = models_ref.Blob + BlobPlacement = models_ref.BlobPlacement # Get or create the blog entry for the digest. try: - blob = get_blob(digest) + blob = get_blob(digest, models_ref) logger.debug('Retrieved blob with digest %s', digest) except Blob.DoesNotExist: blob = Blob.create(digest=_ensure_sha256_header(digest), @@ -38,13 +40,16 @@ def get_or_create_blob(digest, size, media_type_name, locations): BlobPlacement.create(blob=blob, location=location_id) except IntegrityError: logger.debug('Location %s already existing for blob %s', location_name, blob.id) - pass return blob -def get_blob_locations(digest): +def get_blob_locations(digest, models_ref): """ Find all locations names for a blob. """ + Blob = models_ref.Blob + BlobPlacement = models_ref.BlobPlacement + BlobPlacementLocation = models_ref.BlobPlacementLocation + return [x.name for x in BlobPlacementLocation .select() @@ -53,7 +58,9 @@ def get_blob_locations(digest): .where(Blob.digest == _ensure_sha256_header(digest))] -def ensure_blob_locations(*names): +def ensure_blob_locations(models_ref, *names): + BlobPlacementLocation = models_ref.BlobPlacementLocation + with db_transaction(): locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names) diff --git a/data/appr_model/channel.py b/data/appr_model/channel.py index e9e3c2d90..3631d97a5 100644 --- a/data/appr_model/channel.py +++ b/data/appr_model/channel.py @@ -1,11 +1,13 @@ -from data.database import Tag, Channel from data.appr_model import tag as tag_model -def get_channel_releases(repo, channel): +def get_channel_releases(repo, channel, models_ref): """ Return all previously linked tags. This works based upon Tag lifetimes. """ + Channel = models_ref.Channel + Tag = models_ref.Tag + tag_kind_id = Channel.tag_kind.get_id('channel') channel_name = channel.name return (Tag @@ -17,40 +19,46 @@ def get_channel_releases(repo, channel): .order_by(Tag.lifetime_end)) -def get_channel(repo, channel_name): +def get_channel(repo, channel_name, models_ref): """ Find a Channel by name. """ - channel = tag_model.get_tag(repo, channel_name, "channel") + channel = tag_model.get_tag(repo, channel_name, models_ref, "channel") return channel -def get_tag_channels(repo, tag_name, active=True): +def get_tag_channels(repo, tag_name, models_ref, active=True): """ Find the Channels associated with a Tag. """ - tag = tag_model.get_tag(repo, tag_name, "release") + Tag = models_ref.Tag + + tag = tag_model.get_tag(repo, tag_name, models_ref, "release") query = tag.tag_parents if active: - query = tag_model.tag_alive_oci(query) + query = tag_model.tag_is_alive(query, Tag) return query -def delete_channel(repo, channel_name): +def delete_channel(repo, channel_name, models_ref): """ Delete a channel by name. """ - return tag_model.delete_tag(repo, channel_name, "channel") + return tag_model.delete_tag(repo, channel_name, models_ref, "channel") -def create_or_update_channel(repo, channel_name, tag_name): +def create_or_update_channel(repo, channel_name, tag_name, models_ref): """ Creates or updates a channel to include a particular tag. """ - tag = tag_model.get_tag(repo, tag_name, 'release') - return tag_model.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel") + tag = tag_model.get_tag(repo, tag_name, models_ref, 'release') + return tag_model.create_or_update_tag(repo, channel_name, models_ref, linked_tag=tag, + tag_kind="channel") -def get_repo_channels(repo): +def get_repo_channels(repo, models_ref): """ Creates or updates a channel to include a particular tag. """ + Channel = models_ref.Channel + Tag = models_ref.Tag + tag_kind_id = Channel.tag_kind.get_id('channel') query = (Channel .select(Channel, Tag) .join(Tag, on=(Tag.id == Channel.linked_tag)) .where(Channel.repository == repo, Channel.tag_kind == tag_kind_id)) - return tag_model.tag_alive_oci(query, cls=Channel) + return tag_model.tag_is_alive(query, Channel) diff --git a/data/appr_model/manifest.py b/data/appr_model/manifest.py index b6c7341ca..f08be8d9b 100644 --- a/data/appr_model/manifest.py +++ b/data/appr_model/manifest.py @@ -4,7 +4,7 @@ import json from cnr.models.package_base import get_media_type -from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag +from data.database import db_transaction, MediaType from data.appr_model import tag as tag_model @@ -21,20 +21,23 @@ def _digest(manifestjson): return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()) -def get_manifest_query(digest, media_type): +def get_manifest_query(digest, media_type, models_ref): + Manifest = models_ref.Manifest return Manifest.select().where(Manifest.digest == _ensure_sha256_header(digest), Manifest.media_type == Manifest.media_type.get_id(media_type)) -def get_manifest_with_blob(digest, media_type): - query = get_manifest_query(digest, media_type) +def get_manifest_with_blob(digest, media_type, models_ref): + Blob = models_ref.Blob + query = get_manifest_query(digest, media_type, models_ref) return query.join(Blob).get() -def get_or_create_manifest(manifest_json, media_type_name): +def get_or_create_manifest(manifest_json, media_type_name, models_ref): + Manifest = models_ref.Manifest digest = _digest(manifest_json) try: - manifest = get_manifest_query(digest, media_type_name).get() + manifest = get_manifest_query(digest, media_type_name, models_ref).get() except Manifest.DoesNotExist: with db_transaction(): manifest = Manifest.create(digest=digest, @@ -42,16 +45,19 @@ def get_or_create_manifest(manifest_json, media_type_name): media_type=Manifest.media_type.get_id(media_type_name)) return manifest -def get_manifest_types(repo, release=None): +def get_manifest_types(repo, models_ref, release=None): """ Returns an array of MediaTypes.name for a repo, can filter by tag """ - query = tag_model.tag_alive_oci(Tag + Tag = models_ref.Tag + ManifestListManifest = models_ref.ManifestListManifest + + query = tag_model.tag_is_alive(Tag .select(MediaType.name) .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list)) .join(MediaType, on=(ManifestListManifest.media_type == MediaType.id)) .where(Tag.repository == repo, - Tag.tag_kind == Tag.tag_kind.get_id('release'))) + Tag.tag_kind == Tag.tag_kind.get_id('release')), Tag) if release: query = query.where(Tag.name == release) diff --git a/data/appr_model/manifest_list.py b/data/appr_model/manifest_list.py index c1bd35af5..92b10be6e 100644 --- a/data/appr_model/manifest_list.py +++ b/data/appr_model/manifest_list.py @@ -2,7 +2,7 @@ import logging import hashlib import json -from data.database import ManifestList, ManifestListManifest, db_transaction +from data.database import db_transaction logger = logging.getLogger(__name__) @@ -18,16 +18,19 @@ def _digest(manifestjson): return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()) -def get_manifest_list(digest): +def get_manifest_list(digest, models_ref): + ManifestList = models_ref.ManifestList return ManifestList.select().where(ManifestList.digest == _ensure_sha256_header(digest)).get() -def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version): +def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version, models_ref): + ManifestList = models_ref.ManifestList + digest = _digest(manifest_list_json) media_type_id = ManifestList.media_type.get_id(media_type_name) try: - return get_manifest_list(digest) + return get_manifest_list(digest, models_ref) except ManifestList.DoesNotExist: with db_transaction(): manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json, @@ -35,7 +38,7 @@ def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_vers return manifestlist -def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json): +def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json, models_ref): """ From a manifestlist, manifests, and the manifest list blob, create if doesn't exist the manfiestlistmanifest for each manifest """ for pos in xrange(len(manifest_ids)): @@ -43,10 +46,13 @@ def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json): manifest_json = manifest_list_json[pos] get_or_create_manifestlistmanifest(manifest=manifest_id, manifestlist=manifestlist, - media_type_name=manifest_json['mediaType']) + media_type_name=manifest_json['mediaType'], + models_ref=models_ref) -def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name): +def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name, models_ref): + ManifestListManifest = models_ref.ManifestListManifest + media_type_id = ManifestListManifest.media_type.get_id(media_type_name) try: ml = (ManifestListManifest diff --git a/data/appr_model/models.py b/data/appr_model/models.py new file mode 100644 index 000000000..87f41eb8f --- /dev/null +++ b/data/appr_model/models.py @@ -0,0 +1,21 @@ +from collections import namedtuple + +from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, + ManifestListManifest, Manifest, BlobPlacement, Channel) +from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, + ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, + ApprBlobPlacement, ApprChannel) + +ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation', 'ManifestList', + 'ManifestBlob', 'Blob', 'ManifestListManifest', 'Manifest', + 'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name', + 'tag_set_prefetch_name']) + +OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, + ManifestListManifest, Manifest, BlobPlacement, Channel, + 'manifestlistmanifest_set', 'tag_set_prefetch') + +NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, + ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, + ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set', + 'apprtag_set_prefetch') diff --git a/data/appr_model/package.py b/data/appr_model/package.py index 4e41c6c87..745701690 100644 --- a/data/appr_model/package.py +++ b/data/appr_model/package.py @@ -3,12 +3,14 @@ from peewee import prefetch from data import model -from data.database import Repository, Namespace, Tag, ManifestListManifest +from data.database import Repository, Namespace from data.appr_model import tag as tag_model -def list_packages_query(namespace=None, media_type=None, search_query=None, username=None): +def list_packages_query(models_ref, namespace=None, media_type=None, search_query=None, + username=None): """ List and filter repository by search query. """ + Tag = models_ref.Tag fields = [model.repository.SEARCH_FIELDS.name.name] if search_query is not None: @@ -40,9 +42,9 @@ def list_packages_query(namespace=None, media_type=None, search_query=None, user .order_by(Tag.lifetime_start)) if media_type: - tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type) + tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type, models_ref) - tag_query = tag_model.tag_alive_oci(tag_query) + tag_query = tag_model.tag_is_alive(tag_query, Tag) query = prefetch(repo_query, tag_query) return query diff --git a/data/appr_model/release.py b/data/appr_model/release.py index 82b442f02..dcfa455d0 100644 --- a/data/appr_model/release.py +++ b/data/appr_model/release.py @@ -3,8 +3,7 @@ import bisect from cnr.exception import PackageAlreadyExists from cnr.models.package_base import manifest_media_type -from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag, - ManifestListManifest, Blob, ManifestBlob) +from data.database import db_transaction, get_epoch_timestamp from data.appr_model import (blob as blob_model, manifest as manifest_model, manifest_list as manifest_list_model, tag as tag_model) @@ -20,11 +19,17 @@ def _ensure_sha256_header(digest): return 'sha256:' + digest -def get_app_release(repo, tag_name, media_type): +def get_app_release(repo, tag_name, media_type, models_ref): """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """ - tag = tag_model.get_tag(repo, tag_name, tag_kind='release') + ManifestListManifest = models_ref.ManifestListManifest + Manifest = models_ref.Manifest + Blob = models_ref.Blob + ManifestBlob = models_ref.ManifestBlob + manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name + + tag = tag_model.get_tag(repo, tag_name, models_ref, tag_kind='release') media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) - manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set + manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name) .join(Manifest) .where(ManifestListManifest.media_type == media_type_id).get()) manifest = manifestlistmanifest.manifest @@ -32,25 +37,26 @@ def get_app_release(repo, tag_name, media_type): return (tag, manifest, blob) -def delete_app_release(repo, tag_name, media_type): +def delete_app_release(repo, tag_name, media_type, models_ref): """ Terminate a Tag/media-type couple It find the corresponding tag/manifest and remove from the manifestlistmanifest the manifest 1. it terminates the current tag (in all-cases) 2. if the new manifestlist is not empty, it creates a new tag for it """ + ManifestListManifest = models_ref.ManifestListManifest + manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name + media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) with db_transaction(): - tag = tag_model.get_tag(repo, tag_name) + tag = tag_model.get_tag(repo, tag_name, models_ref) manifest_list = tag.manifest_list list_json = manifest_list.manifest_list_json mlm_query = (ManifestListManifest .select() .where(ManifestListManifest.manifest_list == tag.manifest_list)) list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query]) - manifestlistmanifest = (tag - .manifest_list - .manifestlistmanifest_set + manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name) .where(ManifestListManifest.media_type == media_type_id).get()) index = list_manifest_ids.index(manifestlistmanifest.manifest_id) list_manifest_ids.pop(index) @@ -61,36 +67,42 @@ def delete_app_release(repo, tag_name, media_type): tag.save() else: manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, - SCHEMA_VERSION) + SCHEMA_VERSION, models_ref) manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, - list_json) - tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + list_json, models_ref) + tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist, tag_kind="release") return tag -def create_app_release(repo, tag_name, manifest_data, digest, force=False): +def create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False): """ Create a new application release, it includes creating a new Tag, ManifestList, ManifestListManifests, Manifest, ManifestBlob. To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id. To find the insert point in the ManifestList it uses bisect on the manifest-ids list. """ + ManifestList = models_ref.ManifestList + ManifestListManifest = models_ref.ManifestListManifest + Blob = models_ref.Blob + ManifestBlob = models_ref.ManifestBlob + with db_transaction(): # Create/get the package manifest - manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType']) + manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType'], + models_ref) # get the tag - tag = tag_model.get_or_initialize_tag(repo, tag_name) + tag = tag_model.get_or_initialize_tag(repo, tag_name, models_ref) if tag.manifest_list is None: tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE), schema_version=SCHEMA_VERSION, - manifest_list_json=[]) + manifest_list_json=[], ) - elif tag_model.tag_media_type_exists(tag, manifest.media_type): + elif tag_model.tag_media_type_exists(tag, manifest.media_type, models_ref): if force: - delete_app_release(repo, tag_name, manifest.media_type.name) - return create_app_release(repo, tag_name, manifest_data, digest, force=False) + delete_app_release(repo, tag_name, manifest.media_type.name, models_ref) + return create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False) else: raise PackageAlreadyExists("package exists already") @@ -103,10 +115,11 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False): list_json.insert(insert_point, manifest.manifest_json) list_manifest_ids.insert(insert_point, manifest.id) manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, - SCHEMA_VERSION) - manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) + SCHEMA_VERSION, models_ref) + manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json, + models_ref) - tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist, tag_kind="release") blob_digest = digest @@ -117,21 +130,23 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False): .where(ManifestBlob.manifest == manifest, Blob.digest == _ensure_sha256_header(blob_digest)).get()) except ManifestBlob.DoesNotExist: - blob = blob_model.get_blob(blob_digest) + blob = blob_model.get_blob(blob_digest, models_ref) ManifestBlob.create(manifest=manifest, blob=blob) return tag -def get_release_objs(repo, media_type=None): +def get_release_objs(repo, models_ref, media_type=None): """ Returns an array of Tag for a repo, with optional filtering by media_type. """ + Tag = models_ref.Tag + release_query = (Tag .select() .where(Tag.repository == repo, Tag.tag_kind == Tag.tag_kind.get_id("release"))) if media_type: - release_query = tag_model.filter_tags_by_media_type(release_query, media_type) + release_query = tag_model.filter_tags_by_media_type(release_query, media_type, models_ref) - return tag_model.tag_alive_oci(release_query) + return tag_model.tag_is_alive(release_query, Tag) -def get_releases(repo, media_type=None): +def get_releases(repo, model_refs, media_type=None): """ Returns an array of Tag.name for a repo, can filter by media_type. """ - return [t.name for t in get_release_objs(repo, media_type)] + return [t.name for t in get_release_objs(repo, model_refs, media_type)] diff --git a/data/appr_model/tag.py b/data/appr_model/tag.py index 56bea5a29..4903a4572 100644 --- a/data/appr_model/tag.py +++ b/data/appr_model/tag.py @@ -4,32 +4,37 @@ from cnr.models.package_base import manifest_media_type from peewee import IntegrityError from data.model import (db_transaction, TagAlreadyCreatedException) -from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update +from data.database import get_epoch_timestamp_ms, db_for_update logger = logging.getLogger(__name__) -def tag_alive_oci(query, now_ts=None, cls=Tag): +def tag_is_alive(query, cls, now_ts=None): return query.where((cls.lifetime_end >> None) | (cls.lifetime_end > now_ts)) -def tag_media_type_exists(tag, media_type): - return (tag.manifest_list.manifestlistmanifest_set +def tag_media_type_exists(tag, media_type, models_ref): + ManifestListManifest = models_ref.ManifestListManifest + manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name + return (getattr(tag.manifest_list, manifestlistmanifest_set_name) .where(ManifestListManifest.media_type == media_type).count() > 0) -def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"): +def create_or_update_tag(repo, tag_name, models_ref, manifest_list=None, linked_tag=None, + tag_kind="release"): + Tag = models_ref.Tag + now_ts = get_epoch_timestamp_ms() tag_kind_id = Tag.tag_kind.get_id(tag_kind) with db_transaction(): try: - tag = db_for_update(tag_alive_oci(Tag + tag = db_for_update(tag_is_alive(Tag .select() .where(Tag.repository == repo, Tag.name == tag_name, - Tag.tag_kind == tag_kind_id), now_ts)).get() + Tag.tag_kind == tag_kind_id), Tag, now_ts)).get() if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag: return tag tag.lifetime_end = now_ts @@ -46,40 +51,47 @@ def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, ta raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name)) -def get_or_initialize_tag(repo, tag_name, tag_kind="release"): +def get_or_initialize_tag(repo, tag_name, models_ref, tag_kind="release"): + Tag = models_ref.Tag + try: - return tag_alive_oci(Tag.select().where(Tag.repository == repo, Tag.name == tag_name)).get() + return tag_is_alive(Tag.select().where(Tag.repository == repo, Tag.name == tag_name), Tag).get() except Tag.DoesNotExist: return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind)) -def get_tag(repo, tag_name, tag_kind="release"): - return tag_alive_oci(Tag.select() +def get_tag(repo, tag_name, models_ref, tag_kind="release"): + Tag = models_ref.Tag + return tag_is_alive(Tag.select() .where(Tag.repository == repo, Tag.name == tag_name, - Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get() + Tag.tag_kind == Tag.tag_kind.get_id(tag_kind)), Tag).get() -def delete_tag(repo, tag_name, tag_kind="release"): +def delete_tag(repo, tag_name, models_ref, tag_kind="release"): + Tag = models_ref.Tag tag_kind_id = Tag.tag_kind.get_id(tag_kind) - tag = tag_alive_oci(Tag.select() + tag = tag_is_alive(Tag.select() .where(Tag.repository == repo, - Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get() + Tag.name == tag_name, Tag.tag_kind == tag_kind_id), Tag).get() tag.lifetime_end = get_epoch_timestamp_ms() tag.save() return tag -def tag_exists(repo, tag_name, tag_kind="release"): +def tag_exists(repo, tag_name, models_ref, tag_kind="release"): + Tag = models_ref.Tag try: - get_tag(repo, tag_name, tag_kind) + get_tag(repo, tag_name, models_ref, tag_kind) return True except Tag.DoesNotExist: return False -def filter_tags_by_media_type(tag_query, media_type): +def filter_tags_by_media_type(tag_query, media_type, models_ref): """ Return only available tag for a media_type. """ + ManifestListManifest = models_ref.ManifestListManifest + Tag = models_ref.Tag media_type = manifest_media_type(media_type) t = (tag_query .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list)) diff --git a/data/database.py b/data/database.py index e3d249f8b..ef88615b6 100644 --- a/data/database.py +++ b/data/database.py @@ -470,8 +470,8 @@ class User(BaseModel): RepositoryTag, PermissionPrototype, DerivedStorageForImage, TagManifest, AccessToken, OAuthAccessToken, BlobUpload, RepositoryNotification, OAuthAuthorizationCode, - RepositoryActionCount, TagManifestLabel, Tag, - TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes + RepositoryActionCount, TagManifestLabel, + TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes) @@ -619,7 +619,7 @@ class Repository(BaseModel): # are cleaned up directly skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload, Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage, - RepositorySearchScore} | cnr_classes + RepositorySearchScore} | cnr_classes | appr_classes delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes) @@ -1187,7 +1187,6 @@ class ServiceKey(BaseModel): class MediaType(BaseModel): """ MediaType is an enumeration of the possible formats of various objects in the data model. - This model is a part of the new OCI/CNR model set. """ name = CharField(index=True, unique=True) @@ -1201,7 +1200,6 @@ class Messages(BaseModel): class LabelSourceType(BaseModel): """ LabelSourceType is an enumeration of the possible sources for a label. - This model is a part of the new OCI/CNR model set. """ name = CharField(index=True, unique=True) mutable = BooleanField(default=False) @@ -1210,7 +1208,6 @@ class LabelSourceType(BaseModel): class Label(BaseModel): """ Label represents user-facing metadata associated with another entry in the database (e.g. a Manifest). - This model is a part of the new OCI/CNR model set. """ uuid = CharField(default=uuid_generator, index=True, unique=True) key = CharField(index=True) @@ -1221,7 +1218,6 @@ class Label(BaseModel): class TagManifestLabel(BaseModel): """ Mapping from a tag manifest to a label. - This model is a part of the new OCI/CNR model set. """ repository = ForeignKeyField(Repository, index=True) annotated = ForeignKeyField(TagManifest, index=True) @@ -1237,8 +1233,17 @@ class TagManifestLabel(BaseModel): class Blob(BaseModel): """ Blob represents a content-addressable object stored outside of the database. - This model is a part of the new OCI/CNR model set. - CNR + This is deprecated in favor of ApprBlob. + """ + digest = CharField(index=True, unique=True) + media_type = EnumField(MediaType) + size = BigIntegerField() + uncompressed_size = BigIntegerField(null=True) + + + +class ApprBlob(BaseModel): + """ ApprBlob represents a content-addressable object stored outside of the database. """ digest = CharField(index=True, unique=True) media_type = EnumField(MediaType) @@ -1248,16 +1253,20 @@ class Blob(BaseModel): class BlobPlacementLocation(BaseModel): """ BlobPlacementLocation is an enumeration of the possible storage locations for Blobs. - This model is a part of the new OCI/CNR model set. - CNR + This is deprecated in favor of ApprBlobPlacementLocation. + """ + name = CharField(index=True, unique=True) + + +class ApprBlobPlacementLocation(BaseModel): + """ ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs. """ name = CharField(index=True, unique=True) class BlobPlacement(BaseModel): """ BlobPlacement represents the location of a Blob. - This model is a part of the new OCI/CNR model set. - CNR + This is deprecated in favor of ApprBlobPlacement. """ blob = ForeignKeyField(Blob) location = EnumField(BlobPlacementLocation) @@ -1270,10 +1279,31 @@ class BlobPlacement(BaseModel): ) +class ApprBlobPlacement(BaseModel): + """ ApprBlobPlacement represents the location of a Blob. + """ + blob = ForeignKeyField(ApprBlob) + location = EnumField(ApprBlobPlacementLocation) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('blob', 'location'), True), + ) + + class Manifest(BaseModel): """ Manifest represents the metadata and collection of blobs that comprise a container image. - This model is a part of the new OCI/CNR model set. - CNR + This is deprecated in favor of ApprManifest. + """ + digest = CharField(index=True, unique=True) + media_type = EnumField(MediaType) + manifest_json = JSONField() + + +class ApprManifest(BaseModel): + """ ApprManifest represents the metadata and collection of blobs that comprise an Appr image. """ digest = CharField(index=True, unique=True) media_type = EnumField(MediaType) @@ -1282,8 +1312,7 @@ class Manifest(BaseModel): class ManifestBlob(BaseModel): """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs. - This model is a part of the new OCI/CNR model set. - CNR + This is deprecated in favor of ApprManifestBlob. """ manifest = ForeignKeyField(Manifest, index=True) blob = ForeignKeyField(Blob, index=True) @@ -1296,10 +1325,32 @@ class ManifestBlob(BaseModel): ) +class ApprManifestBlob(BaseModel): + """ ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs. + """ + manifest = ForeignKeyField(ApprManifest, index=True) + blob = ForeignKeyField(ApprBlob, index=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('manifest', 'blob'), True), + ) + + class ManifestList(BaseModel): """ ManifestList represents all of the various manifests that compose a Tag. - This model is a part of the new OCI/CNR model set. - CNR + This is deprecated in favor of ApprManifestList. + """ + digest = CharField(index=True, unique=True) + manifest_list_json = JSONField() + schema_version = CharField() + media_type = EnumField(MediaType) + + +class ApprManifestList(BaseModel): + """ ApprManifestList represents all of the various Appr manifests that compose an ApprTag. """ digest = CharField(index=True, unique=True) manifest_list_json = JSONField() @@ -1309,16 +1360,20 @@ class ManifestList(BaseModel): class TagKind(BaseModel): """ TagKind is a enumtable to reference tag kinds. - This model is a part of the new OCI/CNR model set. - CNR + This model is deprecated in favor of ApprTagKind. + """ + name = CharField(index=True, unique=True) + + +class ApprTagKind(BaseModel): + """ ApprTagKind is a enumtable to reference tag kinds. """ name = CharField(index=True, unique=True) class Tag(BaseModel): """ Tag represents a user-facing alias for referencing a ManifestList. - This model is a part of the new OCI/CNR model set. - CNR + This model is deprecated in favor of ApprTag. """ name = CharField() repository = ForeignKeyField(Repository) @@ -1342,13 +1397,37 @@ class Tag(BaseModel): ) +class ApprTag(BaseModel): + """ ApprTag represents a user-facing alias for referencing an ApprManifestList. + """ + name = CharField() + repository = ForeignKeyField(Repository) + manifest_list = ForeignKeyField(ApprManifestList, null=True) + lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms) + lifetime_end = BigIntegerField(null=True, index=True) + hidden = BooleanField(default=False) + reverted = BooleanField(default=False) + protected = BooleanField(default=False) + tag_kind = EnumField(ApprTagKind) + linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents') + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'name'), False), + (('repository', 'name', 'hidden'), False), + # This unique index prevents deadlocks when concurrently moving and deleting tags + (('repository', 'name', 'lifetime_end'), True), + ) Channel = Tag.alias() +ApprChannel = ApprTag.alias() + class ManifestListManifest(BaseModel): """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. - This model is a part of the new OCI/CNR model set. - CNR + This model is deprecated in favor of ApprManifestListManifest. """ manifest_list = ForeignKeyField(ManifestList, index=True) manifest = ForeignKeyField(Manifest, index=True) @@ -1366,6 +1445,25 @@ class ManifestListManifest(BaseModel): ) +class ApprManifestListManifest(BaseModel): + """ ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and + ApprManifests. + """ + manifest_list = ForeignKeyField(ApprManifestList, index=True) + manifest = ForeignKeyField(ApprManifest, index=True) + operating_system = CharField(null=True) + architecture = CharField(null=True) + platform_json = JSONField(null=True) + media_type = EnumField(MediaType) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('manifest_list', 'media_type'), False), + ) + + class AppSpecificAuthToken(BaseModel): """ AppSpecificAuthToken represents a token generated by a user for use with an external application where putting the user's credentials, even encrypted, is deemed too risky. @@ -1388,5 +1486,8 @@ class AppSpecificAuthToken(BaseModel): cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob, ManifestListManifest, Manifest, BlobPlacement]) +appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList, + ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest, + ApprBlobPlacement]) is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index a1d0f8f5d..0c9a0e9a1 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -14,7 +14,7 @@ up_mysql() { sleep 25 # Add the database to mysql. - docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' + docker run --rm --link mysql:mysql mysql:5.7 sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword' } down_mysql() { diff --git a/data/migrations/versions/5cbbfc95bac7_remove_oci_tables_not_used_by_cnr_the_.py b/data/migrations/versions/5cbbfc95bac7_remove_oci_tables_not_used_by_cnr_the_.py index 1deb2ac34..57e504ddc 100644 --- a/data/migrations/versions/5cbbfc95bac7_remove_oci_tables_not_used_by_cnr_the_.py +++ b/data/migrations/versions/5cbbfc95bac7_remove_oci_tables_not_used_by_cnr_the_.py @@ -15,7 +15,7 @@ import sqlalchemy as sa from sqlalchemy.dialects import mysql from util.migrate import UTF8LongText, UTF8CharField -def upgrade(tables): +def upgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('derivedimage') op.drop_table('manifestlabel') @@ -28,7 +28,7 @@ def upgrade(tables): # ### end Alembic commands ### -def downgrade(tables): +def downgrade(tables, tester): # ### commands auto generated by Alembic - please adjust! ### op.create_table( 'manifestlayer', diff --git a/data/migrations/versions/610320e9dacf_add_new_appr_specific_tables.py b/data/migrations/versions/610320e9dacf_add_new_appr_specific_tables.py new file mode 100644 index 000000000..e0ea461f1 --- /dev/null +++ b/data/migrations/versions/610320e9dacf_add_new_appr_specific_tables.py @@ -0,0 +1,203 @@ +"""Add new Appr-specific tables + +Revision ID: 610320e9dacf +Revises: 5cbbfc95bac7 +Create Date: 2018-05-24 16:46:13.514562 + +""" + +# revision identifiers, used by Alembic. +revision = '610320e9dacf' +down_revision = '5cbbfc95bac7' + +from alembic import op +import sqlalchemy as sa + +from util.migrate.table_ops import copy_table_contents + +def upgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('apprblobplacementlocation', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacementlocation')) + ) + op.create_index('apprblobplacementlocation_name', 'apprblobplacementlocation', ['name'], unique=True) + op.create_table('apprtagkind', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtagkind')) + ) + op.create_index('apprtagkind_name', 'apprtagkind', ['name'], unique=True) + op.create_table('apprblob', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('digest', sa.String(length=255), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.Column('size', sa.BigInteger(), nullable=False), + sa.Column('uncompressed_size', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprblob_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblob')) + ) + op.create_index('apprblob_digest', 'apprblob', ['digest'], unique=True) + op.create_index('apprblob_media_type_id', 'apprblob', ['media_type_id'], unique=False) + op.create_table('apprmanifest', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('digest', sa.String(length=255), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.Column('manifest_json', sa.Text(), nullable=False), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifest_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifest')) + ) + op.create_index('apprmanifest_digest', 'apprmanifest', ['digest'], unique=True) + op.create_index('apprmanifest_media_type_id', 'apprmanifest', ['media_type_id'], unique=False) + op.create_table('apprmanifestlist', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('digest', sa.String(length=255), nullable=False), + sa.Column('manifest_list_json', sa.Text(), nullable=False), + sa.Column('schema_version', sa.String(length=255), nullable=False), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlist_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlist')) + ) + op.create_index('apprmanifestlist_digest', 'apprmanifestlist', ['digest'], unique=True) + op.create_index('apprmanifestlist_media_type_id', 'apprmanifestlist', ['media_type_id'], unique=False) + op.create_table('apprblobplacement', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('blob_id', sa.Integer(), nullable=False), + sa.Column('location_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprblobplacement_blob_id_apprblob')), + sa.ForeignKeyConstraint(['location_id'], ['apprblobplacementlocation.id'], name=op.f('fk_apprblobplacement_location_id_apprblobplacementlocation')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacement')) + ) + op.create_index('apprblobplacement_blob_id', 'apprblobplacement', ['blob_id'], unique=False) + op.create_index('apprblobplacement_blob_id_location_id', 'apprblobplacement', ['blob_id', 'location_id'], unique=True) + op.create_index('apprblobplacement_location_id', 'apprblobplacement', ['location_id'], unique=False) + op.create_table('apprmanifestblob', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('manifest_id', sa.Integer(), nullable=False), + sa.Column('blob_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprmanifestblob_blob_id_apprblob')), + sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestblob_manifest_id_apprmanifest')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestblob')) + ) + op.create_index('apprmanifestblob_blob_id', 'apprmanifestblob', ['blob_id'], unique=False) + op.create_index('apprmanifestblob_manifest_id', 'apprmanifestblob', ['manifest_id'], unique=False) + op.create_index('apprmanifestblob_manifest_id_blob_id', 'apprmanifestblob', ['manifest_id', 'blob_id'], unique=True) + op.create_table('apprmanifestlistmanifest', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('manifest_list_id', sa.Integer(), nullable=False), + sa.Column('manifest_id', sa.Integer(), nullable=False), + sa.Column('operating_system', sa.String(length=255), nullable=True), + sa.Column('architecture', sa.String(length=255), nullable=True), + sa.Column('platform_json', sa.Text(), nullable=True), + sa.Column('media_type_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_id_apprmanifest')), + sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_list_id_apprmanifestlist')), + sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlistmanifest_media_type_id_mediatype')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlistmanifest')) + ) + op.create_index('apprmanifestlistmanifest_manifest_id', 'apprmanifestlistmanifest', ['manifest_id'], unique=False) + op.create_index('apprmanifestlistmanifest_manifest_list_id', 'apprmanifestlistmanifest', ['manifest_list_id'], unique=False) + op.create_index('apprmanifestlistmanifest_manifest_list_id_media_type_id', 'apprmanifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False) + op.create_index('apprmanifestlistmanifest_manifest_list_id_operating_system_arch', 'apprmanifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False) + op.create_index('apprmanifestlistmanifest_media_type_id', 'apprmanifestlistmanifest', ['media_type_id'], unique=False) + op.create_table('apprtag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('manifest_list_id', sa.Integer(), nullable=True), + sa.Column('lifetime_start', sa.BigInteger(), nullable=False), + sa.Column('lifetime_end', sa.BigInteger(), nullable=True), + sa.Column('hidden', sa.Boolean(), nullable=False), + sa.Column('reverted', sa.Boolean(), nullable=False), + sa.Column('protected', sa.Boolean(), nullable=False), + sa.Column('tag_kind_id', sa.Integer(), nullable=False), + sa.Column('linked_tag_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['linked_tag_id'], ['apprtag.id'], name=op.f('fk_apprtag_linked_tag_id_apprtag')), + sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprtag_manifest_list_id_apprmanifestlist')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_apprtag_repository_id_repository')), + sa.ForeignKeyConstraint(['tag_kind_id'], ['apprtagkind.id'], name=op.f('fk_apprtag_tag_kind_id_apprtagkind')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtag')) + ) + op.create_index('apprtag_lifetime_end', 'apprtag', ['lifetime_end'], unique=False) + op.create_index('apprtag_linked_tag_id', 'apprtag', ['linked_tag_id'], unique=False) + op.create_index('apprtag_manifest_list_id', 'apprtag', ['manifest_list_id'], unique=False) + op.create_index('apprtag_repository_id', 'apprtag', ['repository_id'], unique=False) + op.create_index('apprtag_repository_id_name', 'apprtag', ['repository_id', 'name'], unique=False) + op.create_index('apprtag_repository_id_name_hidden', 'apprtag', ['repository_id', 'name', 'hidden'], unique=False) + op.create_index('apprtag_repository_id_name_lifetime_end', 'apprtag', ['repository_id', 'name', 'lifetime_end'], unique=True) + op.create_index('apprtag_tag_kind_id', 'apprtag', ['tag_kind_id'], unique=False) + # ### end Alembic commands ### + + conn = op.get_bind() + copy_table_contents('blobplacementlocation', 'apprblobplacementlocation', conn) + copy_table_contents('tagkind', 'apprtagkind', conn) + + # ### population of test data ### # + + tester.populate_table('apprmanifest', [ + ('digest', tester.TestDataType.String), + ('media_type_id', tester.TestDataType.Foreign('mediatype')), + ('manifest_json', tester.TestDataType.JSON), + ]) + + tester.populate_table('apprmanifestlist', [ + ('digest', tester.TestDataType.String), + ('manifest_list_json', tester.TestDataType.JSON), + ('schema_version', tester.TestDataType.String), + ('media_type_id', tester.TestDataType.Foreign('mediatype')), + ]) + + tester.populate_table('apprmanifestlistmanifest', [ + ('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')), + ('manifest_id', tester.TestDataType.Foreign('apprmanifest')), + ('operating_system', tester.TestDataType.String), + ('architecture', tester.TestDataType.String), + ('platform_json', tester.TestDataType.JSON), + ('media_type_id', tester.TestDataType.Foreign('mediatype')), + ]) + + tester.populate_table('apprblob', [ + ('digest', tester.TestDataType.String), + ('media_type_id', tester.TestDataType.Foreign('mediatype')), + ('size', tester.TestDataType.BigInteger), + ('uncompressed_size', tester.TestDataType.BigInteger), + ]) + + tester.populate_table('apprmanifestblob', [ + ('manifest_id', tester.TestDataType.Foreign('apprmanifest')), + ('blob_id', tester.TestDataType.Foreign('apprblob')), + ]) + + tester.populate_table('apprtag', [ + ('name', tester.TestDataType.String), + ('repository_id', tester.TestDataType.Foreign('repository')), + ('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')), + ('lifetime_start', tester.TestDataType.Integer), + ('hidden', tester.TestDataType.Boolean), + ('reverted', tester.TestDataType.Boolean), + ('protected', tester.TestDataType.Boolean), + ('tag_kind_id', tester.TestDataType.Foreign('apprtagkind')), + ]) + + tester.populate_table('apprblobplacement', [ + ('blob_id', tester.TestDataType.Foreign('apprmanifestblob')), + ('location_id', tester.TestDataType.Foreign('apprblobplacementlocation')), + ]) + + # ### end population of test data ### # + + + +def downgrade(tables, tester): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('apprtag') + op.drop_table('apprmanifestlistmanifest') + op.drop_table('apprmanifestblob') + op.drop_table('apprblobplacement') + op.drop_table('apprmanifestlist') + op.drop_table('apprmanifest') + op.drop_table('apprblob') + op.drop_table('apprtagkind') + op.drop_table('apprblobplacementlocation') + # ### end Alembic commands ### diff --git a/data/model/repository.py b/data/model/repository.py index a94a52562..c2c47dd6b 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -10,7 +10,7 @@ from cachetools import ttl_cache from data.model import ( config, DataModelException, tag, db_transaction, storage, permission, _basequery) from data.database import ( - Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, + Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag, RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp, db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind) @@ -87,11 +87,13 @@ def purge_repository(namespace_name, repository_name): except Repository.DoesNotExist: return False - # Delete the repository of all OCI-referenced entries. + # Delete the repository of all Appr-referenced entries. # Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags, # and MySQL is... particular... about such relationships when deleting. Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute() Tag.delete().where(Tag.repository == repo).execute() + ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute() + ApprTag.delete().where(ApprTag.repository == repo).execute() # Delete all tags to allow gc to reclaim storage previously_referenced = tag.purge_all_tags(repo) diff --git a/data/model/sqlalchemybridge.py b/data/model/sqlalchemybridge.py index 9add46ba9..23aaab46f 100644 --- a/data/model/sqlalchemybridge.py +++ b/data/model/sqlalchemybridge.py @@ -15,7 +15,6 @@ OPTION_TRANSLATIONS = { 'null': 'nullable', } - def gen_sqlalchemy_metadata(peewee_model_list): metadata = MetaData(naming_convention={ "ix": 'ix_%(column_0_label)s', diff --git a/data/model/storage.py b/data/model/storage.py index 6a4ff291e..9ab595aff 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -9,7 +9,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation, ImageStorageTransformation, ImageStorageSignature, ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob, - ensure_under_transaction) + ApprBlob, ensure_under_transaction) logger = logging.getLogger(__name__) @@ -105,15 +105,21 @@ def garbage_collect_storage(storage_id_whitelist): logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced', is_referenced_checksums) - # Check the new Blob table as well. + # Check the ApprBlob tables as well. query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums)) blob_referenced_checksums = set([blob.digest for blob in query]) if blob_referenced_checksums: logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced', blob_referenced_checksums) + query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums)) + appr_blob_referenced_checksums = set([blob.digest for blob in query]) + if appr_blob_referenced_checksums: + logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced', + appr_blob_referenced_checksums) + unreferenced_checksums = (content_checksums - blob_referenced_checksums - - is_referenced_checksums) + appr_blob_referenced_checksums - is_referenced_checksums) # Return all placements for all image storages found not at a CAS path or with a content # checksum that is referenced. diff --git a/data/model/test/test_gc.py b/data/model/test/test_gc.py index e3070e4fb..d1ea2b72c 100644 --- a/data/model/test/test_gc.py +++ b/data/model/test/test_gc.py @@ -9,7 +9,8 @@ from contextlib import contextmanager from playhouse.test_utils import assert_query_count from data import model, database -from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, Blob +from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, + Blob, ApprBlob) from test.fixtures import * @@ -187,7 +188,10 @@ def assert_gc_integrity(expect_storage_removed=True): storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum)) for blob_row in Blob.select(): - storage.get_content({preferred}, storage.blob_path(blob_row.digest)) + storage.get_content({preferred}, storage.blob_path(blob_row.digest)) + + for blob_row in ApprBlob.select(): + storage.get_content({preferred}, storage.blob_path(blob_row.digest)) def test_has_garbage(default_tag_policy, initialized_db): @@ -589,6 +593,7 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d is1 = database.ImageStorage.create(content_checksum=digest, uploading=False) database.Blob.create(digest=digest, size=0, media_type=media_type) + database.ApprBlob.create(digest=digest, size=0, media_type=media_type) location = database.ImageStorageLocation.get(name=preferred) database.ImageStoragePlacement.create(location=location, storage=is1) diff --git a/endpoints/api/repository_models_pre_oci.py b/endpoints/api/repository_models_pre_oci.py index d394b6d8e..d9f468b52 100644 --- a/endpoints/api/repository_models_pre_oci.py +++ b/endpoints/api/repository_models_pre_oci.py @@ -3,7 +3,9 @@ from collections import defaultdict from datetime import datetime, timedelta from auth.permissions import ReadRepositoryPermission -from data import model, appr_model +from data import model +from data.appr_model import channel as channel_model, release as release_model +from endpoints.appr.models_cnr import model as appr_model from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \ ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count @@ -143,8 +145,8 @@ class PreOCIModel(RepositoryDataInterface): False, False, False) if base.kind_name == 'application': - channels = appr_model.channel.get_repo_channels(repo) - releases = appr_model.release.get_release_objs(repo) + channels = channel_model.get_repo_channels(repo, appr_model.models_ref) + releases = release_model.get_release_objs(repo, appr_model.models_ref) releases_channels_map = defaultdict(list) return ApplicationRepository( base, [_create_channel(channel, releases_channels_map) for channel in channels], [ diff --git a/endpoints/api/test/test_repository.py b/endpoints/api/test/test_repository.py index 2981d7e22..8d632f132 100644 --- a/endpoints/api/test/test_repository.py +++ b/endpoints/api/test/test_repository.py @@ -2,7 +2,9 @@ import pytest from mock import patch, ANY, MagicMock -from data import model +from data import model, database +from data.appr_model import release, channel, blob +from endpoints.appr.models_cnr import model as appr_model from endpoints.api.test.shared import conduct_api_call from endpoints.api.repository import RepositoryTrust, Repository, RepositoryList from endpoints.test.shared import client_with_identity @@ -117,3 +119,26 @@ def test_create_repository(repo_name, expected_status, client): if expected_status == 201: assert result['name'] == repo_name assert model.repository.get_repository('devtable', repo_name).name == repo_name + + +def test_get_app_repo(client, initialized_db): + with client_with_identity('devtable', client) as cl: + devtable = model.user.get_user('devtable') + repo = model.repository.create_repository('devtable', 'someappr', devtable, + repo_kind='application') + + models_ref = appr_model.models_ref + blob.get_or_create_blob('sha256:somedigest', 0, 'application/vnd.cnr.blob.v0.tar+gzip', + ['local_us'], models_ref) + + release.create_app_release(repo, 'test', + dict(mediaType='application/vnd.cnr.package-manifest.helm.v0.json'), + 'sha256:somedigest', models_ref, False) + + channel.create_or_update_channel(repo, 'somechannel', 'test', models_ref) + + params = {'repository': 'devtable/someappr'} + response = conduct_api_call(cl, Repository, 'GET', params).json + assert response['kind'] == 'application' + assert response['channels'] + assert response['releases'] diff --git a/endpoints/api/test/test_search.py b/endpoints/api/test/test_search.py index b19efa8f3..5e034934c 100644 --- a/endpoints/api/test/test_search.py +++ b/endpoints/api/test/test_search.py @@ -2,6 +2,7 @@ import pytest from playhouse.test_utils import assert_query_count +from data import model, database from endpoints.api.search import ConductRepositorySearch, ConductSearch from endpoints.api.test.shared import conduct_api_call from endpoints.test.shared import client_with_identity @@ -14,6 +15,10 @@ from test.fixtures import * ('repository'), ]) def test_repository_search(query, client): + # Prime the caches. + database.Repository.kind.get_id('image') + database.Repository.kind.get_name(1) + with client_with_identity('devtable', client) as cl: params = {'query': query} with assert_query_count(7): diff --git a/endpoints/appr/models_cnr.py b/endpoints/appr/models_cnr.py index d473e911c..046e89245 100644 --- a/endpoints/appr/models_cnr.py +++ b/endpoints/appr/models_cnr.py @@ -2,13 +2,15 @@ from datetime import datetime import cnr.semver -from cnr.exception import raise_package_not_found, raise_channel_not_found +from cnr.exception import raise_package_not_found, raise_channel_not_found, CnrException +import features import data.model from app import storage, authentication from data import appr_model -from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel +from data.database import Repository, MediaType, db_transaction +from data.appr_model.models import OLD_MODELS, NEW_MODELS from endpoints.appr.models_interface import ( ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface, BlobDescriptor, ChannelView, ChannelReleasesView) @@ -17,6 +19,12 @@ from util.morecollections import AttrDict from util.names import parse_robot_username + +class ReadOnlyException(CnrException): + status_code = 405 + errorcode = "read-only" + + def _strip_sha256_header(digest): if digest.startswith('sha256:'): return digest.split('sha256:')[1] @@ -48,6 +56,10 @@ def _application(package): class CNRAppModel(AppRegistryDataInterface): + def __init__(self, models_ref, is_readonly): + self.models_ref = models_ref + self.is_readonly = is_readonly + def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None, analytics_sample=1, metadata=None): metadata = {} if metadata is None else metadata @@ -70,9 +82,10 @@ class CNRAppModel(AppRegistryDataInterface): """ views = [] - for repo in appr_model.package.list_packages_query(namespace, media_type, search, - username=username): - releases = [t.name for t in repo.tag_set_prefetch] + for repo in appr_model.package.list_packages_query(self.models_ref, namespace, media_type, + search, username=username): + tag_set_prefetch = getattr(repo, self.models_ref.tag_set_prefetch_name) + releases = [t.name for t in tag_set_prefetch] if not releases: continue available_releases = [ @@ -81,7 +94,7 @@ class CNRAppModel(AppRegistryDataInterface): if with_channels: channels = [ ChannelView(name=chan.name, current=chan.linked_tag.name) - for chan in appr_model.channel.get_repo_channels(repo)] + for chan in appr_model.channel.get_repo_channels(repo, self.models_ref)] app_name = _join_package_name(repo.namespace_user.username, repo.name) manifests = self.list_manifests(app_name, available_releases[0]) @@ -93,8 +106,8 @@ class CNRAppModel(AppRegistryDataInterface): channels=channels, manifests=manifests, releases=available_releases, - updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start), - created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start),) + updated_at=_timestamp_to_iso(tag_set_prefetch[-1].lifetime_start), + created_at=_timestamp_to_iso(tag_set_prefetch[0].lifetime_start),) views.append(view) return views @@ -108,6 +121,9 @@ class CNRAppModel(AppRegistryDataInterface): def create_application(self, package_name, visibility, owner): """ Create a new app repository, owner is the user who creates it """ + if self.is_readonly: + raise ReadOnlyException('Currently in read-only mode') + ns, name = _split_package_name(package_name) data.model.repository.create_repository(ns, name, owner, visibility, 'application') @@ -137,7 +153,7 @@ class CNRAppModel(AppRegistryDataInterface): Todo: * Paginate """ - return appr_model.release.get_releases(_application(package_name), media_type) + return appr_model.release.get_releases(_application(package_name), self.models_ref, media_type) def list_manifests(self, package_name, release=None): """ Returns the list of all manifests of an Application. @@ -147,8 +163,8 @@ class CNRAppModel(AppRegistryDataInterface): """ try: repo = _application(package_name) - return list(appr_model.manifest.get_manifest_types(repo, release)) - except (Repository.DoesNotExist, Tag.DoesNotExist): + return list(appr_model.manifest.get_manifest_types(repo, self.models_ref, release)) + except (Repository.DoesNotExist, self.models_ref.Tag.DoesNotExist): raise_package_not_found(package_name, release) def fetch_release(self, package_name, release, media_type): @@ -157,7 +173,8 @@ class CNRAppModel(AppRegistryDataInterface): """ repo = _application(package_name) try: - tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type) + tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type, + self.models_ref) created_at = _timestamp_to_iso(tag.lifetime_start) blob_descriptor = BlobDescriptor(digest=_strip_sha256_header(blob.digest), @@ -169,17 +186,23 @@ class CNRAppModel(AppRegistryDataInterface): app_release = ApplicationRelease(release=tag.name, created_at=created_at, name=package_name, manifest=app_manifest) return app_release - except (Tag.DoesNotExist, Manifest.DoesNotExist, Blob.DoesNotExist, Repository.DoesNotExist, + except (self.models_ref.Tag.DoesNotExist, + self.models_ref.Manifest.DoesNotExist, + self.models_ref.Blob.DoesNotExist, + Repository.DoesNotExist, MediaType.DoesNotExist): raise_package_not_found(package_name, release, media_type) def store_blob(self, cnrblob, content_media_type): + if self.is_readonly: + raise ReadOnlyException('Currently in read-only mode') + fp = cnrblob.packager.io_file path = cnrblob.upload_url(cnrblob.digest) locations = storage.preferred_locations storage.stream_write(locations, path, fp, 'application/x-gzip') db_blob = appr_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type, - locations) + locations, self.models_ref) return BlobDescriptor(mediaType=content_media_type, digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[]) @@ -187,49 +210,60 @@ class CNRAppModel(AppRegistryDataInterface): """ Add an app-release to a repository package is an instance of data.cnr.package.Package """ + if self.is_readonly: + raise ReadOnlyException('Currently in read-only mode') manifest = package.manifest() ns, name = package.namespace, package.name repo = data.model.repository.get_or_create_repository(ns, name, user, visibility=visibility, - repo_kind='application') + repo_kind='application') tag_name = package.release - appr_model.release.create_app_release(repo, tag_name, - package.manifest(), manifest['content']['digest'], force) + appr_model.release.create_app_release(repo, tag_name, package.manifest(), + manifest['content']['digest'], self.models_ref, force) def delete_release(self, package_name, release, media_type): """ Remove/Delete an app-release from an app-repository. It does not delete the entire app-repository, only a single release """ + if self.is_readonly: + raise ReadOnlyException('Currently in read-only mode') + repo = _application(package_name) try: - appr_model.release.delete_app_release(repo, release, media_type) - except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist): + appr_model.release.delete_app_release(repo, release, media_type, self.models_ref) + except (self.models_ref.Channel.DoesNotExist, + self.models_ref.Tag.DoesNotExist, + MediaType.DoesNotExist): raise_package_not_found(package_name, release, media_type) def release_exists(self, package, release): """ Return true if a release with that name already exist or have existed (include deleted ones) """ + # TODO: Figure out why this isn't implemented. def channel_exists(self, package_name, channel_name): """ Returns true if channel exists """ repo = _application(package_name) - return appr_model.tag.tag_exists(repo, channel_name, "channel") + return appr_model.tag.tag_exists(repo, channel_name, self.models_ref, "channel") def delete_channel(self, package_name, channel_name): """ Delete an AppChannel Note: It doesn't delete the AppReleases """ + if self.is_readonly: + raise ReadOnlyException('Currently in read-only mode') + repo = _application(package_name) try: - appr_model.channel.delete_channel(repo, channel_name) - except (Channel.DoesNotExist, Tag.DoesNotExist): + appr_model.channel.delete_channel(repo, channel_name, self.models_ref) + except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist): raise_channel_not_found(package_name, channel_name) def list_channels(self, package_name): """ Returns all AppChannel for a package """ repo = _application(package_name) - channels = appr_model.channel.get_repo_channels(repo) + channels = appr_model.channel.get_repo_channels(repo, self.models_ref) return [ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in channels] def fetch_channel(self, package_name, channel_name, with_releases=True): @@ -237,12 +271,12 @@ class CNRAppModel(AppRegistryDataInterface): repo = _application(package_name) try: - channel = appr_model.channel.get_channel(repo, channel_name) - except (Channel.DoesNotExist, Tag.DoesNotExist): + channel = appr_model.channel.get_channel(repo, channel_name, self.models_ref) + except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist): raise_channel_not_found(package_name, channel_name) if with_releases: - releases = appr_model.channel.get_channel_releases(repo, channel) + releases = appr_model.channel.get_channel_releases(repo, channel, self.models_ref) chanview = ChannelReleasesView( current=channel.linked_tag.name, name=channel.name, releases=[channel.linked_tag.name] + [c.name for c in releases]) @@ -254,9 +288,9 @@ class CNRAppModel(AppRegistryDataInterface): def list_release_channels(self, package_name, release, active=True): repo = _application(package_name) try: - channels = appr_model.channel.get_tag_channels(repo, release, active=active) + channels = appr_model.channel.get_tag_channels(repo, release, self.models_ref, active=active) return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels] - except (Channel.DoesNotExist, Tag.DoesNotExist): + except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist): raise_package_not_found(package_name, release) def update_channel(self, package_name, channel_name, release): @@ -264,12 +298,17 @@ class CNRAppModel(AppRegistryDataInterface): Returns: A new AppChannel with the release """ + if self.is_readonly: + raise ReadOnlyException('Currently in read-only mode') + repo = _application(package_name) - channel = appr_model.channel.create_or_update_channel(repo, channel_name, release) + channel = appr_model.channel.create_or_update_channel(repo, channel_name, release, + self.models_ref) return ChannelView(current=channel.linked_tag.name, name=channel.name) def get_blob_locations(self, digest): - return appr_model.blob.get_blob_locations(digest) + return appr_model.blob.get_blob_locations(digest, self.models_ref) -model = CNRAppModel() +# Phase 1: Read from old tables, disallow writing. +model = CNRAppModel(OLD_MODELS, features.READONLY_APP_REGISTRY) diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py index ae40ea0aa..99af88c2c 100644 --- a/endpoints/appr/test/test_api.py +++ b/endpoints/appr/test/test_api.py @@ -6,13 +6,13 @@ from cnr.tests.conftest import * from cnr.tests.test_apiserver import BaseTestServer from cnr.tests.test_models import CnrTestModels -import data.appr_model.blob as oci_blob +import data.appr_model.blob as appr_blob from data.database import User from data.model import organization, user from endpoints.appr import registry # Needed to register the endpoint from endpoints.appr.cnr_backend import Channel, Package, QuayDB -from endpoints.appr.models_cnr import model as oci_app_model +from endpoints.appr.models_cnr import model as appr_app_model from test.fixtures import * @@ -28,7 +28,7 @@ class ChannelTest(Channel): @classmethod def dump_all(cls, package_class=None): result = [] - for repo in oci_app_model.list_applications(with_channels=True): + for repo in appr_app_model.list_applications(with_channels=True): for chan in repo.channels: result.append({'name': chan.name, 'current': chan.current, 'package': repo.name}) return result @@ -51,17 +51,17 @@ class PackageTest(Package): @classmethod def dump_all(cls, blob_cls): result = [] - for repo in oci_app_model.list_applications(with_channels=True): + for repo in appr_app_model.list_applications(with_channels=True): package_name = repo.name for release in repo.releases: for mtype in cls.manifests(package_name, release): - package = oci_app_model.fetch_release(package_name, release, mtype) + package = appr_app_model.fetch_release(package_name, release, mtype) blob = blob_cls.get(package_name, package.manifest.content.digest) app_data = cls._apptuple_to_dict(package) app_data.pop('digest') app_data['channels'] = [ x.name - for x in oci_app_model.list_release_channels(package_name, package.release, False) + for x in appr_app_model.list_release_channels(package_name, package.release, False) ] app_data['blob'] = blob.b64blob result.append(app_data) @@ -141,11 +141,11 @@ class TestQuayModels(CnrTestModels): assert p.release == "2.0.1" assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80" blob = db_with_data1.Blob.get("titi/rocketchat", p.digest) - bdb = oci_blob.get_blob(p.digest) + bdb = appr_blob.get_blob(p.digest, appr_app_model.models_ref) newblob = db_with_data1.Blob("titi/app2", blob.b64blob) p2 = db_with_data1.Package("titi/app2", "1.0.0", "helm", newblob) p2.save() - b2db = oci_blob.get_blob(p2.digest) + b2db = appr_blob.get_blob(p2.digest, appr_app_model.models_ref) assert b2db.id == bdb.id def test_force_push_different_blob(self, db_with_data1): diff --git a/endpoints/appr/test/test_registry.py b/endpoints/appr/test/test_registry.py index 6d9c92361..7891d718e 100644 --- a/endpoints/appr/test/test_registry.py +++ b/endpoints/appr/test/test_registry.py @@ -1,14 +1,18 @@ import base64 import json -from flask import url_for +from mock import patch + import pytest +from flask import url_for + from data import model from endpoints.appr.registry import appr_bp from test.fixtures import * + @pytest.mark.parametrize('login_data, expected_code', [ ({ "username": "devtable", @@ -62,3 +66,27 @@ def test_invalid_release_name(release_name, app, client): rv = client.open(url, method='POST', data=json.dumps(data), headers=headers) assert rv.status_code == 422 + + +@pytest.mark.parametrize('readonly, expected_status', [ + (True, 405), + (False, 422), +]) +def test_readonly(readonly, expected_status, app, client): + params = { + 'namespace': 'devtable', + 'package_name': 'someapprepo', + } + + url = url_for('appr.push', **params) + auth = base64.b64encode('devtable:password') + headers = {'Content-Type': 'application/json', 'Authorization': 'Basic ' + auth} + data = { + 'release': '1.0', + 'media_type': 'application/vnd.cnr.manifest.v0+json', + 'blob': 'H4sIAFQwWVoAA+3PMQrCQBAF0Bxlb+Bk143nETGIIEoSC29vMMFOu3TvNb/5DH/Ot8f02jWbiohDremT3ZKR90uuUlty7nKJNmqKtkQuTarbzlo8x+k4zFOu4+lyH4afvbnW93/urH98EwAAAAAAAAAAADb0BsdwExIAKAAA', + } + + with patch('endpoints.appr.models_cnr.model.is_readonly', readonly): + rv = client.open(url, method='POST', data=json.dumps(data), headers=headers) + assert rv.status_code == expected_status diff --git a/initdb.py b/initdb.py index f368684f8..cf30aadc7 100644 --- a/initdb.py +++ b/initdb.py @@ -21,7 +21,7 @@ from data.database import (db, all_models, cnr_classes, Role, TeamRole, Visibili QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode, ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind, RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason, - DeletedNamespace) + DeletedNamespace, appr_classes, ApprTagKind, ApprBlobPlacementLocation) from data import model from data.queue import WorkQueue from app import app, storage as store, tf @@ -364,6 +364,9 @@ def initialize_database(): BlobPlacementLocation.create(name='local_eu') BlobPlacementLocation.create(name='local_us') + ApprBlobPlacementLocation.create(name='local_eu') + ApprBlobPlacementLocation.create(name='local_us') + ImageStorageTransformation.create(name='squash') ImageStorageTransformation.create(name='aci') @@ -435,6 +438,10 @@ def initialize_database(): TagKind.create(name='release') TagKind.create(name='channel') + ApprTagKind.create(name='tag') + ApprTagKind.create(name='release') + ApprTagKind.create(name='channel') + DisableReason.create(name='user_toggled') DisableReason.create(name='successive_build_failures') DisableReason.create(name='successive_build_internal_errors') @@ -906,7 +913,7 @@ def find_models_missing_data(): # whitelisted. models_missing_data = set() for one_model in all_models: - if one_model in cnr_classes: + if one_model in cnr_classes or one_model in appr_classes: continue try: diff --git a/test/test_api_usage.py b/test/test_api_usage.py index 6ccd868ac..88d234057 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -27,6 +27,7 @@ from app import app, config_provider, all_queues, dockerfile_build_queue, notifi from buildtrigger.basehandler import BuildTriggerHandler from initdb import setup_database_for_testing, finished_database_for_testing from data import database, model, appr_model +from data.appr_model.models import OLD_MODELS from data.database import RepositoryActionCount, Repository as RepositoryTable from test.helpers import assert_action_logged from util.secscan.fake import fake_security_scanner @@ -2154,10 +2155,10 @@ class TestDeleteRepository(ApiTestCase): repository = model.repository.get_repository(ADMIN_ACCESS_USER, 'complex') # Add some CNR tags and linked tags. - base_tag = appr_model.tag.create_or_update_tag(repository, 'somebasetag') - base_tag2 = appr_model.tag.create_or_update_tag(repository, 'somebasetag2') - appr_model.tag.create_or_update_tag(repository, 'somelinkedtag', linked_tag=base_tag) - appr_model.tag.create_or_update_tag(repository, 'somelinkedtag2', linked_tag=base_tag2) + base_tag = appr_model.tag.create_or_update_tag(repository, 'somebasetag', OLD_MODELS) + base_tag2 = appr_model.tag.create_or_update_tag(repository, 'somebasetag2', OLD_MODELS) + appr_model.tag.create_or_update_tag(repository, 'somelinkedtag', OLD_MODELS, linked_tag=base_tag) + appr_model.tag.create_or_update_tag(repository, 'somelinkedtag2', OLD_MODELS, linked_tag=base_tag2) # Create some access tokens. access_token = model.token.create_access_token(repository, 'read') diff --git a/util/config/database.py b/util/config/database.py index 5180f9401..356e12ce3 100644 --- a/util/config/database.py +++ b/util/config/database.py @@ -1,4 +1,6 @@ -from data import model, appr_model +from data import model +from data.appr_model import blob +from data.appr_model.models import OLD_MODELS def sync_database_with_config(config): @@ -7,4 +9,4 @@ def sync_database_with_config(config): location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys() if location_names: model.image.ensure_image_locations(*location_names) - appr_model.blob.ensure_blob_locations(*location_names) + blob.ensure_blob_locations(OLD_MODELS, *location_names) diff --git a/util/config/schema.py b/util/config/schema.py index dd53894f5..0f0fccaf5 100644 --- a/util/config/schema.py +++ b/util/config/schema.py @@ -881,6 +881,13 @@ CONFIG_SCHEMA = { 'x-example': False, }, + # Feature Flag: Read only app registry. + 'FEATURE_READONLY_APP_REGISTRY': { + 'type': 'boolean', + 'description': 'Whether to App repositories are read-only. Defaults to False', + 'x-example': True, + }, + # Feature Flag: Public Reposiotires in _catalog Endpoint. 'FEATURE_PUBLIC_CATALOG': { 'type': 'boolean', diff --git a/util/migrate/table_ops.py b/util/migrate/table_ops.py new file mode 100644 index 000000000..83abd630b --- /dev/null +++ b/util/migrate/table_ops.py @@ -0,0 +1,11 @@ +def copy_table_contents(source_table, destination_table, conn): + if conn.engine.name == 'postgresql': + conn.execute('INSERT INTO "%s" SELECT * FROM "%s"' % (destination_table, source_table)) + result = list(conn.execute('Select Max(id) from "%s"' % destination_table))[0] + new_start_id = result[0] + 1 + conn.execute('ALTER SEQUENCE "%s_id_seq" RESTART WITH %s' % (destination_table, new_start_id)) + else: + conn.execute("INSERT INTO `%s` SELECT * FROM `%s` WHERE 1" % (destination_table, source_table)) + result = list(conn.execute('Select Max(id) from `%s` WHERE 1' % destination_table))[0] + new_start_id = result[0] + 1 + conn.execute("ALTER TABLE `%s` AUTO_INCREMENT = %s" % (destination_table, new_start_id))