From 9f684fa73f9ef67f8ea6e979b0cebfe1e597deeb Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:51:28 -0400 Subject: [PATCH] data.oci_model: init with app methods --- data/oci_model/__init__.py | 0 data/oci_model/blob.py | 54 +++++++++++++++ data/oci_model/channel.py | 56 +++++++++++++++ data/oci_model/manifest.py | 55 +++++++++++++++ data/oci_model/manifest_list.py | 55 +++++++++++++++ data/oci_model/package.py | 40 +++++++++++ data/oci_model/release.py | 116 ++++++++++++++++++++++++++++++++ data/oci_model/tag.py | 88 ++++++++++++++++++++++++ 8 files changed, 464 insertions(+) create mode 100644 data/oci_model/__init__.py create mode 100644 data/oci_model/blob.py create mode 100644 data/oci_model/channel.py create mode 100644 data/oci_model/manifest.py create mode 100644 data/oci_model/manifest_list.py create mode 100644 data/oci_model/package.py create mode 100644 data/oci_model/release.py create mode 100644 data/oci_model/tag.py diff --git a/data/oci_model/__init__.py b/data/oci_model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/data/oci_model/blob.py b/data/oci_model/blob.py new file mode 100644 index 000000000..521f072ba --- /dev/null +++ b/data/oci_model/blob.py @@ -0,0 +1,54 @@ +from peewee import IntegrityError + +from data.model import db_transaction +from data.database import Blob, BlobPlacementLocation, BlobPlacement + + +def get_blob(digest): + """ Find a blob by its digest. """ + return Blob.select().where(Blob.digest == digest).get() + + +def get_or_create_blob(digest, size, media_type_name, locations): + """ Try to find a blob by its digest or create it. """ + with db_transaction(): + try: + blob = get_blob(digest) + except Blob.DoesNotExist: + blob = Blob.create(digest=digest, + media_type_id=Blob.media_type.get_id(media_type_name), + size=size) + for location_name in locations: + location_id = BlobPlacement.location.get_id(location_name) + try: + BlobPlacement.create(blob=blob, location=location_id) + except IntegrityError: + pass + + return blob + + +def get_blob_locations(digest): + """ Find all locations names for a blob. """ + return [x.name for x in + BlobPlacementLocation + .select() + .join(BlobPlacement) + .join(Blob) + .where(Blob.digest == digest)] + + +def ensure_blob_locations(*names): + with db_transaction(): + locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names) + + insert_names = list(names) + + for location in locations: + insert_names.remove(location.name) + + if not insert_names: + return + + data = [{'name': name} for name in insert_names] + BlobPlacementLocation.insert_many(data).execute() diff --git a/data/oci_model/channel.py b/data/oci_model/channel.py new file mode 100644 index 000000000..f8d169661 --- /dev/null +++ b/data/oci_model/channel.py @@ -0,0 +1,56 @@ +from data.oci_model import tag as tag_model +from data.database import Tag, Channel + + +def get_channel_releases(repo, channel): + """ Return all previously linked tags. + This works based upon Tag lifetimes. + """ + tag_kind_id = Channel.tag_kind.get_id('channel') + channel_name = channel.name + return (Tag + .select(Tag, Channel) + .join(Channel, on=(Tag.id == Channel.linked_tag)) + .where(Channel.repository == repo, + Channel.name == channel_name, + Channel.tag_kind == tag_kind_id, Channel.lifetime_end != None) + .order_by(Tag.lifetime_end)) + + +def get_channel(repo, channel_name): + """ Find a Channel by name. """ + channel = tag_model.get_tag(repo, channel_name, "channel") + return channel + + +def get_tag_channels(repo, tag_name, active=True): + """ Find the Channels associated with a Tag. """ + tag = tag_model.get_tag(repo, tag_name, "release") + query = tag.tag_parents + + if active: + query = tag_model.tag_alive_oci(query) + + return query + + +def delete_channel(repo, channel_name): + """ Delete a channel by name. """ + return tag_model.delete_tag(repo, channel_name, "channel") + + +def create_or_update_channel(repo, channel_name, tag_name): + """ Creates or updates a channel to include a particular tag. """ + tag = tag_model.get_tag(repo, tag_name, 'release') + return tag.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel") + + +def get_repo_channels(repo): + """ Creates or updates a channel to include a particular tag. """ + tag_kind_id = Channel.tag_kind.get_id('channel') + query = (Channel + .select(Channel, Tag) + .join(Tag, on=(Tag.id == Channel.linked_tag)) + .where(Channel.repository == repo, + Channel.tag_kind == tag_kind_id)) + return tag_model.tag_alive_oci(query, cls=Channel) diff --git a/data/oci_model/manifest.py b/data/oci_model/manifest.py new file mode 100644 index 000000000..bb690ddc4 --- /dev/null +++ b/data/oci_model/manifest.py @@ -0,0 +1,55 @@ +import logging +import hashlib +import json + +from cnr.models.package_base import get_media_type + +from data import oci_model +from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag + + +logger = logging.getLogger(__name__) + + +def _digest(manifestjson): + return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest() + + +def get_manifest_query(digest, media_type): + return Manifest.select().where(Manifest.digest == digest, + Manifest.media_type == Manifest.media_type.get_id(media_type)) + + +def get_manifest_with_blob(digest, media_type): + query = get_manifest_query(digest, media_type) + return query.join(Blob).get() + + +def get_or_create_manifest(manifest_json, media_type_name): + digest = _digest(manifest_json) + try: + manifest = get_manifest_query(digest, media_type_name).get() + except Manifest.DoesNotExist: + with db_transaction(): + manifest = Manifest.create(digest=digest, + manifest_json=manifest_json, + media_type=Manifest.media_type.get_id(media_type_name)) + return manifest + +def get_manifest_types(repo, release=None): + """ Returns an array of MediaTypes.name for a repo, can filter by tag """ + query = oci_model.tag.tag_alive_oci(Tag + .select(MediaType.name) + .join(ManifestListManifest, + on=(ManifestListManifest.manifest_list == Tag.manifest_list)) + .join(MediaType, + on=(ManifestListManifest.media_type == MediaType.id)) + .where(Tag.repository == repo, + Tag.tag_kind == Tag.tag_kind.get_id('release'))) + if release: + query = query.where(Tag.name == release) + + manifests = set() + for m in query.distinct().tuples(): + manifests.add(get_media_type(m[0])) + return manifests diff --git a/data/oci_model/manifest_list.py b/data/oci_model/manifest_list.py new file mode 100644 index 000000000..68be2855f --- /dev/null +++ b/data/oci_model/manifest_list.py @@ -0,0 +1,55 @@ +import logging +import hashlib +import json + +from data.database import ManifestList, ManifestListManifest, db_transaction + + +logger = logging.getLogger(__name__) + + +def _digest(manifestjson): + return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest() + + +def get_manifest_list(digest): + return ManifestList.select().where(ManifestList.digest == digest).get() + + +def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version): + digest = _digest(manifest_list_json) + media_type_id = ManifestList.media_type.get_id(media_type_name) + + try: + return get_manifest_list(digest) + except ManifestList.DoesNotExist: + with db_transaction(): + manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json, + schema_version=schema_version, media_type=media_type_id) + return manifestlist + + +def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json): + """ From a manifestlist, manifests, and the manifest list blob, + create if doesn't exist the manfiestlistmanifest for each manifest """ + for pos in xrange(len(manifest_ids)): + manifest_id = manifest_ids[pos] + manifest_json = manifest_list_json[pos] + get_or_create_manifestlistmanifest(manifest=manifest_id, + manifestlist=manifestlist, + media_type_name=manifest_json['mediaType']) + + +def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name): + media_type_id = ManifestListManifest.media_type.get_id(media_type_name) + try: + ml = (ManifestListManifest + .select() + .where(ManifestListManifest.manifest == manifest, + ManifestListManifest.media_type == media_type_id, + ManifestListManifest.manifest_list == manifestlist)).get() + + except ManifestListManifest.DoesNotExist: + ml = ManifestListManifest.create(manifest_list=manifestlist, media_type=media_type_id, + manifest=manifest) + return ml diff --git a/data/oci_model/package.py b/data/oci_model/package.py new file mode 100644 index 000000000..8ebecc722 --- /dev/null +++ b/data/oci_model/package.py @@ -0,0 +1,40 @@ +from cnr.models.package_base import get_media_type, manifest_media_type +from peewee import prefetch + +from data import model, oci_model +from data.database import Repository, Namespace, Tag, ManifestListManifest + + +def list_packages_query(namespace=None, media_type=None, search_query=None, username=None): + """ List and filter repository by search query. """ + if search_query is not None: + repositories = model.repository.get_app_search(search_query, + username=username, + limit=50) + repo_query = (Repository + .select(Repository, Namespace.username) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.id << [repo.id for repo in repositories])) + else: + repo_query = (Repository + .select(Repository, Namespace.username) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.visibility == model.repository.get_public_repo_visibility(), + Repository.kind == Repository.kind.get_id('application'))) + + if namespace: + repo_query = (repo_query + .where(Namespace.username == namespace)) + + tag_query = (Tag + .select() + .where(Tag.tag_kind == Tag.tag_kind.get_id('release')) + .order_by(Tag.lifetime_start)) + + if media_type: + tag_query = oci_model.tag.filter_tags_by_media_type(tag_query, media_type) + + tag_query = oci_model.tag.tag_alive_oci(tag_query) + query = prefetch(repo_query, tag_query) + + return query diff --git a/data/oci_model/release.py b/data/oci_model/release.py new file mode 100644 index 000000000..6624900ab --- /dev/null +++ b/data/oci_model/release.py @@ -0,0 +1,116 @@ +import bisect + +from cnr.exception import PackageAlreadyExists +from cnr.models.package_base import manifest_media_type + +from data import oci_model +from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag, + ManifestListManifest, Blob, ManifestBlob) + + +LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json' +SCHEMA_VERSION = 'v0' + + +def get_app_release(repo, tag_name, media_type): + """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """ + tag = oci_model.tag.get_tag(repo, tag_name, tag_kind='release') + media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) + manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set + .join(Manifest) + .where(ManifestListManifest.media_type == media_type_id).get()) + manifest = manifestlistmanifest.manifest + blob = Blob.select().join(ManifestBlob).where(ManifestBlob.manifest == manifest).get() + return (tag, manifest, blob) + + +def create_app_release(repo, tag_name, manifest, digest): + """ Create a new application release, it includes creating a new Tag, ManifestList, + ManifestListManifests, Manifest, ManifestBlob. + + To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id. + To find the insert point in the ManifestList it uses bisect on the manifest-ids list. + """ + with db_transaction(): + # Create/get the package manifest + manifest = oci_model.manifest.get_or_create_manifest(manifest, manifest['mediaType']) + # get the tag + tag = oci_model.tag.get_or_initialize_tag(repo, tag_name) + + if tag.manifest_list is None: + tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE), + schema_version=SCHEMA_VERSION, + manifest_list_json=[]) + + elif oci_model.tag.tag_media_type_exists(tag, manifest.media_type): + raise PackageAlreadyExists("package exists already") + + list_json = tag.manifest_list.manifest_list_json + mlm_query = (ManifestListManifest + .select() + .where(ManifestListManifest.manifest_list == tag.manifest_list)) + list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query]) + insert_point = bisect.bisect_left(list_manifest_ids, manifest.id) + list_json.insert(insert_point, manifest.manifest_json) + list_manifest_ids.insert(insert_point, manifest.id) + manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, + SCHEMA_VERSION) + oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) + + tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag_kind="release") + blob_digest = digest + + try: + (ManifestBlob + .select() + .join(Blob) + .where(ManifestBlob.manifest == manifest, Blob.digest == blob_digest).get()) + except ManifestBlob.DoesNotExist: + blob = oci_model.blob.get_blob(blob_digest) + ManifestBlob.create(manifest=manifest, blob=blob) + return tag + + +def delete_app_release(repo, tag_name, media_type): + """ Delete a Tag/media-type couple """ + media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) + + with db_transaction(): + tag = oci_model.tag.get_tag(repo, tag_name) + manifest_list = tag.manifest_list + list_json = manifest_list.manifest_list_json + mlm_query = (ManifestListManifest + .select() + .where(ManifestListManifest.manifest_list == tag.manifest_list)) + list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query]) + manifestlistmanifest = (tag + .manifest_list + .manifestlistmanifest_set + .where(ManifestListManifest.media_type == media_type_id).get()) + index = list_manifest_ids.index(manifestlistmanifest.manifest_id) + list_manifest_ids.pop(index) + list_json.pop(index) + + if not list_json: + tag.lifetime_end = get_epoch_timestamp() + tag.save() + else: + manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, + SCHEMA_VERSION) + oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, + list_json) + tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag_kind="release") + return tag + + +def get_releases(repo, media_type=None): + """ Returns an array of Tag.name for a repo, can filter by media_type. """ + release_query = (Tag + .select() + .where(Tag.repository == repo, + Tag.tag_kind == Tag.tag_kind.get_id("release"))) + if media_type: + release_query = oci_model.tag.filter_tags_by_media_type(release_query, media_type) + return [t.name for t in oci_model.tag.tag_alive_oci(release_query)] diff --git a/data/oci_model/tag.py b/data/oci_model/tag.py new file mode 100644 index 000000000..16400ea25 --- /dev/null +++ b/data/oci_model/tag.py @@ -0,0 +1,88 @@ +import logging + +from peewee import IntegrityError + +from data.model import (db_transaction, TagAlreadyCreatedException) +from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update + + +logger = logging.getLogger(__name__) + + +def tag_alive_oci(query, now_ts=None, cls=Tag): + return query.where((cls.lifetime_end >> None) | + (cls.lifetime_end > now_ts)) + + +def tag_media_type_exists(tag, media_type): + return (tag.manifest_list.manifestlistmanifest_set + .where(ManifestListManifest.media_type == media_type).count() > 0) + + +def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"): + now_ts = get_epoch_timestamp_ms() + tag_kind_id = Tag.tag_kind.get_id(tag_kind) + with db_transaction(): + try: + tag = db_for_update(tag_alive_oci(Tag + .select() + .where(Tag.repository == repo, + Tag.name == tag_name, + Tag.tag_kind == tag_kind_id), now_ts)).get() + if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag: + return tag + tag.lifetime_end = now_ts + tag.save() + except Tag.DoesNotExist: + pass + + try: + return Tag.create(repository=repo, manifest_list=manifest_list, linked_tag=linked_tag, + name=tag_name, lifetime_start=now_ts, lifetime_end=None, + tag_kind=tag_kind_id) + except IntegrityError: + msg = 'Tag with name %s and lifetime start %s under repository %s/%s already exists' + raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name)) + + +def get_or_initialize_tag(repo, tag_name, tag_kind="release"): + try: + return Tag.select().where(Tag.repository == repo, Tag.name == tag_name).get() + except Tag.DoesNotExist: + return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind)) + + +def get_tag(repo, tag_name, tag_kind="release"): + return tag_alive_oci(Tag.select() + .where(Tag.repository == repo, + Tag.name == tag_name, + Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get() + + +def delete_tag(repo, tag_name, tag_kind="release"): + tag_kind_id = Tag.tag_kind.get_id(tag_kind) + tag = tag_alive_oci(Tag.select() + .where(Tag.repository == repo, + Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get() + tag.lifetime_end = get_epoch_timestamp_ms() + tag.save() + return tag + + +def tag_exists(repo, tag_name, tag_kind="release"): + try: + get_tag(repo, tag_name, tag_kind) + return True + except Tag.DoesNotExist: + return False + + +def filter_tags_by_media_type(tag_query, media_type): + """ Return only available tag for a media_type. """ + media_type = manifest_media_type(media_type) + t = (tag_query + .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list)) + .where(ManifestListManifest.media_type == ManifestListManifest.media_type.get_id(media_type))) + return t + +