data.oci_model: init with app methods

data/oci_model/blob.py

@@ -0,0 +1,54 @@
+from peewee import IntegrityError
+
+from data.model import db_transaction
+from data.database import Blob, BlobPlacementLocation, BlobPlacement
+
+
+def get_blob(digest):
+  """ Find a blob by its digest. """
+  return Blob.select().where(Blob.digest == digest).get()
+
+
+def get_or_create_blob(digest, size, media_type_name, locations):
+  """ Try to find a blob by its digest or create it. """
+  with db_transaction():
+    try:
+      blob = get_blob(digest)
+    except Blob.DoesNotExist:
+      blob = Blob.create(digest=digest,
+                         media_type_id=Blob.media_type.get_id(media_type_name),
+                         size=size)
+    for location_name in locations:
+      location_id = BlobPlacement.location.get_id(location_name)
+      try:
+        BlobPlacement.create(blob=blob, location=location_id)
+      except IntegrityError:
+        pass
+
+    return blob
+
+
+def get_blob_locations(digest):
+  """ Find all locations names for a blob. """
+  return [x.name for x in
+          BlobPlacementLocation
+          .select()
+          .join(BlobPlacement)
+          .join(Blob)
+          .where(Blob.digest == digest)]
+
+
+def ensure_blob_locations(*names):
+  with db_transaction():
+    locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names)
+
+    insert_names = list(names)
+
+    for location in locations:
+      insert_names.remove(location.name)
+
+    if not insert_names:
+      return
+
+    data = [{'name': name} for name in insert_names]
+    BlobPlacementLocation.insert_many(data).execute()

data/oci_model/channel.py

@@ -0,0 +1,56 @@
+from data.oci_model import tag as tag_model
+from data.database import Tag, Channel
+
+
+def get_channel_releases(repo, channel):
+  """ Return all previously linked tags.
+      This works based upon Tag lifetimes.
+  """
+  tag_kind_id = Channel.tag_kind.get_id('channel')
+  channel_name = channel.name
+  return (Tag
+          .select(Tag, Channel)
+          .join(Channel, on=(Tag.id == Channel.linked_tag))
+          .where(Channel.repository == repo,
+                 Channel.name == channel_name,
+                 Channel.tag_kind == tag_kind_id, Channel.lifetime_end != None)
+          .order_by(Tag.lifetime_end))
+
+
+def get_channel(repo, channel_name):
+  """ Find a Channel by name. """
+  channel = tag_model.get_tag(repo, channel_name, "channel")
+  return channel
+
+
+def get_tag_channels(repo, tag_name, active=True):
+  """ Find the Channels associated with a Tag. """
+  tag = tag_model.get_tag(repo, tag_name, "release")
+  query = tag.tag_parents
+
+  if active:
+    query = tag_model.tag_alive_oci(query)
+
+  return query
+
+
+def delete_channel(repo, channel_name):
+  """ Delete a channel by name. """
+  return tag_model.delete_tag(repo, channel_name, "channel")
+
+
+def create_or_update_channel(repo, channel_name, tag_name):
+  """ Creates or updates a channel to include a particular tag. """
+  tag = tag_model.get_tag(repo, tag_name, 'release')
+  return tag.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel")
+
+
+def get_repo_channels(repo):
+  """ Creates or updates a channel to include a particular tag. """
+  tag_kind_id = Channel.tag_kind.get_id('channel')
+  query = (Channel
+           .select(Channel, Tag)
+           .join(Tag, on=(Tag.id == Channel.linked_tag))
+           .where(Channel.repository == repo,
+                  Channel.tag_kind == tag_kind_id))
+  return tag_model.tag_alive_oci(query, cls=Channel)

data/oci_model/manifest.py

@@ -0,0 +1,55 @@
+import logging
+import hashlib
+import json
+
+from cnr.models.package_base import get_media_type
+
+from data import oci_model
+from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
+
+
+logger = logging.getLogger(__name__)
+
+
+def _digest(manifestjson):
+  return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()
+
+
+def get_manifest_query(digest, media_type):
+  return Manifest.select().where(Manifest.digest == digest,
+                                 Manifest.media_type == Manifest.media_type.get_id(media_type))
+
+
+def get_manifest_with_blob(digest, media_type):
+  query = get_manifest_query(digest, media_type)
+  return query.join(Blob).get()
+
+
+def get_or_create_manifest(manifest_json, media_type_name):
+  digest = _digest(manifest_json)
+  try:
+    manifest = get_manifest_query(digest, media_type_name).get()
+  except Manifest.DoesNotExist:
+    with db_transaction():
+      manifest = Manifest.create(digest=digest,
+                                 manifest_json=manifest_json,
+                                 media_type=Manifest.media_type.get_id(media_type_name))
+  return manifest
+
+def get_manifest_types(repo, release=None):
+  """ Returns an array of MediaTypes.name for a repo, can filter by tag """
+  query = oci_model.tag.tag_alive_oci(Tag
+                                      .select(MediaType.name)
+                                      .join(ManifestListManifest,
+                                            on=(ManifestListManifest.manifest_list == Tag.manifest_list))
+                                      .join(MediaType,
+                                            on=(ManifestListManifest.media_type == MediaType.id))
+                                      .where(Tag.repository == repo,
+                                             Tag.tag_kind == Tag.tag_kind.get_id('release')))
+  if release:
+    query = query.where(Tag.name == release)
+
+  manifests = set()
+  for m in query.distinct().tuples():
+    manifests.add(get_media_type(m[0]))
+  return manifests

data/oci_model/manifest_list.py

@@ -0,0 +1,55 @@
+import logging
+import hashlib
+import json
+
+from data.database import ManifestList, ManifestListManifest, db_transaction
+
+
+logger = logging.getLogger(__name__)
+
+
+def _digest(manifestjson):
+  return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()
+
+
+def get_manifest_list(digest):
+  return ManifestList.select().where(ManifestList.digest == digest).get()
+
+
+def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version):
+  digest = _digest(manifest_list_json)
+  media_type_id = ManifestList.media_type.get_id(media_type_name)
+
+  try:
+    return get_manifest_list(digest)
+  except ManifestList.DoesNotExist:
+    with db_transaction():
+      manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json,
+                                         schema_version=schema_version, media_type=media_type_id)
+  return manifestlist
+
+
+def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
+  """ From a manifestlist, manifests, and the manifest list blob,
+  create if doesn't exist the manfiestlistmanifest for each manifest """
+  for pos in xrange(len(manifest_ids)):
+    manifest_id = manifest_ids[pos]
+    manifest_json = manifest_list_json[pos]
+    get_or_create_manifestlistmanifest(manifest=manifest_id,
+                                       manifestlist=manifestlist,
+                                       media_type_name=manifest_json['mediaType'])
+
+
+def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name):
+  media_type_id = ManifestListManifest.media_type.get_id(media_type_name)
+  try:
+    ml = (ManifestListManifest
+          .select()
+          .where(ManifestListManifest.manifest == manifest,
+                 ManifestListManifest.media_type == media_type_id,
+                 ManifestListManifest.manifest_list == manifestlist)).get()
+
+  except ManifestListManifest.DoesNotExist:
+    ml = ManifestListManifest.create(manifest_list=manifestlist, media_type=media_type_id,
+                                     manifest=manifest)
+    return ml

data/oci_model/package.py

@@ -0,0 +1,40 @@
+from cnr.models.package_base import get_media_type, manifest_media_type
+from peewee import prefetch
+
+from data import model, oci_model
+from data.database import Repository, Namespace, Tag, ManifestListManifest
+
+
+def list_packages_query(namespace=None, media_type=None, search_query=None, username=None):
+  """ List and filter repository by search query. """
+  if search_query is not None:
+    repositories = model.repository.get_app_search(search_query,
+                                                   username=username,
+                                                   limit=50)
+    repo_query = (Repository
+                  .select(Repository, Namespace.username)
+                  .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+                  .where(Repository.id << [repo.id for repo in repositories]))
+  else:
+    repo_query = (Repository
+                  .select(Repository, Namespace.username)
+                  .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+                  .where(Repository.visibility == model.repository.get_public_repo_visibility(),
+                         Repository.kind == Repository.kind.get_id('application')))
+
+  if namespace:
+    repo_query = (repo_query
+                  .where(Namespace.username == namespace))
+
+  tag_query = (Tag
+               .select()
+               .where(Tag.tag_kind == Tag.tag_kind.get_id('release'))
+               .order_by(Tag.lifetime_start))
+
+  if media_type:
+    tag_query = oci_model.tag.filter_tags_by_media_type(tag_query, media_type)
+
+  tag_query = oci_model.tag.tag_alive_oci(tag_query)
+  query = prefetch(repo_query, tag_query)
+
+  return query

data/oci_model/release.py

@@ -0,0 +1,116 @@
+import bisect
+
+from cnr.exception import PackageAlreadyExists
+from cnr.models.package_base import manifest_media_type
+
+from data import oci_model
+from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
+                           ManifestListManifest, Blob, ManifestBlob)
+
+
+LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json'
+SCHEMA_VERSION = 'v0'
+
+
+def get_app_release(repo, tag_name, media_type):
+  """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """
+  tag = oci_model.tag.get_tag(repo, tag_name, tag_kind='release')
+  media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
+  manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set
+                          .join(Manifest)
+                          .where(ManifestListManifest.media_type == media_type_id).get())
+  manifest = manifestlistmanifest.manifest
+  blob = Blob.select().join(ManifestBlob).where(ManifestBlob.manifest == manifest).get()
+  return (tag, manifest, blob)
+
+
+def create_app_release(repo, tag_name, manifest, digest):
+  """ Create a new application release, it includes creating a new Tag, ManifestList,
+      ManifestListManifests, Manifest, ManifestBlob.
+
+      To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id.
+      To find the insert point in the ManifestList it uses bisect on the manifest-ids list.
+  """
+  with db_transaction():
+    # Create/get the package manifest
+    manifest = oci_model.manifest.get_or_create_manifest(manifest, manifest['mediaType'])
+    # get the tag
+    tag = oci_model.tag.get_or_initialize_tag(repo, tag_name)
+
+    if tag.manifest_list is None:
+      tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE),
+                                       schema_version=SCHEMA_VERSION,
+                                       manifest_list_json=[])
+
+    elif oci_model.tag.tag_media_type_exists(tag, manifest.media_type):
+      raise PackageAlreadyExists("package exists already")
+
+    list_json = tag.manifest_list.manifest_list_json
+    mlm_query = (ManifestListManifest
+                 .select()
+                 .where(ManifestListManifest.manifest_list == tag.manifest_list))
+    list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
+    insert_point = bisect.bisect_left(list_manifest_ids, manifest.id)
+    list_json.insert(insert_point, manifest.manifest_json)
+    list_manifest_ids.insert(insert_point, manifest.id)
+    manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
+                                                                       SCHEMA_VERSION)
+    oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json)
+
+    tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
+                                             tag_kind="release")
+    blob_digest = digest
+
+    try:
+      (ManifestBlob
+       .select()
+       .join(Blob)
+       .where(ManifestBlob.manifest == manifest, Blob.digest == blob_digest).get())
+    except ManifestBlob.DoesNotExist:
+      blob = oci_model.blob.get_blob(blob_digest)
+      ManifestBlob.create(manifest=manifest, blob=blob)
+    return tag
+
+
+def delete_app_release(repo, tag_name, media_type):
+  """ Delete a Tag/media-type couple """
+  media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
+
+  with db_transaction():
+    tag = oci_model.tag.get_tag(repo, tag_name)
+    manifest_list = tag.manifest_list
+    list_json = manifest_list.manifest_list_json
+    mlm_query = (ManifestListManifest
+                 .select()
+                 .where(ManifestListManifest.manifest_list == tag.manifest_list))
+    list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
+    manifestlistmanifest = (tag
+                            .manifest_list
+                            .manifestlistmanifest_set
+                            .where(ManifestListManifest.media_type == media_type_id).get())
+    index = list_manifest_ids.index(manifestlistmanifest.manifest_id)
+    list_manifest_ids.pop(index)
+    list_json.pop(index)
+
+    if not list_json:
+      tag.lifetime_end = get_epoch_timestamp()
+      tag.save()
+    else:
+      manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
+                                                                         SCHEMA_VERSION)
+      oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids,
+                                                          list_json)
+      tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
+                                               tag_kind="release")
+    return tag
+
+
+def get_releases(repo, media_type=None):
+  """ Returns an array of Tag.name for a repo, can filter by media_type. """
+  release_query = (Tag
+                   .select()
+                   .where(Tag.repository == repo,
+                          Tag.tag_kind == Tag.tag_kind.get_id("release")))
+  if media_type:
+    release_query = oci_model.tag.filter_tags_by_media_type(release_query, media_type)
+  return [t.name for t in oci_model.tag.tag_alive_oci(release_query)]

data/oci_model/tag.py

@@ -0,0 +1,88 @@
+import logging
+
+from peewee import IntegrityError
+
+from data.model import (db_transaction, TagAlreadyCreatedException)
+from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update
+
+
+logger = logging.getLogger(__name__)
+
+
+def tag_alive_oci(query, now_ts=None, cls=Tag):
+  return query.where((cls.lifetime_end >> None) |
+                     (cls.lifetime_end > now_ts))
+
+
+def tag_media_type_exists(tag, media_type):
+  return (tag.manifest_list.manifestlistmanifest_set
+          .where(ManifestListManifest.media_type == media_type).count() > 0)
+
+
+def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"):
+  now_ts = get_epoch_timestamp_ms()
+  tag_kind_id = Tag.tag_kind.get_id(tag_kind)
+  with db_transaction():
+    try:
+      tag = db_for_update(tag_alive_oci(Tag
+                                        .select()
+                                        .where(Tag.repository == repo,
+                                               Tag.name == tag_name,
+                                               Tag.tag_kind == tag_kind_id), now_ts)).get()
+      if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag:
+        return tag
+      tag.lifetime_end = now_ts
+      tag.save()
+    except Tag.DoesNotExist:
+      pass
+
+    try:
+      return Tag.create(repository=repo, manifest_list=manifest_list, linked_tag=linked_tag,
+                        name=tag_name, lifetime_start=now_ts, lifetime_end=None,
+                        tag_kind=tag_kind_id)
+    except IntegrityError:
+      msg = 'Tag with name %s and lifetime start %s under repository %s/%s already exists'
+      raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name))
+
+
+def get_or_initialize_tag(repo, tag_name, tag_kind="release"):
+  try:
+    return Tag.select().where(Tag.repository == repo, Tag.name == tag_name).get()
+  except Tag.DoesNotExist:
+    return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind))
+
+
+def get_tag(repo, tag_name, tag_kind="release"):
+  return tag_alive_oci(Tag.select()
+                       .where(Tag.repository == repo,
+                              Tag.name == tag_name,
+                              Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get()
+
+
+def delete_tag(repo, tag_name, tag_kind="release"):
+  tag_kind_id = Tag.tag_kind.get_id(tag_kind)
+  tag = tag_alive_oci(Tag.select()
+                      .where(Tag.repository == repo,
+                             Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get()
+  tag.lifetime_end = get_epoch_timestamp_ms()
+  tag.save()
+  return tag
+
+
+def tag_exists(repo, tag_name, tag_kind="release"):
+  try:
+    get_tag(repo, tag_name, tag_kind)
+    return True
+  except Tag.DoesNotExist:
+    return False
+
+
+def filter_tags_by_media_type(tag_query, media_type):
+  """ Return only available tag for a media_type. """
+  media_type = manifest_media_type(media_type)
+  t = (tag_query
+       .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list))
+       .where(ManifestListManifest.media_type == ManifestListManifest.media_type.get_id(media_type)))
+  return t
+
+