Merge pull request #2455 from jzelinskie/cnr-step3

CNR Step 3

config.py

@@ -231,6 +231,9 @@ class DefaultConfig(object):
   # Feature Flag: Whether to support signing
   FEATURE_SIGNING = False
 
+  # Feature Flag: Whether to enable support for App repositories.
+  FEATURE_APP_REGISTRY = False
+
   # The namespace to use for library repositories.
   # Note: This must remain 'library' until Docker removes their hard-coded namespace for libraries.
   # See: https://github.com/docker/docker/blob/master/registry/session.go#L320

data/interfaces/appr.py

@@ -0,0 +1,428 @@
+from abc import ABCMeta, abstractmethod
+from collections import namedtuple
+from datetime import datetime
+
+import cnr.semver
+
+from cnr.exception import raise_package_not_found, raise_channel_not_found
+from six import add_metaclass
+
+
+from app import storage
+from data import model, oci_model
+from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel
+
+
+class BlobDescriptor(namedtuple('Blob', ['mediaType', 'size', 'digest', 'urls'])):
+  """ BlobDescriptor describes a blob with its mediatype, size and digest.
+      A BlobDescriptor is used to retrieves the actual blob.
+  """
+
+
+class ChannelReleasesView(namedtuple('ChannelReleasesView', ['name', 'current', 'releases'])):
+  """ A channel is a pointer to a Release (current).
+      Releases are the previous tags pointed by channel (history).
+  """
+
+
+class ChannelView(namedtuple('ChannelView', ['name', 'current'])):
+  """ A channel is a pointer to a Release (current).
+  """
+
+
+class ApplicationSummaryView(namedtuple('ApplicationSummaryView', ['name',
+                                                                   'namespace',
+                                                                   'visibility',
+                                                                   'default',
+                                                                   'manifests',
+                                                                   'channels',
+                                                                   'releases',
+                                                                   'updated_at',
+                                                                   'created_at'])):
+  """ ApplicationSummaryView is an aggregated view of an application repository.
+  """
+
+
+class ApplicationManifest(namedtuple('ApplicationManifest', ['mediaType',
+                                                             'digest',
+                                                             'content'])):
+  """ ApplicationManifest embed the BlobDescriptor and some metadata around it.
+      An ApplicationManifest is content-addressable.
+  """
+
+
+class ApplicationRelease(namedtuple('ApplicationRelease', ['release',
+                                                           'name',
+                                                           'created_at',
+                                                           'manifest'])):
+  """ The ApplicationRelease associates an ApplicationManifest to a repository and release.
+  """
+
+
+@add_metaclass(ABCMeta)
+class AppRegistryDataInterface(object):
+  """ Interface that represents all data store interactions required by a App Registry.
+  """
+
+  @abstractmethod
+  def _application(self, package_name):
+    pass
+
+  @abstractmethod
+  def list_applications(self, namespace=None, media_type=None, search=None, username=None,
+                        with_channels=False):
+    """ Lists all repositories that contain applications, with optional filtering to a specific
+        namespace and/or to those visible to a specific user.
+
+       Returns: list of ApplicationSummaryView
+    """
+
+  @abstractmethod
+  def application_is_public(self, package_name):
+    """
+      Returns true if the application is public
+    """
+
+  @abstractmethod
+  def create_application(self, package_name, visibility, owner):
+    """ Create a new app repository, owner is the user who creates it """
+
+  @abstractmethod
+  def application_exists(self, package_name):
+    """ Returns true if the application exists """
+
+  @abstractmethod
+  def basic_search(self, query, username=None):
+    """ Returns an array of matching application in the format: 'namespace/name'
+    Note:
+      * Only 'public' repositories are returned
+    """
+
+  # @TODO: Paginate
+  @abstractmethod
+  def list_releases(self, package_name, media_type=None):
+    """ Returns the list of all releases(names) of an AppRepository
+    Example:
+        >>> get_app_releases('ant31/rocketchat')
+        ['1.7.1', '1.7.0', '1.7.2']
+    """
+
+  # @TODO: Paginate
+  @abstractmethod
+  def list_manifests(self, package_name, release=None):
+    """ Returns the list of all available manifests type of an Application across all releases or
+    for a specific one.
+
+    Example:
+        >>> get_app_releases('ant31/rocketchat')
+        ['1.7.1', '1.7.0', '1.7.2']
+    """
+
+  @abstractmethod
+  def fetch_release(self, package_name, release, media_type):
+    """
+     Returns an ApplicationRelease
+    """
+
+  @abstractmethod
+  def store_blob(self, cnrblob, content_media_type):
+    """
+    Upload the blob content to a storage location and creates a Blob entry in the DB.
+
+    Returns a BlobDescriptor
+    """
+
+  @abstractmethod
+  def create_release(self, package, user, visibility, force=False):
+    """ Creates and returns an ApplicationRelease
+    - package is a data.model.Package object
+    - user is the owner of the package
+    - visibility is a string: 'public' or 'private'
+    """
+
+  @abstractmethod
+  def release_exists(self, package, release):
+    """ Return true if a release with that name already exist or
+    has existed (including deleted ones)
+    """
+    pass
+
+  @abstractmethod
+  def delete_release(self, package_name, release, media_type):
+    """ Remove/Delete an app-release from an app-repository.
+        It does not delete the entire app-repository, only a single release
+    """
+
+  @abstractmethod
+  def list_release_channels(self, package_name, release, active=True):
+    """ Returns a list of Channel that are/was pointing to a release.
+        If active is True, returns only active Channel (lifetime_end not null)
+    """
+
+  @abstractmethod
+  def channel_exists(self, package_name, channel_name):
+    """ Returns true if the channel with the given name exists under the matching package """
+
+  @abstractmethod
+  def update_channel(self, package_name, channel_name, release):
+    """ Append a new release to the Channel
+    Returns a new Channel with the release as current
+    """
+
+  @abstractmethod
+  def delete_channel(self, package_name, channel_name):
+    """ Delete a Channel, it doesn't delete/touch the ApplicationRelease pointed by the channel """
+
+  # @TODO: Paginate
+  @abstractmethod
+  def list_channels(self, package_name):
+    """ Returns all AppChannel for a package """
+
+  @abstractmethod
+  def fetch_channel(self, package_name, channel_name, with_releases=True):
+    """ Returns an Channel
+    Raises: ChannelNotFound, PackageNotFound
+    """
+
+
+def _split_package_name(package):
+  """ Returns the namespace and package-name """
+  return package.split("/")
+
+
+def _join_package_name(ns, name):
+  """ Returns a app-name in the 'namespace/name' format """
+  return "%s/%s" % (ns, name)
+
+
+def _timestamp_to_iso(timestamp, in_ms=True):
+  if in_ms:
+    timestamp = timestamp / 1000
+  return datetime.fromtimestamp(timestamp).isoformat()
+
+
+class OCIAppModel(AppRegistryDataInterface):
+  def _application(self, package):
+    ns, name = _split_package_name(package)
+    repo = model.repository.get_app_repository(ns, name)
+    if repo is None:
+      raise_package_not_found(package)
+    return repo
+
+  def list_applications(self, namespace=None, media_type=None, search=None, username=None,
+                        with_channels=False):
+    """ Lists all repositories that contain applications, with optional filtering to a specific
+        namespace and view a specific user.
+    """
+
+    views = []
+    for repo in oci_model.package.list_packages_query(namespace, media_type, search,
+                                                      username=username):
+      releases = [t.name for t in repo.tag_set_prefetch]
+      if not releases:
+        continue
+      available_releases = [str(x) for x in sorted(cnr.semver.versions(releases, False),
+                                                   reverse=True)]
+      channels = None
+      if with_channels:
+        channels = [ChannelView(name=chan.name, current=chan.linked_tag.name)
+                    for chan in oci_model.channel.get_repo_channels(repo)]
+
+      app_name = _join_package_name(repo.namespace_user.username, repo.name)
+      manifests = self.list_manifests(app_name, available_releases[0])
+      view = ApplicationSummaryView(
+        namespace=repo.namespace_user.username,
+        name=app_name,
+        visibility=repo.visibility.name,
+        default=available_releases[0],
+        channels=channels,
+        manifests=manifests,
+        releases=available_releases,
+        updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start),
+        created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start),
+      )
+      views.append(view)
+    return views
+
+  def application_is_public(self, package_name):
+    """
+      Returns:
+        * True if the repository is public
+    """
+    namespace, name = _split_package_name(package_name)
+    return model.repository.repository_is_public(namespace, name)
+
+  def create_application(self, package_name, visibility, owner):
+    """ Create a new app repository, owner is the user who creates it """
+    ns, name = _split_package_name(package_name)
+    model.repository.create_repository(ns, name, owner, visibility, "application")
+
+  def application_exists(self, package_name):
+    """ Create a new app repository, owner is the user who creates it """
+    ns, name = _split_package_name(package_name)
+    return model.repository.get_repository(ns, name, kind_filter='application') is not None
+
+  def basic_search(self, query, username=None):
+    """ Returns an array of matching AppRepositories in the format: 'namespace/name'
+        Note:
+          * Only 'public' repositories are returned
+
+        Todo:
+          * Filter results with readeable reposistory for the user (including visibilitys)
+    """
+    return [_join_package_name(r.namespace_user.username, r.name)
+            for r in model.repository.get_app_search(lookup=query, username=username, limit=50)]
+
+
+  def list_releases(self, package_name, media_type=None):
+    """ Return the list of all releases of an Application
+        Example:
+            >>> get_app_releases('ant31/rocketchat')
+            ['1.7.1', '1.7.0', '1.7.2']
+
+        Todo:
+          * Paginate
+    """
+    return oci_model.release.get_releases(self._application(package_name), media_type)
+
+  def list_manifests(self, package_name, release=None):
+    """ Returns the list of all manifests of an Application.
+
+        Todo:
+          * Paginate
+    """
+    try:
+      repo = self._application(package_name)
+      return list(oci_model.manifest.get_manifest_types(repo, release))
+    except (Repository.DoesNotExist, Tag.DoesNotExist):
+      raise_package_not_found(package_name, release)
+
+  def fetch_release(self, package_name, release, media_type):
+    """
+     Retrieves an AppRelease from it's repository-name and release-name
+    """
+    repo = self._application(package_name)
+    try:
+      tag, manifest, blob = oci_model.release.get_app_release(repo, release, media_type)
+      created_at = _timestamp_to_iso(tag.lifetime_start)
+
+      blob_descriptor = BlobDescriptor(digest=blob.digest, mediaType=blob.media_type.name,
+                                       size=blob.size, urls=[])
+
+      app_manifest = ApplicationManifest(digest=manifest.digest, mediaType=manifest.media_type.name,
+                                         content=blob_descriptor)
+
+      app_release = ApplicationRelease(release=tag.name,
+                                       created_at=created_at,
+                                       name=package_name,
+                                       manifest=app_manifest)
+      return app_release
+    except (Tag.DoesNotExist,
+            Manifest.DoesNotExist,
+            Blob.DoesNotExist,
+            Repository.DoesNotExist,
+            MediaType.DoesNotExist):
+      raise_package_not_found(package_name, release, media_type)
+
+  def store_blob(self, cnrblob, content_media_type):
+    fp = cnrblob.packager.io_file
+    path = cnrblob.upload_url(cnrblob.digest)
+    locations = storage.preferred_locations
+    storage.stream_write(locations, path, fp, 'application/x-gzip')
+    db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest,
+                                                cnrblob.size,
+                                                content_media_type,
+                                                locations)
+    return BlobDescriptor(mediaType=content_media_type, digest=db_blob.digest, size=db_blob.size,
+                          urls=[])
+
+  def create_release(self, package, user, visibility, force=False):
+    """ Add an app-release to a repository
+    package is an instance of data.cnr.package.Package
+    """
+
+    data = package.manifest()
+    ns, name = package.namespace, package.name
+    repo = model.repository.get_or_create_repository(ns, name, user, visibility=visibility,
+                                                     repo_kind='application')
+    tag_name = package.release
+    oci_model.release.create_app_release(repo, tag_name, package.manifest(),
+                                         data['content']['digest'])
+
+  def delete_release(self, package_name, release, media_type):
+    """ Remove/Delete an app-release from an app-repository.
+        It does not delete the entire app-repository, only a single release
+    """
+    repo = self._application(package_name)
+    try:
+      oci_model.release.delete_app_release(repo, release, media_type)
+    except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist):
+      raise_package_not_found(package_name, release, media_type)
+
+  def release_exists(self, package, release):
+    """ Return true if a release with that name already exist or
+    have existed (include deleted ones) """
+
+  def channel_exists(self, package_name, channel_name):
+    """ Returns true if channel exists """
+    repo = self._application(package_name)
+    return oci_model.tag.tag_exists(repo, channel_name, "channel")
+
+  def delete_channel(self, package_name, channel_name):
+    """ Delete an AppChannel
+    Note:
+      It doesn't delete the AppReleases
+    """
+    repo = self._application(package_name)
+    try:
+      oci_model.channel.delete_channel(repo, channel_name)
+    except (Channel.DoesNotExist, Tag.DoesNotExist):
+      raise_channel_not_found(package_name, channel_name)
+
+  def list_channels(self, package_name):
+    """ Returns all AppChannel for a package """
+    repo = self._application(package_name)
+    channels = oci_model.channel.get_repo_channels(repo)
+    return [ChannelView(name=chan.name,
+                        current=chan.linked_tag.name) for chan in channels]
+
+  def fetch_channel(self, package_name, channel_name, with_releases=True):
+    """ Returns an AppChannel """
+    repo = self._application(package_name)
+
+    try:
+      channel = oci_model.channel.get_channel(repo, channel_name)
+    except (Channel.DoesNotExist, Tag.DoesNotExist):
+      raise_channel_not_found(package_name, channel_name)
+
+    if with_releases:
+      releases = oci_model.channel.get_channel_releases(repo, channel)
+      chanview = ChannelReleasesView(current=channel.linked_tag.name,
+                                     name=channel.name,
+                                     releases=[channel.linked_tag.name]+[c.name for c in releases])
+    else:
+      chanview = ChannelView(current=channel.linked_tag.name,
+                             name=channel.name)
+
+    return chanview
+
+  def list_release_channels(self, package_name, release, active=True):
+    repo = self._application(package_name)
+    try:
+      channels = oci_model.channel.get_tag_channels(repo, release, active=active)
+      return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels]
+    except (Channel.DoesNotExist, Tag.DoesNotExist):
+      raise_package_not_found(package_name, release)
+
+  def update_channel(self, package_name, channel_name, release):
+    """ Append a new release to the AppChannel
+    Returns:
+      A new AppChannel with the release
+    """
+    repo = self._application(package_name)
+    channel = oci_model.channel.create_or_update_channel(repo, channel_name, release)
+    return  ChannelView(current=channel.linked_tag.name,
+                        name=channel.name)
+
+
+oci_app_model = OCIAppModel()

data/model/repository.py

@@ -1,6 +1,7 @@
 import logging
 import random
 
+from enum import Enum
 from datetime import timedelta, datetime
 from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError
 from cachetools import ttl_cache
@@ -17,6 +18,7 @@ from util.itertoolrecipes import take
 
 
 logger = logging.getLogger(__name__)
+SEARCH_FIELDS = Enum("SearchFields", ["name", "description"])
 
 
 def get_repo_kind_name(repo):
@@ -58,6 +60,14 @@ def get_repository(namespace_name, repository_name, kind_filter=None):
     return None
 
 
+def get_or_create_repository(namespace, name, creating_user, visibility='private',
+                             repo_kind='image'):
+  repo = get_repository(namespace, name, repo_kind)
+  if repo is None:
+    repo = create_repository(namespace, name, creating_user, visibility, repo_kind)
+  return repo
+
+
 def purge_repository(namespace_name, repository_name):
   """ Completely delete all traces of the repository. Will return True upon
       complete success, and False upon partial or total failure. Garbage
@@ -339,20 +349,42 @@ def get_visible_repositories(username, namespace=None, repo_kind='image', includ
   return query
 
 
+def get_app_repository(namespace_name, repository_name):
+  """ Find an application repository. """
+  try:
+    return _basequery.get_existing_repository(namespace_name, repository_name,
+                                              kind_filter='application')
+  except Repository.DoesNotExist:
+    return None
+
+
+def get_app_search(lookup, search_fields=None, username=None, limit=50):
+  if search_fields is None:
+    search_fields = set([SEARCH_FIELDS.name.name])
+
+  return get_filtered_matching_repositories(lookup, filter_username=username,
+                                            search_fields=search_fields,
+                                            repo_kind='application', offset=0, limit=limit)
+
+
 def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image',
-                                       offset=0, limit=25):
+                                       offset=0, limit=25, search_fields=None):
   """ Returns an iterator of all repositories matching the given lookup value, with optional
       filtering to a specific user. If the user is unspecified, only public repositories will
       be returned.
   """
+  if search_fields is None:
+    search_fields = set([SEARCH_FIELDS.description.name, SEARCH_FIELDS.name.name])
 
   # Build the unfiltered search query.
   unfiltered_query = _get_sorted_matching_repositories(lookup_value, repo_kind=repo_kind,
+                                                       search_fields=search_fields,
                                                        include_private=filter_username is not None)
 
   # Add a filter to the iterator, if necessary.
   if filter_username is not None:
-    iterator = _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit)
+    iterator = _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit,
+                                                        repo_kind)
   else:
     iterator = unfiltered_query
 
@@ -363,7 +395,7 @@ def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_
   return list(take(limit, iterator))
 
 
-def _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit):
+def _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit, repo_kind):
   encountered = set()
   chunk_count = limit * 2
   unfiltered_page = 0
@@ -393,8 +425,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
              .join(RepositoryPermission)
              .where(Repository.id << list(new_unfiltered_ids)))
 
-    filtered = _basequery.filter_to_repos_for_user(query, filter_username)
-
+    filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind)
     for filtered_repo in filtered:
       yield filtered_repo
 
@@ -405,17 +436,28 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username,
     iteration_count = iteration_count + 1
 
 
-def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_private=False):
+def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_private=False,
+                                      search_fields=None):
   """ Returns a query of repositories matching the given lookup string, with optional inclusion of
       private repositories. Note that this method does *not* filter results based on visibility
       to users.
   """
+
+  if search_fields is None:
+    search_fields = set([SEARCH_FIELDS.description.name, SEARCH_FIELDS.name.name])
+
+  # Always search at least on name (init clause)
+  clause = Repository.name.match(lookup_value)
+
+  if SEARCH_FIELDS.description.name in search_fields:
+    clause = Repository.description.match(lookup_value) | clause
+
   last_week = datetime.now() - timedelta(weeks=1)
 
   query = (Repository
            .select(Repository, Namespace)
            .join(Namespace, on=(Namespace.id == Repository.namespace_user))
-           .where(Repository.name.match(lookup_value) | Repository.description.match(lookup_value),
+           .where(clause,
                   Repository.kind == Repository.kind.get_id(repo_kind))
            .group_by(Repository.id, Namespace.id))
 

data/model/storage.py

@@ -8,7 +8,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
                         DataModelException, _basequery)
 from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
                            ImageStorageTransformation, ImageStorageSignature,
-                           ImageStorageSignatureKind, Repository, Namespace, TorrentInfo,
+                           ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
                            ensure_under_transaction)
 
 
@@ -95,15 +95,24 @@ def garbage_collect_storage(storage_id_whitelist):
 
       unreferenced_checksums = set()
       if content_checksums:
+        # Check the current image storage.
         query = (ImageStorage
                  .select(ImageStorage.content_checksum)
                  .where(ImageStorage.content_checksum << list(content_checksums)))
-        referenced_checksums = set([image_storage.content_checksum for image_storage in query])
-        if referenced_checksums:
-          logger.warning('GC attempted to remove CAS checksums %s, which are still referenced',
-                         referenced_checksums)
+        is_referenced_checksums = set([image_storage.content_checksum for image_storage in query])
+        if is_referenced_checksums:
+          logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
+                         is_referenced_checksums)
 
-        unreferenced_checksums = content_checksums - referenced_checksums
+        # Check the new Blob table as well.
+        query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
+        blob_referenced_checksums = set([blob.digest for blob in query])
+        if blob_referenced_checksums:
+          logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
+                         blob_referenced_checksums)
+
+        unreferenced_checksums = (content_checksums - blob_referenced_checksums -
+                                  is_referenced_checksums)
 
       # Return all placements for all image storages found not at a CAS path or with a content
       # checksum that is referenced.

data/oci_model/__init__.py

@@ -0,0 +1,9 @@
+from data.oci_model import (
+  blob,
+  channel,
+  manifest,
+  manifest_list,
+  package,
+  release,
+  tag,
+)

data/oci_model/blob.py

@@ -0,0 +1,54 @@
+from peewee import IntegrityError
+
+from data.model import db_transaction
+from data.database import Blob, BlobPlacementLocation, BlobPlacement
+
+
+def get_blob(digest):
+  """ Find a blob by its digest. """
+  return Blob.select().where(Blob.digest == digest).get()
+
+
+def get_or_create_blob(digest, size, media_type_name, locations):
+  """ Try to find a blob by its digest or create it. """
+  with db_transaction():
+    try:
+      blob = get_blob(digest)
+    except Blob.DoesNotExist:
+      blob = Blob.create(digest=digest,
+                         media_type_id=Blob.media_type.get_id(media_type_name),
+                         size=size)
+    for location_name in locations:
+      location_id = BlobPlacement.location.get_id(location_name)
+      try:
+        BlobPlacement.create(blob=blob, location=location_id)
+      except IntegrityError:
+        pass
+
+    return blob
+
+
+def get_blob_locations(digest):
+  """ Find all locations names for a blob. """
+  return [x.name for x in
+          BlobPlacementLocation
+          .select()
+          .join(BlobPlacement)
+          .join(Blob)
+          .where(Blob.digest == digest)]
+
+
+def ensure_blob_locations(*names):
+  with db_transaction():
+    locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names)
+
+    insert_names = list(names)
+
+    for location in locations:
+      insert_names.remove(location.name)
+
+    if not insert_names:
+      return
+
+    data = [{'name': name} for name in insert_names]
+    BlobPlacementLocation.insert_many(data).execute()

data/oci_model/channel.py

@@ -0,0 +1,56 @@
+from data.database import Tag, Channel
+from data.oci_model import tag as tag_model
+
+
+def get_channel_releases(repo, channel):
+  """ Return all previously linked tags.
+      This works based upon Tag lifetimes.
+  """
+  tag_kind_id = Channel.tag_kind.get_id('channel')
+  channel_name = channel.name
+  return (Tag
+          .select(Tag, Channel)
+          .join(Channel, on=(Tag.id == Channel.linked_tag))
+          .where(Channel.repository == repo,
+                 Channel.name == channel_name,
+                 Channel.tag_kind == tag_kind_id, Channel.lifetime_end != None)
+          .order_by(Tag.lifetime_end))
+
+
+def get_channel(repo, channel_name):
+  """ Find a Channel by name. """
+  channel = tag_model.get_tag(repo, channel_name, "channel")
+  return channel
+
+
+def get_tag_channels(repo, tag_name, active=True):
+  """ Find the Channels associated with a Tag. """
+  tag = tag_model.get_tag(repo, tag_name, "release")
+  query = tag.tag_parents
+
+  if active:
+    query = tag_model.tag_alive_oci(query)
+
+  return query
+
+
+def delete_channel(repo, channel_name):
+  """ Delete a channel by name. """
+  return tag_model.delete_tag(repo, channel_name, "channel")
+
+
+def create_or_update_channel(repo, channel_name, tag_name):
+  """ Creates or updates a channel to include a particular tag. """
+  tag = tag_model.get_tag(repo, tag_name, 'release')
+  return tag_model.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel")
+
+
+def get_repo_channels(repo):
+  """ Creates or updates a channel to include a particular tag. """
+  tag_kind_id = Channel.tag_kind.get_id('channel')
+  query = (Channel
+           .select(Channel, Tag)
+           .join(Tag, on=(Tag.id == Channel.linked_tag))
+           .where(Channel.repository == repo,
+                  Channel.tag_kind == tag_kind_id))
+  return tag_model.tag_alive_oci(query, cls=Channel)

data/oci_model/manifest.py

@@ -0,0 +1,55 @@
+import logging
+import hashlib
+import json
+
+from cnr.models.package_base import get_media_type
+
+from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
+from data.oci_model import tag as tag_model
+
+
+logger = logging.getLogger(__name__)
+
+
+def _digest(manifestjson):
+  return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()
+
+
+def get_manifest_query(digest, media_type):
+  return Manifest.select().where(Manifest.digest == digest,
+                                 Manifest.media_type == Manifest.media_type.get_id(media_type))
+
+
+def get_manifest_with_blob(digest, media_type):
+  query = get_manifest_query(digest, media_type)
+  return query.join(Blob).get()
+
+
+def get_or_create_manifest(manifest_json, media_type_name):
+  digest = _digest(manifest_json)
+  try:
+    manifest = get_manifest_query(digest, media_type_name).get()
+  except Manifest.DoesNotExist:
+    with db_transaction():
+      manifest = Manifest.create(digest=digest,
+                                 manifest_json=manifest_json,
+                                 media_type=Manifest.media_type.get_id(media_type_name))
+  return manifest
+
+def get_manifest_types(repo, release=None):
+  """ Returns an array of MediaTypes.name for a repo, can filter by tag """
+  query = tag_model.tag_alive_oci(Tag
+                                  .select(MediaType.name)
+                                  .join(ManifestListManifest,
+                                        on=(ManifestListManifest.manifest_list == Tag.manifest_list))
+                                  .join(MediaType,
+                                        on=(ManifestListManifest.media_type == MediaType.id))
+                                  .where(Tag.repository == repo,
+                                         Tag.tag_kind == Tag.tag_kind.get_id('release')))
+  if release:
+    query = query.where(Tag.name == release)
+
+  manifests = set()
+  for m in query.distinct().tuples():
+    manifests.add(get_media_type(m[0]))
+  return manifests

data/oci_model/manifest_list.py

@@ -0,0 +1,55 @@
+import logging
+import hashlib
+import json
+
+from data.database import ManifestList, ManifestListManifest, db_transaction
+
+
+logger = logging.getLogger(__name__)
+
+
+def _digest(manifestjson):
+  return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest()
+
+
+def get_manifest_list(digest):
+  return ManifestList.select().where(ManifestList.digest == digest).get()
+
+
+def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version):
+  digest = _digest(manifest_list_json)
+  media_type_id = ManifestList.media_type.get_id(media_type_name)
+
+  try:
+    return get_manifest_list(digest)
+  except ManifestList.DoesNotExist:
+    with db_transaction():
+      manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json,
+                                         schema_version=schema_version, media_type=media_type_id)
+  return manifestlist
+
+
+def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
+  """ From a manifestlist, manifests, and the manifest list blob,
+  create if doesn't exist the manfiestlistmanifest for each manifest """
+  for pos in xrange(len(manifest_ids)):
+    manifest_id = manifest_ids[pos]
+    manifest_json = manifest_list_json[pos]
+    get_or_create_manifestlistmanifest(manifest=manifest_id,
+                                       manifestlist=manifestlist,
+                                       media_type_name=manifest_json['mediaType'])
+
+
+def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name):
+  media_type_id = ManifestListManifest.media_type.get_id(media_type_name)
+  try:
+    ml = (ManifestListManifest
+          .select()
+          .where(ManifestListManifest.manifest == manifest,
+                 ManifestListManifest.media_type == media_type_id,
+                 ManifestListManifest.manifest_list == manifestlist)).get()
+
+  except ManifestListManifest.DoesNotExist:
+    ml = ManifestListManifest.create(manifest_list=manifestlist, media_type=media_type_id,
+                                     manifest=manifest)
+    return ml

data/oci_model/package.py

@@ -0,0 +1,45 @@
+from cnr.models.package_base import get_media_type, manifest_media_type
+from peewee import prefetch
+
+
+from data import model
+from data.database import Repository, Namespace, Tag, ManifestListManifest
+from data.oci_model import tag as tag_model
+
+
+def list_packages_query(namespace=None, media_type=None, search_query=None, username=None):
+  """ List and filter repository by search query. """
+  fields = [model.repository.SEARCH_FIELDS.name.name]
+
+  if search_query is not None:
+    repositories = model.repository.get_app_search(search_query,
+                                                   username=username,
+                                                   search_fields=fields,
+                                                   limit=50)
+    repo_query = (Repository
+                  .select(Repository, Namespace.username)
+                  .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+                  .where(Repository.id << [repo.id for repo in repositories]))
+  else:
+    repo_query = (Repository
+                  .select(Repository, Namespace.username)
+                  .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+                  .where(Repository.visibility == model.repository.get_public_repo_visibility(),
+                         Repository.kind == Repository.kind.get_id('application')))
+
+  if namespace:
+    repo_query = (repo_query
+                  .where(Namespace.username == namespace))
+
+  tag_query = (Tag
+               .select()
+               .where(Tag.tag_kind == Tag.tag_kind.get_id('release'))
+               .order_by(Tag.lifetime_start))
+
+  if media_type:
+    tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type)
+
+  tag_query = tag_model.tag_alive_oci(tag_query)
+  query = prefetch(repo_query, tag_query)
+
+  return query

data/oci_model/release.py

@@ -0,0 +1,118 @@
+import bisect
+
+from cnr.exception import PackageAlreadyExists
+from cnr.models.package_base import manifest_media_type
+
+from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
+                           ManifestListManifest, Blob, ManifestBlob)
+from data.oci_model import (blob as blob_model, manifest as manifest_model,
+                            manifest_list as manifest_list_model,
+                            tag as tag_model)
+
+
+LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json'
+SCHEMA_VERSION = 'v0'
+
+
+def get_app_release(repo, tag_name, media_type):
+  """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """
+  tag = tag_model.get_tag(repo, tag_name, tag_kind='release')
+  media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
+  manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set
+                          .join(Manifest)
+                          .where(ManifestListManifest.media_type == media_type_id).get())
+  manifest = manifestlistmanifest.manifest
+  blob = Blob.select().join(ManifestBlob).where(ManifestBlob.manifest == manifest).get()
+  return (tag, manifest, blob)
+
+
+def create_app_release(repo, tag_name, manifest, digest):
+  """ Create a new application release, it includes creating a new Tag, ManifestList,
+      ManifestListManifests, Manifest, ManifestBlob.
+
+      To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id.
+      To find the insert point in the ManifestList it uses bisect on the manifest-ids list.
+  """
+  with db_transaction():
+    # Create/get the package manifest
+    manifest = manifest_model.get_or_create_manifest(manifest, manifest['mediaType'])
+    # get the tag
+    tag = tag_model.get_or_initialize_tag(repo, tag_name)
+
+    if tag.manifest_list is None:
+      tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE),
+                                       schema_version=SCHEMA_VERSION,
+                                       manifest_list_json=[])
+
+    elif tag_model.tag_media_type_exists(tag, manifest.media_type):
+      raise PackageAlreadyExists("package exists already")
+
+    list_json = tag.manifest_list.manifest_list_json
+    mlm_query = (ManifestListManifest
+                 .select()
+                 .where(ManifestListManifest.manifest_list == tag.manifest_list))
+    list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
+    insert_point = bisect.bisect_left(list_manifest_ids, manifest.id)
+    list_json.insert(insert_point, manifest.manifest_json)
+    list_manifest_ids.insert(insert_point, manifest.id)
+    manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
+                                                                       SCHEMA_VERSION)
+    manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json)
+
+    tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
+                                         tag_kind="release")
+    blob_digest = digest
+
+    try:
+      (ManifestBlob
+       .select()
+       .join(Blob)
+       .where(ManifestBlob.manifest == manifest, Blob.digest == blob_digest).get())
+    except ManifestBlob.DoesNotExist:
+      blob = blob_model.get_blob(blob_digest)
+      ManifestBlob.create(manifest=manifest, blob=blob)
+    return tag
+
+
+def delete_app_release(repo, tag_name, media_type):
+  """ Delete a Tag/media-type couple """
+  media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
+
+  with db_transaction():
+    tag = tag_model.get_tag(repo, tag_name)
+    manifest_list = tag.manifest_list
+    list_json = manifest_list.manifest_list_json
+    mlm_query = (ManifestListManifest
+                 .select()
+                 .where(ManifestListManifest.manifest_list == tag.manifest_list))
+    list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
+    manifestlistmanifest = (tag
+                            .manifest_list
+                            .manifestlistmanifest_set
+                            .where(ManifestListManifest.media_type == media_type_id).get())
+    index = list_manifest_ids.index(manifestlistmanifest.manifest_id)
+    list_manifest_ids.pop(index)
+    list_json.pop(index)
+
+    if not list_json:
+      tag.lifetime_end = get_epoch_timestamp()
+      tag.save()
+    else:
+      manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
+                                                                     SCHEMA_VERSION)
+      manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids,
+                                                      list_json)
+      tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
+                                           tag_kind="release")
+    return tag
+
+
+def get_releases(repo, media_type=None):
+  """ Returns an array of Tag.name for a repo, can filter by media_type. """
+  release_query = (Tag
+                   .select()
+                   .where(Tag.repository == repo,
+                          Tag.tag_kind == Tag.tag_kind.get_id("release")))
+  if media_type:
+    release_query = tag_model.filter_tags_by_media_type(release_query, media_type)
+  return [t.name for t in tag_model.tag_alive_oci(release_query)]

data/oci_model/tag.py

@@ -0,0 +1,89 @@
+import logging
+
+from cnr.models.package_base import manifest_media_type
+from peewee import IntegrityError
+
+from data.model import (db_transaction, TagAlreadyCreatedException)
+from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update
+
+
+logger = logging.getLogger(__name__)
+
+
+def tag_alive_oci(query, now_ts=None, cls=Tag):
+  return query.where((cls.lifetime_end >> None) |
+                     (cls.lifetime_end > now_ts))
+
+
+def tag_media_type_exists(tag, media_type):
+  return (tag.manifest_list.manifestlistmanifest_set
+          .where(ManifestListManifest.media_type == media_type).count() > 0)
+
+
+def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"):
+  now_ts = get_epoch_timestamp_ms()
+  tag_kind_id = Tag.tag_kind.get_id(tag_kind)
+  with db_transaction():
+    try:
+      tag = db_for_update(tag_alive_oci(Tag
+                                        .select()
+                                        .where(Tag.repository == repo,
+                                               Tag.name == tag_name,
+                                               Tag.tag_kind == tag_kind_id), now_ts)).get()
+      if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag:
+        return tag
+      tag.lifetime_end = now_ts
+      tag.save()
+    except Tag.DoesNotExist:
+      pass
+
+    try:
+      return Tag.create(repository=repo, manifest_list=manifest_list, linked_tag=linked_tag,
+                        name=tag_name, lifetime_start=now_ts, lifetime_end=None,
+                        tag_kind=tag_kind_id)
+    except IntegrityError:
+      msg = 'Tag with name %s and lifetime start %s under repository %s/%s already exists'
+      raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name))
+
+
+def get_or_initialize_tag(repo, tag_name, tag_kind="release"):
+  try:
+    return Tag.select().where(Tag.repository == repo, Tag.name == tag_name).get()
+  except Tag.DoesNotExist:
+    return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind))
+
+
+def get_tag(repo, tag_name, tag_kind="release"):
+  return tag_alive_oci(Tag.select()
+                       .where(Tag.repository == repo,
+                              Tag.name == tag_name,
+                              Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get()
+
+
+def delete_tag(repo, tag_name, tag_kind="release"):
+  tag_kind_id = Tag.tag_kind.get_id(tag_kind)
+  tag = tag_alive_oci(Tag.select()
+                      .where(Tag.repository == repo,
+                             Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get()
+  tag.lifetime_end = get_epoch_timestamp_ms()
+  tag.save()
+  return tag
+
+
+def tag_exists(repo, tag_name, tag_kind="release"):
+  try:
+    get_tag(repo, tag_name, tag_kind)
+    return True
+  except Tag.DoesNotExist:
+    return False
+
+
+def filter_tags_by_media_type(tag_query, media_type):
+  """ Return only available tag for a media_type. """
+  media_type = manifest_media_type(media_type)
+  t = (tag_query
+       .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list))
+       .where(ManifestListManifest.media_type == ManifestListManifest.media_type.get_id(media_type)))
+  return t
+
+

endpoints/appr/__init__.py

@@ -0,0 +1,43 @@
+import logging
+
+from functools import wraps
+
+from cnr.exception import UnauthorizedAccess
+from flask import Blueprint
+
+from app import metric_queue
+from auth.permissions import (AdministerRepositoryPermission, ReadRepositoryPermission,
+                              ModifyRepositoryPermission)
+from endpoints.appr.decorators import require_repo_permission
+from util.metrics.metricqueue import time_blueprint
+
+
+appr_bp = Blueprint('appr', __name__)
+time_blueprint(appr_bp, metric_queue)
+logger = logging.getLogger(__name__)
+
+
+def _raise_method(repository, scopes):
+  raise UnauthorizedAccess("Unauthorized access for: %s" % repository,
+                           {"package": repository, "scopes": scopes})
+
+
+def _get_reponame_kwargs(*args, **kwargs):
+  return [kwargs['namespace'], kwargs['package_name']]
+
+
+require_app_repo_read = require_repo_permission(ReadRepositoryPermission,
+                                                scopes=['pull'],
+                                                allow_public=True,
+                                                raise_method=_raise_method,
+                                                get_reponame_method=_get_reponame_kwargs)
+
+require_app_repo_write = require_repo_permission(ModifyRepositoryPermission,
+                                                 scopes=['pull', 'push'],
+                                                 raise_method=_raise_method,
+                                                 get_reponame_method=_get_reponame_kwargs)
+
+require_app_repo_admin = require_repo_permission(AdministerRepositoryPermission,
+                                                 scopes=['pull', 'push'],
+                                                 raise_method=_raise_method,
+                                                 get_reponame_method=_get_reponame_kwargs)

endpoints/appr/cnr_backend.py

@@ -0,0 +1,160 @@
+import base64
+
+from cnr.exception import raise_package_not_found
+from cnr.models.blob_base import BlobBase
+from cnr.models.channel_base import ChannelBase
+from cnr.models.db_base import CnrDB
+from cnr.models.package_base import PackageBase, manifest_media_type
+
+from app import storage
+from data.interfaces.appr import oci_app_model
+from data.oci_model import blob # TODO these calls should be through oci_app_model
+
+
+class Blob(BlobBase):
+  @classmethod
+  def upload_url(cls, digest):
+    return "cnr/blobs/sha256/%s/%s" % (digest[0:2], digest)
+
+  def save(self, content_media_type):
+    oci_app_model.store_blob(self, content_media_type)
+
+  @classmethod
+  def delete(cls, package_name, digest):
+    pass
+
+  @classmethod
+  def _fetch_b64blob(cls, package_name, digest):
+    blobpath = cls.upload_url(digest)
+    locations = blob.get_blob_locations(digest)
+    if not locations:
+      raise_package_not_found(package_name, digest)
+    return base64.b64encode(storage.get_content(locations, blobpath))
+
+  @classmethod
+  def download_url(cls, package_name, digest):
+    blobpath = cls.upload_url(digest)
+    locations = blob.get_blob_locations(digest)
+    if not locations:
+      raise_package_not_found(package_name, digest)
+    return storage.get_direct_download_url(locations, blobpath)
+
+
+class Channel(ChannelBase):
+  """ CNR Channel model implemented against the Quay data model. """
+  def __init__(self, name, package, current=None):
+    super(Channel, self).__init__(name, package, current=current)
+    self._channel_data = None
+
+  def _exists(self):
+    """ Check if the channel is saved already """
+    return oci_app_model.channel_exists(self.package, self.name)
+
+  @classmethod
+  def get(cls, name, package):
+    chanview = oci_app_model.fetch_channel(package, name, with_releases=False)
+    return cls(name, package, chanview.current)
+
+  def save(self):
+    oci_app_model.update_channel(self.package, self.name, self.current)
+
+  def delete(self):
+    oci_app_model.delete_channel(self.package, self.name)
+
+  @classmethod
+  def all(cls, package_name):
+    return [Channel(c.name, package_name, c.current)
+            for c in oci_app_model.list_channels(package_name)]
+
+  @property
+  def _channel(self):
+    if self._channel_data is None:
+      self._channel_data = oci_app_model.fetch_channel(self.package, self.name)
+    return self._channel_data
+
+  def releases(self):
+    """ Returns the list of versions """
+    return self._channel.releases
+
+  def _add_release(self, release):
+    return oci_app_model.update_channel(self.package, self.name, release)._asdict
+
+  def _remove_release(self, release):
+    oci_app_model.delete_channel(self.package, self.name)
+
+
+class Package(PackageBase):
+  """ CNR Package model implemented against the Quay data model. """
+
+  @classmethod
+  def _apptuple_to_dict(cls, apptuple):
+    return {'release': apptuple.release,
+            'created_at': apptuple.created_at,
+            'digest': apptuple.manifest.digest,
+            'mediaType': apptuple.manifest.mediaType,
+            'package': apptuple.name,
+            'content': apptuple.manifest.content._asdict()}
+
+  @classmethod
+  def create_repository(cls, package_name, visibility, owner):
+    oci_app_model.create_application(package_name, visibility, owner)
+
+  @classmethod
+  def exists(cls, package_name):
+    return oci_app_model.application_exists(package_name)
+
+  @classmethod
+  def all(cls, organization=None, media_type=None, search=None, username=None, **kwargs):
+    return [dict(x._asdict()) for x in oci_app_model.list_applications(namespace=organization,
+                                                                       media_type=media_type,
+                                                                       search=search,
+                                                                       username=username)]
+
+  @classmethod
+  def _fetch(cls, package_name, release, media_type):
+    data = oci_app_model.fetch_release(package_name, release, manifest_media_type(media_type))
+    return cls._apptuple_to_dict(data)
+
+  @classmethod
+  def all_releases(cls, package_name, media_type=None):
+    return oci_app_model.list_releases(package_name, media_type)
+
+  @classmethod
+  def search(cls, query, username=None):
+    return oci_app_model.basic_search(query, username=username)
+
+  def _save(self, force=False, **kwargs):
+    user = kwargs['user']
+    visibility = kwargs['visibility']
+    oci_app_model.create_release(self, user, visibility, force)
+
+  @classmethod
+  def _delete(cls, package_name, release, media_type):
+    oci_app_model.delete_release(package_name, release, manifest_media_type(media_type))
+
+  @classmethod
+  def isdeleted_release(cls, package, release):
+    return oci_app_model.release_exists(package, release)
+
+  def channels(self, channel_class, iscurrent=True):
+    return [c.name for c in oci_app_model.list_release_channels(self.package, self.release,
+                                                                active=iscurrent)]
+
+  @classmethod
+  def manifests(cls, package, release=None):
+    return oci_app_model.list_manifests(package, release)
+
+  @classmethod
+  def dump_all(cls, blob_cls):
+    raise NotImplementedError
+
+
+class QuayDB(CnrDB):
+  """ Wrapper Class to embed all CNR Models """
+  Channel = Channel
+  Package = Package
+  Blob = Blob
+
+  @classmethod
+  def reset_db(cls, force=False):
+    pass

endpoints/appr/decorators.py

@@ -0,0 +1,53 @@
+import logging
+
+from functools import wraps
+
+from flask import abort
+
+from data import model
+
+
+logger = logging.getLogger(__name__)
+
+
+def _raise_unauthorized(repository, scopes):
+  raise StandardError("Unauthorized acces to %s", repository)
+
+
+def _get_reponame_kwargs(*args, **kwargs):
+  return [kwargs['namespace'], kwargs['package_name']]
+
+
+def disallow_for_image_repository(get_reponame_method=_get_reponame_kwargs):
+  def wrapper(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+      namespace_name, repo_name = get_reponame_method(*args, **kwargs)
+      image_repo = model.repository.get_repository(namespace_name, repo_name, kind_filter='image')
+      if image_repo is not None:
+        logger.debug('Tried to invoked a CNR method on an image repository')
+        abort(501)
+      return func(*args, **kwargs)
+    return wrapped
+  return wrapper
+
+
+def require_repo_permission(permission_class, scopes=None, allow_public=False,
+                            raise_method=_raise_unauthorized,
+                            get_reponame_method=_get_reponame_kwargs):
+  def wrapper(func):
+    @wraps(func)
+    @disallow_for_image_repository(get_reponame_method=get_reponame_method)
+    def wrapped(*args, **kwargs):
+      namespace_name, repo_name = get_reponame_method(*args, **kwargs)
+      logger.debug('Checking permission %s for repo: %s/%s', permission_class,
+                   namespace_name, repo_name)
+      permission = permission_class(namespace_name, repo_name)
+      if (permission.can() or
+          (allow_public and
+           model.repository.repository_is_public(namespace_name, repo_name))):
+        return func(*args, **kwargs)
+      repository = namespace_name + '/' + repo_name
+      raise_method(repository, scopes)
+    return wrapped
+  return wrapper

endpoints/appr/registry.py

@@ -0,0 +1,299 @@
+import logging
+
+from base64 import b64encode
+
+import cnr
+
+from cnr.api.impl import registry as cnr_registry
+from cnr.api.registry import repo_name, _pull
+from cnr.exception import (CnrException, InvalidUsage, InvalidParams, InvalidRelease,
+                           UnableToLockResource, UnauthorizedAccess, Unsupported, ChannelNotFound,
+                           PackageAlreadyExists, PackageNotFound, PackageReleaseNotFound)
+from flask import request, jsonify
+
+from auth.process import process_auth
+from auth.auth_context import get_authenticated_user
+from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermission
+from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write
+from endpoints.appr.decorators import disallow_for_image_repository
+from endpoints.appr.cnr_backend import Package, Channel, Blob
+from endpoints.decorators import anon_allowed, anon_protect
+from util.names import REPOSITORY_NAME_REGEX, TAG_REGEX
+
+
+logger = logging.getLogger(__name__)
+
+
+@appr_bp.errorhandler(Unsupported)
+@appr_bp.errorhandler(PackageAlreadyExists)
+@appr_bp.errorhandler(InvalidRelease)
+@appr_bp.errorhandler(UnableToLockResource)
+@appr_bp.errorhandler(UnauthorizedAccess)
+@appr_bp.errorhandler(PackageNotFound)
+@appr_bp.errorhandler(PackageReleaseNotFound)
+@appr_bp.errorhandler(CnrException)
+@appr_bp.errorhandler(InvalidUsage)
+@appr_bp.errorhandler(InvalidParams)
+@appr_bp.errorhandler(ChannelNotFound)
+def render_error(error):
+  response = jsonify({"error": error.to_dict()})
+  response.status_code = error.status_code
+  return response
+
+
+@appr_bp.route("/version")
+@anon_allowed
+def version():
+  return jsonify({"cnr-api": cnr.__version__})
+
+
+@appr_bp.route("/api/v1/users/login", methods=['POST'])
+@anon_allowed
+def login():
+  """
+    Todo:
+      * Implement better login protocol
+  """
+  values = request.get_json(force=True, silent=True)
+  return jsonify({'token': "basic " + b64encode("%s:%s" % (values['user']['username'],
+                                                           values['user']['password']))})
+
+
+# @TODO: Redirect to S3 url
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/blobs/sha256/<string:digest>",
+  methods=['GET'],
+  strict_slashes=False,
+)
+def blobs(namespace, package_name, digest):
+  reponame = repo_name(namespace, package_name)
+  data = cnr_registry.pull_blob(reponame, digest, blob_class=Blob)
+  json_format = request.args.get('format', None) == 'json'
+  return _pull(data, json_format=json_format)
+
+
+@appr_bp.route("/api/v1/packages", methods=['GET'], strict_slashes=False)
+@process_auth
+@anon_protect
+def list_packages():
+  namespace = request.args.get('namespace', None)
+  media_type = request.args.get('media_type', None)
+  query = request.args.get('query', None)
+  user = get_authenticated_user()
+  username = None
+  if user:
+    username = user.username
+  result_data = cnr_registry.list_packages(namespace,
+                                           package_class=Package,
+                                           search=query,
+                                           media_type=media_type,
+                                           username=username)
+  return jsonify(result_data)
+
+
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/<string:release>/<string:media_type>",
+  methods=['DELETE'], strict_slashes=False)
+@process_auth
+@require_app_repo_write
+@anon_protect
+def delete_package(namespace, package_name, release, media_type):
+  reponame = repo_name(namespace, package_name)
+  result = cnr_registry.delete_package(reponame,
+                                       release,
+                                       media_type,
+                                       package_class=Package)
+  return jsonify(result)
+
+
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/<string:release>/<string:media_type>",
+  methods=['GET'],
+  strict_slashes=False
+)
+@process_auth
+@require_app_repo_read
+def show_package(namespace, package_name, release, media_type):
+  reponame = repo_name(namespace, package_name)
+  result = cnr_registry.show_package(reponame, release,
+                                     media_type,
+                                     channel_class=Channel,
+                                     package_class=Package)
+  return jsonify(result)
+
+
+
+@appr_bp.route("/api/v1/packages/<string:namespace>/<string:package_name>", methods=['GET'],
+               strict_slashes=False)
+@process_auth
+@require_app_repo_read
+@anon_protect
+def show_package_releases(namespace, package_name):
+  reponame = repo_name(namespace, package_name)
+  media_type = request.args.get('media_type', None)
+  result = cnr_registry.show_package_releases(reponame,
+                                              media_type=media_type,
+                                              package_class=Package)
+  return jsonify(result)
+
+
+@appr_bp.route("/api/v1/packages/<string:namespace>/<string:package_name>/<string:release>",
+               methods=['GET'], strict_slashes=False)
+@process_auth
+@require_app_repo_read
+@anon_protect
+def show_package_releasse_manifests(namespace, package_name, release):
+  reponame = repo_name(namespace, package_name)
+  result = cnr_registry.show_package_manifests(reponame,
+                                               release,
+                                               package_class=Package)
+  return jsonify(result)
+
+
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/<string:release>/<string:media_type>/pull",
+  methods=['GET'],
+  strict_slashes=False,
+)
+@process_auth
+@require_app_repo_read
+@anon_protect
+def pull(namespace, package_name, release, media_type):
+  reponame = repo_name(namespace, package_name)
+  logger.info("pull %s", reponame)
+  data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
+  return _pull(data)
+
+
+@appr_bp.route("/api/v1/packages/<string:namespace>/<string:package_name>", methods=['POST'],
+               strict_slashes=False)
+@disallow_for_image_repository()
+@process_auth
+@anon_protect
+def push(namespace, package_name):
+  reponame = repo_name(namespace, package_name)
+
+  if not REPOSITORY_NAME_REGEX.match(package_name):
+    logger.debug('Found invalid repository name CNR push: %s', reponame)
+    raise InvalidUsage()
+
+  values = request.get_json(force=True, silent=True)
+  release_version = values['release']
+  media_type = values['media_type']
+  force = request.args.get('force', 'false') == 'true'
+  private = values.get('visibility', 'public')
+  owner = get_authenticated_user()
+  if not Package.exists(reponame):
+    if not CreateRepositoryPermission(namespace).can():
+      raise UnauthorizedAccess("Unauthorized access for: %s" % reponame,
+                               {"package": reponame, "scopes": ['create']})
+    Package.create_repository(reponame, private, owner)
+
+
+  if not ModifyRepositoryPermission(namespace, package_name).can():
+    raise UnauthorizedAccess("Unauthorized access for: %s" % reponame,
+                             {"package": reponame, "scopes": ['push']})
+
+  blob = Blob(reponame, values['blob'])
+  app_release = cnr_registry.push(reponame, release_version, media_type, blob, force,
+                                  package_class=Package, user=owner, visibility=private)
+  return jsonify(app_release)
+
+
+@appr_bp.route("/api/v1/packages/search", methods=['GET'], strict_slashes=False)
+@process_auth
+@anon_protect
+def search_packages():
+  query = request.args.get("q")
+  user = get_authenticated_user()
+  username = None
+  if user:
+    username = user.username
+
+  search_results = cnr_registry.search(query, Package, username=username)
+  return jsonify(search_results)
+
+
+# CHANNELS
+@appr_bp.route("/api/v1/packages/<string:namespace>/<string:package_name>/channels",
+               methods=['GET'], strict_slashes=False)
+@process_auth
+@require_app_repo_read
+@anon_protect
+def list_channels(namespace, package_name):
+  reponame = repo_name(namespace, package_name)
+  return jsonify(cnr_registry.list_channels(reponame, channel_class=Channel))
+
+
+@appr_bp.route("/api/v1/packages/<string:namespace>/<string:package_name>/channels/<string:channel_name>", methods=['GET'], strict_slashes=False)
+@process_auth
+@require_app_repo_read
+@anon_protect
+def show_channel(namespace, package_name, channel_name):
+  reponame = repo_name(namespace, package_name)
+  channel = cnr_registry.show_channel(reponame, channel_name, channel_class=Channel)
+  return jsonify(channel)
+
+
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/channels/<string:channel_name>/<string:release>",
+  methods=['POST'],
+  strict_slashes=False,
+)
+@process_auth
+@require_app_repo_write
+@anon_protect
+def add_channel_release(namespace, package_name, channel_name, release):
+  if not TAG_REGEX.match(channel_name):
+    logger.debug('Found invalid channel name CNR add channel release: %s', channel_name)
+    raise InvalidUsage()
+
+  if not TAG_REGEX.match(release):
+    logger.debug('Found invalid release name CNR add channel release: %s', release)
+    raise InvalidUsage()
+
+  reponame = repo_name(namespace, package_name)
+  result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel,
+                                            package_class=Package)
+  return jsonify(result)
+
+
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/channels/<string:channel_name>/<string:release>",
+  methods=['DELETE'],
+  strict_slashes=False,
+)
+@process_auth
+@require_app_repo_write
+@anon_protect
+def delete_channel_release(namespace, package_name, channel_name, release):
+  if not TAG_REGEX.match(channel_name):
+    logger.debug('Found invalid channel name CNR delete channel release: %s', channel_name)
+    raise InvalidUsage()
+
+  if not TAG_REGEX.match(release):
+    logger.debug('Found invalid release name CNR delete channel release: %s', release)
+    raise InvalidUsage()
+
+  reponame = repo_name(namespace, package_name)
+  result = cnr_registry.delete_channel_release(reponame, channel_name, release,
+                                               channel_class=Channel, package_class=Package)
+  return jsonify(result)
+
+
+@appr_bp.route(
+  "/api/v1/packages/<string:namespace>/<string:package_name>/channels/<string:channel_name>",
+  methods=['DELETE'],
+  strict_slashes=False,
+)
+@process_auth
+@require_app_repo_write
+@anon_protect
+def delete_channel(namespace, package_name, channel_name):
+  if not TAG_REGEX.match(channel_name):
+    logger.debug('Found invalid channel name CNR delete channel: %s', channel_name)
+    raise InvalidUsage()
+
+  reponame = repo_name(namespace, package_name)
+  result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel)
+  return jsonify(result)

endpoints/appr/test/test_api.py

@@ -0,0 +1,201 @@
+import os
+import shutil
+import uuid
+
+import pytest
+
+from cnr.models.db_base import CnrDB
+from cnr.tests.test_apiserver import BaseTestServer
+from cnr.tests.conftest import *
+from cnr.tests.test_models import CnrTestModels
+from peewee import SqliteDatabase
+
+from app import app as application
+from data.database import close_db_filter, User, db as database
+from data.model import user, organization
+from data.interfaces.appr import oci_app_model
+from endpoints.appr import appr_bp, registry
+from endpoints.appr.cnr_backend import QuayDB, Channel, Package
+from initdb import wipe_database, initialize_database, populate_database
+
+application.register_blueprint(appr_bp, url_prefix='/cnr')
+
+
+# TODO: avoid direct usage of database
+def create_org(namespace, owner):
+  try:
+    User.get(username=namespace)
+  except User.DoesNotExist:
+    organization.create_organization(namespace, "%s@test.com" % str(uuid.uuid1()), owner)
+
+
+class ChannelTest(Channel):
+  @classmethod
+  def dump_all(cls, package_class=None):
+    result = []
+    for repo in oci_app_model.list_applications(with_channels=True):
+      for chan in repo.channels:
+        result.append({'name': chan.name, 'current': chan.current, 'package': repo.name})
+    return result
+
+
+class PackageTest(Package):
+  def _save(self, force, **kwargs):
+    owner = user.get_user('devtable')
+    create_org(self.namespace, owner)
+    super(PackageTest, self)._save(force, user=owner, visibility="public")
+
+  @classmethod
+  def create_repository(cls, package_name, visibility, owner):
+    ns, _ = package_name.split("/")
+    owner = user.get_user('devtable')
+    visibility = "public"
+    create_org(ns, owner)
+    return super(PackageTest, cls).create_repository(package_name, visibility, owner)
+
+  @classmethod
+  def dump_all(cls, blob_cls):
+    result = []
+    for repo in oci_app_model.list_applications(with_channels=True):
+      package_name = repo.name
+      for release in repo.releases:
+        for mtype in cls.manifests(package_name, release):
+          package = oci_app_model.fetch_release(package_name, release, mtype)
+          blob = blob_cls.get(package_name, package.manifest.content.digest)
+          data = cls._apptuple_to_dict(package)
+          data.pop('digest')
+          data['channels'] = [x.name for x in oci_app_model.list_release_channels(package_name,
+                                                                                  package.release,
+                                                                                  False)]
+          data['blob'] = blob.b64blob
+          result.append(data)
+    return result
+
+
+@pytest.fixture(autouse=True)
+def quaydb(monkeypatch):
+  monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Package', PackageTest)
+  monkeypatch.setattr('endpoints.appr.cnr_backend.Package', PackageTest)
+  monkeypatch.setattr('endpoints.appr.registry.Package', PackageTest)
+  monkeypatch.setattr('cnr.models.Package', PackageTest)
+
+  monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Channel', ChannelTest)
+  # monkeypatch.setattr('data.cnrmodel.channel.Channel', ChannelTest)
+  monkeypatch.setattr('endpoints.appr.registry.Channel', ChannelTest)
+  monkeypatch.setattr('cnr.models.Channel', ChannelTest)
+
+
+def seed_db():
+  create_org("titi", user.get_user("devtable"))
+
+
+@pytest.fixture()
+def sqlitedb_file(tmpdir):
+  test_db_file = tmpdir.mkdir("quaydb").join("test.db")
+  return str(test_db_file)
+
+
+@pytest.fixture(scope="module")
+def init_db_path(tmpdir_factory):
+  sqlitedb_file_loc = str(tmpdir_factory.mktemp("data").join("test.db"))
+  sqlitedb = 'sqlite:///{0}'.format(sqlitedb_file_loc)
+  conf = {"TESTING": True,
+          "DEBUG": True,
+          "DB_URI": sqlitedb}
+  os.environ['TEST_DATABASE_URI'] = str(sqlitedb)
+  os.environ['DB_URI'] = str(sqlitedb)
+  database.initialize(SqliteDatabase(sqlitedb_file_loc))
+  application.config.update(conf)
+  application.config.update({"DB_URI": sqlitedb})
+  wipe_database()
+  initialize_database()
+  populate_database(minimal=True)
+  close_db_filter(None)
+  seed_db()
+  return str(sqlitedb_file_loc)
+
+
+@pytest.fixture()
+def database_uri(monkeypatch, init_db_path, sqlitedb_file):
+  shutil.copy2(init_db_path, sqlitedb_file)
+  database.initialize(SqliteDatabase(sqlitedb_file))
+  db_path = 'sqlite:///{0}'.format(sqlitedb_file)
+  monkeypatch.setenv("DB_URI", db_path)
+  seed_db()
+  return db_path
+
+
+@pytest.fixture()
+def appconfig(database_uri):
+  conf = {"TESTING": True,
+          "DEBUG": True,
+          "DB_URI": database_uri}
+  return conf
+
+
+@pytest.fixture()
+def create_app():
+  try:
+    application.register_blueprint(appr_bp, url_prefix='')
+  except:
+    pass
+  return application
+
+
+@pytest.fixture(autouse=True)
+def app(create_app, appconfig):
+  create_app.config.update(appconfig)
+  return create_app
+
+
+@pytest.fixture()
+def db():
+  return CnrDB
+
+
+class TestServerQuayDB(BaseTestServer):
+  DB_CLASS = QuayDB
+
+  @property
+  def token(self):
+    return "basic ZGV2dGFibGU6cGFzc3dvcmQ="
+
+  def test_search_package_match(self, db_with_data1, client):
+    """ TODO: search cross namespace and package name """
+    BaseTestServer.test_search_package_match(self, db_with_data1, client)
+
+  @pytest.mark.xfail
+  def test_push_package_already_exists_force(self, db_with_data1, package_b64blob, client):
+    """ No force push implemented """
+    BaseTestServer.test_push_package_already_exists_force(self, db_with_data1, package_b64blob,
+                                                          client)
+
+  @pytest.mark.xfail
+  def test_delete_channel_release_absent_release(self, db_with_data1, client):
+    BaseTestServer.test_delete_channel_release_absent_release(self, db_with_data1, client)
+
+
+class TestQuayModels(CnrTestModels):
+  DB_CLASS = QuayDB
+
+  @pytest.fixture(autouse=True)
+  def load_db(self, appconfig):
+    return appconfig
+
+  @pytest.mark.xfail
+  def test_save_package_exists_force(self, newdb, package_b64blob):
+    CnrTestModels.test_save_package_exists_force(self, newdb, package_b64blob)
+
+  @pytest.mark.xfail
+  def test_channel_delete_releases(self, db_with_data1):
+    """ Can't remove a release from the channel, only delete the channel entirely """
+    CnrTestModels.test_channel_delete_releases(self, db_with_data1)
+
+  @pytest.mark.xfail
+  def test_forbiddeb_db_reset(self, db_class):
+    pass
+
+  @pytest.mark.xfail
+  def test_db_restore(self, newdb, dbdata1):
+    # This will fail as long as CNR tests use a mediatype with v1.
+    pass

endpoints/appr/test/test_decorators.py

@@ -0,0 +1,19 @@
+import pytest
+
+from werkzeug.exceptions import NotImplemented as NIE
+
+from data import model
+from endpoints.test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file
+from endpoints.appr import require_app_repo_read
+
+def test_require_app_repo_read(app):
+  called = [False]
+
+  # Ensure that trying to read an *image* repository fails.
+  @require_app_repo_read
+  def empty(**kwargs):
+    called[0] = True
+
+  with pytest.raises(NIE):
+    empty(namespace='devtable', package_name='simple')
+    assert not called[0]

endpoints/decorators.py

@@ -1,10 +1,14 @@
 """ Various decorators for endpoint and API handlers. """
 
-import features
+from functools import wraps
+
 from flask import abort
+
+import features
+
 from auth.auth_context import (get_validated_oauth_token, get_authenticated_user,
                                get_validated_token, get_grant_context)
-from functools import wraps
+from data import model # TODO: stop using model directly
 
 
 def anon_allowed(func):

registry.py

@@ -2,11 +2,12 @@ import logging
 import logging.config
 import os
 
+import endpoints.decorated # Note: We need to import this module to make sure the decorators are registered.
+import features
+
 from app import app as application
 
-# Note: We need to import this module to make sure the decorators are registered.
-import endpoints.decorated
-
+from endpoints.appr import appr_bp, registry # registry needed to ensure routes registered
 from endpoints.v1 import v1_bp
 from endpoints.v2 import v2_bp
 
@@ -15,3 +16,6 @@ if os.environ.get('DEBUGLOG') == 'true':
 
 application.register_blueprint(v1_bp, url_prefix='/v1')
 application.register_blueprint(v2_bp, url_prefix='/v2')
+
+if features.APP_REGISTRY:
+  application.register_blueprint(appr_bp, url_prefix='/cnr')

requirements-nover.txt

@@ -8,6 +8,7 @@
 -e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab
 -e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib
 -e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython
+-e git+https://github.com/app-registry/appr-server.git#egg=cnr-server
 APScheduler==3.0.5
 Flask-Login
 Flask-Mail
@@ -70,3 +71,4 @@ tzlocal
 xhtml2pdf
 recaptcha2
 mockredispy
+cnr

requirements.txt

@@ -2,6 +2,7 @@ aiowsgi==0.6
 alembic==0.8.8
 -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601
 -e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode
+-e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server
 APScheduler==3.0.5
 autobahn==0.9.3.post3
 Babel==2.3.4

test/test_gc.py

@@ -8,7 +8,7 @@ from playhouse.test_utils import assert_query_count
 from app import app, storage
 from initdb import setup_database_for_testing, finished_database_for_testing
 from data import model, database
-from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel
+from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, Blob
 
 
 ADMIN_ACCESS_USER = 'devtable'
@@ -190,6 +190,9 @@ class TestGarbageCollection(unittest.TestCase):
       if storage_row.cas_path:
         storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
 
+    for blob_row in Blob.select():
+        storage.get_content({preferred}, storage.blob_path(blob_row.digest))
+
   def test_has_garbage(self):
     """ Remove all existing repositories, then add one without garbage, check, then add one with
         garbage, and check again.
@@ -502,7 +505,48 @@ class TestGarbageCollection(unittest.TestCase):
       # Ensure the CAS path still exists.
       self.assertTrue(storage.exists({preferred}, storage.blob_path(digest)))
 
+  def test_images_shared_cas_with_new_blob_table(self):
+    """ A repository with a tag and image that shares its CAS path with a record in the new Blob
+        table. Deleting the first tag should delete the first image, and its storage, but not the
+        file in storage, as it shares its CAS path with the blob row.
+    """
+    with self.assert_gc_integrity(expect_storage_removed=True):
+      repository = self.createRepository()
 
+      # Create two image storage records with the same content checksum.
+      content = 'hello world'
+      digest = 'sha256:' + hashlib.sha256(content).hexdigest()
+      preferred = storage.preferred_locations[0]
+      storage.put_content({preferred}, storage.blob_path(digest), content)
+
+      media_type = database.MediaType.get(name='text/plain')
+
+      is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
+      is2 = database.Blob.create(digest=digest, size=0, media_type=media_type)
+
+      location = database.ImageStorageLocation.get(name=preferred)
+      database.ImageStoragePlacement.create(location=location, storage=is1)
+
+      # Ensure the CAS path exists.
+      self.assertTrue(storage.exists({preferred}, storage.blob_path(digest)))
+
+      # Create the image in the repository, and the tag.
+      first_image = Image.create(docker_image_id='i1',
+                                 repository=repository, storage=is1,
+                                 ancestors='/')
+
+      model.tag.store_tag_manifest(repository.namespace_user.username, repository.name,
+                                   'first', first_image.docker_image_id,
+                                   'sha:someshahere1', '{}')
+
+      self.assertNotDeleted(repository, 'i1')
+
+      # Delete the tag.
+      self.deleteTag(repository, 'first')
+      self.assertDeleted(repository, 'i1')
+
+      # Ensure the CAS path still exists, as it is referenced by the Blob table.
+      self.assertTrue(storage.exists({preferred}, storage.blob_path(digest)))
 
 if __name__ == '__main__':
   unittest.main()

test/testconfig.py

@@ -92,3 +92,5 @@ class TestConfig(DefaultConfig):
 
   RECAPTCHA_SITE_KEY = 'somekey'
   RECAPTCHA_SECRET_KEY = 'somesecretkey'
+
+  FEATURE_APP_REGISTRY = True

util/config/database.py

@@ -1,4 +1,4 @@
-from data import model
+from data import model, oci_model
 
 
 def sync_database_with_config(config):
@@ -7,3 +7,4 @@ def sync_database_with_config(config):
   location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
   if location_names:
     model.image.ensure_image_locations(*location_names)
+    oci_model.blob.ensure_blob_locations(*location_names)

web.py

@@ -2,7 +2,6 @@ import os
 import logging.config
 
 from app import app as application
-
 from endpoints.api import api_bp
 from endpoints.bitbuckettrigger import bitbuckettrigger
 from endpoints.githubtrigger import githubtrigger