from abc import ABCMeta, abstractmethod from collections import namedtuple from datetime import datetime import cnr.semver from cnr.exception import raise_package_not_found, raise_channel_not_found from six import add_metaclass from app import storage, authentication from data import model, oci_model from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel from util.audit import track_and_log from util.morecollections import AttrDict from util.names import parse_robot_username class BlobDescriptor(namedtuple('Blob', ['mediaType', 'size', 'digest', 'urls'])): """ BlobDescriptor describes a blob with its mediatype, size and digest. A BlobDescriptor is used to retrieves the actual blob. """ class ChannelReleasesView(namedtuple('ChannelReleasesView', ['name', 'current', 'releases'])): """ A channel is a pointer to a Release (current). Releases are the previous tags pointed by channel (history). """ class ChannelView(namedtuple('ChannelView', ['name', 'current'])): """ A channel is a pointer to a Release (current). """ class ApplicationSummaryView( namedtuple('ApplicationSummaryView', [ 'name', 'namespace', 'visibility', 'default', 'manifests', 'channels', 'releases', 'updated_at', 'created_at' ])): """ ApplicationSummaryView is an aggregated view of an application repository. """ class ApplicationManifest(namedtuple('ApplicationManifest', ['mediaType', 'digest', 'content'])): """ ApplicationManifest embed the BlobDescriptor and some metadata around it. An ApplicationManifest is content-addressable. """ class ApplicationRelease( namedtuple('ApplicationRelease', ['release', 'name', 'created_at', 'manifest'])): """ The ApplicationRelease associates an ApplicationManifest to a repository and release. """ @add_metaclass(ABCMeta) class AppRegistryDataInterface(object): """ Interface that represents all data store interactions required by a App Registry. """ @abstractmethod def list_applications(self, namespace=None, media_type=None, search=None, username=None, with_channels=False): """ Lists all repositories that contain applications, with optional filtering to a specific namespace and/or to those visible to a specific user. Returns: list of ApplicationSummaryView """ @abstractmethod def application_is_public(self, package_name): """ Returns true if the application is public """ @abstractmethod def create_application(self, package_name, visibility, owner): """ Create a new app repository, owner is the user who creates it """ @abstractmethod def application_exists(self, package_name): """ Returns true if the application exists """ @abstractmethod def basic_search(self, query, username=None): """ Returns an array of matching application in the format: 'namespace/name' Note: * Only 'public' repositories are returned """ # @TODO: Paginate @abstractmethod def list_releases(self, package_name, media_type=None): """ Returns the list of all releases(names) of an AppRepository Example: >>> get_app_releases('ant31/rocketchat') ['1.7.1', '1.7.0', '1.7.2'] """ # @TODO: Paginate @abstractmethod def list_manifests(self, package_name, release=None): """ Returns the list of all available manifests type of an Application across all releases or for a specific one. Example: >>> get_app_releases('ant31/rocketchat') ['1.7.1', '1.7.0', '1.7.2'] """ @abstractmethod def fetch_release(self, package_name, release, media_type): """ Returns an ApplicationRelease """ @abstractmethod def store_blob(self, cnrblob, content_media_type): """ Upload the blob content to a storage location and creates a Blob entry in the DB. Returns a BlobDescriptor """ @abstractmethod def create_release(self, package, user, visibility, force=False): """ Creates and returns an ApplicationRelease - package is a data.model.Package object - user is the owner of the package - visibility is a string: 'public' or 'private' """ @abstractmethod def release_exists(self, package, release): """ Return true if a release with that name already exist or has existed (including deleted ones) """ pass @abstractmethod def delete_release(self, package_name, release, media_type): """ Remove/Delete an app-release from an app-repository. It does not delete the entire app-repository, only a single release """ @abstractmethod def list_release_channels(self, package_name, release, active=True): """ Returns a list of Channel that are/was pointing to a release. If active is True, returns only active Channel (lifetime_end not null) """ @abstractmethod def channel_exists(self, package_name, channel_name): """ Returns true if the channel with the given name exists under the matching package """ @abstractmethod def update_channel(self, package_name, channel_name, release): """ Append a new release to the Channel Returns a new Channel with the release as current """ @abstractmethod def delete_channel(self, package_name, channel_name): """ Delete a Channel, it doesn't delete/touch the ApplicationRelease pointed by the channel """ # @TODO: Paginate @abstractmethod def list_channels(self, package_name): """ Returns all AppChannel for a package """ @abstractmethod def fetch_channel(self, package_name, channel_name, with_releases=True): """ Returns an Channel Raises: ChannelNotFound, PackageNotFound """ @abstractmethod def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None, analytics_sample=1, **kwargs): """ Logs an action to the audit log. """ def _split_package_name(package): """ Returns the namespace and package-name """ return package.split("/") def _join_package_name(ns, name): """ Returns a app-name in the 'namespace/name' format """ return "%s/%s" % (ns, name) def _timestamp_to_iso(timestamp, in_ms=True): if in_ms: timestamp = timestamp / 1000 return datetime.fromtimestamp(timestamp).isoformat() class OCIAppModel(AppRegistryDataInterface): def _application(self, package): ns, name = _split_package_name(package) repo = model.repository.get_app_repository(ns, name) if repo is None: raise_package_not_found(package) return repo def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None, analytics_sample=1, metadata=None): metadata = {} if metadata is None else metadata repo = None if repo_name is not None: db_repo = model.repository.get_repository(namespace_name, repo_name, kind_filter='application') repo = AttrDict({ 'id': db_repo.id, 'name': db_repo.name, 'namespace_name': db_repo.namespace_user.username, }) track_and_log(event_name, repo, analytics_name=analytics_name, analytics_sample=analytics_sample, **metadata) def list_applications(self, namespace=None, media_type=None, search=None, username=None, with_channels=False): """ Lists all repositories that contain applications, with optional filtering to a specific namespace and view a specific user. """ views = [] for repo in oci_model.package.list_packages_query(namespace, media_type, search, username=username): releases = [t.name for t in repo.tag_set_prefetch] if not releases: continue available_releases = [ str(x) for x in sorted(cnr.semver.versions(releases, False), reverse=True) ] channels = None if with_channels: channels = [ ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in oci_model.channel.get_repo_channels(repo) ] app_name = _join_package_name(repo.namespace_user.username, repo.name) manifests = self.list_manifests(app_name, available_releases[0]) view = ApplicationSummaryView( namespace=repo.namespace_user.username, name=app_name, visibility=repo.visibility.name, default=available_releases[0], channels=channels, manifests=manifests, releases=available_releases, updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start), created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start),) views.append(view) return views def application_is_public(self, package_name): """ Returns: * True if the repository is public """ namespace, name = _split_package_name(package_name) return model.repository.repository_is_public(namespace, name) def create_application(self, package_name, visibility, owner): """ Create a new app repository, owner is the user who creates it """ ns, name = _split_package_name(package_name) model.repository.create_repository(ns, name, owner, visibility, 'application') def application_exists(self, package_name): """ Create a new app repository, owner is the user who creates it """ ns, name = _split_package_name(package_name) return model.repository.get_repository(ns, name, kind_filter='application') is not None def basic_search(self, query, username=None): """ Returns an array of matching AppRepositories in the format: 'namespace/name' Note: * Only 'public' repositories are returned Todo: * Filter results with readeable reposistory for the user (including visibilitys) """ return [ _join_package_name(r.namespace_user.username, r.name) for r in model.repository.get_app_search(lookup=query, username=username, limit=50) ] def list_releases(self, package_name, media_type=None): """ Return the list of all releases of an Application Example: >>> get_app_releases('ant31/rocketchat') ['1.7.1', '1.7.0', '1.7.2'] Todo: * Paginate """ return oci_model.release.get_releases(self._application(package_name), media_type) def list_manifests(self, package_name, release=None): """ Returns the list of all manifests of an Application. Todo: * Paginate """ try: repo = self._application(package_name) return list(oci_model.manifest.get_manifest_types(repo, release)) except (Repository.DoesNotExist, Tag.DoesNotExist): raise_package_not_found(package_name, release) def fetch_release(self, package_name, release, media_type): """ Retrieves an AppRelease from it's repository-name and release-name """ repo = self._application(package_name) try: tag, manifest, blob = oci_model.release.get_app_release(repo, release, media_type) created_at = _timestamp_to_iso(tag.lifetime_start) blob_descriptor = BlobDescriptor(digest=_strip_sha256_header(blob.digest), mediaType=blob.media_type.name, size=blob.size, urls=[]) app_manifest = ApplicationManifest( digest=manifest.digest, mediaType=manifest.media_type.name, content=blob_descriptor) app_release = ApplicationRelease(release=tag.name, created_at=created_at, name=package_name, manifest=app_manifest) return app_release except (Tag.DoesNotExist, Manifest.DoesNotExist, Blob.DoesNotExist, Repository.DoesNotExist, MediaType.DoesNotExist): raise_package_not_found(package_name, release, media_type) def store_blob(self, cnrblob, content_media_type): fp = cnrblob.packager.io_file path = cnrblob.upload_url(cnrblob.digest) locations = storage.preferred_locations storage.stream_write(locations, path, fp, 'application/x-gzip') db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type, locations) return BlobDescriptor(mediaType=content_media_type, digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[]) def create_release(self, package, user, visibility, force=False): """ Add an app-release to a repository package is an instance of data.cnr.package.Package """ data = package.manifest() ns, name = package.namespace, package.name repo = model.repository.get_or_create_repository(ns, name, user, visibility=visibility, repo_kind='application') tag_name = package.release oci_model.release.create_app_release(repo, tag_name, package.manifest(), data['content']['digest'], force) def delete_release(self, package_name, release, media_type): """ Remove/Delete an app-release from an app-repository. It does not delete the entire app-repository, only a single release """ repo = self._application(package_name) try: oci_model.release.delete_app_release(repo, release, media_type) except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist): raise_package_not_found(package_name, release, media_type) def release_exists(self, package, release): """ Return true if a release with that name already exist or have existed (include deleted ones) """ def channel_exists(self, package_name, channel_name): """ Returns true if channel exists """ repo = self._application(package_name) return oci_model.tag.tag_exists(repo, channel_name, "channel") def delete_channel(self, package_name, channel_name): """ Delete an AppChannel Note: It doesn't delete the AppReleases """ repo = self._application(package_name) try: oci_model.channel.delete_channel(repo, channel_name) except (Channel.DoesNotExist, Tag.DoesNotExist): raise_channel_not_found(package_name, channel_name) def list_channels(self, package_name): """ Returns all AppChannel for a package """ repo = self._application(package_name) channels = oci_model.channel.get_repo_channels(repo) return [ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in channels] def fetch_channel(self, package_name, channel_name, with_releases=True): """ Returns an AppChannel """ repo = self._application(package_name) try: channel = oci_model.channel.get_channel(repo, channel_name) except (Channel.DoesNotExist, Tag.DoesNotExist): raise_channel_not_found(package_name, channel_name) if with_releases: releases = oci_model.channel.get_channel_releases(repo, channel) chanview = ChannelReleasesView( current=channel.linked_tag.name, name=channel.name, releases=[channel.linked_tag.name] + [c.name for c in releases]) else: chanview = ChannelView(current=channel.linked_tag.name, name=channel.name) return chanview def list_release_channels(self, package_name, release, active=True): repo = self._application(package_name) try: channels = oci_model.channel.get_tag_channels(repo, release, active=active) return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels] except (Channel.DoesNotExist, Tag.DoesNotExist): raise_package_not_found(package_name, release) def update_channel(self, package_name, channel_name, release): """ Append a new release to the AppChannel Returns: A new AppChannel with the release """ repo = self._application(package_name) channel = oci_model.channel.create_or_update_channel(repo, channel_name, release) return ChannelView(current=channel.linked_tag.name, name=channel.name) def get_user(self, username, password): err_msg = None if parse_robot_username(username) is not None: try: user = model.user.verify_robot(username, password) except model.InvalidRobotException as exc: return (None, exc.message) else: user, err_msg = authentication.verify_and_link_user(username, password) return (user, err_msg) def _strip_sha256_header(digest): if digest.startswith('sha256:'): return digest.split('sha256:')[1] return digest oci_app_model = OCIAppModel()