From 9f684fa73f9ef67f8ea6e979b0cebfe1e597deeb Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:51:28 -0400 Subject: [PATCH 01/28] data.oci_model: init with app methods --- data/oci_model/__init__.py | 0 data/oci_model/blob.py | 54 +++++++++++++++ data/oci_model/channel.py | 56 +++++++++++++++ data/oci_model/manifest.py | 55 +++++++++++++++ data/oci_model/manifest_list.py | 55 +++++++++++++++ data/oci_model/package.py | 40 +++++++++++ data/oci_model/release.py | 116 ++++++++++++++++++++++++++++++++ data/oci_model/tag.py | 88 ++++++++++++++++++++++++ 8 files changed, 464 insertions(+) create mode 100644 data/oci_model/__init__.py create mode 100644 data/oci_model/blob.py create mode 100644 data/oci_model/channel.py create mode 100644 data/oci_model/manifest.py create mode 100644 data/oci_model/manifest_list.py create mode 100644 data/oci_model/package.py create mode 100644 data/oci_model/release.py create mode 100644 data/oci_model/tag.py diff --git a/data/oci_model/__init__.py b/data/oci_model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/data/oci_model/blob.py b/data/oci_model/blob.py new file mode 100644 index 000000000..521f072ba --- /dev/null +++ b/data/oci_model/blob.py @@ -0,0 +1,54 @@ +from peewee import IntegrityError + +from data.model import db_transaction +from data.database import Blob, BlobPlacementLocation, BlobPlacement + + +def get_blob(digest): + """ Find a blob by its digest. """ + return Blob.select().where(Blob.digest == digest).get() + + +def get_or_create_blob(digest, size, media_type_name, locations): + """ Try to find a blob by its digest or create it. """ + with db_transaction(): + try: + blob = get_blob(digest) + except Blob.DoesNotExist: + blob = Blob.create(digest=digest, + media_type_id=Blob.media_type.get_id(media_type_name), + size=size) + for location_name in locations: + location_id = BlobPlacement.location.get_id(location_name) + try: + BlobPlacement.create(blob=blob, location=location_id) + except IntegrityError: + pass + + return blob + + +def get_blob_locations(digest): + """ Find all locations names for a blob. """ + return [x.name for x in + BlobPlacementLocation + .select() + .join(BlobPlacement) + .join(Blob) + .where(Blob.digest == digest)] + + +def ensure_blob_locations(*names): + with db_transaction(): + locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names) + + insert_names = list(names) + + for location in locations: + insert_names.remove(location.name) + + if not insert_names: + return + + data = [{'name': name} for name in insert_names] + BlobPlacementLocation.insert_many(data).execute() diff --git a/data/oci_model/channel.py b/data/oci_model/channel.py new file mode 100644 index 000000000..f8d169661 --- /dev/null +++ b/data/oci_model/channel.py @@ -0,0 +1,56 @@ +from data.oci_model import tag as tag_model +from data.database import Tag, Channel + + +def get_channel_releases(repo, channel): + """ Return all previously linked tags. + This works based upon Tag lifetimes. + """ + tag_kind_id = Channel.tag_kind.get_id('channel') + channel_name = channel.name + return (Tag + .select(Tag, Channel) + .join(Channel, on=(Tag.id == Channel.linked_tag)) + .where(Channel.repository == repo, + Channel.name == channel_name, + Channel.tag_kind == tag_kind_id, Channel.lifetime_end != None) + .order_by(Tag.lifetime_end)) + + +def get_channel(repo, channel_name): + """ Find a Channel by name. """ + channel = tag_model.get_tag(repo, channel_name, "channel") + return channel + + +def get_tag_channels(repo, tag_name, active=True): + """ Find the Channels associated with a Tag. """ + tag = tag_model.get_tag(repo, tag_name, "release") + query = tag.tag_parents + + if active: + query = tag_model.tag_alive_oci(query) + + return query + + +def delete_channel(repo, channel_name): + """ Delete a channel by name. """ + return tag_model.delete_tag(repo, channel_name, "channel") + + +def create_or_update_channel(repo, channel_name, tag_name): + """ Creates or updates a channel to include a particular tag. """ + tag = tag_model.get_tag(repo, tag_name, 'release') + return tag.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel") + + +def get_repo_channels(repo): + """ Creates or updates a channel to include a particular tag. """ + tag_kind_id = Channel.tag_kind.get_id('channel') + query = (Channel + .select(Channel, Tag) + .join(Tag, on=(Tag.id == Channel.linked_tag)) + .where(Channel.repository == repo, + Channel.tag_kind == tag_kind_id)) + return tag_model.tag_alive_oci(query, cls=Channel) diff --git a/data/oci_model/manifest.py b/data/oci_model/manifest.py new file mode 100644 index 000000000..bb690ddc4 --- /dev/null +++ b/data/oci_model/manifest.py @@ -0,0 +1,55 @@ +import logging +import hashlib +import json + +from cnr.models.package_base import get_media_type + +from data import oci_model +from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag + + +logger = logging.getLogger(__name__) + + +def _digest(manifestjson): + return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest() + + +def get_manifest_query(digest, media_type): + return Manifest.select().where(Manifest.digest == digest, + Manifest.media_type == Manifest.media_type.get_id(media_type)) + + +def get_manifest_with_blob(digest, media_type): + query = get_manifest_query(digest, media_type) + return query.join(Blob).get() + + +def get_or_create_manifest(manifest_json, media_type_name): + digest = _digest(manifest_json) + try: + manifest = get_manifest_query(digest, media_type_name).get() + except Manifest.DoesNotExist: + with db_transaction(): + manifest = Manifest.create(digest=digest, + manifest_json=manifest_json, + media_type=Manifest.media_type.get_id(media_type_name)) + return manifest + +def get_manifest_types(repo, release=None): + """ Returns an array of MediaTypes.name for a repo, can filter by tag """ + query = oci_model.tag.tag_alive_oci(Tag + .select(MediaType.name) + .join(ManifestListManifest, + on=(ManifestListManifest.manifest_list == Tag.manifest_list)) + .join(MediaType, + on=(ManifestListManifest.media_type == MediaType.id)) + .where(Tag.repository == repo, + Tag.tag_kind == Tag.tag_kind.get_id('release'))) + if release: + query = query.where(Tag.name == release) + + manifests = set() + for m in query.distinct().tuples(): + manifests.add(get_media_type(m[0])) + return manifests diff --git a/data/oci_model/manifest_list.py b/data/oci_model/manifest_list.py new file mode 100644 index 000000000..68be2855f --- /dev/null +++ b/data/oci_model/manifest_list.py @@ -0,0 +1,55 @@ +import logging +import hashlib +import json + +from data.database import ManifestList, ManifestListManifest, db_transaction + + +logger = logging.getLogger(__name__) + + +def _digest(manifestjson): + return hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest() + + +def get_manifest_list(digest): + return ManifestList.select().where(ManifestList.digest == digest).get() + + +def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version): + digest = _digest(manifest_list_json) + media_type_id = ManifestList.media_type.get_id(media_type_name) + + try: + return get_manifest_list(digest) + except ManifestList.DoesNotExist: + with db_transaction(): + manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json, + schema_version=schema_version, media_type=media_type_id) + return manifestlist + + +def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json): + """ From a manifestlist, manifests, and the manifest list blob, + create if doesn't exist the manfiestlistmanifest for each manifest """ + for pos in xrange(len(manifest_ids)): + manifest_id = manifest_ids[pos] + manifest_json = manifest_list_json[pos] + get_or_create_manifestlistmanifest(manifest=manifest_id, + manifestlist=manifestlist, + media_type_name=manifest_json['mediaType']) + + +def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name): + media_type_id = ManifestListManifest.media_type.get_id(media_type_name) + try: + ml = (ManifestListManifest + .select() + .where(ManifestListManifest.manifest == manifest, + ManifestListManifest.media_type == media_type_id, + ManifestListManifest.manifest_list == manifestlist)).get() + + except ManifestListManifest.DoesNotExist: + ml = ManifestListManifest.create(manifest_list=manifestlist, media_type=media_type_id, + manifest=manifest) + return ml diff --git a/data/oci_model/package.py b/data/oci_model/package.py new file mode 100644 index 000000000..8ebecc722 --- /dev/null +++ b/data/oci_model/package.py @@ -0,0 +1,40 @@ +from cnr.models.package_base import get_media_type, manifest_media_type +from peewee import prefetch + +from data import model, oci_model +from data.database import Repository, Namespace, Tag, ManifestListManifest + + +def list_packages_query(namespace=None, media_type=None, search_query=None, username=None): + """ List and filter repository by search query. """ + if search_query is not None: + repositories = model.repository.get_app_search(search_query, + username=username, + limit=50) + repo_query = (Repository + .select(Repository, Namespace.username) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.id << [repo.id for repo in repositories])) + else: + repo_query = (Repository + .select(Repository, Namespace.username) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.visibility == model.repository.get_public_repo_visibility(), + Repository.kind == Repository.kind.get_id('application'))) + + if namespace: + repo_query = (repo_query + .where(Namespace.username == namespace)) + + tag_query = (Tag + .select() + .where(Tag.tag_kind == Tag.tag_kind.get_id('release')) + .order_by(Tag.lifetime_start)) + + if media_type: + tag_query = oci_model.tag.filter_tags_by_media_type(tag_query, media_type) + + tag_query = oci_model.tag.tag_alive_oci(tag_query) + query = prefetch(repo_query, tag_query) + + return query diff --git a/data/oci_model/release.py b/data/oci_model/release.py new file mode 100644 index 000000000..6624900ab --- /dev/null +++ b/data/oci_model/release.py @@ -0,0 +1,116 @@ +import bisect + +from cnr.exception import PackageAlreadyExists +from cnr.models.package_base import manifest_media_type + +from data import oci_model +from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag, + ManifestListManifest, Blob, ManifestBlob) + + +LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json' +SCHEMA_VERSION = 'v0' + + +def get_app_release(repo, tag_name, media_type): + """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """ + tag = oci_model.tag.get_tag(repo, tag_name, tag_kind='release') + media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) + manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set + .join(Manifest) + .where(ManifestListManifest.media_type == media_type_id).get()) + manifest = manifestlistmanifest.manifest + blob = Blob.select().join(ManifestBlob).where(ManifestBlob.manifest == manifest).get() + return (tag, manifest, blob) + + +def create_app_release(repo, tag_name, manifest, digest): + """ Create a new application release, it includes creating a new Tag, ManifestList, + ManifestListManifests, Manifest, ManifestBlob. + + To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id. + To find the insert point in the ManifestList it uses bisect on the manifest-ids list. + """ + with db_transaction(): + # Create/get the package manifest + manifest = oci_model.manifest.get_or_create_manifest(manifest, manifest['mediaType']) + # get the tag + tag = oci_model.tag.get_or_initialize_tag(repo, tag_name) + + if tag.manifest_list is None: + tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE), + schema_version=SCHEMA_VERSION, + manifest_list_json=[]) + + elif oci_model.tag.tag_media_type_exists(tag, manifest.media_type): + raise PackageAlreadyExists("package exists already") + + list_json = tag.manifest_list.manifest_list_json + mlm_query = (ManifestListManifest + .select() + .where(ManifestListManifest.manifest_list == tag.manifest_list)) + list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query]) + insert_point = bisect.bisect_left(list_manifest_ids, manifest.id) + list_json.insert(insert_point, manifest.manifest_json) + list_manifest_ids.insert(insert_point, manifest.id) + manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, + SCHEMA_VERSION) + oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) + + tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag_kind="release") + blob_digest = digest + + try: + (ManifestBlob + .select() + .join(Blob) + .where(ManifestBlob.manifest == manifest, Blob.digest == blob_digest).get()) + except ManifestBlob.DoesNotExist: + blob = oci_model.blob.get_blob(blob_digest) + ManifestBlob.create(manifest=manifest, blob=blob) + return tag + + +def delete_app_release(repo, tag_name, media_type): + """ Delete a Tag/media-type couple """ + media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) + + with db_transaction(): + tag = oci_model.tag.get_tag(repo, tag_name) + manifest_list = tag.manifest_list + list_json = manifest_list.manifest_list_json + mlm_query = (ManifestListManifest + .select() + .where(ManifestListManifest.manifest_list == tag.manifest_list)) + list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query]) + manifestlistmanifest = (tag + .manifest_list + .manifestlistmanifest_set + .where(ManifestListManifest.media_type == media_type_id).get()) + index = list_manifest_ids.index(manifestlistmanifest.manifest_id) + list_manifest_ids.pop(index) + list_json.pop(index) + + if not list_json: + tag.lifetime_end = get_epoch_timestamp() + tag.save() + else: + manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, + SCHEMA_VERSION) + oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, + list_json) + tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag_kind="release") + return tag + + +def get_releases(repo, media_type=None): + """ Returns an array of Tag.name for a repo, can filter by media_type. """ + release_query = (Tag + .select() + .where(Tag.repository == repo, + Tag.tag_kind == Tag.tag_kind.get_id("release"))) + if media_type: + release_query = oci_model.tag.filter_tags_by_media_type(release_query, media_type) + return [t.name for t in oci_model.tag.tag_alive_oci(release_query)] diff --git a/data/oci_model/tag.py b/data/oci_model/tag.py new file mode 100644 index 000000000..16400ea25 --- /dev/null +++ b/data/oci_model/tag.py @@ -0,0 +1,88 @@ +import logging + +from peewee import IntegrityError + +from data.model import (db_transaction, TagAlreadyCreatedException) +from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update + + +logger = logging.getLogger(__name__) + + +def tag_alive_oci(query, now_ts=None, cls=Tag): + return query.where((cls.lifetime_end >> None) | + (cls.lifetime_end > now_ts)) + + +def tag_media_type_exists(tag, media_type): + return (tag.manifest_list.manifestlistmanifest_set + .where(ManifestListManifest.media_type == media_type).count() > 0) + + +def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"): + now_ts = get_epoch_timestamp_ms() + tag_kind_id = Tag.tag_kind.get_id(tag_kind) + with db_transaction(): + try: + tag = db_for_update(tag_alive_oci(Tag + .select() + .where(Tag.repository == repo, + Tag.name == tag_name, + Tag.tag_kind == tag_kind_id), now_ts)).get() + if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag: + return tag + tag.lifetime_end = now_ts + tag.save() + except Tag.DoesNotExist: + pass + + try: + return Tag.create(repository=repo, manifest_list=manifest_list, linked_tag=linked_tag, + name=tag_name, lifetime_start=now_ts, lifetime_end=None, + tag_kind=tag_kind_id) + except IntegrityError: + msg = 'Tag with name %s and lifetime start %s under repository %s/%s already exists' + raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name)) + + +def get_or_initialize_tag(repo, tag_name, tag_kind="release"): + try: + return Tag.select().where(Tag.repository == repo, Tag.name == tag_name).get() + except Tag.DoesNotExist: + return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind)) + + +def get_tag(repo, tag_name, tag_kind="release"): + return tag_alive_oci(Tag.select() + .where(Tag.repository == repo, + Tag.name == tag_name, + Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get() + + +def delete_tag(repo, tag_name, tag_kind="release"): + tag_kind_id = Tag.tag_kind.get_id(tag_kind) + tag = tag_alive_oci(Tag.select() + .where(Tag.repository == repo, + Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get() + tag.lifetime_end = get_epoch_timestamp_ms() + tag.save() + return tag + + +def tag_exists(repo, tag_name, tag_kind="release"): + try: + get_tag(repo, tag_name, tag_kind) + return True + except Tag.DoesNotExist: + return False + + +def filter_tags_by_media_type(tag_query, media_type): + """ Return only available tag for a media_type. """ + media_type = manifest_media_type(media_type) + t = (tag_query + .join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list)) + .where(ManifestListManifest.media_type == ManifestListManifest.media_type.get_id(media_type))) + return t + + From 650723430bdd600e60b56e07005eb523edc8de2c Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:51:41 -0400 Subject: [PATCH 02/28] data.interfaces.appr: init --- data/interfaces/appr.py | 427 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 data/interfaces/appr.py diff --git a/data/interfaces/appr.py b/data/interfaces/appr.py new file mode 100644 index 000000000..504293398 --- /dev/null +++ b/data/interfaces/appr.py @@ -0,0 +1,427 @@ +from abc import ABCMeta, abstractmethod +from collections import namedtuple +from datetime import datetime + +import cnr.semver + +from cnr.exception import raise_package_not_found, raise_channel_not_found +from six import add_metaclass + + +from app import storage +from data import model, oci_model +from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel + + +class BlobDescriptor(namedtuple('Blob', ['mediaType', 'size', 'digest', 'urls'])): + """ BlobDescriptor describes a blob with its mediatype, size and digest. + A BlobDescriptor is used to retrieves the actual blob. + """ + + +class ChannelReleasesView(namedtuple('ChannelReleasesView', ['name', 'current', 'releases'])): + """ A channel is a pointer to a Release (current). + Releases are the previous tags pointed by channel (history). + """ + + +class ChannelView(namedtuple('ChannelView', ['name', 'current'])): + """ A channel is a pointer to a Release (current). + """ + + +class ApplicationSummaryView(namedtuple('ApplicationSummaryView', ['name', + 'namespace', + 'visibility', + 'default', + 'manifests', + 'channels', + 'releases', + 'updated_at', + 'created_at'])): + """ ApplicationSummaryView is an aggregated view of an application repository. + """ + + +class ApplicationManifest(namedtuple('ApplicationManifest', ['mediaType', + 'digest', + 'content'])): + """ ApplicationManifest embed the BlobDescriptor and some metadata around it. + An ApplicationManifest is content-addressable. + """ + + +class ApplicationRelease(namedtuple('ApplicationRelease', ['release', + 'name', + 'created_at', + 'manifest'])): + """ The ApplicationRelease associates an ApplicationManifest to a repository and release. + """ + + +@add_metaclass(ABCMeta) +class AppRegistryDataInterface(object): + """ Interface that represents all data store interactions required by a App Registry. + """ + + @abstractmethod + def _application(self, package_name): + pass + + @abstractmethod + def list_applications(self, namespace=None, media_type=None, search=None, username=None, + with_channels=False): + """ Lists all repositories that contain applications, with optional filtering to a specific + namespace and/or to those visible to a specific user. + + Returns: list of ApplicationSummaryView + """ + + @abstractmethod + def application_is_public(self, package_name): + """ + Returns true if the application is public + """ + + @abstractmethod + def create_application(self, package_name, visibility, owner): + """ Create a new app repository, owner is the user who creates it """ + + @abstractmethod + def application_exists(self, package_name): + """ Returns true if the application exists """ + + @abstractmethod + def basic_search(self, query, username=None): + """ Returns an array of matching application in the format: 'namespace/name' + Note: + * Only 'public' repositories are returned + """ + + # @TODO: Paginate + @abstractmethod + def list_releases(self, package_name, media_type=None): + """ Returns the list of all releases(names) of an AppRepository + Example: + >>> get_app_releases('ant31/rocketchat') + ['1.7.1', '1.7.0', '1.7.2'] + """ + + # @TODO: Paginate + @abstractmethod + def list_manifests(self, package_name, release=None): + """ Returns the list of all available manifests type of an Application across all releases or + for a specific one. + + Example: + >>> get_app_releases('ant31/rocketchat') + ['1.7.1', '1.7.0', '1.7.2'] + """ + + @abstractmethod + def fetch_release(self, package_name, release, media_type): + """ + Returns an ApplicationRelease + """ + + @abstractmethod + def store_blob(self, cnrblob, content_media_type): + """ + Upload the blob content to a storage location and creates a Blob entry in the DB. + + Returns a BlobDescriptor + """ + + @abstractmethod + def create_release(self, package, user, visibility, force=False): + """ Creates and returns an ApplicationRelease + - package is a data.model.Package object + - user is the owner of the package + - visibility is a string: 'public' or 'private' + """ + + @abstractmethod + def release_exists(self, package, release): + """ Return true if a release with that name already exist or + has existed (including deleted ones) + """ + pass + + @abstractmethod + def delete_release(self, package_name, release, media_type): + """ Remove/Delete an app-release from an app-repository. + It does not delete the entire app-repository, only a single release + """ + + @abstractmethod + def list_release_channels(self, package_name, release, active=True): + """ Returns a list of Channel that are/was pointing to a release. + If active is True, returns only active Channel (lifetime_end not null) + """ + + @abstractmethod + def channel_exists(self, package_name, channel_name): + """ Returns true if the channel with the given name exists under the matching package """ + + @abstractmethod + def update_channel(self, package_name, channel_name, release): + """ Append a new release to the Channel + Returns a new Channel with the release as current + """ + + @abstractmethod + def delete_channel(self, package_name, channel_name): + """ Delete a Channel, it doesn't delete/touch the ApplicationRelease pointed by the channel """ + + # @TODO: Paginate + @abstractmethod + def list_channels(self, package_name): + """ Returns all AppChannel for a package """ + + @abstractmethod + def fetch_channel(self, package_name, channel_name, with_releases=True): + """ Returns an Channel + Raises: ChannelNotFound, PackageNotFound + """ + + +def _split_package_name(package): + """ Returns the namespace and package-name """ + return package.split("/") + + +def _join_package_name(ns, name): + """ Returns a app-name in the 'namespace/name' format """ + return "%s/%s" % (ns, name) + + +def _timestamp_to_iso(timestamp, in_ms=True): + if in_ms: + timestamp = timestamp / 1000 + return datetime.fromtimestamp(timestamp).isoformat() + + +class OCIAppModel(AppRegistryDataInterface): + def _application(self, package): + ns, name = _split_package_name(package) + repo = model.repository.get_app_repository(ns, name) + if repo is None: + raise_package_not_found(package) + + def list_applications(self, namespace=None, media_type=None, search=None, username=None, + with_channels=False): + """ Lists all repositories that contain applications, with optional filtering to a specific + namespace and view a specific user. + """ + + views = [] + for repo in oci_model.package.list_packages_query(namespace, media_type, search, + username=username): + releases = [t.name for t in repo.tag_set_prefetch] + if not releases: + continue + available_releases = [str(x) for x in sorted(cnr.semver.versions(releases, False), + reverse=True)] + channels = None + if with_channels: + channels = [ChannelView(name=chan.name, current=chan.linked_tag.name) + for chan in oci_model.channel.get_repo_channels(repo)] + + app_name = _join_package_name(repo.namespace_user.username, repo.name) + manifests = self.list_manifests(app_name, available_releases[0]) + view = ApplicationSummaryView( + namespace=repo.namespace_user.username, + name=app_name, + visibility=repo.visibility.name, + default=available_releases[0], + channels=channels, + manifests=manifests, + releases=available_releases, + updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start), + created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start), + ) + views.append(view) + return views + + def application_is_public(self, package_name): + """ + Returns: + * True if the repository is public + """ + namespace, name = _split_package_name(package_name) + return model.repository.repository_is_public(namespace, name) + + def create_application(self, package_name, visibility, owner): + """ Create a new app repository, owner is the user who creates it """ + ns, name = _split_package_name(package_name) + model.repository.create_repository(ns, name, owner, visibility, "application") + + def application_exists(self, package_name): + """ Create a new app repository, owner is the user who creates it """ + ns, name = _split_package_name(package_name) + return model.repository.get_repository(ns, name, kind_filter='application') is not None + + def basic_search(self, query, username=None): + """ Returns an array of matching AppRepositories in the format: 'namespace/name' + Note: + * Only 'public' repositories are returned + + Todo: + * Filter results with readeable reposistory for the user (including visibilitys) + """ + return [_join_package_name(r.namespace_user.username, r.name) + for r in model.repository.get_app_search(lookup=query, username=username, limit=50)] + + + def list_releases(self, package_name, media_type=None): + """ Return the list of all releases of an Application + Example: + >>> get_app_releases('ant31/rocketchat') + ['1.7.1', '1.7.0', '1.7.2'] + + Todo: + * Paginate + """ + return oci_model.release.get_releases(self._application(package_name), media_type) + + def list_manifests(self, package_name, release=None): + """ Returns the list of all manifests of an Application. + + Todo: + * Paginate + """ + try: + repo = self._application(package_name) + return list(oci_model.manifest.get_manifest_types(repo, release)) + except (Repository.DoesNotExist, Tag.DoesNotExist): + raise_package_not_found(package_name, release) + + def fetch_release(self, package_name, release, media_type): + """ + Retrieves an AppRelease from it's repository-name and release-name + """ + repo = self._application(package_name) + try: + tag, manifest, blob = oci_model.release.get_app_release(repo, release, media_type) + created_at = _timestamp_to_iso(tag.lifetime_start) + + blob_descriptor = BlobDescriptor(digest=blob.digest, mediaType=blob.media_type.name, + size=blob.size, urls=[]) + + app_manifest = ApplicationManifest(digest=manifest.digest, mediaType=manifest.media_type.name, + content=blob_descriptor) + + app_release = ApplicationRelease(release=tag.name, + created_at=created_at, + name=package_name, + manifest=app_manifest) + return app_release + except (Tag.DoesNotExist, + Manifest.DoesNotExist, + Blob.DoesNotExist, + Repository.DoesNotExist, + MediaType.DoesNotExist): + raise_package_not_found(package_name, release, media_type) + + def store_blob(self, cnrblob, content_media_type): + fp = cnrblob.packager.io_file + path = cnrblob.upload_url(cnrblob.digest) + locations = storage.preferred_locations + storage.stream_write(locations, path, fp, 'application/x-gzip') + db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest, + cnrblob.size, + content_media_type, + locations) + return BlobDescriptor(mediaType=content_media_type, digest=db_blob.digest, size=db_blob.size, + urls=[]) + + def create_release(self, package, user, visibility, force=False): + """ Add an app-release to a repository + package is an instance of data.cnr.package.Package + """ + + data = package.manifest() + ns, name = package.namespace, package.name + repo = model.repository.get_or_create_repository(ns, name, user, visibility=visibility, + repo_kind='application') + tag_name = package.release + oci_model.release.create_app_release(repo, tag_name, package.manifest(), + data['content']['digest']) + + def delete_release(self, package_name, release, media_type): + """ Remove/Delete an app-release from an app-repository. + It does not delete the entire app-repository, only a single release + """ + repo = self._application(package_name) + try: + oci_model.release.delete_app_release(repo, release, media_type) + except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist): + raise_package_not_found(package_name, release, media_type) + + def release_exists(self, package, release): + """ Return true if a release with that name already exist or + have existed (include deleted ones) """ + + def channel_exists(self, package_name, channel_name): + """ Returns true if channel exists """ + repo = self._application(package_name) + return oci_model.tag.tag_exists(repo, channel_name, "channel") + + def delete_channel(self, package_name, channel_name): + """ Delete an AppChannel + Note: + It doesn't delete the AppReleases + """ + repo = self._application(package_name) + try: + oci_model.channel.delete_channel(repo, channel_name) + except (Channel.DoesNotExist, Tag.DoesNotExist): + raise_channel_not_found(package_name, channel_name) + + def list_channels(self, package_name): + """ Returns all AppChannel for a package """ + repo = self._application(package_name) + channels = oci_model.channel.get_repo_channels(repo) + return [ChannelView(name=chan.name, + current=chan.linked_tag.name) for chan in channels] + + def fetch_channel(self, package_name, channel_name, with_releases=True): + """ Returns an AppChannel """ + repo = self._application(package_name) + + try: + channel = oci_model.channel.get_channel(repo, channel_name) + except (Channel.DoesNotExist, Tag.DoesNotExist): + raise_channel_not_found(package_name, channel_name) + + if with_releases: + releases = oci_model.channel.get_channel_releases(repo, channel) + chanview = ChannelReleasesView(current=channel.linked_tag.name, + name=channel.name, + releases=[channel.linked_tag.name]+[c.name for c in releases]) + else: + chanview = ChannelView(current=channel.linked_tag.name, + name=channel.name) + + return chanview + + def list_release_channels(self, package_name, release, active=True): + repo = self._application(package_name) + try: + channels = oci_model.channel.get_tag_channels(repo, release, active=active) + return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels] + except (Channel.DoesNotExist, Tag.DoesNotExist): + raise_package_not_found(package_name, release) + + def update_channel(self, package_name, channel_name, release): + """ Append a new release to the AppChannel + Returns: + A new AppChannel with the release + """ + repo = self._application(package_name) + channel = oci_model.channel.create_or_update_channel(repo, channel_name, release) + return ChannelView(current=channel.linked_tag.name, + name=channel.name) + + +oci_app_model = OCIAppModel() From ddad957a5606e3b39614175aaa0fa55d4e49dbe5 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:51:55 -0400 Subject: [PATCH 03/28] data.model.repository: add app methods --- data/model/repository.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/data/model/repository.py b/data/model/repository.py index fbd47aeef..ecf5a8d75 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -58,6 +58,14 @@ def get_repository(namespace_name, repository_name, kind_filter=None): return None +def get_or_create_repository(namespace, name, creating_user, visibility='private', + repo_kind='image'): + repo = get_repository(namespace, name, repo_kind) + if repo is None: + repo = create_repository(namespace, name, creating_user, visibility, repo_kind) + return repo + + def purge_repository(namespace_name, repository_name): """ Completely delete all traces of the repository. Will return True upon complete success, and False upon partial or total failure. Garbage @@ -339,6 +347,20 @@ def get_visible_repositories(username, namespace=None, repo_kind='image', includ return query +def get_app_repository(namespace_name, repository_name): + """ Find an application repository. """ + try: + return _basequery.get_existing_repository(namespace_name, repository_name, + kind_filter='application') + except Repository.DoesNotExist: + return None + + +def get_app_search(lookup, username=None, limit=50): + return get_filtered_matching_repositories(lookup, filter_username=username, + repo_kind='application', offset=0, limit=limit) + + def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image', offset=0, limit=25): """ Returns an iterator of all repositories matching the given lookup value, with optional From 6fe6ea0bcbfe2d6d763a3524b712fe521d73a9b2 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:52:14 -0400 Subject: [PATCH 04/28] requirements: add CNR dependency --- requirements-nover.txt | 2 ++ requirements.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/requirements-nover.txt b/requirements-nover.txt index f03fe050d..af5b1f243 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -8,6 +8,7 @@ -e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab -e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib -e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython +-e git+https://github.com/app-registry/appr-server.git#egg=cnr-server APScheduler==3.0.5 Flask-Login Flask-Mail @@ -70,3 +71,4 @@ tzlocal xhtml2pdf recaptcha2 mockredispy +cnr diff --git a/requirements.txt b/requirements.txt index aed0f9659..8f3276c04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ aiowsgi==0.6 alembic==0.8.8 -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601 -e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode +-e git+https://github.com/app-registry/appr-server.git@v0.2.7-1#egg=appr_server APScheduler==3.0.5 autobahn==0.9.3.post3 Babel==2.3.4 From 102c671587a1d87f5ad9fbdb623500335812b3d9 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:52:45 -0400 Subject: [PATCH 05/28] endpoints.appr: init --- endpoints/appr/__init__.py | 72 +++++++++ endpoints/appr/cnr_backend.py | 160 ++++++++++++++++++++ endpoints/appr/registry.py | 269 ++++++++++++++++++++++++++++++++++ 3 files changed, 501 insertions(+) create mode 100644 endpoints/appr/__init__.py create mode 100644 endpoints/appr/cnr_backend.py create mode 100644 endpoints/appr/registry.py diff --git a/endpoints/appr/__init__.py b/endpoints/appr/__init__.py new file mode 100644 index 000000000..83c8bd060 --- /dev/null +++ b/endpoints/appr/__init__.py @@ -0,0 +1,72 @@ +import logging + +from functools import wraps + +from cnr.exception import UnauthorizedAccess +from flask import Blueprint + +from app import metric_queue +from auth.permissions import (AdministerRepositoryPermission, ReadRepositoryPermission, + ModifyRepositoryPermission) +from data import model # TODO: stop using model directly +from util.metrics.metricqueue import time_blueprint + + +appr_bp = Blueprint('appr', __name__) +time_blueprint(appr_bp, metric_queue) +logger = logging.getLogger(__name__) + + +def _raise_unauthorized(repository, scopes): + raise StandardError("Unauthorized acces to %s", repository) + + +def _get_reponame_kwargs(*args, **kwargs): + return [kwargs['namespace_name'], kwargs['repo_name']] + + +def require_repo_permission(permission_class, scopes=None, allow_public=False, + raise_method=_raise_unauthorized, + get_reponame_method=_get_reponame_kwargs): + def wrapper(func): + @wraps(func) + def wrapped(*args, **kwargs): + namespace_name, repo_name = get_reponame_method(*args, **kwargs) + + logger.debug('Checking permission %s for repo: %s/%s', permission_class, + namespace_name, repo_name) + permission = permission_class(namespace_name, repo_name) + if (permission.can() or + (allow_public and + model.repository.repository_is_public(namespace_name, repo_name))): + return func(*args, **kwargs) + repository = namespace_name + '/' + repo_name + raise_method(repository, scopes) + return wrapped + return wrapper + + +def _raise_method(repository, scopes): + raise UnauthorizedAccess("Unauthorized access for: %s" % repository, + {"package": repository, "scopes": scopes}) + + +def _get_reponame_kwargs(*args, **kwargs): + return [kwargs['namespace'], kwargs['package_name']] + + +require_app_repo_read = require_repo_permission(ReadRepositoryPermission, + scopes=['pull'], + allow_public=True, + raise_method=_raise_method, + get_reponame_method=_get_reponame_kwargs) + +require_app_repo_write = require_repo_permission(ModifyRepositoryPermission, + scopes=['pull', 'push'], + raise_method=_raise_method, + get_reponame_method=_get_reponame_kwargs) + +require_app_repo_admin = require_repo_permission(AdministerRepositoryPermission, + scopes=['pull', 'push'], + raise_method=_raise_method, + get_reponame_method=_get_reponame_kwargs) diff --git a/endpoints/appr/cnr_backend.py b/endpoints/appr/cnr_backend.py new file mode 100644 index 000000000..7449dc4c8 --- /dev/null +++ b/endpoints/appr/cnr_backend.py @@ -0,0 +1,160 @@ +import base64 + +from cnr.exception import raise_package_not_found +from cnr.models.blob_base import BlobBase +from cnr.models.channel_base import ChannelBase +from cnr.models.db_base import CnrDB +from cnr.models.package_base import PackageBase, manifest_media_type + +from app import storage +from data.interfaces.appr import oci_app_model +from data.oci_model import blob # TODO these calls should be through oci_app_model + + +class Blob(BlobBase): + @classmethod + def upload_url(cls, digest): + return "cnr/blobs/sha256/%s/%s" % (digest[0:2], digest) + + def save(self, content_media_type): + oci_app_model.store_blob(self, content_media_type) + + @classmethod + def delete(cls, package_name, digest): + pass + + @classmethod + def _fetch_b64blob(cls, package_name, digest): + blobpath = cls.upload_url(digest) + locations = blob.get_blob_locations(digest) + if not locations: + raise_package_not_found(package_name, digest) + return base64.b64encode(storage.get_content(locations, blobpath)) + + @classmethod + def download_url(cls, package_name, digest): + blobpath = cls.upload_url(digest) + locations = blob.get_blob_locations(digest) + if not locations: + raise_package_not_found(package_name, digest) + return storage.get_direct_download_url(locations, blobpath) + + +class Channel(ChannelBase): + """ CNR Channel model implemented against the Quay data model. """ + def __init__(self, name, package, current=None): + super(Channel, self).__init__(name, package, current=current) + self._channel_data = None + + def _exists(self): + """ Check if the channel is saved already """ + return oci_app_model.channel_exists(self.package, self.name) + + @classmethod + def get(cls, name, package): + chanview = oci_app_model.fetch_channel(package, name, with_releases=False) + return cls(name, package, chanview.current) + + def save(self): + oci_app_model.update_channel(self.package, self.name, self.current) + + def delete(self): + oci_app_model.delete_channel(self.package, self.name) + + @classmethod + def all(cls, package_name): + return [Channel(c.name, package_name, c.current) + for c in oci_app_model.list_channels(package_name)] + + @property + def _channel(self): + if self._channel_data is None: + self._channel_data = oci_app_model.fetch_channel(self.package, self.name) + return self._channel_data + + def releases(self): + """ Returns the list of versions """ + return self._channel.releases + + def _add_release(self, release): + return oci_app_model.update_channel(self.package, self.name, release)._asdict + + def _remove_release(self, release): + oci_app_model.delete_channel(self.package, self.name) + + +class Package(PackageBase): + """ CNR Package model implemented against the Quay data model. """ + + @classmethod + def _apptuple_to_dict(cls, apptuple): + return {'release': apptuple.release, + 'created_at': apptuple.created_at, + 'digest': apptuple.manifest.digest, + 'mediaType': apptuple.manifest.mediaType, + 'package': apptuple.name, + 'content': apptuple.manifest.content._asdict()} + + @classmethod + def create_repository(cls, package_name, visibility, owner): + oci_app_model.create_application(package_name, visibility, owner) + + @classmethod + def exists(cls, package_name): + return oci_app_model.application_exists(package_name) + + @classmethod + def all(cls, organization=None, media_type=None, search=None, username=None, **kwargs): + return [dict(x._asdict()) for x in oci_app_model.list_applications(namespace=organization, + media_type=media_type, + search=search, + username=username)] + + @classmethod + def _fetch(cls, package_name, release, media_type): + data = oci_app_model.fetch_release(package_name, release, manifest_media_type(media_type)) + return cls._apptuple_to_dict(data) + + @classmethod + def all_releases(cls, package_name, media_type=None): + return oci_app_model.list_releases(package_name, media_type) + + @classmethod + def search(cls, query, username=None): + return oci_app_model.basic_search(query, username=username) + + def _save(self, force=False, **kwargs): + user = kwargs['user'] + visibility = kwargs['visibility'] + oci_app_model.create_release(self, user, visibility, force) + + @classmethod + def _delete(cls, package_name, release, media_type): + oci_app_model.delete_release(package_name, release, manifest_media_type(media_type)) + + @classmethod + def isdeleted_release(cls, package, release): + return oci_app_model.release_exists(package, release) + + def channels(self, channel_class, iscurrent=True): + return [c.name for c in oci_app_model.list_release_channels(self.package, self.release, + active=iscurrent)] + + @classmethod + def manifests(cls, package, release=None): + return oci_app_model.list_manifests(package, release) + + @classmethod + def dump_all(cls, blob_cls): + raise NotImplementedError + + +class QuayDB(CnrDB): + """ Wrapper Class to embed all CNR Models """ + Channel = Channel + Package = Package + Blob = Blob + + @classmethod + def reset_db(cls, force=False): + pass diff --git a/endpoints/appr/registry.py b/endpoints/appr/registry.py new file mode 100644 index 000000000..a6e39d87c --- /dev/null +++ b/endpoints/appr/registry.py @@ -0,0 +1,269 @@ +import logging + +from base64 import b64encode + +import cnr + +from cnr.api.impl import registry as cnr_registry +from cnr.api.registry import repo_name, _pull +from cnr.exception import (CnrException, InvalidUsage, InvalidParams, InvalidRelease, + UnableToLockResource, UnauthorizedAccess, Unsupported, ChannelNotFound, + PackageAlreadyExists, PackageNotFound, PackageReleaseNotFound) +from flask import request, jsonify + +from auth.process import process_auth +from auth.auth_context import get_authenticated_user +from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermission +from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write +from endpoints.appr.cnr_backend import Package, Channel, Blob +from endpoints.decorators import anon_allowed, anon_protect + + +logger = logging.getLogger(__name__) + + +@appr_bp.errorhandler(Unsupported) +@appr_bp.errorhandler(PackageAlreadyExists) +@appr_bp.errorhandler(InvalidRelease) +@appr_bp.errorhandler(UnableToLockResource) +@appr_bp.errorhandler(UnauthorizedAccess) +@appr_bp.errorhandler(PackageNotFound) +@appr_bp.errorhandler(PackageReleaseNotFound) +@appr_bp.errorhandler(CnrException) +@appr_bp.errorhandler(InvalidUsage) +@appr_bp.errorhandler(InvalidParams) +@appr_bp.errorhandler(ChannelNotFound) +def render_error(error): + response = jsonify({"error": error.to_dict()}) + response.status_code = error.status_code + return response + + +@appr_bp.route("/version") +@anon_allowed +def version(): + return jsonify({"cnr-api": cnr.__version__}) + + +@appr_bp.route("/api/v1/users/login", methods=['POST']) +@anon_allowed +def login(): + """ + Todo: + * Implement better login protocol + """ + values = request.get_json(force=True, silent=True) + return jsonify({'token': "basic " + b64encode("%s:%s" % (values['user']['username'], + values['user']['password']))}) + + +# @TODO: Redirect to S3 url +@appr_bp.route( + "/api/v1/packages///blobs/sha256/", + methods=['GET'], + strict_slashes=False, +) +def blobs(namespace, package_name, digest): + reponame = repo_name(namespace, package_name) + data = cnr_registry.pull_blob(reponame, digest, blob_class=Blob) + json_format = request.args.get('format', None) == 'json' + return _pull(data, json_format=json_format) + + +@appr_bp.route("/api/v1/packages", methods=['GET'], strict_slashes=False) +@process_auth +@anon_protect +def list_packages(): + namespace = request.args.get('namespace', None) + media_type = request.args.get('media_type', None) + query = request.args.get('query', None) + user = get_authenticated_user() + username = None + if user: + username = user.username + result_data = cnr_registry.list_packages(namespace, + package_class=Package, + search=query, + media_type=media_type, + username=username) + return jsonify(result_data) + + +@appr_bp.route( + "/api/v1/packages////", + methods=['DELETE'], strict_slashes=False) +@process_auth +@require_app_repo_write +@anon_protect +def delete_package(namespace, package_name, release, media_type): + reponame = repo_name(namespace, package_name) + result = cnr_registry.delete_package(reponame, + release, + media_type, + package_class=Package) + return jsonify(result) + + +@appr_bp.route( + "/api/v1/packages////", + methods=['GET'], + strict_slashes=False +) +def show_package(namespace, package_name, release, media_type): + reponame = repo_name(namespace, package_name) + result = cnr_registry.show_package(reponame, release, + media_type, + channel_class=Channel, + package_class=Package) + return jsonify(result) + + + +@appr_bp.route("/api/v1/packages//", methods=['GET'], + strict_slashes=False) +@process_auth +@require_app_repo_read +@anon_protect +def show_package_releases(namespace, package_name): + reponame = repo_name(namespace, package_name) + media_type = request.args.get('media_type', None) + result = cnr_registry.show_package_releases(reponame, + media_type=media_type, + package_class=Package) + return jsonify(result) + + +@appr_bp.route("/api/v1/packages///", + methods=['GET'], strict_slashes=False) +@process_auth +@require_app_repo_read +@anon_protect +def show_package_releasse_manifests(namespace, package_name, release): + reponame = repo_name(namespace, package_name) + result = cnr_registry.show_package_manifests(reponame, + release, + package_class=Package) + return jsonify(result) + + +@appr_bp.route( + "/api/v1/packages/////pull", + methods=['GET'], + strict_slashes=False, +) +@process_auth +@require_app_repo_read +@anon_protect +def pull(namespace, package_name, release, media_type): + reponame = repo_name(namespace, package_name) + logger.info("pull %s", reponame) + data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob) + return _pull(data) + + +@appr_bp.route("/api/v1/packages//", methods=['POST'], + strict_slashes=False) +@process_auth +@anon_protect +def push(namespace, package_name): + reponame = repo_name(namespace, package_name) + values = request.get_json(force=True, silent=True) + release_version = values['release'] + media_type = values['media_type'] + force = request.args.get('force', 'false') == 'true' + private = values.get('visibility', 'public') + owner = get_authenticated_user() + if not Package.exists(reponame): + if not CreateRepositoryPermission(namespace).can(): + raise UnauthorizedAccess("Unauthorized access for: %s" % reponame, + {"package": reponame, "scopes": ['create']}) + Package.create_repository(reponame, private, owner) + + + if not ModifyRepositoryPermission(namespace, package_name).can(): + raise UnauthorizedAccess("Unauthorized access for: %s" % reponame, + {"package": reponame, "scopes": ['push']}) + + blob = Blob(reponame, values['blob']) + app_release = cnr_registry.push(reponame, release_version, media_type, blob, force, + package_class=Package, user=owner, visibility=private) + return jsonify(app_release) + + +@appr_bp.route("/api/v1/packages/search", methods=['GET'], strict_slashes=False) +@process_auth +@anon_protect +def search_packages(): + query = request.args.get("q") + user = get_authenticated_user() + username = None + if user: + username = user.username + + search_results = cnr_registry.search(query, Package, username=username) + return jsonify(search_results) + + +# CHANNELS +@appr_bp.route("/api/v1/packages///channels", + methods=['GET'], strict_slashes=False) +@process_auth +@require_app_repo_read +@anon_protect +def list_channels(namespace, package_name): + reponame = repo_name(namespace, package_name) + return jsonify(cnr_registry.list_channels(reponame, channel_class=Channel)) + + +@appr_bp.route("/api/v1/packages///channels/", methods=['GET'], strict_slashes=False) +@process_auth +@require_app_repo_read +@anon_protect +def show_channel(namespace, package_name, channel_name): + reponame = repo_name(namespace, package_name) + channel = cnr_registry.show_channel(reponame, channel_name, channel_class=Channel) + return jsonify(channel) + + +@appr_bp.route( + "/api/v1/packages///channels//", + methods=['POST'], + strict_slashes=False, +) +@process_auth +@require_app_repo_write +@anon_protect +def add_channel_release(namespace, package_name, channel_name, release): + reponame = repo_name(namespace, package_name) + result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel, + package_class=Package) + return jsonify(result) + + +@appr_bp.route( + "/api/v1/packages///channels//", + methods=['DELETE'], + strict_slashes=False, +) +@process_auth +@require_app_repo_write +@anon_protect +def delete_channel_release(namespace, package_name, channel_name, release): + reponame = repo_name(namespace, package_name) + result = cnr_registry.delete_channel_release(reponame, channel_name, release, + channel_class=Channel, package_class=Package) + return jsonify(result) + + +@appr_bp.route( + "/api/v1/packages///channels/", + methods=['DELETE'], + strict_slashes=False, +) +@process_auth +@require_app_repo_write +@anon_protect +def delete_channel(namespace, package_name, channel_name): + reponame = repo_name(namespace, package_name) + result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel) + return jsonify(result) From 4614419e531d74a5f14d39e9b322ccb78f8d1e9d Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:53:05 -0400 Subject: [PATCH 06/28] config: add app registry feature flag --- config.py | 3 +++ registry.py | 10 +++++++--- web.py | 1 - 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config.py b/config.py index 9da6fd30d..79cf034a6 100644 --- a/config.py +++ b/config.py @@ -231,6 +231,9 @@ class DefaultConfig(object): # Feature Flag: Whether to support signing FEATURE_SIGNING = False + # Feature Flag: Whether to enable support for App repositories. + FEATURE_APP_REGISTRY = True + # The namespace to use for library repositories. # Note: This must remain 'library' until Docker removes their hard-coded namespace for libraries. # See: https://github.com/docker/docker/blob/master/registry/session.go#L320 diff --git a/registry.py b/registry.py index df868242c..828bbe657 100644 --- a/registry.py +++ b/registry.py @@ -2,11 +2,12 @@ import logging import logging.config import os +import endpoints.decorated # Note: We need to import this module to make sure the decorators are registered. +import features + from app import app as application -# Note: We need to import this module to make sure the decorators are registered. -import endpoints.decorated - +from endpoints.appr import appr_bp from endpoints.v1 import v1_bp from endpoints.v2 import v2_bp @@ -15,3 +16,6 @@ if os.environ.get('DEBUGLOG') == 'true': application.register_blueprint(v1_bp, url_prefix='/v1') application.register_blueprint(v2_bp, url_prefix='/v2') + +if features.APP_REGISTRY: + application.register_blueprint(appr_bp, url_prefix='/cnr') diff --git a/web.py b/web.py index c07d1eea2..904dd9e98 100644 --- a/web.py +++ b/web.py @@ -2,7 +2,6 @@ import os import logging.config from app import app as application - from endpoints.api import api_bp from endpoints.bitbuckettrigger import bitbuckettrigger from endpoints.githubtrigger import githubtrigger From 23759a159263fda47288e1896c5fa96407d387d1 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 21:53:23 -0400 Subject: [PATCH 07/28] util.config.db: ensure blob locations sync on boot --- util/config/database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/config/database.py b/util/config/database.py index ea04dc5dc..585dfd13d 100644 --- a/util/config/database.py +++ b/util/config/database.py @@ -1,4 +1,4 @@ -from data import model +from data import model, oci_model def sync_database_with_config(config): @@ -7,3 +7,4 @@ def sync_database_with_config(config): location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys() if location_names: model.image.ensure_image_locations(*location_names) + oci_model.blob.ensure_blob_locations(*location_names) From cafde813221da5f088278ac09c8572e5ebc23a00 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 22:25:19 -0400 Subject: [PATCH 08/28] endpoints.appr.test: init --- endpoints/appr/test/test_api.py | 193 ++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 endpoints/appr/test/test_api.py diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py new file mode 100644 index 000000000..c013141fa --- /dev/null +++ b/endpoints/appr/test/test_api.py @@ -0,0 +1,193 @@ +import os +import shutil +import uuid + +import pytest + +from cnr.models.db_base import CnrDB +from cnr.tests.test_apiserver import BaseTestServer +from cnr.tests.test_models import CnrTestModels +from peewee import SqliteDatabase + +from app import app as application +from data.database import close_db_filter, User, db as database +from data.model import user, organization +from data.interfaces.appr import oci_app_model +from endpoints.appr import appr_bp +from endpoints.appr.cnr_backend import QuayDB, Channel, Package +from initdb import wipe_database, initialize_database, populate_database + + +# TODO: avoid direct usage of database +def create_org(namespace, owner): + try: + User.get(username=namespace) + except User.DoesNotExist: + organization.create_organization(namespace, "%s@test.com" % str(uuid.uuid1()), owner) + + +class ChannelTest(Channel): + @classmethod + def dump_all(cls, package_class=None): + result = [] + for repo in oci_app_model.list_applications(with_channels=True): + for chan in repo.channels: + result.append({'name': chan.name, 'current': chan.current, 'package': repo.name}) + return result + + +class PackageTest(Package): + def _save(self, force, **kwargs): + owner = user.get_user('devtable') + create_org(self.namespace, owner) + super(PackageTest, self)._save(force, user=owner, visibility="public") + + @classmethod + def create_repository(cls, package_name, visibility, owner): + ns, _ = package_name.split("/") + owner = user.get_user('devtable') + visibility = "public" + create_org(ns, owner) + return super(PackageTest, cls).create_repository(package_name, visibility, owner) + + @classmethod + def dump_all(cls, blob_cls): + result = [] + for repo in oci_app_model.list_applications(with_channels=True): + package_name = repo.name + for release in repo.releases: + for mtype in cls.manifests(package_name, release): + package = oci_app_model.fetch_release(package_name, release, mtype) + blob = blob_cls.get(package_name, package.manifest.content.digest) + data = cls._apptuple_to_dict(package) + data.pop('digest') + data['channels'] = [x.name for x in oci_app_model.list_release_channels(package_name, + package.release, + False)] + data['blob'] = blob.b64blob + result.append(data) + return result + + +@pytest.fixture(autouse=True) +def quaydb(monkeypatch): + monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Package', PackageTest) + monkeypatch.setattr('endpoints.appr.cnr_backend.Package', PackageTest) + monkeypatch.setattr('endpoints.appr.registry.Package', PackageTest) + monkeypatch.setattr('cnr.models.Package', PackageTest) + + monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Channel', ChannelTest) + # monkeypatch.setattr('data.cnrmodel.channel.Channel', ChannelTest) + monkeypatch.setattr('endpoints.appr.registry.Channel', ChannelTest) + monkeypatch.setattr('cnr.models.Channel', ChannelTest) + + +def seed_db(): + create_org("titi", user.get_user("devtable")) + + +@pytest.fixture() +def sqlitedb_file(tmpdir): + test_db_file = tmpdir.mkdir("quaydb").join("test.db") + return str(test_db_file) + + +@pytest.fixture(scope="module") +def init_db_path(tmpdir_factory): + sqlitedb_file_loc = str(tmpdir_factory.mktemp("data").join("test.db")) + sqlitedb = 'sqlite:///{0}'.format(sqlitedb_file_loc) + conf = {"TESTING": True, + "DEBUG": True, + "DB_URI": sqlitedb} + os.environ['TEST_DATABASE_URI'] = str(sqlitedb) + os.environ['DB_URI'] = str(sqlitedb) + database.initialize(SqliteDatabase(sqlitedb_file_loc)) + application.config.update(conf) + application.config.update({"DB_URI": sqlitedb}) + wipe_database() + initialize_database() + populate_database(minimal=True) + close_db_filter(None) + seed_db() + return str(sqlitedb_file_loc) + + +@pytest.fixture() +def database_uri(monkeypatch, init_db_path, sqlitedb_file): + shutil.copy2(init_db_path, sqlitedb_file) + database.initialize(SqliteDatabase(sqlitedb_file)) + db_path = 'sqlite:///{0}'.format(sqlitedb_file) + monkeypatch.setenv("DB_URI", db_path) + seed_db() + return db_path + + +@pytest.fixture() +def appconfig(database_uri): + conf = {"TESTING": True, + "DEBUG": True, + "DB_URI": database_uri} + return conf + + +@pytest.fixture() +def create_app(): + try: + application.register_blueprint(appr_bp, url_prefix='') + except: + pass + return application + + +@pytest.fixture(autouse=True) +def app(create_app, appconfig): + create_app.config.update(appconfig) + return create_app + + +@pytest.fixture() +def db(): + return CnrDB + + +class TestServerQuayDB(BaseTestServer): + DB_CLASS = QuayDB + + @property + def token(self): + return "basic ZGV2dGFibGU6cGFzc3dvcmQ=" + + def test_search_package_match(self, db_with_data1, client): + """ TODO: search cross namespace and package name """ + BaseTestServer.test_search_package_match(self, db_with_data1, client) + + @pytest.mark.xfail + def test_push_package_already_exists_force(self, db_with_data1, package_b64blob, client): + """ No force push implemented """ + BaseTestServer.test_push_package_already_exists_force(self, db_with_data1, package_b64blob, + client) + + @pytest.mark.xfail + def test_delete_channel_release_absent_release(self, db_with_data1, client): + BaseTestServer.test_delete_channel_release_absent_release(self, db_with_data1, client) + + +class TestQuayModels(CnrTestModels): + DB_CLASS = QuayDB + + @pytest.fixture(autouse=True) + def load_db(self, appconfig): + return appconfig + + @pytest.mark.xfail + def test_save_package_exists_force(self, newdb, package_b64blob): + CnrTestModels.test_save_package_exists_force(self, newdb, package_b64blob) + + @pytest.mark.xfail + def test_channel_delete_releases(self, db_with_data1): + """ Can't remove a release from the channel, only delete the channel entirely """ + CnrTestModels.test_channel_delete_releases(self, db_with_data1) + + @pytest.mark.xfail + def test_forbidden_db_reset(self, db_class): + pass From 3d1c1f9f39329a8a8e17b466fe2b6a5924164c3c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 22 Mar 2017 23:16:41 -0400 Subject: [PATCH 09/28] Add missing import for registry module --- registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry.py b/registry.py index 828bbe657..861420f08 100644 --- a/registry.py +++ b/registry.py @@ -7,7 +7,7 @@ import features from app import app as application -from endpoints.appr import appr_bp +from endpoints.appr import appr_bp, registry # registry needed to ensure routes registered from endpoints.v1 import v1_bp from endpoints.v2 import v2_bp From 82bcd45727a72513ff052ddf0cfe892c1726d590 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 23:41:31 -0400 Subject: [PATCH 10/28] endpoints: clarify repo access decorators --- endpoints/appr/__init__.py | 31 +--------------------------- endpoints/decorators.py | 41 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/endpoints/appr/__init__.py b/endpoints/appr/__init__.py index 83c8bd060..eda407b2e 100644 --- a/endpoints/appr/__init__.py +++ b/endpoints/appr/__init__.py @@ -8,7 +8,7 @@ from flask import Blueprint from app import metric_queue from auth.permissions import (AdministerRepositoryPermission, ReadRepositoryPermission, ModifyRepositoryPermission) -from data import model # TODO: stop using model directly +from endpoints.decorators import require_repo_permission from util.metrics.metricqueue import time_blueprint @@ -17,35 +17,6 @@ time_blueprint(appr_bp, metric_queue) logger = logging.getLogger(__name__) -def _raise_unauthorized(repository, scopes): - raise StandardError("Unauthorized acces to %s", repository) - - -def _get_reponame_kwargs(*args, **kwargs): - return [kwargs['namespace_name'], kwargs['repo_name']] - - -def require_repo_permission(permission_class, scopes=None, allow_public=False, - raise_method=_raise_unauthorized, - get_reponame_method=_get_reponame_kwargs): - def wrapper(func): - @wraps(func) - def wrapped(*args, **kwargs): - namespace_name, repo_name = get_reponame_method(*args, **kwargs) - - logger.debug('Checking permission %s for repo: %s/%s', permission_class, - namespace_name, repo_name) - permission = permission_class(namespace_name, repo_name) - if (permission.can() or - (allow_public and - model.repository.repository_is_public(namespace_name, repo_name))): - return func(*args, **kwargs) - repository = namespace_name + '/' + repo_name - raise_method(repository, scopes) - return wrapped - return wrapper - - def _raise_method(repository, scopes): raise UnauthorizedAccess("Unauthorized access for: %s" % repository, {"package": repository, "scopes": scopes}) diff --git a/endpoints/decorators.py b/endpoints/decorators.py index b032b624a..8c8af2a52 100644 --- a/endpoints/decorators.py +++ b/endpoints/decorators.py @@ -1,10 +1,19 @@ """ Various decorators for endpoint and API handlers. """ -import features +import logging + +from functools import wraps + from flask import abort + +import features + from auth.auth_context import (get_validated_oauth_token, get_authenticated_user, get_validated_token, get_grant_context) -from functools import wraps +from data import model # TODO: stop using model directly + + +logger = logging.getLogger(__name__) def anon_allowed(func): @@ -34,3 +43,31 @@ def check_anon_protection(func): abort(401) return wrapper + +def _raise_unauthorized(repository, scopes): + raise StandardError("Unauthorized acces to %s", repository) + + +def _get_reponame_kwargs(*args, **kwargs): + return [kwargs['namespace_name'], kwargs['repo_name']] + + +def require_repo_permission(permission_class, scopes=None, allow_public=False, + raise_method=_raise_unauthorized, + get_reponame_method=_get_reponame_kwargs): + def wrapper(func): + @wraps(func) + def wrapped(*args, **kwargs): + namespace_name, repo_name = get_reponame_method(*args, **kwargs) + + logger.debug('Checking permission %s for repo: %s/%s', permission_class, + namespace_name, repo_name) + permission = permission_class(namespace_name, repo_name) + if (permission.can() or + (allow_public and + model.repository.repository_is_public(namespace_name, repo_name))): + return func(*args, **kwargs) + repository = namespace_name + '/' + repo_name + raise_method(repository, scopes) + return wrapped + return wrapper From 959549c597f39debf0c0277766990dce8771366c Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 23:42:02 -0400 Subject: [PATCH 11/28] requirements: use HEAD of CNR for proper mimetype --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f3276c04..3c5f851ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiowsgi==0.6 alembic==0.8.8 -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601 -e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode --e git+https://github.com/app-registry/appr-server.git@v0.2.7-1#egg=appr_server +-e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server APScheduler==3.0.5 autobahn==0.9.3.post3 Babel==2.3.4 From 6dfd1ef660798173eebf8a85059a4070a6a29720 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 23:42:19 -0400 Subject: [PATCH 12/28] endpoints.appr.test: include CNR fixtures --- endpoints/appr/test/test_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py index c013141fa..988f38f2c 100644 --- a/endpoints/appr/test/test_api.py +++ b/endpoints/appr/test/test_api.py @@ -6,6 +6,7 @@ import pytest from cnr.models.db_base import CnrDB from cnr.tests.test_apiserver import BaseTestServer +from cnr.tests.conftest import * from cnr.tests.test_models import CnrTestModels from peewee import SqliteDatabase @@ -13,7 +14,7 @@ from app import app as application from data.database import close_db_filter, User, db as database from data.model import user, organization from data.interfaces.appr import oci_app_model -from endpoints.appr import appr_bp +from endpoints.appr import appr_bp, registry from endpoints.appr.cnr_backend import QuayDB, Channel, Package from initdb import wipe_database, initialize_database, populate_database From 3d0e63d8e5da4885cb9ca51c4abf8ab25eb9b15e Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 22 Mar 2017 23:53:03 -0400 Subject: [PATCH 13/28] endpoints.appr.decorators: isolate appr decorators --- endpoints/appr/__init__.py | 2 +- endpoints/appr/decorators.py | 37 ++++++++++++++++++++++++++++++++++++ endpoints/decorators.py | 33 -------------------------------- 3 files changed, 38 insertions(+), 34 deletions(-) create mode 100644 endpoints/appr/decorators.py diff --git a/endpoints/appr/__init__.py b/endpoints/appr/__init__.py index eda407b2e..cbf02e4c7 100644 --- a/endpoints/appr/__init__.py +++ b/endpoints/appr/__init__.py @@ -8,7 +8,7 @@ from flask import Blueprint from app import metric_queue from auth.permissions import (AdministerRepositoryPermission, ReadRepositoryPermission, ModifyRepositoryPermission) -from endpoints.decorators import require_repo_permission +from endpoints.appr.decorators import require_repo_permission from util.metrics.metricqueue import time_blueprint diff --git a/endpoints/appr/decorators.py b/endpoints/appr/decorators.py new file mode 100644 index 000000000..4d3efd783 --- /dev/null +++ b/endpoints/appr/decorators.py @@ -0,0 +1,37 @@ +import logging + +from functools import wraps + +from data import model + + +logger = logging.getLogger(__name__) + + +def _raise_unauthorized(repository, scopes): + raise StandardError("Unauthorized acces to %s", repository) + + +def _get_reponame_kwargs(*args, **kwargs): + return [kwargs['namespace_name'], kwargs['repo_name']] + + +def require_repo_permission(permission_class, scopes=None, allow_public=False, + raise_method=_raise_unauthorized, + get_reponame_method=_get_reponame_kwargs): + def wrapper(func): + @wraps(func) + def wrapped(*args, **kwargs): + namespace_name, repo_name = get_reponame_method(*args, **kwargs) + + logger.debug('Checking permission %s for repo: %s/%s', permission_class, + namespace_name, repo_name) + permission = permission_class(namespace_name, repo_name) + if (permission.can() or + (allow_public and + model.repository.repository_is_public(namespace_name, repo_name))): + return func(*args, **kwargs) + repository = namespace_name + '/' + repo_name + raise_method(repository, scopes) + return wrapped + return wrapper diff --git a/endpoints/decorators.py b/endpoints/decorators.py index 8c8af2a52..3cc374db3 100644 --- a/endpoints/decorators.py +++ b/endpoints/decorators.py @@ -1,7 +1,5 @@ """ Various decorators for endpoint and API handlers. """ -import logging - from functools import wraps from flask import abort @@ -13,9 +11,6 @@ from auth.auth_context import (get_validated_oauth_token, get_authenticated_user from data import model # TODO: stop using model directly -logger = logging.getLogger(__name__) - - def anon_allowed(func): """ Marks a method to allow anonymous access where it would otherwise be disallowed. """ func.__anon_allowed = True @@ -43,31 +38,3 @@ def check_anon_protection(func): abort(401) return wrapper - -def _raise_unauthorized(repository, scopes): - raise StandardError("Unauthorized acces to %s", repository) - - -def _get_reponame_kwargs(*args, **kwargs): - return [kwargs['namespace_name'], kwargs['repo_name']] - - -def require_repo_permission(permission_class, scopes=None, allow_public=False, - raise_method=_raise_unauthorized, - get_reponame_method=_get_reponame_kwargs): - def wrapper(func): - @wraps(func) - def wrapped(*args, **kwargs): - namespace_name, repo_name = get_reponame_method(*args, **kwargs) - - logger.debug('Checking permission %s for repo: %s/%s', permission_class, - namespace_name, repo_name) - permission = permission_class(namespace_name, repo_name) - if (permission.can() or - (allow_public and - model.repository.repository_is_public(namespace_name, repo_name))): - return func(*args, **kwargs) - repository = namespace_name + '/' + repo_name - raise_method(repository, scopes) - return wrapped - return wrapper From bdda74d6dfa641d82e96b31a4db4162af95e62b7 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 22 Mar 2017 23:45:46 -0400 Subject: [PATCH 14/28] Make sure GC checks new Blob table as well before deleting CAS storage --- data/model/storage.py | 21 ++++++++++++++------ test/test_gc.py | 46 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/data/model/storage.py b/data/model/storage.py index 79ab529bb..112b9cc5f 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -8,7 +8,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn DataModelException, _basequery) from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation, ImageStorageTransformation, ImageStorageSignature, - ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, + ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob, ensure_under_transaction) @@ -95,15 +95,24 @@ def garbage_collect_storage(storage_id_whitelist): unreferenced_checksums = set() if content_checksums: + # Check the current image storage. query = (ImageStorage .select(ImageStorage.content_checksum) .where(ImageStorage.content_checksum << list(content_checksums))) - referenced_checksums = set([image_storage.content_checksum for image_storage in query]) - if referenced_checksums: - logger.warning('GC attempted to remove CAS checksums %s, which are still referenced', - referenced_checksums) + is_referenced_checksums = set([image_storage.content_checksum for image_storage in query]) + if is_referenced_checksums: + logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced', + is_referenced_checksums) - unreferenced_checksums = content_checksums - referenced_checksums + # Check the new Blob table as well. + query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums)) + blob_referenced_checksums = set([blob.digest for blob in query]) + if blob_referenced_checksums: + logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced', + blob_referenced_checksums) + + unreferenced_checksums = (content_checksums - blob_referenced_checksums - + is_referenced_checksums) # Return all placements for all image storages found not at a CAS path or with a content # checksum that is referenced. diff --git a/test/test_gc.py b/test/test_gc.py index c8ed5ecaa..1e5369675 100644 --- a/test/test_gc.py +++ b/test/test_gc.py @@ -8,7 +8,7 @@ from playhouse.test_utils import assert_query_count from app import app, storage from initdb import setup_database_for_testing, finished_database_for_testing from data import model, database -from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel +from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, Blob ADMIN_ACCESS_USER = 'devtable' @@ -190,6 +190,9 @@ class TestGarbageCollection(unittest.TestCase): if storage_row.cas_path: storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum)) + for blob_row in Blob.select(): + storage.get_content({preferred}, storage.blob_path(blob_row.digest)) + def test_has_garbage(self): """ Remove all existing repositories, then add one without garbage, check, then add one with garbage, and check again. @@ -502,7 +505,48 @@ class TestGarbageCollection(unittest.TestCase): # Ensure the CAS path still exists. self.assertTrue(storage.exists({preferred}, storage.blob_path(digest))) + def test_images_shared_cas_with_new_blob_table(self): + """ A repository with a tag and image that shares its CAS path with a record in the new Blob + table. Deleting the first tag should delete the first image, and its storage, but not the + file in storage, as it shares its CAS path with the blob row. + """ + with self.assert_gc_integrity(expect_storage_removed=True): + repository = self.createRepository() + # Create two image storage records with the same content checksum. + content = 'hello world' + digest = 'sha256:' + hashlib.sha256(content).hexdigest() + preferred = storage.preferred_locations[0] + storage.put_content({preferred}, storage.blob_path(digest), content) + + media_type = database.MediaType.get(name='text/plain') + + is1 = database.ImageStorage.create(content_checksum=digest, uploading=False) + is2 = database.Blob.create(digest=digest, size=0, media_type=media_type) + + location = database.ImageStorageLocation.get(name=preferred) + database.ImageStoragePlacement.create(location=location, storage=is1) + + # Ensure the CAS path exists. + self.assertTrue(storage.exists({preferred}, storage.blob_path(digest))) + + # Create the image in the repository, and the tag. + first_image = Image.create(docker_image_id='i1', + repository=repository, storage=is1, + ancestors='/') + + model.tag.store_tag_manifest(repository.namespace_user.username, repository.name, + 'first', first_image.docker_image_id, + 'sha:someshahere1', '{}') + + self.assertNotDeleted(repository, 'i1') + + # Delete the tag. + self.deleteTag(repository, 'first') + self.assertDeleted(repository, 'i1') + + # Ensure the CAS path still exists, as it is referenced by the Blob table. + self.assertTrue(storage.exists({preferred}, storage.blob_path(digest))) if __name__ == '__main__': unittest.main() From 4c34b00b38848358ef24ef01667be9c2d46b6c1c Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 22 Mar 2017 23:46:05 -0400 Subject: [PATCH 15/28] Prevent CNR methods from auth-ing on non-app repos --- endpoints/appr/decorators.py | 7 +++++++ endpoints/appr/test/test_decorators.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 endpoints/appr/test/test_decorators.py diff --git a/endpoints/appr/decorators.py b/endpoints/appr/decorators.py index 4d3efd783..857c3f18e 100644 --- a/endpoints/appr/decorators.py +++ b/endpoints/appr/decorators.py @@ -2,6 +2,8 @@ import logging from functools import wraps +from flask import abort + from data import model @@ -24,6 +26,11 @@ def require_repo_permission(permission_class, scopes=None, allow_public=False, def wrapped(*args, **kwargs): namespace_name, repo_name = get_reponame_method(*args, **kwargs) + image_repo = model.repository.get_repository(namespace_name, repo_name, kind_filter='image') + if image_repo is not None: + logger.debug('Tried to invoked a CNR method on an image repository') + abort(501) + logger.debug('Checking permission %s for repo: %s/%s', permission_class, namespace_name, repo_name) permission = permission_class(namespace_name, repo_name) diff --git a/endpoints/appr/test/test_decorators.py b/endpoints/appr/test/test_decorators.py new file mode 100644 index 000000000..0e5565da3 --- /dev/null +++ b/endpoints/appr/test/test_decorators.py @@ -0,0 +1,19 @@ +import pytest + +from werkzeug.exceptions import NotImplemented as NIE + +from data import model +from endpoints.test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file +from endpoints.appr import require_app_repo_read + +def test_require_app_repo_read(app): + called = [False] + + # Ensure that trying to read an *image* repository fails. + @require_app_repo_read + def empty(**kwargs): + called[0] = True + + with pytest.raises(NIE): + empty(namespace='devtable', package_name='simple') + assert not called[0] From 069208f2f1edcbcf317350ad4ee045a0512e06c5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 00:01:37 -0400 Subject: [PATCH 16/28] Break out repo kind checking into its own decorator We then use that decorator both in the API and in the permissions check decorator --- endpoints/appr/decorators.py | 21 +++++++++++++++------ endpoints/appr/registry.py | 4 ++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/endpoints/appr/decorators.py b/endpoints/appr/decorators.py index 857c3f18e..54754c05d 100644 --- a/endpoints/appr/decorators.py +++ b/endpoints/appr/decorators.py @@ -18,19 +18,28 @@ def _get_reponame_kwargs(*args, **kwargs): return [kwargs['namespace_name'], kwargs['repo_name']] +def disallow_for_image_repository(get_reponame_method=_get_reponame_kwargs): + def wrapper(func): + @wraps(func) + def wrapped(*args, **kwargs): + namespace_name, repo_name = get_reponame_method(*args, **kwargs) + image_repo = model.repository.get_repository(namespace_name, repo_name, kind_filter='image') + if image_repo is not None: + logger.debug('Tried to invoked a CNR method on an image repository') + abort(501) + return func(*args, **kwargs) + return wrapped + return wrapper + + def require_repo_permission(permission_class, scopes=None, allow_public=False, raise_method=_raise_unauthorized, get_reponame_method=_get_reponame_kwargs): def wrapper(func): @wraps(func) + @disallow_for_image_repository(get_reponame_method=get_reponame_method) def wrapped(*args, **kwargs): namespace_name, repo_name = get_reponame_method(*args, **kwargs) - - image_repo = model.repository.get_repository(namespace_name, repo_name, kind_filter='image') - if image_repo is not None: - logger.debug('Tried to invoked a CNR method on an image repository') - abort(501) - logger.debug('Checking permission %s for repo: %s/%s', permission_class, namespace_name, repo_name) permission = permission_class(namespace_name, repo_name) diff --git a/endpoints/appr/registry.py b/endpoints/appr/registry.py index a6e39d87c..c44e30cb7 100644 --- a/endpoints/appr/registry.py +++ b/endpoints/appr/registry.py @@ -15,6 +15,7 @@ from auth.process import process_auth from auth.auth_context import get_authenticated_user from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermission from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write +from endpoints.appr.decorators import disallow_for_image_repository from endpoints.appr.cnr_backend import Package, Channel, Blob from endpoints.decorators import anon_allowed, anon_protect @@ -109,6 +110,8 @@ def delete_package(namespace, package_name, release, media_type): methods=['GET'], strict_slashes=False ) +@process_auth +@require_app_repo_read def show_package(namespace, package_name, release, media_type): reponame = repo_name(namespace, package_name) result = cnr_registry.show_package(reponame, release, @@ -163,6 +166,7 @@ def pull(namespace, package_name, release, media_type): @appr_bp.route("/api/v1/packages//", methods=['POST'], strict_slashes=False) +@disallow_for_image_repository() @process_auth @anon_protect def push(namespace, package_name): From e872c310d0f7186cb59df875a8c15bc652dae189 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 23 Mar 2017 00:21:21 -0400 Subject: [PATCH 17/28] data.oci_model: fix imports --- data/oci_model/__init__.py | 9 +++++++++ data/oci_model/channel.py | 2 +- data/oci_model/manifest.py | 18 ++++++++--------- data/oci_model/package.py | 8 +++++--- data/oci_model/release.py | 40 ++++++++++++++++++++------------------ 5 files changed, 45 insertions(+), 32 deletions(-) diff --git a/data/oci_model/__init__.py b/data/oci_model/__init__.py index e69de29bb..94b4f9bb8 100644 --- a/data/oci_model/__init__.py +++ b/data/oci_model/__init__.py @@ -0,0 +1,9 @@ +from data.oci_model import ( + blob, + channel, + manifest, + manifest_list, + package, + release, + tag, +) diff --git a/data/oci_model/channel.py b/data/oci_model/channel.py index f8d169661..42548ba8f 100644 --- a/data/oci_model/channel.py +++ b/data/oci_model/channel.py @@ -1,5 +1,5 @@ -from data.oci_model import tag as tag_model from data.database import Tag, Channel +from data.oci_model import tag as tag_model def get_channel_releases(repo, channel): diff --git a/data/oci_model/manifest.py b/data/oci_model/manifest.py index bb690ddc4..247324256 100644 --- a/data/oci_model/manifest.py +++ b/data/oci_model/manifest.py @@ -4,8 +4,8 @@ import json from cnr.models.package_base import get_media_type -from data import oci_model from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag +from data.oci_model import tag as tag_model logger = logging.getLogger(__name__) @@ -38,14 +38,14 @@ def get_or_create_manifest(manifest_json, media_type_name): def get_manifest_types(repo, release=None): """ Returns an array of MediaTypes.name for a repo, can filter by tag """ - query = oci_model.tag.tag_alive_oci(Tag - .select(MediaType.name) - .join(ManifestListManifest, - on=(ManifestListManifest.manifest_list == Tag.manifest_list)) - .join(MediaType, - on=(ManifestListManifest.media_type == MediaType.id)) - .where(Tag.repository == repo, - Tag.tag_kind == Tag.tag_kind.get_id('release'))) + query = tag_model.tag_alive_oci(Tag + .select(MediaType.name) + .join(ManifestListManifest, + on=(ManifestListManifest.manifest_list == Tag.manifest_list)) + .join(MediaType, + on=(ManifestListManifest.media_type == MediaType.id)) + .where(Tag.repository == repo, + Tag.tag_kind == Tag.tag_kind.get_id('release'))) if release: query = query.where(Tag.name == release) diff --git a/data/oci_model/package.py b/data/oci_model/package.py index 8ebecc722..cf01256f6 100644 --- a/data/oci_model/package.py +++ b/data/oci_model/package.py @@ -1,8 +1,10 @@ from cnr.models.package_base import get_media_type, manifest_media_type from peewee import prefetch -from data import model, oci_model + +from data import model from data.database import Repository, Namespace, Tag, ManifestListManifest +from data.oci_model import tag as tag_model def list_packages_query(namespace=None, media_type=None, search_query=None, username=None): @@ -32,9 +34,9 @@ def list_packages_query(namespace=None, media_type=None, search_query=None, user .order_by(Tag.lifetime_start)) if media_type: - tag_query = oci_model.tag.filter_tags_by_media_type(tag_query, media_type) + tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type) - tag_query = oci_model.tag.tag_alive_oci(tag_query) + tag_query = tag_model.tag_alive_oci(tag_query) query = prefetch(repo_query, tag_query) return query diff --git a/data/oci_model/release.py b/data/oci_model/release.py index 6624900ab..8bdbbb16f 100644 --- a/data/oci_model/release.py +++ b/data/oci_model/release.py @@ -3,9 +3,11 @@ import bisect from cnr.exception import PackageAlreadyExists from cnr.models.package_base import manifest_media_type -from data import oci_model from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag, ManifestListManifest, Blob, ManifestBlob) +from data.oci_model import (blob as blob_model, manifest as manifest_model, + manifest_list as manifest_list_model, + tag as tag_model) LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json' @@ -14,7 +16,7 @@ SCHEMA_VERSION = 'v0' def get_app_release(repo, tag_name, media_type): """ Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """ - tag = oci_model.tag.get_tag(repo, tag_name, tag_kind='release') + tag = tag_model.get_tag(repo, tag_name, tag_kind='release') media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set .join(Manifest) @@ -33,16 +35,16 @@ def create_app_release(repo, tag_name, manifest, digest): """ with db_transaction(): # Create/get the package manifest - manifest = oci_model.manifest.get_or_create_manifest(manifest, manifest['mediaType']) + manifest = manifest_model.get_or_create_manifest(manifest, manifest['mediaType']) # get the tag - tag = oci_model.tag.get_or_initialize_tag(repo, tag_name) + tag = tag_model.get_or_initialize_tag(repo, tag_name) if tag.manifest_list is None: tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE), schema_version=SCHEMA_VERSION, manifest_list_json=[]) - elif oci_model.tag.tag_media_type_exists(tag, manifest.media_type): + elif tag_model.tag_media_type_exists(tag, manifest.media_type): raise PackageAlreadyExists("package exists already") list_json = tag.manifest_list.manifest_list_json @@ -53,12 +55,12 @@ def create_app_release(repo, tag_name, manifest, digest): insert_point = bisect.bisect_left(list_manifest_ids, manifest.id) list_json.insert(insert_point, manifest.manifest_json) list_manifest_ids.insert(insert_point, manifest.id) - manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, + manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, SCHEMA_VERSION) - oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) + manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) - tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, - tag_kind="release") + tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag_kind="release") blob_digest = digest try: @@ -67,7 +69,7 @@ def create_app_release(repo, tag_name, manifest, digest): .join(Blob) .where(ManifestBlob.manifest == manifest, Blob.digest == blob_digest).get()) except ManifestBlob.DoesNotExist: - blob = oci_model.blob.get_blob(blob_digest) + blob = blob_model.get_blob(blob_digest) ManifestBlob.create(manifest=manifest, blob=blob) return tag @@ -77,7 +79,7 @@ def delete_app_release(repo, tag_name, media_type): media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type)) with db_transaction(): - tag = oci_model.tag.get_tag(repo, tag_name) + tag = tag_model.get_tag(repo, tag_name) manifest_list = tag.manifest_list list_json = manifest_list.manifest_list_json mlm_query = (ManifestListManifest @@ -96,12 +98,12 @@ def delete_app_release(repo, tag_name, media_type): tag.lifetime_end = get_epoch_timestamp() tag.save() else: - manifestlist = oci_model.manifest_list.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, - SCHEMA_VERSION) - oci_model.manifest_list.create_manifestlistmanifest(manifestlist, list_manifest_ids, - list_json) - tag = oci_model.tag.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, - tag_kind="release") + manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, + SCHEMA_VERSION) + manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, + list_json) + tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, + tag_kind="release") return tag @@ -112,5 +114,5 @@ def get_releases(repo, media_type=None): .where(Tag.repository == repo, Tag.tag_kind == Tag.tag_kind.get_id("release"))) if media_type: - release_query = oci_model.tag.filter_tags_by_media_type(release_query, media_type) - return [t.name for t in oci_model.tag.tag_alive_oci(release_query)] + release_query = tag_model.filter_tags_by_media_type(release_query, media_type) + return [t.name for t in tag_model.tag_alive_oci(release_query)] From 1145651b7a76d1f3eac33f0ee02751e35ccbd303 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 00:37:39 -0400 Subject: [PATCH 18/28] Work towards fixing tests --- endpoints/appr/decorators.py | 2 +- endpoints/appr/test/test_api.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/endpoints/appr/decorators.py b/endpoints/appr/decorators.py index 54754c05d..5d6e78589 100644 --- a/endpoints/appr/decorators.py +++ b/endpoints/appr/decorators.py @@ -15,7 +15,7 @@ def _raise_unauthorized(repository, scopes): def _get_reponame_kwargs(*args, **kwargs): - return [kwargs['namespace_name'], kwargs['repo_name']] + return [kwargs['namespace'], kwargs['package_name']] def disallow_for_image_repository(get_reponame_method=_get_reponame_kwargs): diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py index 988f38f2c..2433588d6 100644 --- a/endpoints/appr/test/test_api.py +++ b/endpoints/appr/test/test_api.py @@ -18,6 +18,8 @@ from endpoints.appr import appr_bp, registry from endpoints.appr.cnr_backend import QuayDB, Channel, Package from initdb import wipe_database, initialize_database, populate_database +application.register_blueprint(appr_bp, url_prefix='/cnr') + # TODO: avoid direct usage of database def create_org(namespace, owner): From 3277fe9b4e834ddd5c5013da77572bc8b35863ba Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 00:51:54 -0400 Subject: [PATCH 19/28] Make sure repository names in APPR match regex --- endpoints/appr/registry.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/endpoints/appr/registry.py b/endpoints/appr/registry.py index c44e30cb7..731ecc8d7 100644 --- a/endpoints/appr/registry.py +++ b/endpoints/appr/registry.py @@ -18,6 +18,7 @@ from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_writ from endpoints.appr.decorators import disallow_for_image_repository from endpoints.appr.cnr_backend import Package, Channel, Blob from endpoints.decorators import anon_allowed, anon_protect +from util.names import REPOSITORY_NAME_REGEX logger = logging.getLogger(__name__) @@ -171,6 +172,11 @@ def pull(namespace, package_name, release, media_type): @anon_protect def push(namespace, package_name): reponame = repo_name(namespace, package_name) + + if not REPOSITORY_NAME_REGEX.match(package_name): + logger.debug('Found invalid repository name CNR push: %s', reponame) + raise InvalidUsage() + values = request.get_json(force=True, silent=True) release_version = values['release'] media_type = values['media_type'] From e7d78499371de9a65f080a71528935cfb2aa2c24 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 00:55:36 -0400 Subject: [PATCH 20/28] Make sure channels and releases match the tag regex --- endpoints/appr/registry.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/endpoints/appr/registry.py b/endpoints/appr/registry.py index 731ecc8d7..2a6c08d60 100644 --- a/endpoints/appr/registry.py +++ b/endpoints/appr/registry.py @@ -18,7 +18,7 @@ from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_writ from endpoints.appr.decorators import disallow_for_image_repository from endpoints.appr.cnr_backend import Package, Channel, Blob from endpoints.decorators import anon_allowed, anon_protect -from util.names import REPOSITORY_NAME_REGEX +from util.names import REPOSITORY_NAME_REGEX, TAG_REGEX logger = logging.getLogger(__name__) @@ -244,6 +244,14 @@ def show_channel(namespace, package_name, channel_name): @require_app_repo_write @anon_protect def add_channel_release(namespace, package_name, channel_name, release): + if not TAG_REGEX.match(channel_name): + logger.debug('Found invalid channel name CNR add channel release: %s', channel_name) + raise InvalidUsage() + + if not TAG_REGEX.match(release): + logger.debug('Found invalid release name CNR add channel release: %s', release) + raise InvalidUsage() + reponame = repo_name(namespace, package_name) result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel, package_class=Package) @@ -259,6 +267,14 @@ def add_channel_release(namespace, package_name, channel_name, release): @require_app_repo_write @anon_protect def delete_channel_release(namespace, package_name, channel_name, release): + if not TAG_REGEX.match(channel_name): + logger.debug('Found invalid channel name CNR delete channel release: %s', channel_name) + raise InvalidUsage() + + if not TAG_REGEX.match(release): + logger.debug('Found invalid release name CNR delete channel release: %s', release) + raise InvalidUsage() + reponame = repo_name(namespace, package_name) result = cnr_registry.delete_channel_release(reponame, channel_name, release, channel_class=Channel, package_class=Package) @@ -274,6 +290,10 @@ def delete_channel_release(namespace, package_name, channel_name, release): @require_app_repo_write @anon_protect def delete_channel(namespace, package_name, channel_name): + if not TAG_REGEX.match(channel_name): + logger.debug('Found invalid channel name CNR delete channel: %s', channel_name) + raise InvalidUsage() + reponame = repo_name(namespace, package_name) result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel) return jsonify(result) From 2bdd3d4fa1c7acd86396fb4282cafc46b41182f4 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 23 Mar 2017 00:58:31 -0400 Subject: [PATCH 21/28] data.oci_model.tag: add missing import --- data/oci_model/tag.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/oci_model/tag.py b/data/oci_model/tag.py index 16400ea25..4128ddc7d 100644 --- a/data/oci_model/tag.py +++ b/data/oci_model/tag.py @@ -1,5 +1,6 @@ import logging +from cnr.models.package_base import manifest_media_type from peewee import IntegrityError from data.model import (db_transaction, TagAlreadyCreatedException) From d20ff785e68b825342d97ad6a0f61a605e3025b4 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 23 Mar 2017 10:46:04 -0400 Subject: [PATCH 22/28] data.model.repository: add back search fields --- data/model/repository.py | 29 ++++++++++++++++++++++++----- data/oci_model/package.py | 3 +++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/data/model/repository.py b/data/model/repository.py index ecf5a8d75..d70b79213 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -1,6 +1,7 @@ import logging import random +from enum import Enum from datetime import timedelta, datetime from peewee import JOIN_LEFT_OUTER, fn, SQL, IntegrityError from cachetools import ttl_cache @@ -17,6 +18,7 @@ from util.itertoolrecipes import take logger = logging.getLogger(__name__) +SEARCH_FIELDS = Enum("SearchFields", ["name", "description"]) def get_repo_kind_name(repo): @@ -356,20 +358,26 @@ def get_app_repository(namespace_name, repository_name): return None -def get_app_search(lookup, username=None, limit=50): +def get_app_search(lookup, search_fields=None, username=None, limit=50): + if search_fields is None: + search_fields = set([SEARCH_FIELDS.description.name]) return get_filtered_matching_repositories(lookup, filter_username=username, + search_fields=search_fields, repo_kind='application', offset=0, limit=limit) def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_kind='image', - offset=0, limit=25): + offset=0, limit=25, search_fields=None): """ Returns an iterator of all repositories matching the given lookup value, with optional filtering to a specific user. If the user is unspecified, only public repositories will be returned. """ + if search_fields is None: + search_fields = set([SEARCH_FIELDS.description.name, SEARCH_FIELDS.name.name]) # Build the unfiltered search query. unfiltered_query = _get_sorted_matching_repositories(lookup_value, repo_kind=repo_kind, + search_fields=search_fields, include_private=filter_username is not None) # Add a filter to the iterator, if necessary. @@ -427,18 +435,29 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username, iteration_count = iteration_count + 1 -def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_private=False): +def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_private=False, + search_fields=None): """ Returns a query of repositories matching the given lookup string, with optional inclusion of private repositories. Note that this method does *not* filter results based on visibility to users. """ + + if search_fields is None: + search_fields = set([SEARCH_FIELDS.description.name, SEARCH_FIELDS.name.name]) + + # Always search at least on name (init clause) + clause = Repository.name.match(lookup_value) + + if SEARCH_FIELDS.description.name in search_fields: + clause = Repository.description.match(lookup_value) | clause + last_week = datetime.now() - timedelta(weeks=1) query = (Repository .select(Repository, Namespace) .join(Namespace, on=(Namespace.id == Repository.namespace_user)) - .where(Repository.name.match(lookup_value) | Repository.description.match(lookup_value), - Repository.kind == Repository.kind.get_id(repo_kind)) + .where(clause, + Repository.repo_kind == Repository.repo_kind.get_id(repo_kind)) .group_by(Repository.id, Namespace.id)) if not include_private: diff --git a/data/oci_model/package.py b/data/oci_model/package.py index cf01256f6..c91affe7a 100644 --- a/data/oci_model/package.py +++ b/data/oci_model/package.py @@ -9,9 +9,12 @@ from data.oci_model import tag as tag_model def list_packages_query(namespace=None, media_type=None, search_query=None, username=None): """ List and filter repository by search query. """ + fields = [model.repository.SEARCH_FIELDS.name.name] + if search_query is not None: repositories = model.repository.get_app_search(search_query, username=username, + search_fields=fields, limit=50) repo_query = (Repository .select(Repository, Namespace.username) From 05ce571e3e191046ebd8251aca95511e4d6a7f02 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 11:11:21 -0400 Subject: [PATCH 23/28] Add missing return statement --- data/interfaces/appr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data/interfaces/appr.py b/data/interfaces/appr.py index 504293398..fcf42b2e6 100644 --- a/data/interfaces/appr.py +++ b/data/interfaces/appr.py @@ -207,6 +207,7 @@ class OCIAppModel(AppRegistryDataInterface): repo = model.repository.get_app_repository(ns, name) if repo is None: raise_package_not_found(package) + return repo def list_applications(self, namespace=None, media_type=None, search=None, username=None, with_channels=False): From 917d5e2550f3060c1c2feb7b430cc450b2bea3f6 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 11:14:08 -0400 Subject: [PATCH 24/28] Fix typos in data model --- data/model/repository.py | 2 +- data/oci_model/channel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/model/repository.py b/data/model/repository.py index d70b79213..8eb693c6e 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -457,7 +457,7 @@ def _get_sorted_matching_repositories(lookup_value, repo_kind='image', include_p .select(Repository, Namespace) .join(Namespace, on=(Namespace.id == Repository.namespace_user)) .where(clause, - Repository.repo_kind == Repository.repo_kind.get_id(repo_kind)) + Repository.kind == Repository.kind.get_id(repo_kind)) .group_by(Repository.id, Namespace.id)) if not include_private: diff --git a/data/oci_model/channel.py b/data/oci_model/channel.py index 42548ba8f..d7e340eb0 100644 --- a/data/oci_model/channel.py +++ b/data/oci_model/channel.py @@ -42,7 +42,7 @@ def delete_channel(repo, channel_name): def create_or_update_channel(repo, channel_name, tag_name): """ Creates or updates a channel to include a particular tag. """ tag = tag_model.get_tag(repo, tag_name, 'release') - return tag.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel") + return tag_model.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel") def get_repo_channels(repo): From 35b500aa2a1e6a19b255301f44a51db705b9b46e Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 11:17:05 -0400 Subject: [PATCH 25/28] Fix test override --- endpoints/appr/test/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py index 2433588d6..6ed59b963 100644 --- a/endpoints/appr/test/test_api.py +++ b/endpoints/appr/test/test_api.py @@ -192,5 +192,5 @@ class TestQuayModels(CnrTestModels): CnrTestModels.test_channel_delete_releases(self, db_with_data1) @pytest.mark.xfail - def test_forbidden_db_reset(self, db_class): + def test_forbiddeb_db_reset(self, db_class): pass From 77d2b9b2905863f288b3c8c70509fe7afcfe32ad Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Thu, 23 Mar 2017 11:23:46 -0400 Subject: [PATCH 26/28] endpoints.appr.test: mark failing db restore test This test should fail as long as the CNR tests use 'v1' in the mediatype. --- endpoints/appr/test/test_api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py index 6ed59b963..a96ec1569 100644 --- a/endpoints/appr/test/test_api.py +++ b/endpoints/appr/test/test_api.py @@ -194,3 +194,8 @@ class TestQuayModels(CnrTestModels): @pytest.mark.xfail def test_forbiddeb_db_reset(self, db_class): pass + + @pytest.mark.xfail + def test_db_restore(self, newdb, dbdata1): + # This will fail as long as CNR tests use a mediatype with v1. + pass From 7d66f30d522b5c799796b75dee6d9aa028c0dadd Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 11:35:17 -0400 Subject: [PATCH 27/28] Fix filtering of repositories in search --- data/model/repository.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/data/model/repository.py b/data/model/repository.py index 8eb693c6e..17404554b 100644 --- a/data/model/repository.py +++ b/data/model/repository.py @@ -360,7 +360,8 @@ def get_app_repository(namespace_name, repository_name): def get_app_search(lookup, search_fields=None, username=None, limit=50): if search_fields is None: - search_fields = set([SEARCH_FIELDS.description.name]) + search_fields = set([SEARCH_FIELDS.name.name]) + return get_filtered_matching_repositories(lookup, filter_username=username, search_fields=search_fields, repo_kind='application', offset=0, limit=limit) @@ -382,7 +383,8 @@ def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_ # Add a filter to the iterator, if necessary. if filter_username is not None: - iterator = _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit) + iterator = _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit, + repo_kind) else: iterator = unfiltered_query @@ -393,7 +395,7 @@ def get_filtered_matching_repositories(lookup_value, filter_username=None, repo_ return list(take(limit, iterator)) -def _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit): +def _filter_repositories_visible_to_username(unfiltered_query, filter_username, limit, repo_kind): encountered = set() chunk_count = limit * 2 unfiltered_page = 0 @@ -423,8 +425,7 @@ def _filter_repositories_visible_to_username(unfiltered_query, filter_username, .join(RepositoryPermission) .where(Repository.id << list(new_unfiltered_ids))) - filtered = _basequery.filter_to_repos_for_user(query, filter_username) - + filtered = _basequery.filter_to_repos_for_user(query, filter_username, repo_kind=repo_kind) for filtered_repo in filtered: yield filtered_repo From e204f7784c08f7ad2c278acef7eef5cb561e900d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Thu, 23 Mar 2017 12:01:59 -0400 Subject: [PATCH 28/28] Make app registry off by default --- config.py | 2 +- test/testconfig.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.py b/config.py index 79cf034a6..d8482b59f 100644 --- a/config.py +++ b/config.py @@ -232,7 +232,7 @@ class DefaultConfig(object): FEATURE_SIGNING = False # Feature Flag: Whether to enable support for App repositories. - FEATURE_APP_REGISTRY = True + FEATURE_APP_REGISTRY = False # The namespace to use for library repositories. # Note: This must remain 'library' until Docker removes their hard-coded namespace for libraries. diff --git a/test/testconfig.py b/test/testconfig.py index ae7af6f93..b870a787a 100644 --- a/test/testconfig.py +++ b/test/testconfig.py @@ -92,3 +92,5 @@ class TestConfig(DefaultConfig): RECAPTCHA_SITE_KEY = 'somekey' RECAPTCHA_SECRET_KEY = 'somesecretkey' + + FEATURE_APP_REGISTRY = True