diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 18f69eb0f..c822ad7e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ -# Generated from .gitlab-ci.jsonnet -# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN +# Generated from .gitlab-ci.jsonnet +# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN --- appr_e2e: allow_failure: true diff --git a/data/migrations/versions/5d4ae648155c_add_mediatypes.py b/data/migrations/versions/5d4ae648155c_add_mediatypes.py new file mode 100644 index 000000000..3a2666897 --- /dev/null +++ b/data/migrations/versions/5d4ae648155c_add_mediatypes.py @@ -0,0 +1,35 @@ +"""add_mediatypes + +Revision ID: 5d4ae648155c +Revises: d8989249f8f6 +Create Date: 2017-08-04 16:01:47.573800 + +""" + +# revision identifiers, used by Alembic. +revision = '5d4ae648155c' +down_revision = 'd8989249f8f6' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + mtypes = ['helm', 'kpm', 'docker-compose', 'ksonnet', 'appr', 'kubernetes'] + data = [] + for mtype in mtypes: + data.append({'name': 'application/vnd.appr.package-manifest.%s.v0.json' % mtype}) + data.append({'name': 'application/vnd.appr.package.%s.v0.tar+gzip' % mtype}) + + op.bulk_insert( + tables.mediatype, + [ + {'name': 'application/vnd.appr.blob.v0.tar+gzip'}, + {'name': 'application/vnd.appr.manifests.v0.json'}, + {'name': 'application/vnd.appr.manifest.list.v0.json'}, + ] + data, + ) + + +def downgrade(tables): + pass diff --git a/data/oci_model/manifest.py b/data/oci_model/manifest.py index a1af680b5..7fe1c0857 100644 --- a/data/oci_model/manifest.py +++ b/data/oci_model/manifest.py @@ -2,7 +2,7 @@ import logging import hashlib import json -from cnr.models.package_base import get_media_type +from appr.models.package_base import get_media_type from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag from data.oci_model import tag as tag_model diff --git a/data/oci_model/package.py b/data/oci_model/package.py index 61eae8e20..9f72884a2 100644 --- a/data/oci_model/package.py +++ b/data/oci_model/package.py @@ -1,4 +1,4 @@ -from cnr.models.package_base import get_media_type, manifest_media_type +from appr.models.package_base import get_media_type, manifest_media_type from peewee import prefetch diff --git a/data/oci_model/release.py b/data/oci_model/release.py index 890e3e459..d9c8156f1 100644 --- a/data/oci_model/release.py +++ b/data/oci_model/release.py @@ -1,7 +1,7 @@ import bisect -from cnr.exception import PackageAlreadyExists -from cnr.models.package_base import manifest_media_type +from appr.exception import PackageAlreadyExists +from appr.models.package_base import manifest_media_type from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag, ManifestListManifest, Blob, ManifestBlob) @@ -10,6 +10,7 @@ from data.oci_model import (blob as blob_model, manifest as manifest_model, tag as tag_model) +# @TODO(ant31): cnr for retro-compat prepare a data migration LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json' SCHEMA_VERSION = 'v0' @@ -103,7 +104,7 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False): list_json.insert(insert_point, manifest.manifest_json) list_manifest_ids.insert(insert_point, manifest.id) manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE, - SCHEMA_VERSION) + SCHEMA_VERSION) manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json) tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist, diff --git a/data/oci_model/tag.py b/data/oci_model/tag.py index 56bea5a29..69412e998 100644 --- a/data/oci_model/tag.py +++ b/data/oci_model/tag.py @@ -1,6 +1,6 @@ import logging -from cnr.models.package_base import manifest_media_type +from appr.models.package_base import manifest_media_type from peewee import IntegrityError from data.model import (db_transaction, TagAlreadyCreatedException) diff --git a/endpoints/appr/__init__.py b/endpoints/appr/__init__.py index c998d8a95..91c2cf851 100644 --- a/endpoints/appr/__init__.py +++ b/endpoints/appr/__init__.py @@ -2,7 +2,7 @@ import logging from functools import wraps -from cnr.exception import Forbidden +from appr.exception import Forbidden from flask import Blueprint from app import metric_queue diff --git a/endpoints/appr/cnr_backend.py b/endpoints/appr/appr_backend.py similarity index 94% rename from endpoints/appr/cnr_backend.py rename to endpoints/appr/appr_backend.py index 86a5195d8..7c84a5e6a 100644 --- a/endpoints/appr/cnr_backend.py +++ b/endpoints/appr/appr_backend.py @@ -1,10 +1,10 @@ import base64 -from cnr.exception import raise_package_not_found -from cnr.models.blob_base import BlobBase -from cnr.models.channel_base import ChannelBase -from cnr.models.db_base import CnrDB -from cnr.models.package_base import PackageBase, manifest_media_type +from appr.exception import raise_package_not_found +from appr.models.blob_base import BlobBase +from appr.models.channel_base import ChannelBase +from appr.models.db_base import ApprDB +from appr.models.package_base import PackageBase, manifest_media_type from app import storage from endpoints.appr.models_oci import model @@ -13,6 +13,7 @@ from endpoints.appr.models_oci import model class Blob(BlobBase): @classmethod def upload_url(cls, digest): + """ S3 bucket path """ return "cnr/blobs/sha256/%s/%s" % (digest[0:2], digest) def save(self, content_media_type): @@ -164,7 +165,7 @@ class Package(PackageBase): raise NotImplementedError -class QuayDB(CnrDB): +class QuayDB(ApprDB): """ Wrapper Class to embed all CNR Models """ Channel = Channel Package = Package diff --git a/endpoints/appr/models_interface.py b/endpoints/appr/models_interface.py index 6ebf949ac..5ba5b270e 100644 --- a/endpoints/appr/models_interface.py +++ b/endpoints/appr/models_interface.py @@ -112,7 +112,7 @@ class AppRegistryDataInterface(object): pass @abstractmethod - def store_blob(self, cnrblob, content_media_type): + def store_blob(self, apprblob, content_media_type): """ Upload the blob content to a storage location and creates a Blob entry in the DB. diff --git a/endpoints/appr/models_oci.py b/endpoints/appr/models_oci.py index 75d5050fa..c89145bc0 100644 --- a/endpoints/appr/models_oci.py +++ b/endpoints/appr/models_oci.py @@ -1,8 +1,8 @@ from datetime import datetime -import cnr.semver +import appr.semver -from cnr.exception import raise_package_not_found, raise_channel_not_found +from appr.exception import raise_package_not_found, raise_channel_not_found import data.model @@ -76,7 +76,7 @@ class OCIAppModel(AppRegistryDataInterface): if not releases: continue available_releases = [ - str(x) for x in sorted(cnr.semver.versions(releases, False), reverse=True)] + str(x) for x in sorted(appr.semver.versions(releases, False), reverse=True)] channels = None if with_channels: channels = [ @@ -173,19 +173,19 @@ class OCIAppModel(AppRegistryDataInterface): MediaType.DoesNotExist): raise_package_not_found(package_name, release, media_type) - def store_blob(self, cnrblob, content_media_type): - fp = cnrblob.packager.io_file - path = cnrblob.upload_url(cnrblob.digest) + def store_blob(self, apprblob, content_media_type): + fp = apprblob.packager.io_file + path = apprblob.upload_url(apprblob.digest) locations = storage.preferred_locations storage.stream_write(locations, path, fp, 'application/x-gzip') - db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type, + db_blob = oci_model.blob.get_or_create_blob(apprblob.digest, apprblob.size, content_media_type, locations) return BlobDescriptor(mediaType=content_media_type, digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[]) def create_release(self, package, user, visibility, force=False): """ Add an app-release to a repository - package is an instance of data.cnr.package.Package + package is an instance of data.appr.package.Package """ manifest = package.manifest() diff --git a/endpoints/appr/registry.py b/endpoints/appr/registry.py index 8d7e988b3..3e686cd39 100644 --- a/endpoints/appr/registry.py +++ b/endpoints/appr/registry.py @@ -1,11 +1,11 @@ import logging from base64 import b64encode -import cnr -from cnr.api.impl import registry as cnr_registry -from cnr.api.registry import _pull, repo_name -from cnr.exception import ( - ChannelNotFound, CnrException, Forbidden, InvalidParams, InvalidRelease, InvalidUsage, +import appr +from appr.api.impl import registry as appr_registry +from appr.api.registry import _pull, repo_name +from appr.exception import ( + ChannelNotFound, ApprException, Forbidden, InvalidParams, InvalidRelease, InvalidUsage, PackageAlreadyExists, PackageNotFound, PackageReleaseNotFound, UnableToLockResource, UnauthorizedAccess, Unsupported) from flask import jsonify, request @@ -14,7 +14,7 @@ from auth.auth_context import get_authenticated_user from auth.decorators import process_auth from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermission from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write -from endpoints.appr.cnr_backend import Blob, Channel, Package, User +from endpoints.appr.appr_backend import Blob, Channel, Package, User from endpoints.appr.decorators import disallow_for_image_repository from endpoints.appr.models_oci import model from endpoints.decorators import anon_allowed, anon_protect @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) @appr_bp.errorhandler(UnauthorizedAccess) @appr_bp.errorhandler(PackageNotFound) @appr_bp.errorhandler(PackageReleaseNotFound) -@appr_bp.errorhandler(CnrException) +@appr_bp.errorhandler(ApprException) @appr_bp.errorhandler(InvalidUsage) @appr_bp.errorhandler(InvalidParams) @appr_bp.errorhandler(ChannelNotFound) @@ -44,7 +44,7 @@ def render_error(error): @appr_bp.route("/version") @anon_allowed def version(): - return jsonify({"cnr-api": cnr.__version__}) + return jsonify({"appr-api": appr.__version__}) @appr_bp.route("/api/v1/users/login", methods=['POST']) @@ -73,7 +73,7 @@ def login(): @anon_protect def blobs(namespace, package_name, digest): reponame = repo_name(namespace, package_name) - data = cnr_registry.pull_blob(reponame, digest, blob_class=Blob) + data = appr_registry.pull_blob(reponame, digest, blob_class=Blob) json_format = request.args.get('format', None) == 'json' return _pull(data, json_format=json_format) @@ -89,7 +89,7 @@ def list_packages(): username = None if user: username = user.username - result_data = cnr_registry.list_packages(namespace, package_class=Package, search=query, + result_data = appr_registry.list_packages(namespace, package_class=Package, search=query, media_type=media_type, username=username) return jsonify(result_data) @@ -102,7 +102,7 @@ def list_packages(): @anon_protect def delete_package(namespace, package_name, release, media_type): reponame = repo_name(namespace, package_name) - result = cnr_registry.delete_package(reponame, release, media_type, package_class=Package) + result = appr_registry.delete_package(reponame, release, media_type, package_class=Package) model.log_action('delete_tag', namespace, repo_name=package_name, metadata={'release': release, 'mediatype': media_type}) return jsonify(result) @@ -116,7 +116,7 @@ def delete_package(namespace, package_name, release, media_type): @anon_protect def show_package(namespace, package_name, release, media_type): reponame = repo_name(namespace, package_name) - result = cnr_registry.show_package(reponame, release, media_type, channel_class=Channel, + result = appr_registry.show_package(reponame, release, media_type, channel_class=Channel, package_class=Package) return jsonify(result) @@ -129,7 +129,7 @@ def show_package(namespace, package_name, release, media_type): def show_package_releases(namespace, package_name): reponame = repo_name(namespace, package_name) media_type = request.args.get('media_type', None) - result = cnr_registry.show_package_releases(reponame, media_type=media_type, + result = appr_registry.show_package_releases(reponame, media_type=media_type, package_class=Package) return jsonify(result) @@ -141,7 +141,7 @@ def show_package_releases(namespace, package_name): @anon_protect def show_package_release_manifests(namespace, package_name, release): reponame = repo_name(namespace, package_name) - result = cnr_registry.show_package_manifests(reponame, release, package_class=Package) + result = appr_registry.show_package_manifests(reponame, release, package_class=Package) return jsonify(result) @@ -155,7 +155,7 @@ def show_package_release_manifests(namespace, package_name, release): def pull(namespace, package_name, release, media_type): reponame = repo_name(namespace, package_name) logger.info("pull %s", reponame) - data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob) + data = appr_registry.pull(reponame, release, media_type, Package, blob_class=Blob) model.log_action('pull_repo', namespace, repo_name=package_name, metadata={'release': release, 'mediatype': media_type}) json_format = request.args.get('format', None) == 'json' @@ -199,7 +199,7 @@ def push(namespace, package_name): force = request.args.get('force', 'false') == 'true' blob = Blob(reponame, values['blob']) - app_release = cnr_registry.push(reponame, release_version, media_type, blob, force, + app_release = appr_registry.push(reponame, release_version, media_type, blob, force, package_class=Package, user=owner, visibility=private) model.log_action('push_repo', namespace, repo_name=package_name, metadata={'release': release_version}) @@ -216,7 +216,7 @@ def search_packages(): if user: username = user.username - search_results = cnr_registry.search(query, Package, username=username) + search_results = appr_registry.search(query, Package, username=username) return jsonify(search_results) @@ -228,7 +228,7 @@ def search_packages(): @anon_protect def list_channels(namespace, package_name): reponame = repo_name(namespace, package_name) - return jsonify(cnr_registry.list_channels(reponame, channel_class=Channel)) + return jsonify(appr_registry.list_channels(reponame, channel_class=Channel)) @appr_bp.route( @@ -239,7 +239,7 @@ def list_channels(namespace, package_name): @anon_protect def show_channel(namespace, package_name, channel_name): reponame = repo_name(namespace, package_name) - channel = cnr_registry.show_channel(reponame, channel_name, channel_class=Channel) + channel = appr_registry.show_channel(reponame, channel_name, channel_class=Channel) return jsonify(channel) @@ -253,7 +253,7 @@ def show_channel(namespace, package_name, channel_name): def add_channel_release(namespace, package_name, channel_name, release): _check_channel_name(channel_name, release) reponame = repo_name(namespace, package_name) - result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel, + result = appr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel, package_class=Package) model.log_action('create_tag', namespace, repo_name=package_name, metadata={'channel': channel_name, 'release': release}) @@ -284,7 +284,7 @@ def _check_channel_name(channel_name, release=None): def delete_channel_release(namespace, package_name, channel_name, release): _check_channel_name(channel_name, release) reponame = repo_name(namespace, package_name) - result = cnr_registry.delete_channel_release(reponame, channel_name, release, + result = appr_registry.delete_channel_release(reponame, channel_name, release, channel_class=Channel, package_class=Package) model.log_action('delete_tag', namespace, repo_name=package_name, metadata={'channel': channel_name, 'release': release}) @@ -301,7 +301,7 @@ def delete_channel_release(namespace, package_name, channel_name, release): def delete_channel(namespace, package_name, channel_name): _check_channel_name(channel_name) reponame = repo_name(namespace, package_name) - result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel) + result = appr_registry.delete_channel(reponame, channel_name, channel_class=Channel) model.log_action('delete_tag', namespace, repo_name=package_name, metadata={'channel': channel_name}) return jsonify(result) diff --git a/endpoints/appr/test/test_api.py b/endpoints/appr/test/test_api.py index b3064e681..642477c06 100644 --- a/endpoints/appr/test/test_api.py +++ b/endpoints/appr/test/test_api.py @@ -2,16 +2,16 @@ import uuid import pytest -from cnr.tests.conftest import * -from cnr.tests.test_apiserver import BaseTestServer -from cnr.tests.test_models import CnrTestModels +from appr.tests.conftest import * +from appr.tests.test_apiserver import BaseTestServer +from appr.tests.test_models import ApprTestModels import data.oci_model.blob as oci_blob from data.database import User from data.model import organization, user from endpoints.appr import registry # Needed to register the endpoint -from endpoints.appr.cnr_backend import Channel, Package, QuayDB +from endpoints.appr.appr_backend import Channel, Package, QuayDB from endpoints.appr.models_oci import model as oci_app_model from test.fixtures import * @@ -70,14 +70,14 @@ class PackageTest(Package): @pytest.fixture(autouse=True) def quaydb(monkeypatch, app): - monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Package', PackageTest) - monkeypatch.setattr('endpoints.appr.cnr_backend.Package', PackageTest) + monkeypatch.setattr('endpoints.appr.appr_backend.QuayDB.Package', PackageTest) + monkeypatch.setattr('endpoints.appr.appr_backend.Package', PackageTest) monkeypatch.setattr('endpoints.appr.registry.Package', PackageTest) - monkeypatch.setattr('cnr.models.Package', PackageTest) + monkeypatch.setattr('appr.models.Package', PackageTest) - monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Channel', ChannelTest) + monkeypatch.setattr('endpoints.appr.appr_backend.QuayDB.Channel', ChannelTest) monkeypatch.setattr('endpoints.appr.registry.Channel', ChannelTest) - monkeypatch.setattr('cnr.models.Channel', ChannelTest) + monkeypatch.setattr('appr.models.Channel', ChannelTest) class TestServerQuayDB(BaseTestServer): @@ -118,13 +118,13 @@ class TestServerQuayDB(BaseTestServer): pass -class TestQuayModels(CnrTestModels): +class TestQuayModels(ApprTestModels): DB_CLASS = QuayDB @pytest.mark.xfail def test_channel_delete_releases(self, db_with_data1): """ Can't remove a release from the channel, only delete the channel entirely """ - CnrTestModels.test_channel_delete_releases(self, db_with_data1) + ApprTestModels.test_channel_delete_releases(self, db_with_data1) @pytest.mark.xfail def test_forbiddeb_db_reset(self, db_class): diff --git a/initdb.py b/initdb.py index c6b36e18e..ee05c28d3 100644 --- a/initdb.py +++ b/initdb.py @@ -404,15 +404,16 @@ def initialize_database(): MediaType.create(name='text/plain') MediaType.create(name='application/json') MediaType.create(name='text/markdown') - MediaType.create(name='application/vnd.cnr.blob.v0.tar+gzip') - MediaType.create(name='application/vnd.cnr.package-manifest.helm.v0.json') - MediaType.create(name='application/vnd.cnr.package-manifest.kpm.v0.json') - MediaType.create(name='application/vnd.cnr.package-manifest.docker-compose.v0.json') - MediaType.create(name='application/vnd.cnr.package.kpm.v0.tar+gzip') - MediaType.create(name='application/vnd.cnr.package.helm.v0.tar+gzip') - MediaType.create(name='application/vnd.cnr.package.docker-compose.v0.tar+gzip') - MediaType.create(name='application/vnd.cnr.manifests.v0.json') - MediaType.create(name='application/vnd.cnr.manifest.list.v0.json') + for mediatype in ['appr', 'cnr']: + MediaType.create(name='application/vnd.%s.blob.v0.tar+gzip' % mediatype) + MediaType.create(name='application/vnd.%s.package-manifest.helm.v0.json' % mediatype) + MediaType.create(name='application/vnd.%s.package-manifest.kpm.v0.json' % mediatype) + MediaType.create(name='application/vnd.%s.package-manifest.docker-compose.v0.json' % mediatype) + MediaType.create(name='application/vnd.%s.package.kpm.v0.tar+gzip' % mediatype) + MediaType.create(name='application/vnd.%s.package.helm.v0.tar+gzip' % mediatype) + MediaType.create(name='application/vnd.%s.package.docker-compose.v0.tar+gzip' % mediatype) + MediaType.create(name='application/vnd.%s.manifests.v0.json' % mediatype) + MediaType.create(name='application/vnd.%s.manifest.list.v0.json' % mediatype) LabelSourceType.create(name='manifest') LabelSourceType.create(name='api', mutable=True) diff --git a/requirements-nover.txt b/requirements-nover.txt index 6bad4c9b9..3aac5b650 100644 --- a/requirements-nover.txt +++ b/requirements-nover.txt @@ -8,7 +8,7 @@ -e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab -e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib -e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython --e git+https://github.com/app-registry/appr-server.git#egg=cnr-server +-e git+https://github.com/app-registry/appr.git@b80028a1fa6f52c314405690b2d4ffeaccccae5a#egg=appr APScheduler==3.0.5 Flask-Login Flask-Mail diff --git a/requirements.txt b/requirements.txt index 96ef9a3c6..756b34689 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ alembic==0.9.1 -e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601 -e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode --e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server +-e git+https://github.com/app-registry/appr.git@b80028a1fa6f52c314405690b2d4ffeaccccae5a#egg=appr -e git+https://github.com/DevTable/container-cloud-config.git@bce675537904175f6975024a4c89269027ea6792#egg=container_cloud_config -e git+https://github.com/DevTable/python-etcd.git@f1168cb02a2a8c83bec1108c6fcd8615ef463b14#egg=python_etcd -e git+https://github.com/jarus/flask-testing.git@18baff32969a0634a414ce61d2dd4a77433817a8#egg=Flask_Testing diff --git a/test/data/test.db b/test/data/test.db index e109172d6..a7c3b4fef 100644 Binary files a/test/data/test.db and b/test/data/test.db differ