Merge pull request #2844 from coreos-inc/alegrand/use_latest_appr_server_code

Migrate from cnr -> appr
This commit is contained in:
Antoine Legrand 2017-08-09 00:01:25 +02:00 committed by GitHub
commit 646fafb2fd
16 changed files with 106 additions and 68 deletions

View file

@ -1,5 +1,5 @@
# Generated from .gitlab-ci.jsonnet
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
# Generated from .gitlab-ci.jsonnet
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
---
appr_e2e:
allow_failure: true

View file

@ -0,0 +1,35 @@
"""add_mediatypes
Revision ID: 5d4ae648155c
Revises: d8989249f8f6
Create Date: 2017-08-04 16:01:47.573800
"""
# revision identifiers, used by Alembic.
revision = '5d4ae648155c'
down_revision = 'd8989249f8f6'
from alembic import op
import sqlalchemy as sa
def upgrade(tables):
mtypes = ['helm', 'kpm', 'docker-compose', 'ksonnet', 'appr', 'kubernetes']
data = []
for mtype in mtypes:
data.append({'name': 'application/vnd.appr.package-manifest.%s.v0.json' % mtype})
data.append({'name': 'application/vnd.appr.package.%s.v0.tar+gzip' % mtype})
op.bulk_insert(
tables.mediatype,
[
{'name': 'application/vnd.appr.blob.v0.tar+gzip'},
{'name': 'application/vnd.appr.manifests.v0.json'},
{'name': 'application/vnd.appr.manifest.list.v0.json'},
] + data,
)
def downgrade(tables):
pass

View file

@ -2,7 +2,7 @@ import logging
import hashlib
import json
from cnr.models.package_base import get_media_type
from appr.models.package_base import get_media_type
from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
from data.oci_model import tag as tag_model

View file

@ -1,4 +1,4 @@
from cnr.models.package_base import get_media_type, manifest_media_type
from appr.models.package_base import get_media_type, manifest_media_type
from peewee import prefetch

View file

@ -1,7 +1,7 @@
import bisect
from cnr.exception import PackageAlreadyExists
from cnr.models.package_base import manifest_media_type
from appr.exception import PackageAlreadyExists
from appr.models.package_base import manifest_media_type
from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
ManifestListManifest, Blob, ManifestBlob)
@ -10,6 +10,7 @@ from data.oci_model import (blob as blob_model, manifest as manifest_model,
tag as tag_model)
# @TODO(ant31): cnr for retro-compat prepare a data migration
LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json'
SCHEMA_VERSION = 'v0'
@ -103,7 +104,7 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
list_json.insert(insert_point, manifest.manifest_json)
list_manifest_ids.insert(insert_point, manifest.id)
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
SCHEMA_VERSION)
SCHEMA_VERSION)
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json)
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,

View file

@ -1,6 +1,6 @@
import logging
from cnr.models.package_base import manifest_media_type
from appr.models.package_base import manifest_media_type
from peewee import IntegrityError
from data.model import (db_transaction, TagAlreadyCreatedException)

View file

@ -2,7 +2,7 @@ import logging
from functools import wraps
from cnr.exception import Forbidden
from appr.exception import Forbidden
from flask import Blueprint
from app import metric_queue

View file

@ -1,10 +1,10 @@
import base64
from cnr.exception import raise_package_not_found
from cnr.models.blob_base import BlobBase
from cnr.models.channel_base import ChannelBase
from cnr.models.db_base import CnrDB
from cnr.models.package_base import PackageBase, manifest_media_type
from appr.exception import raise_package_not_found
from appr.models.blob_base import BlobBase
from appr.models.channel_base import ChannelBase
from appr.models.db_base import ApprDB
from appr.models.package_base import PackageBase, manifest_media_type
from app import storage
from endpoints.appr.models_oci import model
@ -13,6 +13,7 @@ from endpoints.appr.models_oci import model
class Blob(BlobBase):
@classmethod
def upload_url(cls, digest):
""" S3 bucket path """
return "cnr/blobs/sha256/%s/%s" % (digest[0:2], digest)
def save(self, content_media_type):
@ -164,7 +165,7 @@ class Package(PackageBase):
raise NotImplementedError
class QuayDB(CnrDB):
class QuayDB(ApprDB):
""" Wrapper Class to embed all CNR Models """
Channel = Channel
Package = Package

View file

@ -112,7 +112,7 @@ class AppRegistryDataInterface(object):
pass
@abstractmethod
def store_blob(self, cnrblob, content_media_type):
def store_blob(self, apprblob, content_media_type):
"""
Upload the blob content to a storage location and creates a Blob entry in the DB.

View file

@ -1,8 +1,8 @@
from datetime import datetime
import cnr.semver
import appr.semver
from cnr.exception import raise_package_not_found, raise_channel_not_found
from appr.exception import raise_package_not_found, raise_channel_not_found
import data.model
@ -76,7 +76,7 @@ class OCIAppModel(AppRegistryDataInterface):
if not releases:
continue
available_releases = [
str(x) for x in sorted(cnr.semver.versions(releases, False), reverse=True)]
str(x) for x in sorted(appr.semver.versions(releases, False), reverse=True)]
channels = None
if with_channels:
channels = [
@ -173,19 +173,19 @@ class OCIAppModel(AppRegistryDataInterface):
MediaType.DoesNotExist):
raise_package_not_found(package_name, release, media_type)
def store_blob(self, cnrblob, content_media_type):
fp = cnrblob.packager.io_file
path = cnrblob.upload_url(cnrblob.digest)
def store_blob(self, apprblob, content_media_type):
fp = apprblob.packager.io_file
path = apprblob.upload_url(apprblob.digest)
locations = storage.preferred_locations
storage.stream_write(locations, path, fp, 'application/x-gzip')
db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type,
db_blob = oci_model.blob.get_or_create_blob(apprblob.digest, apprblob.size, content_media_type,
locations)
return BlobDescriptor(mediaType=content_media_type,
digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[])
def create_release(self, package, user, visibility, force=False):
""" Add an app-release to a repository
package is an instance of data.cnr.package.Package
package is an instance of data.appr.package.Package
"""
manifest = package.manifest()

View file

@ -1,11 +1,11 @@
import logging
from base64 import b64encode
import cnr
from cnr.api.impl import registry as cnr_registry
from cnr.api.registry import _pull, repo_name
from cnr.exception import (
ChannelNotFound, CnrException, Forbidden, InvalidParams, InvalidRelease, InvalidUsage,
import appr
from appr.api.impl import registry as appr_registry
from appr.api.registry import _pull, repo_name
from appr.exception import (
ChannelNotFound, ApprException, Forbidden, InvalidParams, InvalidRelease, InvalidUsage,
PackageAlreadyExists, PackageNotFound, PackageReleaseNotFound, UnableToLockResource,
UnauthorizedAccess, Unsupported)
from flask import jsonify, request
@ -14,7 +14,7 @@ from auth.auth_context import get_authenticated_user
from auth.decorators import process_auth
from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermission
from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write
from endpoints.appr.cnr_backend import Blob, Channel, Package, User
from endpoints.appr.appr_backend import Blob, Channel, Package, User
from endpoints.appr.decorators import disallow_for_image_repository
from endpoints.appr.models_oci import model
from endpoints.decorators import anon_allowed, anon_protect
@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
@appr_bp.errorhandler(UnauthorizedAccess)
@appr_bp.errorhandler(PackageNotFound)
@appr_bp.errorhandler(PackageReleaseNotFound)
@appr_bp.errorhandler(CnrException)
@appr_bp.errorhandler(ApprException)
@appr_bp.errorhandler(InvalidUsage)
@appr_bp.errorhandler(InvalidParams)
@appr_bp.errorhandler(ChannelNotFound)
@ -44,7 +44,7 @@ def render_error(error):
@appr_bp.route("/version")
@anon_allowed
def version():
return jsonify({"cnr-api": cnr.__version__})
return jsonify({"appr-api": appr.__version__})
@appr_bp.route("/api/v1/users/login", methods=['POST'])
@ -73,7 +73,7 @@ def login():
@anon_protect
def blobs(namespace, package_name, digest):
reponame = repo_name(namespace, package_name)
data = cnr_registry.pull_blob(reponame, digest, blob_class=Blob)
data = appr_registry.pull_blob(reponame, digest, blob_class=Blob)
json_format = request.args.get('format', None) == 'json'
return _pull(data, json_format=json_format)
@ -89,7 +89,7 @@ def list_packages():
username = None
if user:
username = user.username
result_data = cnr_registry.list_packages(namespace, package_class=Package, search=query,
result_data = appr_registry.list_packages(namespace, package_class=Package, search=query,
media_type=media_type, username=username)
return jsonify(result_data)
@ -102,7 +102,7 @@ def list_packages():
@anon_protect
def delete_package(namespace, package_name, release, media_type):
reponame = repo_name(namespace, package_name)
result = cnr_registry.delete_package(reponame, release, media_type, package_class=Package)
result = appr_registry.delete_package(reponame, release, media_type, package_class=Package)
model.log_action('delete_tag', namespace, repo_name=package_name,
metadata={'release': release, 'mediatype': media_type})
return jsonify(result)
@ -116,7 +116,7 @@ def delete_package(namespace, package_name, release, media_type):
@anon_protect
def show_package(namespace, package_name, release, media_type):
reponame = repo_name(namespace, package_name)
result = cnr_registry.show_package(reponame, release, media_type, channel_class=Channel,
result = appr_registry.show_package(reponame, release, media_type, channel_class=Channel,
package_class=Package)
return jsonify(result)
@ -129,7 +129,7 @@ def show_package(namespace, package_name, release, media_type):
def show_package_releases(namespace, package_name):
reponame = repo_name(namespace, package_name)
media_type = request.args.get('media_type', None)
result = cnr_registry.show_package_releases(reponame, media_type=media_type,
result = appr_registry.show_package_releases(reponame, media_type=media_type,
package_class=Package)
return jsonify(result)
@ -141,7 +141,7 @@ def show_package_releases(namespace, package_name):
@anon_protect
def show_package_release_manifests(namespace, package_name, release):
reponame = repo_name(namespace, package_name)
result = cnr_registry.show_package_manifests(reponame, release, package_class=Package)
result = appr_registry.show_package_manifests(reponame, release, package_class=Package)
return jsonify(result)
@ -155,7 +155,7 @@ def show_package_release_manifests(namespace, package_name, release):
def pull(namespace, package_name, release, media_type):
reponame = repo_name(namespace, package_name)
logger.info("pull %s", reponame)
data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
data = appr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
model.log_action('pull_repo', namespace, repo_name=package_name,
metadata={'release': release, 'mediatype': media_type})
json_format = request.args.get('format', None) == 'json'
@ -199,7 +199,7 @@ def push(namespace, package_name):
force = request.args.get('force', 'false') == 'true'
blob = Blob(reponame, values['blob'])
app_release = cnr_registry.push(reponame, release_version, media_type, blob, force,
app_release = appr_registry.push(reponame, release_version, media_type, blob, force,
package_class=Package, user=owner, visibility=private)
model.log_action('push_repo', namespace, repo_name=package_name,
metadata={'release': release_version})
@ -216,7 +216,7 @@ def search_packages():
if user:
username = user.username
search_results = cnr_registry.search(query, Package, username=username)
search_results = appr_registry.search(query, Package, username=username)
return jsonify(search_results)
@ -228,7 +228,7 @@ def search_packages():
@anon_protect
def list_channels(namespace, package_name):
reponame = repo_name(namespace, package_name)
return jsonify(cnr_registry.list_channels(reponame, channel_class=Channel))
return jsonify(appr_registry.list_channels(reponame, channel_class=Channel))
@appr_bp.route(
@ -239,7 +239,7 @@ def list_channels(namespace, package_name):
@anon_protect
def show_channel(namespace, package_name, channel_name):
reponame = repo_name(namespace, package_name)
channel = cnr_registry.show_channel(reponame, channel_name, channel_class=Channel)
channel = appr_registry.show_channel(reponame, channel_name, channel_class=Channel)
return jsonify(channel)
@ -253,7 +253,7 @@ def show_channel(namespace, package_name, channel_name):
def add_channel_release(namespace, package_name, channel_name, release):
_check_channel_name(channel_name, release)
reponame = repo_name(namespace, package_name)
result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel,
result = appr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel,
package_class=Package)
model.log_action('create_tag', namespace, repo_name=package_name,
metadata={'channel': channel_name, 'release': release})
@ -284,7 +284,7 @@ def _check_channel_name(channel_name, release=None):
def delete_channel_release(namespace, package_name, channel_name, release):
_check_channel_name(channel_name, release)
reponame = repo_name(namespace, package_name)
result = cnr_registry.delete_channel_release(reponame, channel_name, release,
result = appr_registry.delete_channel_release(reponame, channel_name, release,
channel_class=Channel, package_class=Package)
model.log_action('delete_tag', namespace, repo_name=package_name,
metadata={'channel': channel_name, 'release': release})
@ -301,7 +301,7 @@ def delete_channel_release(namespace, package_name, channel_name, release):
def delete_channel(namespace, package_name, channel_name):
_check_channel_name(channel_name)
reponame = repo_name(namespace, package_name)
result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel)
result = appr_registry.delete_channel(reponame, channel_name, channel_class=Channel)
model.log_action('delete_tag', namespace, repo_name=package_name,
metadata={'channel': channel_name})
return jsonify(result)

View file

@ -2,16 +2,16 @@ import uuid
import pytest
from cnr.tests.conftest import *
from cnr.tests.test_apiserver import BaseTestServer
from cnr.tests.test_models import CnrTestModels
from appr.tests.conftest import *
from appr.tests.test_apiserver import BaseTestServer
from appr.tests.test_models import ApprTestModels
import data.oci_model.blob as oci_blob
from data.database import User
from data.model import organization, user
from endpoints.appr import registry # Needed to register the endpoint
from endpoints.appr.cnr_backend import Channel, Package, QuayDB
from endpoints.appr.appr_backend import Channel, Package, QuayDB
from endpoints.appr.models_oci import model as oci_app_model
from test.fixtures import *
@ -70,14 +70,14 @@ class PackageTest(Package):
@pytest.fixture(autouse=True)
def quaydb(monkeypatch, app):
monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Package', PackageTest)
monkeypatch.setattr('endpoints.appr.cnr_backend.Package', PackageTest)
monkeypatch.setattr('endpoints.appr.appr_backend.QuayDB.Package', PackageTest)
monkeypatch.setattr('endpoints.appr.appr_backend.Package', PackageTest)
monkeypatch.setattr('endpoints.appr.registry.Package', PackageTest)
monkeypatch.setattr('cnr.models.Package', PackageTest)
monkeypatch.setattr('appr.models.Package', PackageTest)
monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Channel', ChannelTest)
monkeypatch.setattr('endpoints.appr.appr_backend.QuayDB.Channel', ChannelTest)
monkeypatch.setattr('endpoints.appr.registry.Channel', ChannelTest)
monkeypatch.setattr('cnr.models.Channel', ChannelTest)
monkeypatch.setattr('appr.models.Channel', ChannelTest)
class TestServerQuayDB(BaseTestServer):
@ -118,13 +118,13 @@ class TestServerQuayDB(BaseTestServer):
pass
class TestQuayModels(CnrTestModels):
class TestQuayModels(ApprTestModels):
DB_CLASS = QuayDB
@pytest.mark.xfail
def test_channel_delete_releases(self, db_with_data1):
""" Can't remove a release from the channel, only delete the channel entirely """
CnrTestModels.test_channel_delete_releases(self, db_with_data1)
ApprTestModels.test_channel_delete_releases(self, db_with_data1)
@pytest.mark.xfail
def test_forbiddeb_db_reset(self, db_class):

View file

@ -404,15 +404,16 @@ def initialize_database():
MediaType.create(name='text/plain')
MediaType.create(name='application/json')
MediaType.create(name='text/markdown')
MediaType.create(name='application/vnd.cnr.blob.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package-manifest.helm.v0.json')
MediaType.create(name='application/vnd.cnr.package-manifest.kpm.v0.json')
MediaType.create(name='application/vnd.cnr.package-manifest.docker-compose.v0.json')
MediaType.create(name='application/vnd.cnr.package.kpm.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package.helm.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.package.docker-compose.v0.tar+gzip')
MediaType.create(name='application/vnd.cnr.manifests.v0.json')
MediaType.create(name='application/vnd.cnr.manifest.list.v0.json')
for mediatype in ['appr', 'cnr']:
MediaType.create(name='application/vnd.%s.blob.v0.tar+gzip' % mediatype)
MediaType.create(name='application/vnd.%s.package-manifest.helm.v0.json' % mediatype)
MediaType.create(name='application/vnd.%s.package-manifest.kpm.v0.json' % mediatype)
MediaType.create(name='application/vnd.%s.package-manifest.docker-compose.v0.json' % mediatype)
MediaType.create(name='application/vnd.%s.package.kpm.v0.tar+gzip' % mediatype)
MediaType.create(name='application/vnd.%s.package.helm.v0.tar+gzip' % mediatype)
MediaType.create(name='application/vnd.%s.package.docker-compose.v0.tar+gzip' % mediatype)
MediaType.create(name='application/vnd.%s.manifests.v0.json' % mediatype)
MediaType.create(name='application/vnd.%s.manifest.list.v0.json' % mediatype)
LabelSourceType.create(name='manifest')
LabelSourceType.create(name='api', mutable=True)

View file

@ -8,7 +8,7 @@
-e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab
-e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib
-e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython
-e git+https://github.com/app-registry/appr-server.git#egg=cnr-server
-e git+https://github.com/app-registry/appr.git@b80028a1fa6f52c314405690b2d4ffeaccccae5a#egg=appr
APScheduler==3.0.5
Flask-Login
Flask-Mail

View file

@ -6,7 +6,7 @@ alembic==0.9.1
-e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib
-e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601
-e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode
-e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server
-e git+https://github.com/app-registry/appr.git@b80028a1fa6f52c314405690b2d4ffeaccccae5a#egg=appr
-e git+https://github.com/DevTable/container-cloud-config.git@bce675537904175f6975024a4c89269027ea6792#egg=container_cloud_config
-e git+https://github.com/DevTable/python-etcd.git@f1168cb02a2a8c83bec1108c6fcd8615ef463b14#egg=python_etcd
-e git+https://github.com/jarus/flask-testing.git@18baff32969a0634a414ce61d2dd4a77433817a8#egg=Flask_Testing

Binary file not shown.