Merge pull request #2844 from coreos-inc/alegrand/use_latest_appr_server_code

Migrate from cnr -> appr

data/migrations/versions/5d4ae648155c_add_mediatypes.py

@@ -0,0 +1,35 @@
+"""add_mediatypes
+
+Revision ID: 5d4ae648155c
+Revises: d8989249f8f6
+Create Date: 2017-08-04 16:01:47.573800
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5d4ae648155c'
+down_revision = 'd8989249f8f6'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+  mtypes = ['helm', 'kpm', 'docker-compose', 'ksonnet', 'appr', 'kubernetes']
+  data = []
+  for mtype in mtypes:
+    data.append({'name': 'application/vnd.appr.package-manifest.%s.v0.json' % mtype})
+    data.append({'name': 'application/vnd.appr.package.%s.v0.tar+gzip' % mtype})
+
+  op.bulk_insert(
+    tables.mediatype,
+    [
+      {'name': 'application/vnd.appr.blob.v0.tar+gzip'},
+      {'name': 'application/vnd.appr.manifests.v0.json'},
+      {'name': 'application/vnd.appr.manifest.list.v0.json'},
+    ] + data,
+  )
+
+
+def downgrade(tables):
+    pass

data/oci_model/manifest.py

@@ -2,7 +2,7 @@ import logging
 import hashlib
 import json
 
-from cnr.models.package_base import get_media_type
+from appr.models.package_base import get_media_type
 
 from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
 from data.oci_model import tag as tag_model

data/oci_model/package.py

@@ -1,4 +1,4 @@
-from cnr.models.package_base import get_media_type, manifest_media_type
+from appr.models.package_base import get_media_type, manifest_media_type
 from peewee import prefetch
 
 

data/oci_model/release.py

@@ -1,7 +1,7 @@
 import bisect
 
-from cnr.exception import PackageAlreadyExists
-from cnr.models.package_base import manifest_media_type
+from appr.exception import PackageAlreadyExists
+from appr.models.package_base import manifest_media_type
 
 from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
                            ManifestListManifest, Blob, ManifestBlob)
@@ -10,6 +10,7 @@ from data.oci_model import (blob as blob_model, manifest as manifest_model,
                             tag as tag_model)
 
 
+# @TODO(ant31): cnr for retro-compat prepare a data migration
 LIST_MEDIA_TYPE = 'application/vnd.cnr.manifest.list.v0.json'
 SCHEMA_VERSION = 'v0'
 
@@ -103,7 +104,7 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
     list_json.insert(insert_point, manifest.manifest_json)
     list_manifest_ids.insert(insert_point, manifest.id)
     manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
-                                                                       SCHEMA_VERSION)
+                                                                   SCHEMA_VERSION)
     manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json)
 
     tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,

data/oci_model/tag.py

@@ -1,6 +1,6 @@
 import logging
 
-from cnr.models.package_base import manifest_media_type
+from appr.models.package_base import manifest_media_type
 from peewee import IntegrityError
 
 from data.model import (db_transaction, TagAlreadyCreatedException)

endpoints/appr/__init__.py

@@ -2,7 +2,7 @@ import logging
 
 from functools import wraps
 
-from cnr.exception import Forbidden
+from appr.exception import Forbidden
 from flask import Blueprint
 
 from app import metric_queue

endpoints/appr/appr_backend.py

@@ -1,10 +1,10 @@
 import base64
 
-from cnr.exception import raise_package_not_found
-from cnr.models.blob_base import BlobBase
-from cnr.models.channel_base import ChannelBase
-from cnr.models.db_base import CnrDB
-from cnr.models.package_base import PackageBase, manifest_media_type
+from appr.exception import raise_package_not_found
+from appr.models.blob_base import BlobBase
+from appr.models.channel_base import ChannelBase
+from appr.models.db_base import ApprDB
+from appr.models.package_base import PackageBase, manifest_media_type
 
 from app import storage
 from endpoints.appr.models_oci import model
@@ -13,6 +13,7 @@ from endpoints.appr.models_oci import model
 class Blob(BlobBase):
   @classmethod
   def upload_url(cls, digest):
+    """ S3 bucket path """
     return "cnr/blobs/sha256/%s/%s" % (digest[0:2], digest)
 
   def save(self, content_media_type):
@@ -164,7 +165,7 @@ class Package(PackageBase):
     raise NotImplementedError
 
 
-class QuayDB(CnrDB):
+class QuayDB(ApprDB):
   """ Wrapper Class to embed all CNR Models """
   Channel = Channel
   Package = Package

endpoints/appr/models_interface.py

@@ -112,7 +112,7 @@ class AppRegistryDataInterface(object):
     pass
 
   @abstractmethod
-  def store_blob(self, cnrblob, content_media_type):
+  def store_blob(self, apprblob, content_media_type):
     """
     Upload the blob content to a storage location and creates a Blob entry in the DB.
 

endpoints/appr/models_oci.py

@@ -1,8 +1,8 @@
 from datetime import datetime
 
-import cnr.semver
+import appr.semver
 
-from cnr.exception import raise_package_not_found, raise_channel_not_found
+from appr.exception import raise_package_not_found, raise_channel_not_found
 
 import data.model
 
@@ -76,7 +76,7 @@ class OCIAppModel(AppRegistryDataInterface):
       if not releases:
         continue
       available_releases = [
-        str(x) for x in sorted(cnr.semver.versions(releases, False), reverse=True)]
+        str(x) for x in sorted(appr.semver.versions(releases, False), reverse=True)]
       channels = None
       if with_channels:
         channels = [
@@ -173,19 +173,19 @@ class OCIAppModel(AppRegistryDataInterface):
             MediaType.DoesNotExist):
       raise_package_not_found(package_name, release, media_type)
 
-  def store_blob(self, cnrblob, content_media_type):
-    fp = cnrblob.packager.io_file
-    path = cnrblob.upload_url(cnrblob.digest)
+  def store_blob(self, apprblob, content_media_type):
+    fp = apprblob.packager.io_file
+    path = apprblob.upload_url(apprblob.digest)
     locations = storage.preferred_locations
     storage.stream_write(locations, path, fp, 'application/x-gzip')
-    db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type,
+    db_blob = oci_model.blob.get_or_create_blob(apprblob.digest, apprblob.size, content_media_type,
                                                 locations)
     return BlobDescriptor(mediaType=content_media_type,
                           digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[])
 
   def create_release(self, package, user, visibility, force=False):
     """ Add an app-release to a repository
-    package is an instance of data.cnr.package.Package
+    package is an instance of data.appr.package.Package
     """
 
     manifest = package.manifest()

endpoints/appr/registry.py

@@ -1,11 +1,11 @@
 import logging
 from base64 import b64encode
 
-import cnr
-from cnr.api.impl import registry as cnr_registry
-from cnr.api.registry import _pull, repo_name
-from cnr.exception import (
-  ChannelNotFound, CnrException, Forbidden, InvalidParams, InvalidRelease, InvalidUsage,
+import appr
+from appr.api.impl import registry as appr_registry
+from appr.api.registry import _pull, repo_name
+from appr.exception import (
+  ChannelNotFound, ApprException, Forbidden, InvalidParams, InvalidRelease, InvalidUsage,
   PackageAlreadyExists, PackageNotFound, PackageReleaseNotFound, UnableToLockResource,
   UnauthorizedAccess, Unsupported)
 from flask import jsonify, request
@@ -14,7 +14,7 @@ from auth.auth_context import get_authenticated_user
 from auth.decorators import process_auth
 from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermission
 from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write
-from endpoints.appr.cnr_backend import Blob, Channel, Package, User
+from endpoints.appr.appr_backend import Blob, Channel, Package, User
 from endpoints.appr.decorators import disallow_for_image_repository
 from endpoints.appr.models_oci import model
 from endpoints.decorators import anon_allowed, anon_protect
@@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
 @appr_bp.errorhandler(UnauthorizedAccess)
 @appr_bp.errorhandler(PackageNotFound)
 @appr_bp.errorhandler(PackageReleaseNotFound)
-@appr_bp.errorhandler(CnrException)
+@appr_bp.errorhandler(ApprException)
 @appr_bp.errorhandler(InvalidUsage)
 @appr_bp.errorhandler(InvalidParams)
 @appr_bp.errorhandler(ChannelNotFound)
@@ -44,7 +44,7 @@ def render_error(error):
 @appr_bp.route("/version")
 @anon_allowed
 def version():
-  return jsonify({"cnr-api": cnr.__version__})
+  return jsonify({"appr-api": appr.__version__})
 
 
 @appr_bp.route("/api/v1/users/login", methods=['POST'])
@@ -73,7 +73,7 @@ def login():
 @anon_protect
 def blobs(namespace, package_name, digest):
   reponame = repo_name(namespace, package_name)
-  data = cnr_registry.pull_blob(reponame, digest, blob_class=Blob)
+  data = appr_registry.pull_blob(reponame, digest, blob_class=Blob)
   json_format = request.args.get('format', None) == 'json'
   return _pull(data, json_format=json_format)
 
@@ -89,7 +89,7 @@ def list_packages():
   username = None
   if user:
     username = user.username
-  result_data = cnr_registry.list_packages(namespace, package_class=Package, search=query,
+  result_data = appr_registry.list_packages(namespace, package_class=Package, search=query,
                                            media_type=media_type, username=username)
   return jsonify(result_data)
 
@@ -102,7 +102,7 @@ def list_packages():
 @anon_protect
 def delete_package(namespace, package_name, release, media_type):
   reponame = repo_name(namespace, package_name)
-  result = cnr_registry.delete_package(reponame, release, media_type, package_class=Package)
+  result = appr_registry.delete_package(reponame, release, media_type, package_class=Package)
   model.log_action('delete_tag', namespace, repo_name=package_name,
                    metadata={'release': release, 'mediatype': media_type})
   return jsonify(result)
@@ -116,7 +116,7 @@ def delete_package(namespace, package_name, release, media_type):
 @anon_protect
 def show_package(namespace, package_name, release, media_type):
   reponame = repo_name(namespace, package_name)
-  result = cnr_registry.show_package(reponame, release, media_type, channel_class=Channel,
+  result = appr_registry.show_package(reponame, release, media_type, channel_class=Channel,
                                      package_class=Package)
   return jsonify(result)
 
@@ -129,7 +129,7 @@ def show_package(namespace, package_name, release, media_type):
 def show_package_releases(namespace, package_name):
   reponame = repo_name(namespace, package_name)
   media_type = request.args.get('media_type', None)
-  result = cnr_registry.show_package_releases(reponame, media_type=media_type,
+  result = appr_registry.show_package_releases(reponame, media_type=media_type,
                                               package_class=Package)
   return jsonify(result)
 
@@ -141,7 +141,7 @@ def show_package_releases(namespace, package_name):
 @anon_protect
 def show_package_release_manifests(namespace, package_name, release):
   reponame = repo_name(namespace, package_name)
-  result = cnr_registry.show_package_manifests(reponame, release, package_class=Package)
+  result = appr_registry.show_package_manifests(reponame, release, package_class=Package)
   return jsonify(result)
 
 
@@ -155,7 +155,7 @@ def show_package_release_manifests(namespace, package_name, release):
 def pull(namespace, package_name, release, media_type):
   reponame = repo_name(namespace, package_name)
   logger.info("pull %s", reponame)
-  data = cnr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
+  data = appr_registry.pull(reponame, release, media_type, Package, blob_class=Blob)
   model.log_action('pull_repo', namespace, repo_name=package_name,
                    metadata={'release': release, 'mediatype': media_type})
   json_format = request.args.get('format', None) == 'json'
@@ -199,7 +199,7 @@ def push(namespace, package_name):
   force = request.args.get('force', 'false') == 'true'
 
   blob = Blob(reponame, values['blob'])
-  app_release = cnr_registry.push(reponame, release_version, media_type, blob, force,
+  app_release = appr_registry.push(reponame, release_version, media_type, blob, force,
                                   package_class=Package, user=owner, visibility=private)
   model.log_action('push_repo', namespace, repo_name=package_name,
                    metadata={'release': release_version})
@@ -216,7 +216,7 @@ def search_packages():
   if user:
     username = user.username
 
-  search_results = cnr_registry.search(query, Package, username=username)
+  search_results = appr_registry.search(query, Package, username=username)
   return jsonify(search_results)
 
 
@@ -228,7 +228,7 @@ def search_packages():
 @anon_protect
 def list_channels(namespace, package_name):
   reponame = repo_name(namespace, package_name)
-  return jsonify(cnr_registry.list_channels(reponame, channel_class=Channel))
+  return jsonify(appr_registry.list_channels(reponame, channel_class=Channel))
 
 
 @appr_bp.route(
@@ -239,7 +239,7 @@ def list_channels(namespace, package_name):
 @anon_protect
 def show_channel(namespace, package_name, channel_name):
   reponame = repo_name(namespace, package_name)
-  channel = cnr_registry.show_channel(reponame, channel_name, channel_class=Channel)
+  channel = appr_registry.show_channel(reponame, channel_name, channel_class=Channel)
   return jsonify(channel)
 
 
@@ -253,7 +253,7 @@ def show_channel(namespace, package_name, channel_name):
 def add_channel_release(namespace, package_name, channel_name, release):
   _check_channel_name(channel_name, release)
   reponame = repo_name(namespace, package_name)
-  result = cnr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel,
+  result = appr_registry.add_channel_release(reponame, channel_name, release, channel_class=Channel,
                                             package_class=Package)
   model.log_action('create_tag', namespace, repo_name=package_name,
                    metadata={'channel': channel_name, 'release': release})
@@ -284,7 +284,7 @@ def _check_channel_name(channel_name, release=None):
 def delete_channel_release(namespace, package_name, channel_name, release):
   _check_channel_name(channel_name, release)
   reponame = repo_name(namespace, package_name)
-  result = cnr_registry.delete_channel_release(reponame, channel_name, release,
+  result = appr_registry.delete_channel_release(reponame, channel_name, release,
                                                channel_class=Channel, package_class=Package)
   model.log_action('delete_tag', namespace, repo_name=package_name,
                    metadata={'channel': channel_name, 'release': release})
@@ -301,7 +301,7 @@ def delete_channel_release(namespace, package_name, channel_name, release):
 def delete_channel(namespace, package_name, channel_name):
   _check_channel_name(channel_name)
   reponame = repo_name(namespace, package_name)
-  result = cnr_registry.delete_channel(reponame, channel_name, channel_class=Channel)
+  result = appr_registry.delete_channel(reponame, channel_name, channel_class=Channel)
   model.log_action('delete_tag', namespace, repo_name=package_name,
                    metadata={'channel': channel_name})
   return jsonify(result)

endpoints/appr/test/test_api.py

@@ -2,16 +2,16 @@ import uuid
 
 import pytest
 
-from cnr.tests.conftest import *
-from cnr.tests.test_apiserver import BaseTestServer
-from cnr.tests.test_models import CnrTestModels
+from appr.tests.conftest import *
+from appr.tests.test_apiserver import BaseTestServer
+from appr.tests.test_models import ApprTestModels
 
 import data.oci_model.blob as oci_blob
 
 from data.database import User
 from data.model import organization, user
 from endpoints.appr import registry # Needed to register the endpoint
-from endpoints.appr.cnr_backend import Channel, Package, QuayDB
+from endpoints.appr.appr_backend import Channel, Package, QuayDB
 from endpoints.appr.models_oci import model as oci_app_model
 
 from test.fixtures import *
@@ -70,14 +70,14 @@ class PackageTest(Package):
 
 @pytest.fixture(autouse=True)
 def quaydb(monkeypatch, app):
-  monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Package', PackageTest)
-  monkeypatch.setattr('endpoints.appr.cnr_backend.Package', PackageTest)
+  monkeypatch.setattr('endpoints.appr.appr_backend.QuayDB.Package', PackageTest)
+  monkeypatch.setattr('endpoints.appr.appr_backend.Package', PackageTest)
   monkeypatch.setattr('endpoints.appr.registry.Package', PackageTest)
-  monkeypatch.setattr('cnr.models.Package', PackageTest)
+  monkeypatch.setattr('appr.models.Package', PackageTest)
 
-  monkeypatch.setattr('endpoints.appr.cnr_backend.QuayDB.Channel', ChannelTest)
+  monkeypatch.setattr('endpoints.appr.appr_backend.QuayDB.Channel', ChannelTest)
   monkeypatch.setattr('endpoints.appr.registry.Channel', ChannelTest)
-  monkeypatch.setattr('cnr.models.Channel', ChannelTest)
+  monkeypatch.setattr('appr.models.Channel', ChannelTest)
 
 
 class TestServerQuayDB(BaseTestServer):
@@ -118,13 +118,13 @@ class TestServerQuayDB(BaseTestServer):
     pass
 
 
-class TestQuayModels(CnrTestModels):
+class TestQuayModels(ApprTestModels):
   DB_CLASS = QuayDB
 
   @pytest.mark.xfail
   def test_channel_delete_releases(self, db_with_data1):
     """ Can't remove a release from the channel, only delete the channel entirely """
-    CnrTestModels.test_channel_delete_releases(self, db_with_data1)
+    ApprTestModels.test_channel_delete_releases(self, db_with_data1)
 
   @pytest.mark.xfail
   def test_forbiddeb_db_reset(self, db_class):

initdb.py

@@ -404,15 +404,16 @@ def initialize_database():
   MediaType.create(name='text/plain')
   MediaType.create(name='application/json')
   MediaType.create(name='text/markdown')
-  MediaType.create(name='application/vnd.cnr.blob.v0.tar+gzip')
-  MediaType.create(name='application/vnd.cnr.package-manifest.helm.v0.json')
-  MediaType.create(name='application/vnd.cnr.package-manifest.kpm.v0.json')
-  MediaType.create(name='application/vnd.cnr.package-manifest.docker-compose.v0.json')
-  MediaType.create(name='application/vnd.cnr.package.kpm.v0.tar+gzip')
-  MediaType.create(name='application/vnd.cnr.package.helm.v0.tar+gzip')
-  MediaType.create(name='application/vnd.cnr.package.docker-compose.v0.tar+gzip')
-  MediaType.create(name='application/vnd.cnr.manifests.v0.json')
-  MediaType.create(name='application/vnd.cnr.manifest.list.v0.json')
+  for mediatype in ['appr', 'cnr']:
+    MediaType.create(name='application/vnd.%s.blob.v0.tar+gzip' % mediatype)
+    MediaType.create(name='application/vnd.%s.package-manifest.helm.v0.json' % mediatype)
+    MediaType.create(name='application/vnd.%s.package-manifest.kpm.v0.json' % mediatype)
+    MediaType.create(name='application/vnd.%s.package-manifest.docker-compose.v0.json' % mediatype)
+    MediaType.create(name='application/vnd.%s.package.kpm.v0.tar+gzip' % mediatype)
+    MediaType.create(name='application/vnd.%s.package.helm.v0.tar+gzip' % mediatype)
+    MediaType.create(name='application/vnd.%s.package.docker-compose.v0.tar+gzip' % mediatype)
+    MediaType.create(name='application/vnd.%s.manifests.v0.json' % mediatype)
+    MediaType.create(name='application/vnd.%s.manifest.list.v0.json' % mediatype)
 
   LabelSourceType.create(name='manifest')
   LabelSourceType.create(name='api', mutable=True)

requirements-nover.txt

@@ -8,7 +8,7 @@
 -e git+https://github.com/coreos/pyapi-gitlab.git@timeout#egg=pyapi-gitlab
 -e git+https://github.com/coreos/resumablehashlib.git#egg=resumablehashlib
 -e git+https://github.com/jepcastelein/marketo-rest-python.git#egg=marketorestpython
--e git+https://github.com/app-registry/appr-server.git#egg=cnr-server
+-e git+https://github.com/app-registry/appr.git@b80028a1fa6f52c314405690b2d4ffeaccccae5a#egg=appr
 APScheduler==3.0.5
 Flask-Login
 Flask-Mail

requirements.txt

@@ -6,7 +6,7 @@ alembic==0.9.1
 -e git+https://github.com/coreos/resumablehashlib.git@b1b631249589b07adf40e0ee545b323a501340b4#egg=resumablehashlib
 -e git+https://github.com/DevTable/aniso8601-fake.git@bd7762c7dea0498706d3f57db60cd8a8af44ba90#egg=aniso8601
 -e git+https://github.com/DevTable/anunidecode.git@d59236a822e578ba3a0e5e5abbd3855873fa7a88#egg=anunidecode
--e git+https://github.com/app-registry/appr-server.git@c2ef3b88afe926a92ef5f2e11e7d4a259e286a17#egg=cnr_server
+-e git+https://github.com/app-registry/appr.git@b80028a1fa6f52c314405690b2d4ffeaccccae5a#egg=appr
 -e git+https://github.com/DevTable/container-cloud-config.git@bce675537904175f6975024a4c89269027ea6792#egg=container_cloud_config
 -e git+https://github.com/DevTable/python-etcd.git@f1168cb02a2a8c83bec1108c6fcd8615ef463b14#egg=python_etcd
 -e git+https://github.com/jarus/flask-testing.git@18baff32969a0634a414ce61d2dd4a77433817a8#egg=Flask_Testing