Phase 1 of migrating APPR-specific tables to tables with the Appr prefix
Fixes https://jira.coreos.com/browse/QUAY-950
This commit is contained in:
Joseph Schorr
parent
6622f27c93
commit
113bb96f29
28 changed files with 699 additions and 176 deletions
|
|
@ -2,13 +2,15 @@ from datetime import datetime
|
|||
|
||||
import cnr.semver
|
||||
|
||||
from cnr.exception import raise_package_not_found, raise_channel_not_found
|
||||
from cnr.exception import raise_package_not_found, raise_channel_not_found, CnrException
|
||||
|
||||
import features
|
||||
import data.model
|
||||
|
||||
from app import storage, authentication
|
||||
from data import appr_model
|
||||
from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel
|
||||
from data.database import Repository, MediaType, db_transaction
|
||||
from data.appr_model.models import OLD_MODELS, NEW_MODELS
|
||||
from endpoints.appr.models_interface import (
|
||||
ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface,
|
||||
BlobDescriptor, ChannelView, ChannelReleasesView)
|
||||
|
|
@ -17,6 +19,12 @@ from util.morecollections import AttrDict
|
|||
from util.names import parse_robot_username
|
||||
|
||||
|
||||
|
||||
class ReadOnlyException(CnrException):
|
||||
status_code = 405
|
||||
errorcode = "read-only"
|
||||
|
||||
|
||||
def _strip_sha256_header(digest):
|
||||
if digest.startswith('sha256:'):
|
||||
return digest.split('sha256:')[1]
|
||||
|
|
@ -48,6 +56,10 @@ def _application(package):
|
|||
|
||||
|
||||
class CNRAppModel(AppRegistryDataInterface):
|
||||
def __init__(self, models_ref, is_readonly):
|
||||
self.models_ref = models_ref
|
||||
self.is_readonly = is_readonly
|
||||
|
||||
def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None,
|
||||
analytics_sample=1, metadata=None):
|
||||
metadata = {} if metadata is None else metadata
|
||||
|
|
@ -70,9 +82,10 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
"""
|
||||
|
||||
views = []
|
||||
for repo in appr_model.package.list_packages_query(namespace, media_type, search,
|
||||
username=username):
|
||||
releases = [t.name for t in repo.tag_set_prefetch]
|
||||
for repo in appr_model.package.list_packages_query(self.models_ref, namespace, media_type,
|
||||
search, username=username):
|
||||
tag_set_prefetch = getattr(repo, self.models_ref.tag_set_prefetch_name)
|
||||
releases = [t.name for t in tag_set_prefetch]
|
||||
if not releases:
|
||||
continue
|
||||
available_releases = [
|
||||
|
|
@ -81,7 +94,7 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
if with_channels:
|
||||
channels = [
|
||||
ChannelView(name=chan.name, current=chan.linked_tag.name)
|
||||
for chan in appr_model.channel.get_repo_channels(repo)]
|
||||
for chan in appr_model.channel.get_repo_channels(repo, self.models_ref)]
|
||||
|
||||
app_name = _join_package_name(repo.namespace_user.username, repo.name)
|
||||
manifests = self.list_manifests(app_name, available_releases[0])
|
||||
|
|
@ -93,8 +106,8 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
channels=channels,
|
||||
manifests=manifests,
|
||||
releases=available_releases,
|
||||
updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start),
|
||||
created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start),)
|
||||
updated_at=_timestamp_to_iso(tag_set_prefetch[-1].lifetime_start),
|
||||
created_at=_timestamp_to_iso(tag_set_prefetch[0].lifetime_start),)
|
||||
views.append(view)
|
||||
return views
|
||||
|
||||
|
|
@ -108,6 +121,9 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
|
||||
def create_application(self, package_name, visibility, owner):
|
||||
""" Create a new app repository, owner is the user who creates it """
|
||||
if self.is_readonly:
|
||||
raise ReadOnlyException('Currently in read-only mode')
|
||||
|
||||
ns, name = _split_package_name(package_name)
|
||||
data.model.repository.create_repository(ns, name, owner, visibility, 'application')
|
||||
|
||||
|
|
@ -137,7 +153,7 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
Todo:
|
||||
* Paginate
|
||||
"""
|
||||
return appr_model.release.get_releases(_application(package_name), media_type)
|
||||
return appr_model.release.get_releases(_application(package_name), self.models_ref, media_type)
|
||||
|
||||
def list_manifests(self, package_name, release=None):
|
||||
""" Returns the list of all manifests of an Application.
|
||||
|
|
@ -147,8 +163,8 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
"""
|
||||
try:
|
||||
repo = _application(package_name)
|
||||
return list(appr_model.manifest.get_manifest_types(repo, release))
|
||||
except (Repository.DoesNotExist, Tag.DoesNotExist):
|
||||
return list(appr_model.manifest.get_manifest_types(repo, self.models_ref, release))
|
||||
except (Repository.DoesNotExist, self.models_ref.Tag.DoesNotExist):
|
||||
raise_package_not_found(package_name, release)
|
||||
|
||||
def fetch_release(self, package_name, release, media_type):
|
||||
|
|
@ -157,7 +173,8 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
"""
|
||||
repo = _application(package_name)
|
||||
try:
|
||||
tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type)
|
||||
tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type,
|
||||
self.models_ref)
|
||||
created_at = _timestamp_to_iso(tag.lifetime_start)
|
||||
|
||||
blob_descriptor = BlobDescriptor(digest=_strip_sha256_header(blob.digest),
|
||||
|
|
@ -169,17 +186,23 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
app_release = ApplicationRelease(release=tag.name, created_at=created_at, name=package_name,
|
||||
manifest=app_manifest)
|
||||
return app_release
|
||||
except (Tag.DoesNotExist, Manifest.DoesNotExist, Blob.DoesNotExist, Repository.DoesNotExist,
|
||||
except (self.models_ref.Tag.DoesNotExist,
|
||||
self.models_ref.Manifest.DoesNotExist,
|
||||
self.models_ref.Blob.DoesNotExist,
|
||||
Repository.DoesNotExist,
|
||||
MediaType.DoesNotExist):
|
||||
raise_package_not_found(package_name, release, media_type)
|
||||
|
||||
def store_blob(self, cnrblob, content_media_type):
|
||||
if self.is_readonly:
|
||||
raise ReadOnlyException('Currently in read-only mode')
|
||||
|
||||
fp = cnrblob.packager.io_file
|
||||
path = cnrblob.upload_url(cnrblob.digest)
|
||||
locations = storage.preferred_locations
|
||||
storage.stream_write(locations, path, fp, 'application/x-gzip')
|
||||
db_blob = appr_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type,
|
||||
locations)
|
||||
locations, self.models_ref)
|
||||
return BlobDescriptor(mediaType=content_media_type,
|
||||
digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[])
|
||||
|
||||
|
|
@ -187,49 +210,60 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
""" Add an app-release to a repository
|
||||
package is an instance of data.cnr.package.Package
|
||||
"""
|
||||
if self.is_readonly:
|
||||
raise ReadOnlyException('Currently in read-only mode')
|
||||
|
||||
manifest = package.manifest()
|
||||
ns, name = package.namespace, package.name
|
||||
repo = data.model.repository.get_or_create_repository(ns, name, user, visibility=visibility,
|
||||
repo_kind='application')
|
||||
repo_kind='application')
|
||||
tag_name = package.release
|
||||
appr_model.release.create_app_release(repo, tag_name,
|
||||
package.manifest(), manifest['content']['digest'], force)
|
||||
appr_model.release.create_app_release(repo, tag_name, package.manifest(),
|
||||
manifest['content']['digest'], self.models_ref, force)
|
||||
|
||||
def delete_release(self, package_name, release, media_type):
|
||||
""" Remove/Delete an app-release from an app-repository.
|
||||
It does not delete the entire app-repository, only a single release
|
||||
"""
|
||||
if self.is_readonly:
|
||||
raise ReadOnlyException('Currently in read-only mode')
|
||||
|
||||
repo = _application(package_name)
|
||||
try:
|
||||
appr_model.release.delete_app_release(repo, release, media_type)
|
||||
except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist):
|
||||
appr_model.release.delete_app_release(repo, release, media_type, self.models_ref)
|
||||
except (self.models_ref.Channel.DoesNotExist,
|
||||
self.models_ref.Tag.DoesNotExist,
|
||||
MediaType.DoesNotExist):
|
||||
raise_package_not_found(package_name, release, media_type)
|
||||
|
||||
def release_exists(self, package, release):
|
||||
""" Return true if a release with that name already exist or
|
||||
have existed (include deleted ones) """
|
||||
# TODO: Figure out why this isn't implemented.
|
||||
|
||||
def channel_exists(self, package_name, channel_name):
|
||||
""" Returns true if channel exists """
|
||||
repo = _application(package_name)
|
||||
return appr_model.tag.tag_exists(repo, channel_name, "channel")
|
||||
return appr_model.tag.tag_exists(repo, channel_name, self.models_ref, "channel")
|
||||
|
||||
def delete_channel(self, package_name, channel_name):
|
||||
""" Delete an AppChannel
|
||||
Note:
|
||||
It doesn't delete the AppReleases
|
||||
"""
|
||||
if self.is_readonly:
|
||||
raise ReadOnlyException('Currently in read-only mode')
|
||||
|
||||
repo = _application(package_name)
|
||||
try:
|
||||
appr_model.channel.delete_channel(repo, channel_name)
|
||||
except (Channel.DoesNotExist, Tag.DoesNotExist):
|
||||
appr_model.channel.delete_channel(repo, channel_name, self.models_ref)
|
||||
except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
|
||||
raise_channel_not_found(package_name, channel_name)
|
||||
|
||||
def list_channels(self, package_name):
|
||||
""" Returns all AppChannel for a package """
|
||||
repo = _application(package_name)
|
||||
channels = appr_model.channel.get_repo_channels(repo)
|
||||
channels = appr_model.channel.get_repo_channels(repo, self.models_ref)
|
||||
return [ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in channels]
|
||||
|
||||
def fetch_channel(self, package_name, channel_name, with_releases=True):
|
||||
|
|
@ -237,12 +271,12 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
repo = _application(package_name)
|
||||
|
||||
try:
|
||||
channel = appr_model.channel.get_channel(repo, channel_name)
|
||||
except (Channel.DoesNotExist, Tag.DoesNotExist):
|
||||
channel = appr_model.channel.get_channel(repo, channel_name, self.models_ref)
|
||||
except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
|
||||
raise_channel_not_found(package_name, channel_name)
|
||||
|
||||
if with_releases:
|
||||
releases = appr_model.channel.get_channel_releases(repo, channel)
|
||||
releases = appr_model.channel.get_channel_releases(repo, channel, self.models_ref)
|
||||
chanview = ChannelReleasesView(
|
||||
current=channel.linked_tag.name, name=channel.name,
|
||||
releases=[channel.linked_tag.name] + [c.name for c in releases])
|
||||
|
|
@ -254,9 +288,9 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
def list_release_channels(self, package_name, release, active=True):
|
||||
repo = _application(package_name)
|
||||
try:
|
||||
channels = appr_model.channel.get_tag_channels(repo, release, active=active)
|
||||
channels = appr_model.channel.get_tag_channels(repo, release, self.models_ref, active=active)
|
||||
return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels]
|
||||
except (Channel.DoesNotExist, Tag.DoesNotExist):
|
||||
except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
|
||||
raise_package_not_found(package_name, release)
|
||||
|
||||
def update_channel(self, package_name, channel_name, release):
|
||||
|
|
@ -264,12 +298,17 @@ class CNRAppModel(AppRegistryDataInterface):
|
|||
Returns:
|
||||
A new AppChannel with the release
|
||||
"""
|
||||
if self.is_readonly:
|
||||
raise ReadOnlyException('Currently in read-only mode')
|
||||
|
||||
repo = _application(package_name)
|
||||
channel = appr_model.channel.create_or_update_channel(repo, channel_name, release)
|
||||
channel = appr_model.channel.create_or_update_channel(repo, channel_name, release,
|
||||
self.models_ref)
|
||||
return ChannelView(current=channel.linked_tag.name, name=channel.name)
|
||||
|
||||
def get_blob_locations(self, digest):
|
||||
return appr_model.blob.get_blob_locations(digest)
|
||||
return appr_model.blob.get_blob_locations(digest, self.models_ref)
|
||||
|
||||
|
||||
model = CNRAppModel()
|
||||
# Phase 1: Read from old tables, disallow writing.
|
||||
model = CNRAppModel(OLD_MODELS, features.READONLY_APP_REGISTRY)
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ from cnr.tests.conftest import *
|
|||
from cnr.tests.test_apiserver import BaseTestServer
|
||||
from cnr.tests.test_models import CnrTestModels
|
||||
|
||||
import data.appr_model.blob as oci_blob
|
||||
import data.appr_model.blob as appr_blob
|
||||
|
||||
from data.database import User
|
||||
from data.model import organization, user
|
||||
from endpoints.appr import registry # Needed to register the endpoint
|
||||
from endpoints.appr.cnr_backend import Channel, Package, QuayDB
|
||||
from endpoints.appr.models_cnr import model as oci_app_model
|
||||
from endpoints.appr.models_cnr import model as appr_app_model
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
|
|
@ -28,7 +28,7 @@ class ChannelTest(Channel):
|
|||
@classmethod
|
||||
def dump_all(cls, package_class=None):
|
||||
result = []
|
||||
for repo in oci_app_model.list_applications(with_channels=True):
|
||||
for repo in appr_app_model.list_applications(with_channels=True):
|
||||
for chan in repo.channels:
|
||||
result.append({'name': chan.name, 'current': chan.current, 'package': repo.name})
|
||||
return result
|
||||
|
|
@ -51,17 +51,17 @@ class PackageTest(Package):
|
|||
@classmethod
|
||||
def dump_all(cls, blob_cls):
|
||||
result = []
|
||||
for repo in oci_app_model.list_applications(with_channels=True):
|
||||
for repo in appr_app_model.list_applications(with_channels=True):
|
||||
package_name = repo.name
|
||||
for release in repo.releases:
|
||||
for mtype in cls.manifests(package_name, release):
|
||||
package = oci_app_model.fetch_release(package_name, release, mtype)
|
||||
package = appr_app_model.fetch_release(package_name, release, mtype)
|
||||
blob = blob_cls.get(package_name, package.manifest.content.digest)
|
||||
app_data = cls._apptuple_to_dict(package)
|
||||
app_data.pop('digest')
|
||||
app_data['channels'] = [
|
||||
x.name
|
||||
for x in oci_app_model.list_release_channels(package_name, package.release, False)
|
||||
for x in appr_app_model.list_release_channels(package_name, package.release, False)
|
||||
]
|
||||
app_data['blob'] = blob.b64blob
|
||||
result.append(app_data)
|
||||
|
|
@ -141,11 +141,11 @@ class TestQuayModels(CnrTestModels):
|
|||
assert p.release == "2.0.1"
|
||||
assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80"
|
||||
blob = db_with_data1.Blob.get("titi/rocketchat", p.digest)
|
||||
bdb = oci_blob.get_blob(p.digest)
|
||||
bdb = appr_blob.get_blob(p.digest, appr_app_model.models_ref)
|
||||
newblob = db_with_data1.Blob("titi/app2", blob.b64blob)
|
||||
p2 = db_with_data1.Package("titi/app2", "1.0.0", "helm", newblob)
|
||||
p2.save()
|
||||
b2db = oci_blob.get_blob(p2.digest)
|
||||
b2db = appr_blob.get_blob(p2.digest, appr_app_model.models_ref)
|
||||
assert b2db.id == bdb.id
|
||||
|
||||
def test_force_push_different_blob(self, db_with_data1):
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
import base64
|
||||
import json
|
||||
|
||||
from flask import url_for
|
||||
from mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from flask import url_for
|
||||
|
||||
from data import model
|
||||
from endpoints.appr.registry import appr_bp
|
||||
|
||||
from test.fixtures import *
|
||||
|
||||
|
||||
@pytest.mark.parametrize('login_data, expected_code', [
|
||||
({
|
||||
"username": "devtable",
|
||||
|
|
@ -62,3 +66,27 @@ def test_invalid_release_name(release_name, app, client):
|
|||
|
||||
rv = client.open(url, method='POST', data=json.dumps(data), headers=headers)
|
||||
assert rv.status_code == 422
|
||||
|
||||
|
||||
@pytest.mark.parametrize('readonly, expected_status', [
|
||||
(True, 405),
|
||||
(False, 422),
|
||||
])
|
||||
def test_readonly(readonly, expected_status, app, client):
|
||||
params = {
|
||||
'namespace': 'devtable',
|
||||
'package_name': 'someapprepo',
|
||||
}
|
||||
|
||||
url = url_for('appr.push', **params)
|
||||
auth = base64.b64encode('devtable:password')
|
||||
headers = {'Content-Type': 'application/json', 'Authorization': 'Basic ' + auth}
|
||||
data = {
|
||||
'release': '1.0',
|
||||
'media_type': 'application/vnd.cnr.manifest.v0+json',
|
||||
'blob': 'H4sIAFQwWVoAA+3PMQrCQBAF0Bxlb+Bk143nETGIIEoSC29vMMFOu3TvNb/5DH/Ot8f02jWbiohDremT3ZKR90uuUlty7nKJNmqKtkQuTarbzlo8x+k4zFOu4+lyH4afvbnW93/urH98EwAAAAAAAAAAADb0BsdwExIAKAAA',
|
||||
}
|
||||
|
||||
with patch('endpoints.appr.models_cnr.model.is_readonly', readonly):
|
||||
rv = client.open(url, method='POST', data=json.dumps(data), headers=headers)
|
||||
assert rv.status_code == expected_status
|
||||
|
|
|
|||
Reference in a new issue