vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3093 from quay/joseph.schorr/QUAY-950/appr-step-1

Browse source 
Phase 1 of Appr migration
This commit is contained in:
Joseph Schorr 2018-07-21 14:58:14 -04:00 committed by GitHub
commit d75e932d09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 855 additions and 281 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
config.py
View file

@ -264,6 +264,9 @@ class DefaultConfig(ImmutableConfig):
# Feature Flag: Whether to enable support for App repositories.
FEATURE_APP_REGISTRY = False
# Feature Flag: Whether app registry is in a read-only mode.
FEATURE_READONLY_APP_REGISTRY = False
# Feature Flag: If set to true, the _catalog endpoint returns public repositories. Otherwise,
# only private repositories can be returned.
FEATURE_PUBLIC_CATALOG = False

2
data/oci_model/__init__.py → data/appr_model/__init__.py
View file

@ -1,4 +1,4 @@
from data.oci_model import (
from data.appr_model import (
blob,
channel,
manifest,

21
data/oci_model/blob.py → data/appr_model/blob.py
View file

@ -3,7 +3,6 @@ import logging
from peewee import IntegrityError
from data.model import db_transaction
from data.database import Blob, BlobPlacementLocation, BlobPlacement
logger = logging.getLogger(__name__)
@ -13,17 +12,20 @@ def _ensure_sha256_header(digest):
return 'sha256:' + digest
def get_blob(digest):
def get_blob(digest, models_ref):
""" Find a blob by its digest. """
Blob = models_ref.Blob
return Blob.select().where(Blob.digest == _ensure_sha256_header(digest)).get()
def get_or_create_blob(digest, size, media_type_name, locations):
def get_or_create_blob(digest, size, media_type_name, locations, models_ref):
""" Try to find a blob by its digest or create it. """
Blob = models_ref.Blob
BlobPlacement = models_ref.BlobPlacement
# Get or create the blog entry for the digest.
try:
blob = get_blob(digest)
blob = get_blob(digest, models_ref)
logger.debug('Retrieved blob with digest %s', digest)
except Blob.DoesNotExist:
blob = Blob.create(digest=_ensure_sha256_header(digest),
@ -38,13 +40,16 @@ def get_or_create_blob(digest, size, media_type_name, locations):
BlobPlacement.create(blob=blob, location=location_id)
except IntegrityError:
logger.debug('Location %s already existing for blob %s', location_name, blob.id)
pass
return blob
def get_blob_locations(digest):
def get_blob_locations(digest, models_ref):
""" Find all locations names for a blob. """
Blob = models_ref.Blob
BlobPlacement = models_ref.BlobPlacement
BlobPlacementLocation = models_ref.BlobPlacementLocation
return [x.name for x in
BlobPlacementLocation
.select()
@ -53,7 +58,9 @@ def get_blob_locations(digest):
.where(Blob.digest == _ensure_sha256_header(digest))]
def ensure_blob_locations(*names):
def ensure_blob_locations(models_ref, *names):
BlobPlacementLocation = models_ref.BlobPlacementLocation
with db_transaction():
locations = BlobPlacementLocation.select().where(BlobPlacementLocation.name << names)

38
data/oci_model/channel.py → data/appr_model/channel.py
View file

@ -1,11 +1,13 @@
from data.database import Tag, Channel
from data.oci_model import tag as tag_model
from data.appr_model import tag as tag_model
def get_channel_releases(repo, channel):
def get_channel_releases(repo, channel, models_ref):
""" Return all previously linked tags.
This works based upon Tag lifetimes.
"""
Channel = models_ref.Channel
Tag = models_ref.Tag
tag_kind_id = Channel.tag_kind.get_id('channel')
channel_name = channel.name
return (Tag
@ -17,40 +19,46 @@ def get_channel_releases(repo, channel):
.order_by(Tag.lifetime_end))
def get_channel(repo, channel_name):
def get_channel(repo, channel_name, models_ref):
""" Find a Channel by name. """
channel = tag_model.get_tag(repo, channel_name, "channel")
channel = tag_model.get_tag(repo, channel_name, models_ref, "channel")
return channel
def get_tag_channels(repo, tag_name, active=True):
def get_tag_channels(repo, tag_name, models_ref, active=True):
""" Find the Channels associated with a Tag. """
tag = tag_model.get_tag(repo, tag_name, "release")
Tag = models_ref.Tag
tag = tag_model.get_tag(repo, tag_name, models_ref, "release")
query = tag.tag_parents
if active:
query = tag_model.tag_alive_oci(query)
query = tag_model.tag_is_alive(query, Tag)
return query
def delete_channel(repo, channel_name):
def delete_channel(repo, channel_name, models_ref):
""" Delete a channel by name. """
return tag_model.delete_tag(repo, channel_name, "channel")
return tag_model.delete_tag(repo, channel_name, models_ref, "channel")
def create_or_update_channel(repo, channel_name, tag_name):
def create_or_update_channel(repo, channel_name, tag_name, models_ref):
""" Creates or updates a channel to include a particular tag. """
tag = tag_model.get_tag(repo, tag_name, 'release')
return tag_model.create_or_update_tag(repo, channel_name, linked_tag=tag, tag_kind="channel")
tag = tag_model.get_tag(repo, tag_name, models_ref, 'release')
return tag_model.create_or_update_tag(repo, channel_name, models_ref, linked_tag=tag,
tag_kind="channel")
def get_repo_channels(repo):
def get_repo_channels(repo, models_ref):
""" Creates or updates a channel to include a particular tag. """
Channel = models_ref.Channel
Tag = models_ref.Tag
tag_kind_id = Channel.tag_kind.get_id('channel')
query = (Channel
.select(Channel, Tag)
.join(Tag, on=(Tag.id == Channel.linked_tag))
.where(Channel.repository == repo,
Channel.tag_kind == tag_kind_id))
return tag_model.tag_alive_oci(query, cls=Channel)
return tag_model.tag_is_alive(query, Channel)

26
data/oci_model/manifest.py → data/appr_model/manifest.py
View file

@ -4,8 +4,8 @@ import json
from cnr.models.package_base import get_media_type
from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
from data.oci_model import tag as tag_model
from data.database import db_transaction, MediaType
from data.appr_model import tag as tag_model
logger = logging.getLogger(__name__)
@ -21,20 +21,23 @@ def _digest(manifestjson):
return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest())
def get_manifest_query(digest, media_type):
def get_manifest_query(digest, media_type, models_ref):
Manifest = models_ref.Manifest
return Manifest.select().where(Manifest.digest == _ensure_sha256_header(digest),
Manifest.media_type == Manifest.media_type.get_id(media_type))
def get_manifest_with_blob(digest, media_type):
query = get_manifest_query(digest, media_type)
def get_manifest_with_blob(digest, media_type, models_ref):
Blob = models_ref.Blob
query = get_manifest_query(digest, media_type, models_ref)
return query.join(Blob).get()
def get_or_create_manifest(manifest_json, media_type_name):
def get_or_create_manifest(manifest_json, media_type_name, models_ref):
Manifest = models_ref.Manifest
digest = _digest(manifest_json)
try:
manifest = get_manifest_query(digest, media_type_name).get()
manifest = get_manifest_query(digest, media_type_name, models_ref).get()
except Manifest.DoesNotExist:
with db_transaction():
manifest = Manifest.create(digest=digest,
@ -42,16 +45,19 @@ def get_or_create_manifest(manifest_json, media_type_name):
media_type=Manifest.media_type.get_id(media_type_name))
return manifest
def get_manifest_types(repo, release=None):
def get_manifest_types(repo, models_ref, release=None):
""" Returns an array of MediaTypes.name for a repo, can filter by tag """
query = tag_model.tag_alive_oci(Tag
Tag = models_ref.Tag
ManifestListManifest = models_ref.ManifestListManifest
query = tag_model.tag_is_alive(Tag
.select(MediaType.name)
.join(ManifestListManifest,
on=(ManifestListManifest.manifest_list == Tag.manifest_list))
.join(MediaType,
on=(ManifestListManifest.media_type == MediaType.id))
.where(Tag.repository == repo,
Tag.tag_kind == Tag.tag_kind.get_id('release')))
Tag.tag_kind == Tag.tag_kind.get_id('release')), Tag)
if release:
query = query.where(Tag.name == release)

20
data/oci_model/manifest_list.py → data/appr_model/manifest_list.py
View file

@ -2,7 +2,7 @@ import logging
import hashlib
import json
from data.database import ManifestList, ManifestListManifest, db_transaction
from data.database import db_transaction
logger = logging.getLogger(__name__)
@ -18,16 +18,19 @@ def _digest(manifestjson):
return _ensure_sha256_header(hashlib.sha256(json.dumps(manifestjson, sort_keys=True)).hexdigest())
def get_manifest_list(digest):
def get_manifest_list(digest, models_ref):
ManifestList = models_ref.ManifestList
return ManifestList.select().where(ManifestList.digest == _ensure_sha256_header(digest)).get()
def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version):
def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_version, models_ref):
ManifestList = models_ref.ManifestList
digest = _digest(manifest_list_json)
media_type_id = ManifestList.media_type.get_id(media_type_name)
try:
return get_manifest_list(digest)
return get_manifest_list(digest, models_ref)
except ManifestList.DoesNotExist:
with db_transaction():
manifestlist = ManifestList.create(digest=digest, manifest_list_json=manifest_list_json,
@ -35,7 +38,7 @@ def get_or_create_manifest_list(manifest_list_json, media_type_name, schema_vers
return manifestlist
def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json, models_ref):
""" From a manifestlist, manifests, and the manifest list blob,
create if doesn't exist the manfiestlistmanifest for each manifest """
for pos in xrange(len(manifest_ids)):
@ -43,10 +46,13 @@ def create_manifestlistmanifest(manifestlist, manifest_ids, manifest_list_json):
manifest_json = manifest_list_json[pos]
get_or_create_manifestlistmanifest(manifest=manifest_id,
manifestlist=manifestlist,
media_type_name=manifest_json['mediaType'])
media_type_name=manifest_json['mediaType'],
models_ref=models_ref)
def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name):
def get_or_create_manifestlistmanifest(manifest, manifestlist, media_type_name, models_ref):
ManifestListManifest = models_ref.ManifestListManifest
media_type_id = ManifestListManifest.media_type.get_id(media_type_name)
try:
ml = (ManifestListManifest

21
data/appr_model/models.py Normal file
View file

@ -0,0 +1,21 @@
from collections import namedtuple
from data.database import (Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement, Channel)
from data.database import (ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement, ApprChannel)
ModelsRef = namedtuple('ModelsRef', ['Tag', 'TagKind', 'BlobPlacementLocation', 'ManifestList',
'ManifestBlob', 'Blob', 'ManifestListManifest', 'Manifest',
'BlobPlacement', 'Channel', 'manifestlistmanifest_set_name',
'tag_set_prefetch_name'])
OLD_MODELS = ModelsRef(Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement, Channel,
'manifestlistmanifest_set', 'tag_set_prefetch')
NEW_MODELS = ModelsRef(ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement, ApprChannel, 'apprmanifestlistmanifest_set',
'apprtag_set_prefetch')

12
data/oci_model/package.py → data/appr_model/package.py
View file

@ -3,12 +3,14 @@ from peewee import prefetch
from data import model
from data.database import Repository, Namespace, Tag, ManifestListManifest
from data.oci_model import tag as tag_model
from data.database import Repository, Namespace
from data.appr_model import tag as tag_model
def list_packages_query(namespace=None, media_type=None, search_query=None, username=None):
def list_packages_query(models_ref, namespace=None, media_type=None, search_query=None,
username=None):
""" List and filter repository by search query. """
Tag = models_ref.Tag
fields = [model.repository.SEARCH_FIELDS.name.name]
if search_query is not None:
@ -40,9 +42,9 @@ def list_packages_query(namespace=None, media_type=None, search_query=None, user
.order_by(Tag.lifetime_start))
if media_type:
tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type)
tag_query = tag_model.filter_tags_by_media_type(tag_query, media_type, models_ref)
tag_query = tag_model.tag_alive_oci(tag_query)
tag_query = tag_model.tag_is_alive(tag_query, Tag)
query = prefetch(repo_query, tag_query)
return query

75
data/oci_model/release.py → data/appr_model/release.py
View file

@ -3,9 +3,8 @@ import bisect
from cnr.exception import PackageAlreadyExists
from cnr.models.package_base import manifest_media_type
from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
ManifestListManifest, Blob, ManifestBlob)
from data.oci_model import (blob as blob_model, manifest as manifest_model,
from data.database import db_transaction, get_epoch_timestamp
from data.appr_model import (blob as blob_model, manifest as manifest_model,
manifest_list as manifest_list_model,
tag as tag_model)
@ -20,11 +19,17 @@ def _ensure_sha256_header(digest):
return 'sha256:' + digest
def get_app_release(repo, tag_name, media_type):
def get_app_release(repo, tag_name, media_type, models_ref):
""" Returns (tag, manifest, blob) given a repo object, tag_name, and media_type). """
tag = tag_model.get_tag(repo, tag_name, tag_kind='release')
ManifestListManifest = models_ref.ManifestListManifest
Manifest = models_ref.Manifest
Blob = models_ref.Blob
ManifestBlob = models_ref.ManifestBlob
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
tag = tag_model.get_tag(repo, tag_name, models_ref, tag_kind='release')
media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
manifestlistmanifest = (tag.manifest_list.manifestlistmanifest_set
manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name)
.join(Manifest)
.where(ManifestListManifest.media_type == media_type_id).get())
manifest = manifestlistmanifest.manifest
@ -32,25 +37,26 @@ def get_app_release(repo, tag_name, media_type):
return (tag, manifest, blob)
def delete_app_release(repo, tag_name, media_type):
def delete_app_release(repo, tag_name, media_type, models_ref):
""" Terminate a Tag/media-type couple
It find the corresponding tag/manifest and remove from the manifestlistmanifest the manifest
1. it terminates the current tag (in all-cases)
2. if the new manifestlist is not empty, it creates a new tag for it
"""
ManifestListManifest = models_ref.ManifestListManifest
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
media_type_id = ManifestListManifest.media_type.get_id(manifest_media_type(media_type))
with db_transaction():
tag = tag_model.get_tag(repo, tag_name)
tag = tag_model.get_tag(repo, tag_name, models_ref)
manifest_list = tag.manifest_list
list_json = manifest_list.manifest_list_json
mlm_query = (ManifestListManifest
.select()
.where(ManifestListManifest.manifest_list == tag.manifest_list))
list_manifest_ids = sorted([mlm.manifest_id for mlm in mlm_query])
manifestlistmanifest = (tag
.manifest_list
.manifestlistmanifest_set
manifestlistmanifest = (getattr(tag.manifest_list, manifestlistmanifest_set_name)
.where(ManifestListManifest.media_type == media_type_id).get())
index = list_manifest_ids.index(manifestlistmanifest.manifest_id)
list_manifest_ids.pop(index)
@ -61,36 +67,42 @@ def delete_app_release(repo, tag_name, media_type):
tag.save()
else:
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
SCHEMA_VERSION)
SCHEMA_VERSION, models_ref)
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids,
list_json)
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
list_json, models_ref)
tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist,
tag_kind="release")
return tag
def create_app_release(repo, tag_name, manifest_data, digest, force=False):
def create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False):
""" Create a new application release, it includes creating a new Tag, ManifestList,
ManifestListManifests, Manifest, ManifestBlob.
To deduplicate the ManifestList, the manifestlist_json is kept ordered by the manifest.id.
To find the insert point in the ManifestList it uses bisect on the manifest-ids list.
"""
ManifestList = models_ref.ManifestList
ManifestListManifest = models_ref.ManifestListManifest
Blob = models_ref.Blob
ManifestBlob = models_ref.ManifestBlob
with db_transaction():
# Create/get the package manifest
manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType'])
manifest = manifest_model.get_or_create_manifest(manifest_data, manifest_data['mediaType'],
models_ref)
# get the tag
tag = tag_model.get_or_initialize_tag(repo, tag_name)
tag = tag_model.get_or_initialize_tag(repo, tag_name, models_ref)
if tag.manifest_list is None:
tag.manifest_list = ManifestList(media_type=ManifestList.media_type.get_id(LIST_MEDIA_TYPE),
schema_version=SCHEMA_VERSION,
manifest_list_json=[])
manifest_list_json=[], )
elif tag_model.tag_media_type_exists(tag, manifest.media_type):
elif tag_model.tag_media_type_exists(tag, manifest.media_type, models_ref):
if force:
delete_app_release(repo, tag_name, manifest.media_type.name)
return create_app_release(repo, tag_name, manifest_data, digest, force=False)
delete_app_release(repo, tag_name, manifest.media_type.name, models_ref)
return create_app_release(repo, tag_name, manifest_data, digest, models_ref, force=False)
else:
raise PackageAlreadyExists("package exists already")
@ -103,10 +115,11 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
list_json.insert(insert_point, manifest.manifest_json)
list_manifest_ids.insert(insert_point, manifest.id)
manifestlist = manifest_list_model.get_or_create_manifest_list(list_json, LIST_MEDIA_TYPE,
SCHEMA_VERSION)
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json)
SCHEMA_VERSION, models_ref)
manifest_list_model.create_manifestlistmanifest(manifestlist, list_manifest_ids, list_json,
models_ref)
tag = tag_model.create_or_update_tag(repo, tag_name, manifest_list=manifestlist,
tag = tag_model.create_or_update_tag(repo, tag_name, models_ref, manifest_list=manifestlist,
tag_kind="release")
blob_digest = digest
@ -117,21 +130,23 @@ def create_app_release(repo, tag_name, manifest_data, digest, force=False):
.where(ManifestBlob.manifest == manifest,
Blob.digest == _ensure_sha256_header(blob_digest)).get())
except ManifestBlob.DoesNotExist:
blob = blob_model.get_blob(blob_digest)
blob = blob_model.get_blob(blob_digest, models_ref)
ManifestBlob.create(manifest=manifest, blob=blob)
return tag
def get_release_objs(repo, media_type=None):
def get_release_objs(repo, models_ref, media_type=None):
""" Returns an array of Tag for a repo, with optional filtering by media_type. """
Tag = models_ref.Tag
release_query = (Tag
.select()
.where(Tag.repository == repo,
Tag.tag_kind == Tag.tag_kind.get_id("release")))
if media_type:
release_query = tag_model.filter_tags_by_media_type(release_query, media_type)
release_query = tag_model.filter_tags_by_media_type(release_query, media_type, models_ref)
return tag_model.tag_alive_oci(release_query)
return tag_model.tag_is_alive(release_query, Tag)
def get_releases(repo, media_type=None):
def get_releases(repo, model_refs, media_type=None):
""" Returns an array of Tag.name for a repo, can filter by media_type. """
return [t.name for t in get_release_objs(repo, media_type)]
return [t.name for t in get_release_objs(repo, model_refs, media_type)]

48
data/oci_model/tag.py → data/appr_model/tag.py
View file

@ -4,32 +4,37 @@ from cnr.models.package_base import manifest_media_type
from peewee import IntegrityError
from data.model import (db_transaction, TagAlreadyCreatedException)
from data.database import Tag, ManifestListManifest, get_epoch_timestamp_ms, db_for_update
from data.database import get_epoch_timestamp_ms, db_for_update
logger = logging.getLogger(__name__)
def tag_alive_oci(query, now_ts=None, cls=Tag):
def tag_is_alive(query, cls, now_ts=None):
return query.where((cls.lifetime_end >> None) |
(cls.lifetime_end > now_ts))
def tag_media_type_exists(tag, media_type):
return (tag.manifest_list.manifestlistmanifest_set
def tag_media_type_exists(tag, media_type, models_ref):
ManifestListManifest = models_ref.ManifestListManifest
manifestlistmanifest_set_name = models_ref.manifestlistmanifest_set_name
return (getattr(tag.manifest_list, manifestlistmanifest_set_name)
.where(ManifestListManifest.media_type == media_type).count() > 0)
def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, tag_kind="release"):
def create_or_update_tag(repo, tag_name, models_ref, manifest_list=None, linked_tag=None,
tag_kind="release"):
Tag = models_ref.Tag
now_ts = get_epoch_timestamp_ms()
tag_kind_id = Tag.tag_kind.get_id(tag_kind)
with db_transaction():
try:
tag = db_for_update(tag_alive_oci(Tag
tag = db_for_update(tag_is_alive(Tag
.select()
.where(Tag.repository == repo,
Tag.name == tag_name,
Tag.tag_kind == tag_kind_id), now_ts)).get()
Tag.tag_kind == tag_kind_id), Tag, now_ts)).get()
if tag.manifest_list == manifest_list and tag.linked_tag == linked_tag:
return tag
tag.lifetime_end = now_ts
@ -46,40 +51,47 @@ def create_or_update_tag(repo, tag_name, manifest_list=None, linked_tag=None, ta
raise TagAlreadyCreatedException(msg % (tag_name, now_ts, repo.namespace_user, repo.name))
def get_or_initialize_tag(repo, tag_name, tag_kind="release"):
def get_or_initialize_tag(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
try:
return tag_alive_oci(Tag.select().where(Tag.repository == repo, Tag.name == tag_name)).get()
return tag_is_alive(Tag.select().where(Tag.repository == repo, Tag.name == tag_name), Tag).get()
except Tag.DoesNotExist:
return Tag(repo=repo, name=tag_name, tag_kind=Tag.tag_kind.get_id(tag_kind))
def get_tag(repo, tag_name, tag_kind="release"):
return tag_alive_oci(Tag.select()
def get_tag(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
return tag_is_alive(Tag.select()
.where(Tag.repository == repo,
Tag.name == tag_name,
Tag.tag_kind == Tag.tag_kind.get_id(tag_kind))).get()
Tag.tag_kind == Tag.tag_kind.get_id(tag_kind)), Tag).get()
def delete_tag(repo, tag_name, tag_kind="release"):
def delete_tag(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
tag_kind_id = Tag.tag_kind.get_id(tag_kind)
tag = tag_alive_oci(Tag.select()
tag = tag_is_alive(Tag.select()
.where(Tag.repository == repo,
Tag.name == tag_name, Tag.tag_kind == tag_kind_id)).get()
Tag.name == tag_name, Tag.tag_kind == tag_kind_id), Tag).get()
tag.lifetime_end = get_epoch_timestamp_ms()
tag.save()
return tag
def tag_exists(repo, tag_name, tag_kind="release"):
def tag_exists(repo, tag_name, models_ref, tag_kind="release"):
Tag = models_ref.Tag
try:
get_tag(repo, tag_name, tag_kind)
get_tag(repo, tag_name, models_ref, tag_kind)
return True
except Tag.DoesNotExist:
return False
def filter_tags_by_media_type(tag_query, media_type):
def filter_tags_by_media_type(tag_query, media_type, models_ref):
""" Return only available tag for a media_type. """
ManifestListManifest = models_ref.ManifestListManifest
Tag = models_ref.Tag
media_type = manifest_media_type(media_type)
t = (tag_query
.join(ManifestListManifest, on=(ManifestListManifest.manifest_list == Tag.manifest_list))

220
data/database.py
View file

@ -470,9 +470,8 @@ class User(BaseModel):
RepositoryTag, PermissionPrototype, DerivedStorageForImage,
TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
RepositoryNotification, OAuthAuthorizationCode,
RepositoryActionCount, TagManifestLabel, Tag,
ManifestLabel, BlobUploading, TeamSync,
RepositorySearchScore, DeletedNamespace} | beta_classes
RepositoryActionCount, TagManifestLabel,
TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes | appr_classes
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
@ -620,7 +619,7 @@ class Repository(BaseModel):
# are cleaned up directly
skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
RepositorySearchScore} | beta_classes
RepositorySearchScore} | cnr_classes | appr_classes
delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
@ -1188,7 +1187,6 @@ class ServiceKey(BaseModel):
class MediaType(BaseModel):
""" MediaType is an enumeration of the possible formats of various objects in the data model.
This model is a part of the new OCI/CNR model set.
"""
name = CharField(index=True, unique=True)
@ -1202,7 +1200,6 @@ class Messages(BaseModel):
class LabelSourceType(BaseModel):
""" LabelSourceType is an enumeration of the possible sources for a label.
This model is a part of the new OCI/CNR model set.
"""
name = CharField(index=True, unique=True)
mutable = BooleanField(default=False)
@ -1211,7 +1208,6 @@ class LabelSourceType(BaseModel):
class Label(BaseModel):
""" Label represents user-facing metadata associated with another entry in the database (e.g. a
Manifest).
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(default=uuid_generator, index=True, unique=True)
key = CharField(index=True)
@ -1222,7 +1218,6 @@ class Label(BaseModel):
class TagManifestLabel(BaseModel):
""" Mapping from a tag manifest to a label.
This model is a part of the new OCI/CNR model set.
"""
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(TagManifest, index=True)
@ -1238,7 +1233,17 @@ class TagManifestLabel(BaseModel):
class Blob(BaseModel):
""" Blob represents a content-addressable object stored outside of the database.
This model is a part of the new OCI/CNR model set.
This is deprecated in favor of ApprBlob.
"""
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
size = BigIntegerField()
uncompressed_size = BigIntegerField(null=True)
class ApprBlob(BaseModel):
""" ApprBlob represents a content-addressable object stored outside of the database.
"""
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
@ -1248,22 +1253,20 @@ class Blob(BaseModel):
class BlobPlacementLocation(BaseModel):
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
This model is a part of the new OCI/CNR model set.
This is deprecated in favor of ApprBlobPlacementLocation.
"""
name = CharField(index=True, unique=True)
class BlobPlacementLocationPreference(BaseModel):
""" BlobPlacementLocationPreference is a location to which a user's data will be replicated.
This model is a part of the new OCI/CNR model set.
class ApprBlobPlacementLocation(BaseModel):
""" ApprBlobPlacementLocation is an enumeration of the possible storage locations for ApprBlobs.
"""
user = QuayUserField(index=True, allows_robots=False)
location = EnumField(BlobPlacementLocation)
name = CharField(index=True, unique=True)
class BlobPlacement(BaseModel):
""" BlobPlacement represents the location of a Blob.
This model is a part of the new OCI/CNR model set.
This is deprecated in favor of ApprBlobPlacement.
"""
blob = ForeignKeyField(Blob)
location = EnumField(BlobPlacementLocation)
@ -1276,58 +1279,40 @@ class BlobPlacement(BaseModel):
)
class BlobUploading(BaseModel):
""" BlobUploading represents the state of a Blob currently being uploaded.
This model is a part of the new OCI/CNR model set.
class ApprBlobPlacement(BaseModel):
""" ApprBlobPlacement represents the location of a Blob.
"""
uuid = CharField(index=True, unique=True)
created = DateTimeField(default=datetime.now, index=True)
repository = ForeignKeyField(Repository, index=True)
location = ForeignKeyField(BlobPlacementLocation)
byte_count = IntegerField(default=0)
uncompressed_byte_count = IntegerField(null=True)
chunk_count = IntegerField(default=0)
storage_metadata = JSONField(null=True, default={})
sha_state = ResumableSHA256Field(null=True, default=resumablehashlib.sha256)
piece_sha_state = ResumableSHA1Field(null=True)
piece_hashes = Base64BinaryField(null=True)
blob = ForeignKeyField(ApprBlob)
location = EnumField(ApprBlobPlacementLocation)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'uuid'), True),
(('blob', 'location'), True),
)
class Manifest(BaseModel):
""" Manifest represents the metadata and collection of blobs that comprise a container image.
This model is a part of the new OCI/CNR model set.
This is deprecated in favor of ApprManifest.
"""
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
manifest_json = JSONField()
class ManifestLabel(BaseModel):
""" ManifestLabel represents label metadata annotating a Manifest.
This model is a part of the new OCI/CNR model set.
class ApprManifest(BaseModel):
""" ApprManifest represents the metadata and collection of blobs that comprise an Appr image.
"""
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(Manifest, index=True)
label = ForeignKeyField(Label)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'annotated', 'label'), True),
)
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
manifest_json = JSONField()
class ManifestBlob(BaseModel):
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
This model is a part of the new OCI/CNR model set.
This is deprecated in favor of ApprManifestBlob.
"""
manifest = ForeignKeyField(Manifest, index=True)
blob = ForeignKeyField(Blob, index=True)
@ -1340,9 +1325,32 @@ class ManifestBlob(BaseModel):
)
class ApprManifestBlob(BaseModel):
""" ApprManifestBlob is a many-to-many relation table linking ApprManifests and ApprBlobs.
"""
manifest = ForeignKeyField(ApprManifest, index=True)
blob = ForeignKeyField(ApprBlob, index=True)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest', 'blob'), True),
)
class ManifestList(BaseModel):
""" ManifestList represents all of the various manifests that compose a Tag.
This model is a part of the new OCI/CNR model set.
This is deprecated in favor of ApprManifestList.
"""
digest = CharField(index=True, unique=True)
manifest_list_json = JSONField()
schema_version = CharField()
media_type = EnumField(MediaType)
class ApprManifestList(BaseModel):
""" ApprManifestList represents all of the various Appr manifests that compose an ApprTag.
"""
digest = CharField(index=True, unique=True)
manifest_list_json = JSONField()
@ -1352,14 +1360,20 @@ class ManifestList(BaseModel):
class TagKind(BaseModel):
""" TagKind is a enumtable to reference tag kinds.
This model is a part of the new OCI/CNR model set.
This model is deprecated in favor of ApprTagKind.
"""
name = CharField(index=True, unique=True)
class ApprTagKind(BaseModel):
""" ApprTagKind is a enumtable to reference tag kinds.
"""
name = CharField(index=True, unique=True)
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList.
This model is a part of the new OCI/CNR model set.
This model is deprecated in favor of ApprTag.
"""
name = CharField()
repository = ForeignKeyField(Repository)
@ -1383,12 +1397,37 @@ class Tag(BaseModel):
)
class ApprTag(BaseModel):
""" ApprTag represents a user-facing alias for referencing an ApprManifestList.
"""
name = CharField()
repository = ForeignKeyField(Repository)
manifest_list = ForeignKeyField(ApprManifestList, null=True)
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reverted = BooleanField(default=False)
protected = BooleanField(default=False)
tag_kind = EnumField(ApprTagKind)
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'name'), False),
(('repository', 'name', 'hidden'), False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end'), True),
)
Channel = Tag.alias()
ApprChannel = ApprTag.alias()
class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
This model is a part of the new OCI/CNR model set.
This model is deprecated in favor of ApprManifestListManifest.
"""
manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True)
@ -1406,76 +1445,22 @@ class ManifestListManifest(BaseModel):
)
class ManifestLayer(BaseModel):
""" ManifestLayer represents one of the layers that compose a Manifest.
This model is a part of the new OCI/CNR model set.
class ApprManifestListManifest(BaseModel):
""" ApprManifestListManifest is a many-to-many relation table linking ApprManifestLists and
ApprManifests.
"""
blob = ForeignKeyField(Blob, index=True)
manifest = ForeignKeyField(Manifest)
manifest_index = IntegerField(index=True) # index 0 is the last command in a Dockerfile
metadata_json = JSONField()
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest', 'manifest_index'), True),
)
class ManifestLayerDockerV1(BaseModel):
""" ManifestLayerDockerV1 is the Docker v1 registry protocol metadata for a ManifestLayer.
This model is a part of the new OCI/CNR model set.
"""
manifest_layer = ForeignKeyField(ManifestLayer)
image_id = CharField(index=True)
checksum = CharField()
compat_json = JSONField()
class ManifestLayerScan(BaseModel):
""" ManifestLayerScan represents the state of security scanning for a ManifestLayer.
This model is a part of the new OCI/CNR model set.
"""
layer = ForeignKeyField(ManifestLayer, unique=True)
scannable = BooleanField()
scanned_by = CharField()
class DerivedImage(BaseModel):
""" DerivedImage represents a Manifest transcoded into an alternative format.
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(default=uuid_generator, unique=True)
source_manifest = ForeignKeyField(Manifest)
derived_manifest_json = JSONField()
manifest_list = ForeignKeyField(ApprManifestList, index=True)
manifest = ForeignKeyField(ApprManifest, index=True)
operating_system = CharField(null=True)
architecture = CharField(null=True)
platform_json = JSONField(null=True)
media_type = EnumField(MediaType)
blob = ForeignKeyField(Blob, related_name='blob')
uniqueness_hash = CharField(index=True, unique=True)
signature_blob = ForeignKeyField(Blob, null=True, related_name='signature_blob')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('source_manifest', 'blob'), True),
(('source_manifest', 'media_type', 'uniqueness_hash'), True),
)
class BitTorrentPieces(BaseModel):
""" BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob.
This model is a part of the new OCI/CNR model set.
"""
blob = ForeignKeyField(Blob)
pieces = Base64BinaryField()
piece_length = IntegerField()
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('blob', 'piece_length'), True),
(('manifest_list', 'media_type'), False),
)
@ -1499,9 +1484,10 @@ class AppSpecificAuthToken(BaseModel):
)
beta_classes = set([ManifestLayerScan, Tag, TagKind, BlobPlacementLocation, ManifestLayer, ManifestList,
BitTorrentPieces, MediaType, Label, ManifestBlob, BlobUploading, Blob,
ManifestLayerDockerV1, BlobPlacementLocationPreference, ManifestListManifest,
Manifest, DerivedImage, BlobPlacement, ManifestLabel])
cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement])
appr_classes = set([ApprTag, ApprTagKind, ApprBlobPlacementLocation, ApprManifestList,
ApprManifestBlob, ApprBlob, ApprManifestListManifest, ApprManifest,
ApprBlobPlacement])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

2
data/migrations/migration.sh
View file

@ -14,7 +14,7 @@ up_mysql() {
sleep 25
# Add the database to mysql.
docker run --rm --link mysql:mysql mysql sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
docker run --rm --link mysql:mysql mysql:5.7 sh -c 'echo "create database genschema" | mysql -h"$MYSQL_PORT_3306_TCP_ADDR" -P"$MYSQL_PORT_3306_TCP_PORT" -uroot -ppassword'
}
down_mysql() {

167
data/migrations/versions/5cbbfc95bac7_remove_oci_tables_not_used_by_cnr_the_.py Normal file
View file

@ -0,0 +1,167 @@
"""Remove 'oci' tables not used by CNR. The rest will be migrated and renamed.
Revision ID: 5cbbfc95bac7
Revises: 1783530bee68
Create Date: 2018-05-23 17:28:40.114433
"""
# revision identifiers, used by Alembic.
revision = '5cbbfc95bac7'
down_revision = '1783530bee68'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from util.migrate import UTF8LongText, UTF8CharField
def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('derivedimage')
op.drop_table('manifestlabel')
op.drop_table('blobplacementlocationpreference')
op.drop_table('blobuploading')
op.drop_table('bittorrentpieces')
op.drop_table('manifestlayerdockerv1')
op.drop_table('manifestlayerscan')
op.drop_table('manifestlayer')
# ### end Alembic commands ###
def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
'manifestlayer',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('manifest_index', sa.BigInteger(), nullable=False),
sa.Column('metadata_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestlayer_blob_id_blob')),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlayer_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayer'))
)
op.create_index('manifestlayer_manifest_index', 'manifestlayer', ['manifest_index'], unique=False)
op.create_index('manifestlayer_manifest_id_manifest_index', 'manifestlayer', ['manifest_id', 'manifest_index'], unique=True)
op.create_index('manifestlayer_manifest_id', 'manifestlayer', ['manifest_id'], unique=False)
op.create_index('manifestlayer_blob_id', 'manifestlayer', ['blob_id'], unique=False)
op.create_table(
'manifestlayerscan',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('layer_id', sa.Integer(), nullable=False),
sa.Column('scannable', sa.Boolean(), nullable=False),
sa.Column('scanned_by', UTF8CharField(length=255), nullable=False),
sa.ForeignKeyConstraint(['layer_id'], ['manifestlayer.id'], name=op.f('fk_manifestlayerscan_layer_id_manifestlayer')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayerscan'))
)
op.create_index('manifestlayerscan_layer_id', 'manifestlayerscan', ['layer_id'], unique=True)
op.create_table(
'bittorrentpieces',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('pieces', UTF8LongText, nullable=False),
sa.Column('piece_length', sa.BigInteger(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_bittorrentpieces_blob_id_blob')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_bittorrentpieces'))
)
op.create_index('bittorrentpieces_blob_id_piece_length', 'bittorrentpieces', ['blob_id', 'piece_length'], unique=True)
op.create_index('bittorrentpieces_blob_id', 'bittorrentpieces', ['blob_id'], unique=False)
op.create_table(
'blobuploading',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.Column('byte_count', sa.BigInteger(), nullable=False),
sa.Column('uncompressed_byte_count', sa.BigInteger(), nullable=True),
sa.Column('chunk_count', sa.BigInteger(), nullable=False),
sa.Column('storage_metadata', UTF8LongText, nullable=True),
sa.Column('sha_state', UTF8LongText, nullable=True),
sa.Column('piece_sha_state', UTF8LongText, nullable=True),
sa.Column('piece_hashes', UTF8LongText, nullable=True),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobuploading_location_id_blobplacementlocation')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_blobuploading_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobuploading'))
)
op.create_index('blobuploading_uuid', 'blobuploading', ['uuid'], unique=True)
op.create_index('blobuploading_repository_id_uuid', 'blobuploading', ['repository_id', 'uuid'], unique=True)
op.create_index('blobuploading_repository_id', 'blobuploading', ['repository_id'], unique=False)
op.create_index('blobuploading_location_id', 'blobuploading', ['location_id'], unique=False)
op.create_index('blobuploading_created', 'blobuploading', ['created'], unique=False)
op.create_table(
'manifestlayerdockerv1',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_layer_id', sa.Integer(), nullable=False),
sa.Column('image_id', UTF8CharField(length=255), nullable=False),
sa.Column('checksum', UTF8CharField(length=255), nullable=False),
sa.Column('compat_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['manifest_layer_id'], ['manifestlayer.id'], name=op.f('fk_manifestlayerdockerv1_manifest_layer_id_manifestlayer')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayerdockerv1'))
)
op.create_index('manifestlayerdockerv1_manifest_layer_id', 'manifestlayerdockerv1', ['manifest_layer_id'], unique=False)
op.create_index('manifestlayerdockerv1_image_id', 'manifestlayerdockerv1', ['image_id'], unique=False)
op.create_table(
'manifestlabel',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('annotated_id', sa.Integer(), nullable=False),
sa.Column('label_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['annotated_id'], ['manifest.id'], name=op.f('fk_manifestlabel_annotated_id_manifest')),
sa.ForeignKeyConstraint(['label_id'], ['label.id'], name=op.f('fk_manifestlabel_label_id_label')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_manifestlabel_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlabel'))
)
op.create_index('manifestlabel_repository_id_annotated_id_label_id', 'manifestlabel', ['repository_id', 'annotated_id', 'label_id'], unique=True)
op.create_index('manifestlabel_repository_id', 'manifestlabel', ['repository_id'], unique=False)
op.create_index('manifestlabel_label_id', 'manifestlabel', ['label_id'], unique=False)
op.create_index('manifestlabel_annotated_id', 'manifestlabel', ['annotated_id'], unique=False)
op.create_table(
'blobplacementlocationpreference',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacementlocpref_locid_blobplacementlocation')),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_blobplacementlocationpreference_user_id_user')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocationpreference'))
)
op.create_index('blobplacementlocationpreference_user_id', 'blobplacementlocationpreference', ['user_id'], unique=False)
op.create_index('blobplacementlocationpreference_location_id', 'blobplacementlocationpreference', ['location_id'], unique=False)
op.create_table(
'derivedimage',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('source_manifest_id', sa.Integer(), nullable=False),
sa.Column('derived_manifest_json', UTF8LongText, nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('uniqueness_hash', sa.String(length=255), nullable=False),
sa.Column('signature_blob_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_derivedimage_blob_id_blob')),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_derivedimage_media_type_id_mediatype')),
sa.ForeignKeyConstraint(['signature_blob_id'], ['blob.id'], name=op.f('fk_derivedimage_signature_blob_id_blob')),
sa.ForeignKeyConstraint(['source_manifest_id'], ['manifest.id'], name=op.f('fk_derivedimage_source_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_derivedimage'))
)
op.create_index('derivedimage_uuid', 'derivedimage', ['uuid'], unique=True)
op.create_index('derivedimage_uniqueness_hash', 'derivedimage', ['uniqueness_hash'], unique=True)
op.create_index('derivedimage_source_manifest_id_media_type_id_uniqueness_hash', 'derivedimage', ['source_manifest_id', 'media_type_id', 'uniqueness_hash'], unique=True)
op.create_index('derivedimage_source_manifest_id_blob_id', 'derivedimage', ['source_manifest_id', 'blob_id'], unique=True)
op.create_index('derivedimage_source_manifest_id', 'derivedimage', ['source_manifest_id'], unique=False)
op.create_index('derivedimage_signature_blob_id', 'derivedimage', ['signature_blob_id'], unique=False)
op.create_index('derivedimage_media_type_id', 'derivedimage', ['media_type_id'], unique=False)
op.create_index('derivedimage_blob_id', 'derivedimage', ['blob_id'], unique=False)
# ### end Alembic commands ###

203
data/migrations/versions/610320e9dacf_add_new_appr_specific_tables.py Normal file
View file

@ -0,0 +1,203 @@
"""Add new Appr-specific tables
Revision ID: 610320e9dacf
Revises: 5cbbfc95bac7
Create Date: 2018-05-24 16:46:13.514562
"""
# revision identifiers, used by Alembic.
revision = '610320e9dacf'
down_revision = '5cbbfc95bac7'
from alembic import op
import sqlalchemy as sa
from util.migrate.table_ops import copy_table_contents
def upgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('apprblobplacementlocation',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacementlocation'))
)
op.create_index('apprblobplacementlocation_name', 'apprblobplacementlocation', ['name'], unique=True)
op.create_table('apprtagkind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtagkind'))
)
op.create_index('apprtagkind_name', 'apprtagkind', ['name'], unique=True)
op.create_table('apprblob',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('size', sa.BigInteger(), nullable=False),
sa.Column('uncompressed_size', sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprblob_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblob'))
)
op.create_index('apprblob_digest', 'apprblob', ['digest'], unique=True)
op.create_index('apprblob_media_type_id', 'apprblob', ['media_type_id'], unique=False)
op.create_table('apprmanifest',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('manifest_json', sa.Text(), nullable=False),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifest_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifest'))
)
op.create_index('apprmanifest_digest', 'apprmanifest', ['digest'], unique=True)
op.create_index('apprmanifest_media_type_id', 'apprmanifest', ['media_type_id'], unique=False)
op.create_table('apprmanifestlist',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('manifest_list_json', sa.Text(), nullable=False),
sa.Column('schema_version', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlist_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlist'))
)
op.create_index('apprmanifestlist_digest', 'apprmanifestlist', ['digest'], unique=True)
op.create_index('apprmanifestlist_media_type_id', 'apprmanifestlist', ['media_type_id'], unique=False)
op.create_table('apprblobplacement',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprblobplacement_blob_id_apprblob')),
sa.ForeignKeyConstraint(['location_id'], ['apprblobplacementlocation.id'], name=op.f('fk_apprblobplacement_location_id_apprblobplacementlocation')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprblobplacement'))
)
op.create_index('apprblobplacement_blob_id', 'apprblobplacement', ['blob_id'], unique=False)
op.create_index('apprblobplacement_blob_id_location_id', 'apprblobplacement', ['blob_id', 'location_id'], unique=True)
op.create_index('apprblobplacement_location_id', 'apprblobplacement', ['location_id'], unique=False)
op.create_table('apprmanifestblob',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['apprblob.id'], name=op.f('fk_apprmanifestblob_blob_id_apprblob')),
sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestblob_manifest_id_apprmanifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestblob'))
)
op.create_index('apprmanifestblob_blob_id', 'apprmanifestblob', ['blob_id'], unique=False)
op.create_index('apprmanifestblob_manifest_id', 'apprmanifestblob', ['manifest_id'], unique=False)
op.create_index('apprmanifestblob_manifest_id_blob_id', 'apprmanifestblob', ['manifest_id', 'blob_id'], unique=True)
op.create_table('apprmanifestlistmanifest',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_list_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('operating_system', sa.String(length=255), nullable=True),
sa.Column('architecture', sa.String(length=255), nullable=True),
sa.Column('platform_json', sa.Text(), nullable=True),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['manifest_id'], ['apprmanifest.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_id_apprmanifest')),
sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprmanifestlistmanifest_manifest_list_id_apprmanifestlist')),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_apprmanifestlistmanifest_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprmanifestlistmanifest'))
)
op.create_index('apprmanifestlistmanifest_manifest_id', 'apprmanifestlistmanifest', ['manifest_id'], unique=False)
op.create_index('apprmanifestlistmanifest_manifest_list_id', 'apprmanifestlistmanifest', ['manifest_list_id'], unique=False)
op.create_index('apprmanifestlistmanifest_manifest_list_id_media_type_id', 'apprmanifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False)
op.create_index('apprmanifestlistmanifest_manifest_list_id_operating_system_arch', 'apprmanifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False)
op.create_index('apprmanifestlistmanifest_media_type_id', 'apprmanifestlistmanifest', ['media_type_id'], unique=False)
op.create_table('apprtag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('manifest_list_id', sa.Integer(), nullable=True),
sa.Column('lifetime_start', sa.BigInteger(), nullable=False),
sa.Column('lifetime_end', sa.BigInteger(), nullable=True),
sa.Column('hidden', sa.Boolean(), nullable=False),
sa.Column('reverted', sa.Boolean(), nullable=False),
sa.Column('protected', sa.Boolean(), nullable=False),
sa.Column('tag_kind_id', sa.Integer(), nullable=False),
sa.Column('linked_tag_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['linked_tag_id'], ['apprtag.id'], name=op.f('fk_apprtag_linked_tag_id_apprtag')),
sa.ForeignKeyConstraint(['manifest_list_id'], ['apprmanifestlist.id'], name=op.f('fk_apprtag_manifest_list_id_apprmanifestlist')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_apprtag_repository_id_repository')),
sa.ForeignKeyConstraint(['tag_kind_id'], ['apprtagkind.id'], name=op.f('fk_apprtag_tag_kind_id_apprtagkind')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_apprtag'))
)
op.create_index('apprtag_lifetime_end', 'apprtag', ['lifetime_end'], unique=False)
op.create_index('apprtag_linked_tag_id', 'apprtag', ['linked_tag_id'], unique=False)
op.create_index('apprtag_manifest_list_id', 'apprtag', ['manifest_list_id'], unique=False)
op.create_index('apprtag_repository_id', 'apprtag', ['repository_id'], unique=False)
op.create_index('apprtag_repository_id_name', 'apprtag', ['repository_id', 'name'], unique=False)
op.create_index('apprtag_repository_id_name_hidden', 'apprtag', ['repository_id', 'name', 'hidden'], unique=False)
op.create_index('apprtag_repository_id_name_lifetime_end', 'apprtag', ['repository_id', 'name', 'lifetime_end'], unique=True)
op.create_index('apprtag_tag_kind_id', 'apprtag', ['tag_kind_id'], unique=False)
# ### end Alembic commands ###
conn = op.get_bind()
copy_table_contents('blobplacementlocation', 'apprblobplacementlocation', conn)
copy_table_contents('tagkind', 'apprtagkind', conn)
# ### population of test data ### #
tester.populate_table('apprmanifest', [
('digest', tester.TestDataType.String),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
('manifest_json', tester.TestDataType.JSON),
])
tester.populate_table('apprmanifestlist', [
('digest', tester.TestDataType.String),
('manifest_list_json', tester.TestDataType.JSON),
('schema_version', tester.TestDataType.String),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
])
tester.populate_table('apprmanifestlistmanifest', [
('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')),
('manifest_id', tester.TestDataType.Foreign('apprmanifest')),
('operating_system', tester.TestDataType.String),
('architecture', tester.TestDataType.String),
('platform_json', tester.TestDataType.JSON),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
])
tester.populate_table('apprblob', [
('digest', tester.TestDataType.String),
('media_type_id', tester.TestDataType.Foreign('mediatype')),
('size', tester.TestDataType.BigInteger),
('uncompressed_size', tester.TestDataType.BigInteger),
])
tester.populate_table('apprmanifestblob', [
('manifest_id', tester.TestDataType.Foreign('apprmanifest')),
('blob_id', tester.TestDataType.Foreign('apprblob')),
])
tester.populate_table('apprtag', [
('name', tester.TestDataType.String),
('repository_id', tester.TestDataType.Foreign('repository')),
('manifest_list_id', tester.TestDataType.Foreign('apprmanifestlist')),
('lifetime_start', tester.TestDataType.Integer),
('hidden', tester.TestDataType.Boolean),
('reverted', tester.TestDataType.Boolean),
('protected', tester.TestDataType.Boolean),
('tag_kind_id', tester.TestDataType.Foreign('apprtagkind')),
])
tester.populate_table('apprblobplacement', [
('blob_id', tester.TestDataType.Foreign('apprmanifestblob')),
('location_id', tester.TestDataType.Foreign('apprblobplacementlocation')),
])
# ### end population of test data ### #
def downgrade(tables, tester):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('apprtag')
op.drop_table('apprmanifestlistmanifest')
op.drop_table('apprmanifestblob')
op.drop_table('apprblobplacement')
op.drop_table('apprmanifestlist')
op.drop_table('apprmanifest')
op.drop_table('apprblob')
op.drop_table('apprtagkind')
op.drop_table('apprblobplacementlocation')
# ### end Alembic commands ###

6
data/model/repository.py
View file

@ -10,7 +10,7 @@ from cachetools import ttl_cache
from data.model import (
config, DataModelException, tag, db_transaction, storage, permission, _basequery)
from data.database import (
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag,
Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User, Visibility, Tag, ApprTag,
RepositoryPermission, RepositoryActionCount, Role, RepositoryAuthorizedEmail, TagManifest,
DerivedStorageForImage, Label, TagManifestLabel, db_for_update, get_epoch_timestamp,
db_random_func, db_concat_func, RepositorySearchScore, RepositoryKind)
@ -87,11 +87,13 @@ def purge_repository(namespace_name, repository_name):
except Repository.DoesNotExist:
return False
# Delete the repository of all OCI-referenced entries.
# Delete the repository of all Appr-referenced entries.
# Note that new-model Tag's must be deleted in *two* passes, as they can reference parent tags,
# and MySQL is... particular... about such relationships when deleting.
Tag.delete().where(Tag.repository == repo, ~(Tag.linked_tag >> None)).execute()
Tag.delete().where(Tag.repository == repo).execute()
ApprTag.delete().where(ApprTag.repository == repo, ~(ApprTag.linked_tag >> None)).execute()
ApprTag.delete().where(ApprTag.repository == repo).execute()
# Delete all tags to allow gc to reclaim storage
previously_referenced = tag.purge_all_tags(repo)

1
data/model/sqlalchemybridge.py
View file

@ -15,7 +15,6 @@ OPTION_TRANSLATIONS = {
'null': 'nullable',
}
def gen_sqlalchemy_metadata(peewee_model_list):
metadata = MetaData(naming_convention={
"ix": 'ix_%(column_0_label)s',

12
data/model/storage.py
View file

@ -9,7 +9,7 @@ from data.model import (config, db_transaction, InvalidImageException, TorrentIn
from data.database import (ImageStorage, Image, ImageStoragePlacement, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignature,
ImageStorageSignatureKind, Repository, Namespace, TorrentInfo, Blob,
ensure_under_transaction)
ApprBlob, ensure_under_transaction)
logger = logging.getLogger(__name__)
@ -105,15 +105,21 @@ def garbage_collect_storage(storage_id_whitelist):
logger.warning('GC attempted to remove CAS checksums %s, which are still IS referenced',
is_referenced_checksums)
# Check the new Blob table as well.
# Check the ApprBlob tables as well.
query = Blob.select(Blob.digest).where(Blob.digest << list(content_checksums))
blob_referenced_checksums = set([blob.digest for blob in query])
if blob_referenced_checksums:
logger.warning('GC attempted to remove CAS checksums %s, which are still Blob referenced',
blob_referenced_checksums)
query = ApprBlob.select(ApprBlob.digest).where(ApprBlob.digest << list(content_checksums))
appr_blob_referenced_checksums = set([blob.digest for blob in query])
if appr_blob_referenced_checksums:
logger.warning('GC attempted to remove CAS checksums %s, which are ApprBlob referenced',
appr_blob_referenced_checksums)
unreferenced_checksums = (content_checksums - blob_referenced_checksums -
is_referenced_checksums)
appr_blob_referenced_checksums - is_referenced_checksums)
# Return all placements for all image storages found not at a CAS path or with a content
# checksum that is referenced.

9
data/model/test/test_gc.py
View file

@ -9,7 +9,8 @@ from contextlib import contextmanager
from playhouse.test_utils import assert_query_count
from data import model, database
from data.database import Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel, Blob
from data.database import (Image, ImageStorage, DerivedStorageForImage, Label, TagManifestLabel,
Blob, ApprBlob)
from test.fixtures import *
@ -187,7 +188,10 @@ def assert_gc_integrity(expect_storage_removed=True):
storage.get_content({preferred}, storage.blob_path(storage_row.content_checksum))
for blob_row in Blob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
for blob_row in ApprBlob.select():
storage.get_content({preferred}, storage.blob_path(blob_row.digest))
def test_has_garbage(default_tag_policy, initialized_db):
@ -589,6 +593,7 @@ def test_images_shared_cas_with_new_blob_table(default_tag_policy, initialized_d
is1 = database.ImageStorage.create(content_checksum=digest, uploading=False)
database.Blob.create(digest=digest, size=0, media_type=media_type)
database.ApprBlob.create(digest=digest, size=0, media_type=media_type)
location = database.ImageStorageLocation.get(name=preferred)
database.ImageStoragePlacement.create(location=location, storage=is1)

9
endpoints/api/repository_models_pre_oci.py
View file

@ -3,7 +3,9 @@ from collections import defaultdict
from datetime import datetime, timedelta
from auth.permissions import ReadRepositoryPermission
from data import model, oci_model
from data import model
from data.appr_model import channel as channel_model, release as release_model
from endpoints.appr.models_cnr import model as appr_model
from endpoints.api.repository_models_interface import RepositoryDataInterface, RepositoryBaseElement, Repository, \
ApplicationRepository, ImageRepositoryRepository, Tag, Channel, Release, Count
@ -142,10 +144,9 @@ class PreOCIModel(RepositoryDataInterface):
repo.namespace_user.organization, repo.namespace_user.removed_tag_expiration_s, None, None,
False, False, False)
# Note: This is *temporary* code for the new OCI model stuff.
if base.kind_name == 'application':
channels = oci_model.channel.get_repo_channels(repo)
releases = oci_model.release.get_release_objs(repo)
channels = channel_model.get_repo_channels(repo, appr_model.models_ref)
releases = release_model.get_release_objs(repo, appr_model.models_ref)
releases_channels_map = defaultdict(list)
return ApplicationRepository(
base, [_create_channel(channel, releases_channels_map) for channel in channels], [

27
endpoints/api/test/test_repository.py
View file

@ -2,7 +2,9 @@ import pytest
from mock import patch, ANY, MagicMock
from data import model
from data import model, database
from data.appr_model import release, channel, blob
from endpoints.appr.models_cnr import model as appr_model
from endpoints.api.test.shared import conduct_api_call
from endpoints.api.repository import RepositoryTrust, Repository, RepositoryList
from endpoints.test.shared import client_with_identity
@ -117,3 +119,26 @@ def test_create_repository(repo_name, expected_status, client):
if expected_status == 201:
assert result['name'] == repo_name
assert model.repository.get_repository('devtable', repo_name).name == repo_name
def test_get_app_repo(client, initialized_db):
with client_with_identity('devtable', client) as cl:
devtable = model.user.get_user('devtable')
repo = model.repository.create_repository('devtable', 'someappr', devtable,
repo_kind='application')
models_ref = appr_model.models_ref
blob.get_or_create_blob('sha256:somedigest', 0, 'application/vnd.cnr.blob.v0.tar+gzip',
['local_us'], models_ref)
release.create_app_release(repo, 'test',
dict(mediaType='application/vnd.cnr.package-manifest.helm.v0.json'),
'sha256:somedigest', models_ref, False)
channel.create_or_update_channel(repo, 'somechannel', 'test', models_ref)
params = {'repository': 'devtable/someappr'}
response = conduct_api_call(cl, Repository, 'GET', params).json
assert response['kind'] == 'application'
assert response['channels']
assert response['releases']

5
endpoints/api/test/test_search.py
View file

@ -2,6 +2,7 @@ import pytest
from playhouse.test_utils import assert_query_count
from data import model, database
from endpoints.api.search import ConductRepositorySearch, ConductSearch
from endpoints.api.test.shared import conduct_api_call
from endpoints.test.shared import client_with_identity
@ -14,6 +15,10 @@ from test.fixtures import *
('repository'),
])
def test_repository_search(query, client):
# Prime the caches.
database.Repository.kind.get_id('image')
database.Repository.kind.get_name(1)
with client_with_identity('devtable', client) as cl:
params = {'query': query}
with assert_query_count(7):

2
endpoints/appr/cnr_backend.py
View file

@ -8,7 +8,7 @@ from cnr.models.package_base import PackageBase, manifest_media_type
from flask import request
from app import storage
from endpoints.appr.models_oci import model
from endpoints.appr.models_cnr import model
class Blob(BlobBase):

107
endpoints/appr/models_oci.py → endpoints/appr/models_cnr.py
View file

@ -2,13 +2,15 @@ from datetime import datetime
import cnr.semver
from cnr.exception import raise_package_not_found, raise_channel_not_found
from cnr.exception import raise_package_not_found, raise_channel_not_found, CnrException
import features
import data.model
from app import storage, authentication
from data import oci_model
from data.database import Tag, Manifest, MediaType, Blob, Repository, Channel
from data import appr_model
from data.database import Repository, MediaType, db_transaction
from data.appr_model.models import OLD_MODELS, NEW_MODELS
from endpoints.appr.models_interface import (
ApplicationManifest, ApplicationRelease, ApplicationSummaryView, AppRegistryDataInterface,
BlobDescriptor, ChannelView, ChannelReleasesView)
@ -17,6 +19,12 @@ from util.morecollections import AttrDict
from util.names import parse_robot_username
class ReadOnlyException(CnrException):
status_code = 405
errorcode = "read-only"
def _strip_sha256_header(digest):
if digest.startswith('sha256:'):
return digest.split('sha256:')[1]
@ -47,7 +55,11 @@ def _application(package):
return repo
class OCIAppModel(AppRegistryDataInterface):
class CNRAppModel(AppRegistryDataInterface):
def __init__(self, models_ref, is_readonly):
self.models_ref = models_ref
self.is_readonly = is_readonly
def log_action(self, event_name, namespace_name, repo_name=None, analytics_name=None,
analytics_sample=1, metadata=None):
metadata = {} if metadata is None else metadata
@ -70,9 +82,10 @@ class OCIAppModel(AppRegistryDataInterface):
"""
views = []
for repo in oci_model.package.list_packages_query(namespace, media_type, search,
username=username):
releases = [t.name for t in repo.tag_set_prefetch]
for repo in appr_model.package.list_packages_query(self.models_ref, namespace, media_type,
search, username=username):
tag_set_prefetch = getattr(repo, self.models_ref.tag_set_prefetch_name)
releases = [t.name for t in tag_set_prefetch]
if not releases:
continue
available_releases = [
@ -81,7 +94,7 @@ class OCIAppModel(AppRegistryDataInterface):
if with_channels:
channels = [
ChannelView(name=chan.name, current=chan.linked_tag.name)
for chan in oci_model.channel.get_repo_channels(repo)]
for chan in appr_model.channel.get_repo_channels(repo, self.models_ref)]
app_name = _join_package_name(repo.namespace_user.username, repo.name)
manifests = self.list_manifests(app_name, available_releases[0])
@ -93,8 +106,8 @@ class OCIAppModel(AppRegistryDataInterface):
channels=channels,
manifests=manifests,
releases=available_releases,
updated_at=_timestamp_to_iso(repo.tag_set_prefetch[-1].lifetime_start),
created_at=_timestamp_to_iso(repo.tag_set_prefetch[0].lifetime_start),)
updated_at=_timestamp_to_iso(tag_set_prefetch[-1].lifetime_start),
created_at=_timestamp_to_iso(tag_set_prefetch[0].lifetime_start),)
views.append(view)
return views
@ -108,6 +121,9 @@ class OCIAppModel(AppRegistryDataInterface):
def create_application(self, package_name, visibility, owner):
""" Create a new app repository, owner is the user who creates it """
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
ns, name = _split_package_name(package_name)
data.model.repository.create_repository(ns, name, owner, visibility, 'application')
@ -137,7 +153,7 @@ class OCIAppModel(AppRegistryDataInterface):
Todo:
* Paginate
"""
return oci_model.release.get_releases(_application(package_name), media_type)
return appr_model.release.get_releases(_application(package_name), self.models_ref, media_type)
def list_manifests(self, package_name, release=None):
""" Returns the list of all manifests of an Application.
@ -147,8 +163,8 @@ class OCIAppModel(AppRegistryDataInterface):
"""
try:
repo = _application(package_name)
return list(oci_model.manifest.get_manifest_types(repo, release))
except (Repository.DoesNotExist, Tag.DoesNotExist):
return list(appr_model.manifest.get_manifest_types(repo, self.models_ref, release))
except (Repository.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_package_not_found(package_name, release)
def fetch_release(self, package_name, release, media_type):
@ -157,7 +173,8 @@ class OCIAppModel(AppRegistryDataInterface):
"""
repo = _application(package_name)
try:
tag, manifest, blob = oci_model.release.get_app_release(repo, release, media_type)
tag, manifest, blob = appr_model.release.get_app_release(repo, release, media_type,
self.models_ref)
created_at = _timestamp_to_iso(tag.lifetime_start)
blob_descriptor = BlobDescriptor(digest=_strip_sha256_header(blob.digest),
@ -169,17 +186,23 @@ class OCIAppModel(AppRegistryDataInterface):
app_release = ApplicationRelease(release=tag.name, created_at=created_at, name=package_name,
manifest=app_manifest)
return app_release
except (Tag.DoesNotExist, Manifest.DoesNotExist, Blob.DoesNotExist, Repository.DoesNotExist,
except (self.models_ref.Tag.DoesNotExist,
self.models_ref.Manifest.DoesNotExist,
self.models_ref.Blob.DoesNotExist,
Repository.DoesNotExist,
MediaType.DoesNotExist):
raise_package_not_found(package_name, release, media_type)
def store_blob(self, cnrblob, content_media_type):
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
fp = cnrblob.packager.io_file
path = cnrblob.upload_url(cnrblob.digest)
locations = storage.preferred_locations
storage.stream_write(locations, path, fp, 'application/x-gzip')
db_blob = oci_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type,
locations)
db_blob = appr_model.blob.get_or_create_blob(cnrblob.digest, cnrblob.size, content_media_type,
locations, self.models_ref)
return BlobDescriptor(mediaType=content_media_type,
digest=_strip_sha256_header(db_blob.digest), size=db_blob.size, urls=[])
@ -187,49 +210,60 @@ class OCIAppModel(AppRegistryDataInterface):
""" Add an app-release to a repository
package is an instance of data.cnr.package.Package
"""
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
manifest = package.manifest()
ns, name = package.namespace, package.name
repo = data.model.repository.get_or_create_repository(ns, name, user, visibility=visibility,
repo_kind='application')
repo_kind='application')
tag_name = package.release
oci_model.release.create_app_release(repo, tag_name,
package.manifest(), manifest['content']['digest'], force)
appr_model.release.create_app_release(repo, tag_name, package.manifest(),
manifest['content']['digest'], self.models_ref, force)
def delete_release(self, package_name, release, media_type):
""" Remove/Delete an app-release from an app-repository.
It does not delete the entire app-repository, only a single release
"""
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
repo = _application(package_name)
try:
oci_model.release.delete_app_release(repo, release, media_type)
except (Channel.DoesNotExist, Tag.DoesNotExist, MediaType.DoesNotExist):
appr_model.release.delete_app_release(repo, release, media_type, self.models_ref)
except (self.models_ref.Channel.DoesNotExist,
self.models_ref.Tag.DoesNotExist,
MediaType.DoesNotExist):
raise_package_not_found(package_name, release, media_type)
def release_exists(self, package, release):
""" Return true if a release with that name already exist or
have existed (include deleted ones) """
# TODO: Figure out why this isn't implemented.
def channel_exists(self, package_name, channel_name):
""" Returns true if channel exists """
repo = _application(package_name)
return oci_model.tag.tag_exists(repo, channel_name, "channel")
return appr_model.tag.tag_exists(repo, channel_name, self.models_ref, "channel")
def delete_channel(self, package_name, channel_name):
""" Delete an AppChannel
Note:
It doesn't delete the AppReleases
"""
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
repo = _application(package_name)
try:
oci_model.channel.delete_channel(repo, channel_name)
except (Channel.DoesNotExist, Tag.DoesNotExist):
appr_model.channel.delete_channel(repo, channel_name, self.models_ref)
except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_channel_not_found(package_name, channel_name)
def list_channels(self, package_name):
""" Returns all AppChannel for a package """
repo = _application(package_name)
channels = oci_model.channel.get_repo_channels(repo)
channels = appr_model.channel.get_repo_channels(repo, self.models_ref)
return [ChannelView(name=chan.name, current=chan.linked_tag.name) for chan in channels]
def fetch_channel(self, package_name, channel_name, with_releases=True):
@ -237,12 +271,12 @@ class OCIAppModel(AppRegistryDataInterface):
repo = _application(package_name)
try:
channel = oci_model.channel.get_channel(repo, channel_name)
except (Channel.DoesNotExist, Tag.DoesNotExist):
channel = appr_model.channel.get_channel(repo, channel_name, self.models_ref)
except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_channel_not_found(package_name, channel_name)
if with_releases:
releases = oci_model.channel.get_channel_releases(repo, channel)
releases = appr_model.channel.get_channel_releases(repo, channel, self.models_ref)
chanview = ChannelReleasesView(
current=channel.linked_tag.name, name=channel.name,
releases=[channel.linked_tag.name] + [c.name for c in releases])
@ -254,9 +288,9 @@ class OCIAppModel(AppRegistryDataInterface):
def list_release_channels(self, package_name, release, active=True):
repo = _application(package_name)
try:
channels = oci_model.channel.get_tag_channels(repo, release, active=active)
channels = appr_model.channel.get_tag_channels(repo, release, self.models_ref, active=active)
return [ChannelView(name=c.name, current=c.linked_tag.name) for c in channels]
except (Channel.DoesNotExist, Tag.DoesNotExist):
except (self.models_ref.Channel.DoesNotExist, self.models_ref.Tag.DoesNotExist):
raise_package_not_found(package_name, release)
def update_channel(self, package_name, channel_name, release):
@ -264,12 +298,17 @@ class OCIAppModel(AppRegistryDataInterface):
Returns:
A new AppChannel with the release
"""
if self.is_readonly:
raise ReadOnlyException('Currently in read-only mode')
repo = _application(package_name)
channel = oci_model.channel.create_or_update_channel(repo, channel_name, release)
channel = appr_model.channel.create_or_update_channel(repo, channel_name, release,
self.models_ref)
return ChannelView(current=channel.linked_tag.name, name=channel.name)
def get_blob_locations(self, digest):
return oci_model.blob.get_blob_locations(digest)
return appr_model.blob.get_blob_locations(digest, self.models_ref)
model = OCIAppModel()
# Phase 1: Read from old tables, disallow writing.
model = CNRAppModel(OLD_MODELS, features.READONLY_APP_REGISTRY)

2
endpoints/appr/registry.py
View file

@ -17,7 +17,7 @@ from auth.permissions import CreateRepositoryPermission, ModifyRepositoryPermiss
from endpoints.appr import appr_bp, require_app_repo_read, require_app_repo_write
from endpoints.appr.cnr_backend import Blob, Channel, Package, User
from endpoints.appr.decorators import disallow_for_image_repository
from endpoints.appr.models_oci import model
from endpoints.appr.models_cnr import model
from endpoints.decorators import anon_allowed, anon_protect
from util.names import REPOSITORY_NAME_REGEX, TAG_REGEX

16
endpoints/appr/test/test_api.py
View file

@ -6,13 +6,13 @@ from cnr.tests.conftest import *
from cnr.tests.test_apiserver import BaseTestServer
from cnr.tests.test_models import CnrTestModels
import data.oci_model.blob as oci_blob
import data.appr_model.blob as appr_blob
from data.database import User
from data.model import organization, user
from endpoints.appr import registry # Needed to register the endpoint
from endpoints.appr.cnr_backend import Channel, Package, QuayDB
from endpoints.appr.models_oci import model as oci_app_model
from endpoints.appr.models_cnr import model as appr_app_model
from test.fixtures import *
@ -28,7 +28,7 @@ class ChannelTest(Channel):
@classmethod
def dump_all(cls, package_class=None):
result = []
for repo in oci_app_model.list_applications(with_channels=True):
for repo in appr_app_model.list_applications(with_channels=True):
for chan in repo.channels:
result.append({'name': chan.name, 'current': chan.current, 'package': repo.name})
return result
@ -51,17 +51,17 @@ class PackageTest(Package):
@classmethod
def dump_all(cls, blob_cls):
result = []
for repo in oci_app_model.list_applications(with_channels=True):
for repo in appr_app_model.list_applications(with_channels=True):
package_name = repo.name
for release in repo.releases:
for mtype in cls.manifests(package_name, release):
package = oci_app_model.fetch_release(package_name, release, mtype)
package = appr_app_model.fetch_release(package_name, release, mtype)
blob = blob_cls.get(package_name, package.manifest.content.digest)
app_data = cls._apptuple_to_dict(package)
app_data.pop('digest')
app_data['channels'] = [
x.name
for x in oci_app_model.list_release_channels(package_name, package.release, False)
for x in appr_app_model.list_release_channels(package_name, package.release, False)
]
app_data['blob'] = blob.b64blob
result.append(app_data)
@ -141,11 +141,11 @@ class TestQuayModels(CnrTestModels):
assert p.release == "2.0.1"
assert p.digest == "d3b54b7912fe770a61b59ab612a442eac52a8a5d8d05dbe92bf8f212d68aaa80"
blob = db_with_data1.Blob.get("titi/rocketchat", p.digest)
bdb = oci_blob.get_blob(p.digest)
bdb = appr_blob.get_blob(p.digest, appr_app_model.models_ref)
newblob = db_with_data1.Blob("titi/app2", blob.b64blob)
p2 = db_with_data1.Package("titi/app2", "1.0.0", "helm", newblob)
p2.save()
b2db = oci_blob.get_blob(p2.digest)
b2db = appr_blob.get_blob(p2.digest, appr_app_model.models_ref)
assert b2db.id == bdb.id
def test_force_push_different_blob(self, db_with_data1):

2
endpoints/appr/test/test_digest_prefix.py
View file

@ -1,5 +1,5 @@
import pytest
from endpoints.appr.models_oci import _strip_sha256_header
from endpoints.appr.models_cnr import _strip_sha256_header
@pytest.mark.parametrize('digest,expected', [

30
endpoints/appr/test/test_registry.py
View file

@ -1,14 +1,18 @@
import base64
import json
from flask import url_for
from mock import patch
import pytest
from flask import url_for
from data import model
from endpoints.appr.registry import appr_bp
from test.fixtures import *
@pytest.mark.parametrize('login_data, expected_code', [
({
"username": "devtable",
@ -62,3 +66,27 @@ def test_invalid_release_name(release_name, app, client):
rv = client.open(url, method='POST', data=json.dumps(data), headers=headers)
assert rv.status_code == 422
@pytest.mark.parametrize('readonly, expected_status', [
(True, 405),
(False, 422),
])
def test_readonly(readonly, expected_status, app, client):
params = {
'namespace': 'devtable',
'package_name': 'someapprepo',
}
url = url_for('appr.push', **params)
auth = base64.b64encode('devtable:password')
headers = {'Content-Type': 'application/json', 'Authorization': 'Basic ' + auth}
data = {
'release': '1.0',
'media_type': 'application/vnd.cnr.manifest.v0+json',
'blob': 'H4sIAFQwWVoAA+3PMQrCQBAF0Bxlb+Bk143nETGIIEoSC29vMMFOu3TvNb/5DH/Ot8f02jWbiohDremT3ZKR90uuUlty7nKJNmqKtkQuTarbzlo8x+k4zFOu4+lyH4afvbnW93/urH98EwAAAAAAAAAAADb0BsdwExIAKAAA',
}
with patch('endpoints.appr.models_cnr.model.is_readonly', readonly):
rv = client.open(url, method='POST', data=json.dumps(data), headers=headers)
assert rv.status_code == expected_status

13
initdb.py
View file

@ -14,14 +14,14 @@ from uuid import UUID, uuid4
from threading import Event
from email.utils import formatdate
from data.database import (db, all_models, beta_classes, Role, TeamRole, Visibility, LoginService,
from data.database import (db, all_models, cnr_classes, Role, TeamRole, Visibility, LoginService,
BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignatureKind,
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
QuayRegion, QuayService, UserRegion, OAuthAuthorizationCode,
ServiceKeyApprovalType, MediaType, LabelSourceType, UserPromptKind,
RepositoryKind, TagKind, BlobPlacementLocation, User, DisableReason,
DeletedNamespace)
DeletedNamespace, appr_classes, ApprTagKind, ApprBlobPlacementLocation)
from data import model
from data.queue import WorkQueue
from app import app, storage as store, tf
@ -364,6 +364,9 @@ def initialize_database():
BlobPlacementLocation.create(name='local_eu')
BlobPlacementLocation.create(name='local_us')
ApprBlobPlacementLocation.create(name='local_eu')
ApprBlobPlacementLocation.create(name='local_us')
ImageStorageTransformation.create(name='squash')
ImageStorageTransformation.create(name='aci')
@ -435,6 +438,10 @@ def initialize_database():
TagKind.create(name='release')
TagKind.create(name='channel')
ApprTagKind.create(name='tag')
ApprTagKind.create(name='release')
ApprTagKind.create(name='channel')
DisableReason.create(name='user_toggled')
DisableReason.create(name='successive_build_failures')
DisableReason.create(name='successive_build_internal_errors')
@ -906,7 +913,7 @@ def find_models_missing_data():
# whitelisted.
models_missing_data = set()
for one_model in all_models:
if one_model in beta_classes:
if one_model in cnr_classes or one_model in appr_classes:
continue
try:

13
test/test_api_usage.py
View file

@ -26,7 +26,8 @@ from endpoints.webhooks import webhooks
from app import app, config_provider, all_queues, dockerfile_build_queue, notification_queue
from buildtrigger.basehandler import BuildTriggerHandler
from initdb import setup_database_for_testing, finished_database_for_testing
from data import database, model, oci_model
from data import database, model, appr_model
from data.appr_model.models import OLD_MODELS
from data.database import RepositoryActionCount, Repository as RepositoryTable
from test.helpers import assert_action_logged
from util.secscan.fake import fake_security_scanner
@ -2153,11 +2154,11 @@ class TestDeleteRepository(ApiTestCase):
# Add some data for the repository, in addition to is already existing images and tags.
repository = model.repository.get_repository(ADMIN_ACCESS_USER, 'complex')
# Add some new-style tags and linked tags.
base_tag = oci_model.tag.create_or_update_tag(repository, 'somebasetag')
base_tag2 = oci_model.tag.create_or_update_tag(repository, 'somebasetag2')
oci_model.tag.create_or_update_tag(repository, 'somelinkedtag', linked_tag=base_tag)
oci_model.tag.create_or_update_tag(repository, 'somelinkedtag2', linked_tag=base_tag2)
# Add some CNR tags and linked tags.
base_tag = appr_model.tag.create_or_update_tag(repository, 'somebasetag', OLD_MODELS)
base_tag2 = appr_model.tag.create_or_update_tag(repository, 'somebasetag2', OLD_MODELS)
appr_model.tag.create_or_update_tag(repository, 'somelinkedtag', OLD_MODELS, linked_tag=base_tag)
appr_model.tag.create_or_update_tag(repository, 'somelinkedtag2', OLD_MODELS, linked_tag=base_tag2)
# Create some access tokens.
access_token = model.token.create_access_token(repository, 'read')

6
util/config/database.py
View file

@ -1,4 +1,6 @@
from data import model, oci_model
from data import model
from data.appr_model import blob
from data.appr_model.models import OLD_MODELS
def sync_database_with_config(config):
@ -7,4 +9,4 @@ def sync_database_with_config(config):
location_names = config.get('DISTRIBUTED_STORAGE_CONFIG', {}).keys()
if location_names:
model.image.ensure_image_locations(*location_names)
oci_model.blob.ensure_blob_locations(*location_names)
blob.ensure_blob_locations(OLD_MODELS, *location_names)

7
util/config/schema.py
View file

@ -881,6 +881,13 @@ CONFIG_SCHEMA = {
'x-example': False,
},
# Feature Flag: Read only app registry.
'FEATURE_READONLY_APP_REGISTRY': {
'type': 'boolean',
'description': 'Whether to App repositories are read-only. Defaults to False',
'x-example': True,
},
# Feature Flag: Public Reposiotires in _catalog Endpoint.
'FEATURE_PUBLIC_CATALOG': {
'type': 'boolean',

11
util/migrate/table_ops.py Normal file
View file

@ -0,0 +1,11 @@
def copy_table_contents(source_table, destination_table, conn):
if conn.engine.name == 'postgresql':
conn.execute('INSERT INTO "%s" SELECT * FROM "%s"' % (destination_table, source_table))
result = list(conn.execute('Select Max(id) from "%s"' % destination_table))[0]
new_start_id = result[0] + 1
conn.execute('ALTER SEQUENCE "%s_id_seq" RESTART WITH %s' % (destination_table, new_start_id))
else:
conn.execute("INSERT INTO `%s` SELECT * FROM `%s` WHERE 1" % (destination_table, source_table))
result = list(conn.execute('Select Max(id) from `%s` WHERE 1' % destination_table))[0]
new_start_id = result[0] + 1
conn.execute("ALTER TABLE `%s` AUTO_INCREMENT = %s" % (destination_table, new_start_id))