Rename oci_model to appr_model

This commit is contained in:
Joseph Schorr 2018-05-23 17:08:00 -04:00
parent f0d4841155
commit 6622f27c93
19 changed files with 222 additions and 171 deletions

View file

@ -1,4 +1,4 @@
from data.oci_model import (
from data.appr_model import (
blob,
channel,
manifest,

View file

@ -1,5 +1,5 @@
from data.database import Tag, Channel
from data.oci_model import tag as tag_model
from data.appr_model import tag as tag_model
def get_channel_releases(repo, channel):

View file

@ -5,7 +5,7 @@ import json
from cnr.models.package_base import get_media_type
from data.database import db_transaction, Manifest, ManifestListManifest, MediaType, Blob, Tag
from data.oci_model import tag as tag_model
from data.appr_model import tag as tag_model
logger = logging.getLogger(__name__)

View file

@ -4,7 +4,7 @@ from peewee import prefetch
from data import model
from data.database import Repository, Namespace, Tag, ManifestListManifest
from data.oci_model import tag as tag_model
from data.appr_model import tag as tag_model
def list_packages_query(namespace=None, media_type=None, search_query=None, username=None):

View file

@ -5,7 +5,7 @@ from cnr.models.package_base import manifest_media_type
from data.database import (db_transaction, get_epoch_timestamp, Manifest, ManifestList, Tag,
ManifestListManifest, Blob, ManifestBlob)
from data.oci_model import (blob as blob_model, manifest as manifest_model,
from data.appr_model import (blob as blob_model, manifest as manifest_model,
manifest_list as manifest_list_model,
tag as tag_model)

View file

@ -471,8 +471,7 @@ class User(BaseModel):
TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
RepositoryNotification, OAuthAuthorizationCode,
RepositoryActionCount, TagManifestLabel, Tag,
ManifestLabel, BlobUploading, TeamSync,
RepositorySearchScore, DeletedNamespace} | beta_classes
TeamSync, RepositorySearchScore, DeletedNamespace} | cnr_classes
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
@ -620,7 +619,7 @@ class Repository(BaseModel):
# are cleaned up directly
skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload,
Image, TagManifest, TagManifestLabel, Label, DerivedStorageForImage,
RepositorySearchScore} | beta_classes
RepositorySearchScore} | cnr_classes
delete_instance_filtered(self, Repository, delete_nullable, skip_transitive_deletes)
@ -1239,6 +1238,7 @@ class TagManifestLabel(BaseModel):
class Blob(BaseModel):
""" Blob represents a content-addressable object stored outside of the database.
This model is a part of the new OCI/CNR model set.
CNR
"""
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
@ -1249,21 +1249,15 @@ class Blob(BaseModel):
class BlobPlacementLocation(BaseModel):
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
This model is a part of the new OCI/CNR model set.
CNR
"""
name = CharField(index=True, unique=True)
class BlobPlacementLocationPreference(BaseModel):
""" BlobPlacementLocationPreference is a location to which a user's data will be replicated.
This model is a part of the new OCI/CNR model set.
"""
user = QuayUserField(index=True, allows_robots=False)
location = EnumField(BlobPlacementLocation)
class BlobPlacement(BaseModel):
""" BlobPlacement represents the location of a Blob.
This model is a part of the new OCI/CNR model set.
CNR
"""
blob = ForeignKeyField(Blob)
location = EnumField(BlobPlacementLocation)
@ -1276,58 +1270,20 @@ class BlobPlacement(BaseModel):
)
class BlobUploading(BaseModel):
""" BlobUploading represents the state of a Blob currently being uploaded.
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(index=True, unique=True)
created = DateTimeField(default=datetime.now, index=True)
repository = ForeignKeyField(Repository, index=True)
location = ForeignKeyField(BlobPlacementLocation)
byte_count = IntegerField(default=0)
uncompressed_byte_count = IntegerField(null=True)
chunk_count = IntegerField(default=0)
storage_metadata = JSONField(null=True, default={})
sha_state = ResumableSHA256Field(null=True, default=resumablehashlib.sha256)
piece_sha_state = ResumableSHA1Field(null=True)
piece_hashes = Base64BinaryField(null=True)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'uuid'), True),
)
class Manifest(BaseModel):
""" Manifest represents the metadata and collection of blobs that comprise a container image.
This model is a part of the new OCI/CNR model set.
CNR
"""
digest = CharField(index=True, unique=True)
media_type = EnumField(MediaType)
manifest_json = JSONField()
class ManifestLabel(BaseModel):
""" ManifestLabel represents label metadata annotating a Manifest.
This model is a part of the new OCI/CNR model set.
"""
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(Manifest, index=True)
label = ForeignKeyField(Label)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'annotated', 'label'), True),
)
class ManifestBlob(BaseModel):
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
This model is a part of the new OCI/CNR model set.
CNR
"""
manifest = ForeignKeyField(Manifest, index=True)
blob = ForeignKeyField(Blob, index=True)
@ -1343,6 +1299,7 @@ class ManifestBlob(BaseModel):
class ManifestList(BaseModel):
""" ManifestList represents all of the various manifests that compose a Tag.
This model is a part of the new OCI/CNR model set.
CNR
"""
digest = CharField(index=True, unique=True)
manifest_list_json = JSONField()
@ -1353,6 +1310,7 @@ class ManifestList(BaseModel):
class TagKind(BaseModel):
""" TagKind is a enumtable to reference tag kinds.
This model is a part of the new OCI/CNR model set.
CNR
"""
name = CharField(index=True, unique=True)
@ -1360,6 +1318,7 @@ class TagKind(BaseModel):
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList.
This model is a part of the new OCI/CNR model set.
CNR
"""
name = CharField()
repository = ForeignKeyField(Repository)
@ -1389,6 +1348,7 @@ Channel = Tag.alias()
class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
This model is a part of the new OCI/CNR model set.
CNR
"""
manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True)
@ -1406,79 +1366,6 @@ class ManifestListManifest(BaseModel):
)
class ManifestLayer(BaseModel):
""" ManifestLayer represents one of the layers that compose a Manifest.
This model is a part of the new OCI/CNR model set.
"""
blob = ForeignKeyField(Blob, index=True)
manifest = ForeignKeyField(Manifest)
manifest_index = IntegerField(index=True) # index 0 is the last command in a Dockerfile
metadata_json = JSONField()
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest', 'manifest_index'), True),
)
class ManifestLayerDockerV1(BaseModel):
""" ManifestLayerDockerV1 is the Docker v1 registry protocol metadata for a ManifestLayer.
This model is a part of the new OCI/CNR model set.
"""
manifest_layer = ForeignKeyField(ManifestLayer)
image_id = CharField(index=True)
checksum = CharField()
compat_json = JSONField()
class ManifestLayerScan(BaseModel):
""" ManifestLayerScan represents the state of security scanning for a ManifestLayer.
This model is a part of the new OCI/CNR model set.
"""
layer = ForeignKeyField(ManifestLayer, unique=True)
scannable = BooleanField()
scanned_by = CharField()
class DerivedImage(BaseModel):
""" DerivedImage represents a Manifest transcoded into an alternative format.
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(default=uuid_generator, unique=True)
source_manifest = ForeignKeyField(Manifest)
derived_manifest_json = JSONField()
media_type = EnumField(MediaType)
blob = ForeignKeyField(Blob, related_name='blob')
uniqueness_hash = CharField(index=True, unique=True)
signature_blob = ForeignKeyField(Blob, null=True, related_name='signature_blob')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('source_manifest', 'blob'), True),
(('source_manifest', 'media_type', 'uniqueness_hash'), True),
)
class BitTorrentPieces(BaseModel):
""" BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob.
This model is a part of the new OCI/CNR model set.
"""
blob = ForeignKeyField(Blob)
pieces = Base64BinaryField()
piece_length = IntegerField()
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('blob', 'piece_length'), True),
)
class AppSpecificAuthToken(BaseModel):
""" AppSpecificAuthToken represents a token generated by a user for use with an external
application where putting the user's credentials, even encrypted, is deemed too risky.
@ -1499,9 +1386,7 @@ class AppSpecificAuthToken(BaseModel):
)
beta_classes = set([ManifestLayerScan, Tag, TagKind, BlobPlacementLocation, ManifestLayer, ManifestList,
BitTorrentPieces, MediaType, Label, ManifestBlob, BlobUploading, Blob,
ManifestLayerDockerV1, BlobPlacementLocationPreference, ManifestListManifest,
Manifest, DerivedImage, BlobPlacement, ManifestLabel])
cnr_classes = set([Tag, TagKind, BlobPlacementLocation, ManifestList, ManifestBlob, Blob,
ManifestListManifest, Manifest, BlobPlacement])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

View file

@ -0,0 +1,167 @@
"""Remove 'oci' tables not used by CNR. The rest will be migrated and renamed.
Revision ID: 5cbbfc95bac7
Revises: 1783530bee68
Create Date: 2018-05-23 17:28:40.114433
"""
# revision identifiers, used by Alembic.
revision = '5cbbfc95bac7'
down_revision = '1783530bee68'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from util.migrate import UTF8LongText, UTF8CharField
def upgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('derivedimage')
op.drop_table('manifestlabel')
op.drop_table('blobplacementlocationpreference')
op.drop_table('blobuploading')
op.drop_table('bittorrentpieces')
op.drop_table('manifestlayerdockerv1')
op.drop_table('manifestlayerscan')
op.drop_table('manifestlayer')
# ### end Alembic commands ###
def downgrade(tables):
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
'manifestlayer',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('manifest_index', sa.BigInteger(), nullable=False),
sa.Column('metadata_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestlayer_blob_id_blob')),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlayer_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayer'))
)
op.create_index('manifestlayer_manifest_index', 'manifestlayer', ['manifest_index'], unique=False)
op.create_index('manifestlayer_manifest_id_manifest_index', 'manifestlayer', ['manifest_id', 'manifest_index'], unique=True)
op.create_index('manifestlayer_manifest_id', 'manifestlayer', ['manifest_id'], unique=False)
op.create_index('manifestlayer_blob_id', 'manifestlayer', ['blob_id'], unique=False)
op.create_table(
'manifestlayerscan',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('layer_id', sa.Integer(), nullable=False),
sa.Column('scannable', sa.Boolean(), nullable=False),
sa.Column('scanned_by', UTF8CharField(length=255), nullable=False),
sa.ForeignKeyConstraint(['layer_id'], ['manifestlayer.id'], name=op.f('fk_manifestlayerscan_layer_id_manifestlayer')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayerscan'))
)
op.create_index('manifestlayerscan_layer_id', 'manifestlayerscan', ['layer_id'], unique=True)
op.create_table(
'bittorrentpieces',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('pieces', UTF8LongText, nullable=False),
sa.Column('piece_length', sa.BigInteger(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_bittorrentpieces_blob_id_blob')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_bittorrentpieces'))
)
op.create_index('bittorrentpieces_blob_id_piece_length', 'bittorrentpieces', ['blob_id', 'piece_length'], unique=True)
op.create_index('bittorrentpieces_blob_id', 'bittorrentpieces', ['blob_id'], unique=False)
op.create_table(
'blobuploading',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.Column('byte_count', sa.BigInteger(), nullable=False),
sa.Column('uncompressed_byte_count', sa.BigInteger(), nullable=True),
sa.Column('chunk_count', sa.BigInteger(), nullable=False),
sa.Column('storage_metadata', UTF8LongText, nullable=True),
sa.Column('sha_state', UTF8LongText, nullable=True),
sa.Column('piece_sha_state', UTF8LongText, nullable=True),
sa.Column('piece_hashes', UTF8LongText, nullable=True),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobuploading_location_id_blobplacementlocation')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_blobuploading_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobuploading'))
)
op.create_index('blobuploading_uuid', 'blobuploading', ['uuid'], unique=True)
op.create_index('blobuploading_repository_id_uuid', 'blobuploading', ['repository_id', 'uuid'], unique=True)
op.create_index('blobuploading_repository_id', 'blobuploading', ['repository_id'], unique=False)
op.create_index('blobuploading_location_id', 'blobuploading', ['location_id'], unique=False)
op.create_index('blobuploading_created', 'blobuploading', ['created'], unique=False)
op.create_table(
'manifestlayerdockerv1',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_layer_id', sa.Integer(), nullable=False),
sa.Column('image_id', UTF8CharField(length=255), nullable=False),
sa.Column('checksum', UTF8CharField(length=255), nullable=False),
sa.Column('compat_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['manifest_layer_id'], ['manifestlayer.id'], name=op.f('fk_manifestlayerdockerv1_manifest_layer_id_manifestlayer')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayerdockerv1'))
)
op.create_index('manifestlayerdockerv1_manifest_layer_id', 'manifestlayerdockerv1', ['manifest_layer_id'], unique=False)
op.create_index('manifestlayerdockerv1_image_id', 'manifestlayerdockerv1', ['image_id'], unique=False)
op.create_table(
'manifestlabel',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('annotated_id', sa.Integer(), nullable=False),
sa.Column('label_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['annotated_id'], ['manifest.id'], name=op.f('fk_manifestlabel_annotated_id_manifest')),
sa.ForeignKeyConstraint(['label_id'], ['label.id'], name=op.f('fk_manifestlabel_label_id_label')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_manifestlabel_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlabel'))
)
op.create_index('manifestlabel_repository_id_annotated_id_label_id', 'manifestlabel', ['repository_id', 'annotated_id', 'label_id'], unique=True)
op.create_index('manifestlabel_repository_id', 'manifestlabel', ['repository_id'], unique=False)
op.create_index('manifestlabel_label_id', 'manifestlabel', ['label_id'], unique=False)
op.create_index('manifestlabel_annotated_id', 'manifestlabel', ['annotated_id'], unique=False)
op.create_table(
'blobplacementlocationpreference',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacementlocpref_locid_blobplacementlocation')),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_blobplacementlocationpreference_user_id_user')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocationpreference'))
)
op.create_index('blobplacementlocationpreference_user_id', 'blobplacementlocationpreference', ['user_id'], unique=False)
op.create_index('blobplacementlocationpreference_location_id', 'blobplacementlocationpreference', ['location_id'], unique=False)
op.create_table(
'derivedimage',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('source_manifest_id', sa.Integer(), nullable=False),
sa.Column('derived_manifest_json', UTF8LongText, nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('uniqueness_hash', sa.String(length=255), nullable=False),
sa.Column('signature_blob_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_derivedimage_blob_id_blob')),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_derivedimage_media_type_id_mediatype')),
sa.ForeignKeyConstraint(['signature_blob_id'], ['blob.id'], name=op.f('fk_derivedimage_signature_blob_id_blob')),
sa.ForeignKeyConstraint(['source_manifest_id'], ['manifest.id'], name=op.f('fk_derivedimage_source_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_derivedimage'))
)
op.create_index('derivedimage_uuid', 'derivedimage', ['uuid'], unique=True)
op.create_index('derivedimage_uniqueness_hash', 'derivedimage', ['uniqueness_hash'], unique=True)
op.create_index('derivedimage_source_manifest_id_media_type_id_uniqueness_hash', 'derivedimage', ['source_manifest_id', 'media_type_id', 'uniqueness_hash'], unique=True)
op.create_index('derivedimage_source_manifest_id_blob_id', 'derivedimage', ['source_manifest_id', 'blob_id'], unique=True)
op.create_index('derivedimage_source_manifest_id', 'derivedimage', ['source_manifest_id'], unique=False)
op.create_index('derivedimage_signature_blob_id', 'derivedimage', ['signature_blob_id'], unique=False)
op.create_index('derivedimage_media_type_id', 'derivedimage', ['media_type_id'], unique=False)
op.create_index('derivedimage_blob_id', 'derivedimage', ['blob_id'], unique=False)
# ### end Alembic commands ###