vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #2436 from jzelinskie/cnr-step1

Browse source 
CNR - Step 1
This commit is contained in:
Jimmy Zelinskie 2017-03-17 15:37:29 -04:00 committed by GitHub
commit 0ea600628b
4 changed files with 505 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

250
data/database.py
View file

@ -21,6 +21,7 @@ from playhouse.shortcuts import RetryOperationalError
from sqlalchemy.engine.url import make_url
import resumablehashlib
from cachetools import lru_cache
from data.fields import (ResumableSHA256Field, ResumableSHA1Field, JSONField, Base64BinaryField,
FullIndexedTextField, FullIndexedCharField)
@ -150,6 +151,7 @@ SCHEME_SPECIALIZED_FOR_UPDATE = {
'sqlite': null_for_update,
}
class CallableProxy(Proxy):
def __call__(self, *args, **kwargs):
if self.obj is None:
@ -336,10 +338,54 @@ class QuayUserField(ForeignKeyField):
self.robot_null_delete = robot_null_delete
if 'rel_model' not in kwargs:
kwargs['rel_model'] = User
super(QuayUserField, self).__init__(*args, **kwargs)
# @TODO: Generates client-side enum
class EnumField(ForeignKeyField):
""" Create a cached python Enum from an EnumTable """
def __init__(self, rel_model, enum_key_field='name', *args, **kwargs):
"""
rel_model is the EnumTable model-class (see ForeignKeyField)
enum_key_field is the field from the EnumTable to use as the enum name
"""
self.enum_key_field = enum_key_field
super(EnumField, self).__init__(rel_model, *args, **kwargs)
@property
@lru_cache(maxsize=1)
def enum(self):
""" Returns a python enun.Enum generated from the associated EnumTable """
values = []
for row in self.rel_model.select():
key = getattr(row, self.enum_key_field)
value = getattr(row, 'id')
values.append((key, value))
return Enum(self.rel_model.__name__, values)
def get_id(self, name):
""" Returns the ForeignKeyId from the name field
Example:
>>> Repository.repo_kind.get_id("application")
2
"""
try:
return self.enum[name].value
except KeyError:
raise self.rel_model.DoesNotExist
def get_name(self, value):
""" Returns the name value from the ForeignKeyId
Example:
>>> Repository.repo_kind.get_name(2)
"application"
"""
try:
return self.enum(value).name
except ValueError:
raise self.rel_model.DoesNotExist
class BaseModel(ReadSlaveModel):
class Meta:
database = db
@ -405,7 +451,8 @@ class User(BaseModel):
RepositoryTag, PermissionPrototype, DerivedStorageForImage,
TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
RepositoryNotification, OAuthAuthorizationCode,
RepositoryActionCount, TagManifestLabel}
RepositoryActionCount, TagManifestLabel, Tag,
ManifestLabel, BlobUploading}
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
@ -1046,7 +1093,9 @@ class ServiceKey(BaseModel):
class MediaType(BaseModel):
""" MediaType is an enumeration of the possible formats of various objects in the data model. """
""" MediaType is an enumeration of the possible formats of various objects in the data model.
This model is a part of the new OCI/CNR model set.
"""
name = CharField(index=True, unique=True)
@ -1058,23 +1107,29 @@ class Messages(BaseModel):
class LabelSourceType(BaseModel):
""" LabelSourceType is an enumeration of the possible sources for a label. """
""" LabelSourceType is an enumeration of the possible sources for a label.
This model is a part of the new OCI/CNR model set.
"""
name = CharField(index=True, unique=True)
mutable = BooleanField(default=False)
class Label(BaseModel):
""" Label represents user-facing metadata associated with another entry in the
database (e.g. a Manifest). """
""" Label represents user-facing metadata associated with another entry in the database (e.g. a
Manifest).
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(default=uuid_generator, index=True, unique=True)
key = CharField(index=True)
value = TextField()
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
source_type = ForeignKeyField(LabelSourceType)
class TagManifestLabel(BaseModel):
""" Mapping from a tag manifest to a label. """
""" Mapping from a tag manifest to a label.
This model is a part of the new OCI/CNR model set.
"""
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(TagManifest, index=True)
label = ForeignKeyField(Label)
@ -1087,44 +1142,37 @@ class TagManifestLabel(BaseModel):
)
'''
class ManifestLabel(BaseModel):
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(Manifest, index=True)
label = ForeignKeyField(Label)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'annotated', 'label'), True),
)
class Blob(BaseModel):
""" Blob represents a content-addressable object stored outside of the database. """
""" Blob represents a content-addressable object stored outside of the database.
This model is a part of the new OCI/CNR model set.
"""
digest = CharField(index=True, unique=True)
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
size = BigIntegerField()
uncompressed_size = BigIntegerField(null=True)
class BlobPlacementLocation(BaseModel):
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs. """
""" BlobPlacementLocation is an enumeration of the possible storage locations for Blobs.
This model is a part of the new OCI/CNR model set.
"""
name = CharField(index=True, unique=True)
class BlobPlacementLocationPreference(BaseModel):
""" BlobPlacementLocationPreference is a location to which a user's data will be replicated. """
""" BlobPlacementLocationPreference is a location to which a user's data will be replicated.
This model is a part of the new OCI/CNR model set.
"""
user = QuayUserField(index=True, allows_robots=False)
location = ForeignKeyField(BlobPlacementLocation)
location = EnumField(BlobPlacementLocation)
class BlobPlacement(BaseModel):
""" BlobPlacement represents the location of a Blob. """
""" BlobPlacement represents the location of a Blob.
This model is a part of the new OCI/CNR model set.
"""
blob = ForeignKeyField(Blob)
location = ForeignKeyField(BlobPlacementLocation)
location = EnumField(BlobPlacementLocation)
class Meta:
database = db
@ -1135,7 +1183,9 @@ class BlobPlacement(BaseModel):
class BlobUploading(BaseModel):
""" BlobUploading represents the state of a Blob currently being uploaded. """
""" BlobUploading represents the state of a Blob currently being uploaded.
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(index=True, unique=True)
created = DateTimeField(default=datetime.now, index=True)
repository = ForeignKeyField(Repository, index=True)
@ -1157,14 +1207,34 @@ class BlobUploading(BaseModel):
class Manifest(BaseModel):
""" Manifest represents the metadata and collection of blobs that comprise a container image. """
""" Manifest represents the metadata and collection of blobs that comprise a container image.
This model is a part of the new OCI/CNR model set.
"""
digest = CharField(index=True, unique=True)
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
manifest_json = JSONField()
class ManifestLabel(BaseModel):
""" ManifestLabel represents label metadata annotating a Manifest.
This model is a part of the new OCI/CNR model set.
"""
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(Manifest, index=True)
label = ForeignKeyField(Label)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'annotated', 'label'), True),
)
class ManifestBlob(BaseModel):
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs. """
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs.
This model is a part of the new OCI/CNR model set.
"""
manifest = ForeignKeyField(Manifest, index=True)
blob = ForeignKeyField(Blob, index=True)
@ -1177,31 +1247,75 @@ class ManifestBlob(BaseModel):
class ManifestList(BaseModel):
""" ManifestList represents all of the various manifests that compose a Tag. """
""" ManifestList represents all of the various manifests that compose a Tag.
This model is a part of the new OCI/CNR model set.
"""
digest = CharField(index=True, unique=True)
manifest_list_json = JSONField()
schema_version = CharField()
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """
manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True)
operating_system = CharField()
architecture = CharField()
platform_json = JSONField()
class TagKind(BaseModel):
""" TagKind is a enumtable to reference tag kinds.
This model is a part of the new OCI/CNR model set.
"""
name = CharField(index=True, unique=True)
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList.
This model is a part of the new OCI/CNR model set.
"""
name = CharField()
repository = ForeignKeyField(Repository)
manifest_list = ForeignKeyField(ManifestList, null=True)
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reverted = BooleanField(default=False)
protected = BooleanField(default=False)
tag_kind = EnumField(TagKind)
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest_list', 'operating_system', 'architecture'), False),
(('repository', 'name'), False),
(('repository', 'name', 'hidden'), False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end'), True),
)
Channel = Tag.alias()
class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests.
This model is a part of the new OCI/CNR model set.
"""
manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True)
operating_system = CharField(null=True)
architecture = CharField(null=True)
platform_json = JSONField(null=True)
media_type = EnumField(MediaType)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest_list', 'operating_system', 'architecture', 'media_type'), False),
(('manifest_list', 'media_type'), False),
)
class ManifestLayer(BaseModel):
""" ManifestLayer represents one of the layers that compose a Manifest. """
""" ManifestLayer represents one of the layers that compose a Manifest.
This model is a part of the new OCI/CNR model set.
"""
blob = ForeignKeyField(Blob, index=True)
manifest = ForeignKeyField(Manifest)
manifest_index = IntegerField(index=True) # index 0 is the last command in a Dockerfile
@ -1216,7 +1330,9 @@ class ManifestLayer(BaseModel):
class ManifestLayerDockerV1(BaseModel):
""" ManifestLayerDockerV1 is the Docker v1 registry protocol metadata for a ManifestLayer. """
""" ManifestLayerDockerV1 is the Docker v1 registry protocol metadata for a ManifestLayer.
This model is a part of the new OCI/CNR model set.
"""
manifest_layer = ForeignKeyField(ManifestLayer)
image_id = CharField(index=True)
checksum = CharField()
@ -1224,18 +1340,22 @@ class ManifestLayerDockerV1(BaseModel):
class ManifestLayerScan(BaseModel):
""" ManifestLayerScan represents the state of security scanning for a ManifestLayer. """
""" ManifestLayerScan represents the state of security scanning for a ManifestLayer.
This model is a part of the new OCI/CNR model set.
"""
layer = ForeignKeyField(ManifestLayer, unique=True)
scannable = BooleanField()
scanned_by = CharField()
class DerivedImage(BaseModel):
""" DerivedImage represents a Manifest transcoded into an alternative format. """
""" DerivedImage represents a Manifest transcoded into an alternative format.
This model is a part of the new OCI/CNR model set.
"""
uuid = CharField(default=uuid_generator, unique=True)
source_manifest = ForeignKeyField(Manifest)
derived_manifest_json = JSONField()
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
blob = ForeignKeyField(Blob, related_name='blob')
uniqueness_hash = CharField(index=True, unique=True)
signature_blob = ForeignKeyField(Blob, null=True, related_name='signature_blob')
@ -1249,30 +1369,10 @@ class DerivedImage(BaseModel):
)
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList. """
name = CharField()
repository = ForeignKeyField(Repository)
manifest_list = ForeignKeyField(ManifestList)
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reverted = BooleanField(default=False)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'name'), False),
(('repository', 'name', 'hidden') False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end'), True),
)
class BitTorrentPieces(BaseModel):
""" BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob. """
""" BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob.
This model is a part of the new OCI/CNR model set.
"""
blob = ForeignKeyField(Blob)
pieces = Base64BinaryField()
piece_length = IntegerField()
@ -1285,11 +1385,9 @@ class BitTorrentPieces(BaseModel):
)
beta_classes = set([ManifestLayerScan, Tag, BlobPlacementLocation, ManifestLayer, ManifestList,
beta_classes = set([ManifestLayerScan, Tag, TagKind, BlobPlacementLocation, ManifestLayer, ManifestList,
BitTorrentPieces, MediaType, Label, ManifestBlob, BlobUploading, Blob,
ManifestLayerDockerV1, BlobPlacementLocationPreference, ManifestListManifest,
Manifest, DerivedImage, BlobPlacement])
'''
Manifest, DerivedImage, BlobPlacement, ManifestLabel])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

327
data/migrations/versions/7a525c68eb13_add_oci_app_models.py Normal file
View file

@ -0,0 +1,327 @@
"""Add OCI/App models
Revision ID: 7a525c68eb13
Revises: e2894a3a3c19
Create Date: 2017-01-24 16:25:52.170277
"""
# revision identifiers, used by Alembic.
revision = '7a525c68eb13'
down_revision = 'e2894a3a3c19'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from util.migrate import UTF8LongText, UTF8CharField
def upgrade(tables):
op.create_table(
'tagkind',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_tagkind'))
)
op.create_index('tagkind_name', 'tagkind', ['name'], unique=True)
op.create_table(
'blobplacementlocation',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocation'))
)
op.create_index('blobplacementlocation_name', 'blobplacementlocation', ['name'], unique=True)
op.create_table(
'blob',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('size', sa.BigInteger(), nullable=False),
sa.Column('uncompressed_size', sa.BigInteger(), nullable=True),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_blob_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blob'))
)
op.create_index('blob_digest', 'blob', ['digest'], unique=True)
op.create_index('blob_media_type_id', 'blob', ['media_type_id'], unique=False)
op.create_table(
'blobplacementlocationpreference',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacementlocpref_locid_blobplacementlocation')),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], name=op.f('fk_blobplacementlocationpreference_user_id_user')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacementlocationpreference'))
)
op.create_index('blobplacementlocationpreference_location_id', 'blobplacementlocationpreference', ['location_id'], unique=False)
op.create_index('blobplacementlocationpreference_user_id', 'blobplacementlocationpreference', ['user_id'], unique=False)
op.create_table(
'manifest',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('manifest_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifest_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifest'))
)
op.create_index('manifest_digest', 'manifest', ['digest'], unique=True)
op.create_index('manifest_media_type_id', 'manifest', ['media_type_id'], unique=False)
op.create_table(
'manifestlist',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('digest', sa.String(length=255), nullable=False),
sa.Column('manifest_list_json', UTF8LongText, nullable=False),
sa.Column('schema_version', UTF8CharField(length=255), nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlist_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlist'))
)
op.create_index('manifestlist_digest', 'manifestlist', ['digest'], unique=True)
op.create_index('manifestlist_media_type_id', 'manifestlist', ['media_type_id'], unique=False)
op.create_table(
'bittorrentpieces',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('pieces', UTF8LongText, nullable=False),
sa.Column('piece_length', sa.BigInteger(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_bittorrentpieces_blob_id_blob')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_bittorrentpieces'))
)
op.create_index('bittorrentpieces_blob_id', 'bittorrentpieces', ['blob_id'], unique=False)
op.create_index('bittorrentpieces_blob_id_piece_length', 'bittorrentpieces', ['blob_id', 'piece_length'], unique=True)
op.create_table(
'blobplacement',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_blobplacement_blob_id_blob')),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobplacement_location_id_blobplacementlocation')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobplacement'))
)
op.create_index('blobplacement_blob_id', 'blobplacement', ['blob_id'], unique=False)
op.create_index('blobplacement_blob_id_location_id', 'blobplacement', ['blob_id', 'location_id'], unique=True)
op.create_index('blobplacement_location_id', 'blobplacement', ['location_id'], unique=False)
op.create_table(
'blobuploading',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('created', sa.DateTime(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('location_id', sa.Integer(), nullable=False),
sa.Column('byte_count', sa.BigInteger(), nullable=False),
sa.Column('uncompressed_byte_count', sa.BigInteger(), nullable=True),
sa.Column('chunk_count', sa.BigInteger(), nullable=False),
sa.Column('storage_metadata', UTF8LongText, nullable=True),
sa.Column('sha_state', UTF8LongText, nullable=True),
sa.Column('piece_sha_state', UTF8LongText, nullable=True),
sa.Column('piece_hashes', UTF8LongText, nullable=True),
sa.ForeignKeyConstraint(['location_id'], ['blobplacementlocation.id'], name=op.f('fk_blobuploading_location_id_blobplacementlocation')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_blobuploading_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_blobuploading'))
)
op.create_index('blobuploading_created', 'blobuploading', ['created'], unique=False)
op.create_index('blobuploading_location_id', 'blobuploading', ['location_id'], unique=False)
op.create_index('blobuploading_repository_id', 'blobuploading', ['repository_id'], unique=False)
op.create_index('blobuploading_repository_id_uuid', 'blobuploading', ['repository_id', 'uuid'], unique=True)
op.create_index('blobuploading_uuid', 'blobuploading', ['uuid'], unique=True)
op.create_table(
'derivedimage',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=255), nullable=False),
sa.Column('source_manifest_id', sa.Integer(), nullable=False),
sa.Column('derived_manifest_json', UTF8LongText, nullable=False),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('uniqueness_hash', sa.String(length=255), nullable=False),
sa.Column('signature_blob_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_derivedimage_blob_id_blob')),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_derivedimage_media_type_id_mediatype')),
sa.ForeignKeyConstraint(['signature_blob_id'], ['blob.id'], name=op.f('fk_derivedimage_signature_blob_id_blob')),
sa.ForeignKeyConstraint(['source_manifest_id'], ['manifest.id'], name=op.f('fk_derivedimage_source_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_derivedimage'))
)
op.create_index('derivedimage_blob_id', 'derivedimage', ['blob_id'], unique=False)
op.create_index('derivedimage_media_type_id', 'derivedimage', ['media_type_id'], unique=False)
op.create_index('derivedimage_signature_blob_id', 'derivedimage', ['signature_blob_id'], unique=False)
op.create_index('derivedimage_source_manifest_id', 'derivedimage', ['source_manifest_id'], unique=False)
op.create_index('derivedimage_source_manifest_id_blob_id', 'derivedimage', ['source_manifest_id', 'blob_id'], unique=True)
op.create_index('derivedimage_source_manifest_id_media_type_id_uniqueness_hash', 'derivedimage', ['source_manifest_id', 'media_type_id', 'uniqueness_hash'], unique=True)
op.create_index('derivedimage_uniqueness_hash', 'derivedimage', ['uniqueness_hash'], unique=True)
op.create_index('derivedimage_uuid', 'derivedimage', ['uuid'], unique=True)
op.create_table(
'manifestblob',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestblob_blob_id_blob')),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestblob_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestblob'))
)
op.create_index('manifestblob_blob_id', 'manifestblob', ['blob_id'], unique=False)
op.create_index('manifestblob_manifest_id', 'manifestblob', ['manifest_id'], unique=False)
op.create_index('manifestblob_manifest_id_blob_id', 'manifestblob', ['manifest_id', 'blob_id'], unique=True)
op.create_table(
'manifestlabel',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('annotated_id', sa.Integer(), nullable=False),
sa.Column('label_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['annotated_id'], ['manifest.id'], name=op.f('fk_manifestlabel_annotated_id_manifest')),
sa.ForeignKeyConstraint(['label_id'], ['label.id'], name=op.f('fk_manifestlabel_label_id_label')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_manifestlabel_repository_id_repository')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlabel'))
)
op.create_index('manifestlabel_annotated_id', 'manifestlabel', ['annotated_id'], unique=False)
op.create_index('manifestlabel_label_id', 'manifestlabel', ['label_id'], unique=False)
op.create_index('manifestlabel_repository_id', 'manifestlabel', ['repository_id'], unique=False)
op.create_index('manifestlabel_repository_id_annotated_id_label_id', 'manifestlabel', ['repository_id', 'annotated_id', 'label_id'], unique=True)
op.create_table(
'manifestlayer',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('blob_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('manifest_index', sa.BigInteger(), nullable=False),
sa.Column('metadata_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['blob_id'], ['blob.id'], name=op.f('fk_manifestlayer_blob_id_blob')),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlayer_manifest_id_manifest')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayer'))
)
op.create_index('manifestlayer_blob_id', 'manifestlayer', ['blob_id'], unique=False)
op.create_index('manifestlayer_manifest_id', 'manifestlayer', ['manifest_id'], unique=False)
op.create_index('manifestlayer_manifest_id_manifest_index', 'manifestlayer', ['manifest_id', 'manifest_index'], unique=True)
op.create_index('manifestlayer_manifest_index', 'manifestlayer', ['manifest_index'], unique=False)
op.create_table(
'manifestlistmanifest',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_list_id', sa.Integer(), nullable=False),
sa.Column('manifest_id', sa.Integer(), nullable=False),
sa.Column('operating_system', UTF8CharField(length=255), nullable=True),
sa.Column('architecture', UTF8CharField(length=255), nullable=True),
sa.Column('platform_json', UTF8LongText, nullable=True),
sa.Column('media_type_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['manifest_id'], ['manifest.id'], name=op.f('fk_manifestlistmanifest_manifest_id_manifest')),
sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_manifestlistmanifest_manifest_list_id_manifestlist')),
sa.ForeignKeyConstraint(['media_type_id'], ['mediatype.id'], name=op.f('fk_manifestlistmanifest_media_type_id_mediatype')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlistmanifest'))
)
op.create_index('manifestlistmanifest_manifest_id', 'manifestlistmanifest', ['manifest_id'], unique=False)
op.create_index('manifestlistmanifest_manifest_list_id', 'manifestlistmanifest', ['manifest_list_id'], unique=False)
op.create_index('manifestlistmanifest_manifest_listid_os_arch_mtid', 'manifestlistmanifest', ['manifest_list_id', 'operating_system', 'architecture', 'media_type_id'], unique=False)
op.create_index('manifestlistmanifest_manifest_listid_mtid', 'manifestlistmanifest', ['manifest_list_id', 'media_type_id'], unique=False)
op.create_index('manifestlistmanifest_media_type_id', 'manifestlistmanifest', ['media_type_id'], unique=False)
op.create_table(
'tag',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', UTF8CharField(length=190), nullable=False),
sa.Column('repository_id', sa.Integer(), nullable=False),
sa.Column('manifest_list_id', sa.Integer(), nullable=True),
sa.Column('lifetime_start', sa.BigInteger(), nullable=False),
sa.Column('lifetime_end', sa.BigInteger(), nullable=True),
sa.Column('hidden', sa.Boolean(), nullable=False),
sa.Column('reverted', sa.Boolean(), nullable=False),
sa.Column('protected', sa.Boolean(), nullable=False),
sa.Column('tag_kind_id', sa.Integer(), nullable=False),
sa.Column('linked_tag_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['linked_tag_id'], ['tag.id'], name=op.f('fk_tag_linked_tag_id_tag')),
sa.ForeignKeyConstraint(['manifest_list_id'], ['manifestlist.id'], name=op.f('fk_tag_manifest_list_id_manifestlist')),
sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_tag_repository_id_repository')),
sa.ForeignKeyConstraint(['tag_kind_id'], ['tagkind.id'], name=op.f('fk_tag_tag_kind_id_tagkind')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
)
op.create_index('tag_lifetime_end', 'tag', ['lifetime_end'], unique=False)
op.create_index('tag_linked_tag_id', 'tag', ['linked_tag_id'], unique=False)
op.create_index('tag_manifest_list_id', 'tag', ['manifest_list_id'], unique=False)
op.create_index('tag_repository_id', 'tag', ['repository_id'], unique=False)
op.create_index('tag_repository_id_name_hidden', 'tag', ['repository_id', 'name', 'hidden'], unique=False)
op.create_index('tag_repository_id_name_lifetime_end', 'tag', ['repository_id', 'name', 'lifetime_end'], unique=True)
op.create_index('tag_repository_id_name', 'tag', ['repository_id', 'name'], unique=False)
op.create_index('tag_tag_kind_id', 'tag', ['tag_kind_id'], unique=False)
op.create_table(
'manifestlayerdockerv1',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('manifest_layer_id', sa.Integer(), nullable=False),
sa.Column('image_id', UTF8CharField(length=255), nullable=False),
sa.Column('checksum', UTF8CharField(length=255), nullable=False),
sa.Column('compat_json', UTF8LongText, nullable=False),
sa.ForeignKeyConstraint(['manifest_layer_id'], ['manifestlayer.id'], name=op.f('fk_manifestlayerdockerv1_manifest_layer_id_manifestlayer')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayerdockerv1'))
)
op.create_index('manifestlayerdockerv1_image_id', 'manifestlayerdockerv1', ['image_id'], unique=False)
op.create_index('manifestlayerdockerv1_manifest_layer_id', 'manifestlayerdockerv1', ['manifest_layer_id'], unique=False)
op.create_table(
'manifestlayerscan',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('layer_id', sa.Integer(), nullable=False),
sa.Column('scannable', sa.Boolean(), nullable=False),
sa.Column('scanned_by', UTF8CharField(length=255), nullable=False),
sa.ForeignKeyConstraint(['layer_id'], ['manifestlayer.id'], name=op.f('fk_manifestlayerscan_layer_id_manifestlayer')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_manifestlayerscan'))
)
op.create_index('manifestlayerscan_layer_id', 'manifestlayerscan', ['layer_id'], unique=True)
op.bulk_insert(
tables.blobplacementlocation,
[
{'name': 'local_eu'},
{'name': 'local_us'},
],
)
op.bulk_insert(
tables.mediatype,
[
{'name': 'application/vnd.cnr.blob.v0.tar+gzip'},
{'name': 'application/vnd.cnr.package-manifest.helm.v0.json'},
{'name': 'application/vnd.cnr.package-manifest.kpm.v0.json'},
{'name': 'application/vnd.cnr.package-manifest.docker-compose.v0.json'},
{'name': 'application/vnd.cnr.package.kpm.v0.tar+gzip'},
{'name': 'application/vnd.cnr.package.helm.v0.tar+gzip'},
{'name': 'application/vnd.cnr.package.docker-compose.v0.tar+gzip'},
{'name': 'application/vnd.cnr.manifests.v0.json'},
{'name': 'application/vnd.cnr.manifest.list.v0.json'},
],
)
op.bulk_insert(
tables.tagkind,
[
{'name': 'tag', 'id': 1},
{'name': 'release', 'id': 2},
{'name': 'channel', 'id': 3},
]
)
def downgrade(tables):
op.drop_table('manifestlayerscan')
op.drop_table('manifestlayerdockerv1')
op.drop_table('tag')
op.drop_table('manifestlistmanifest')
op.drop_table('manifestlayer')
op.drop_table('manifestlabel')
op.drop_table('manifestblob')
op.drop_table('derivedimage')
op.drop_table('blobuploading')
op.drop_table('blobplacement')
op.drop_table('bittorrentpieces')
op.drop_table('manifestlist')
op.drop_table('manifest')
op.drop_table('blobplacementlocationpreference')
op.drop_table('blob')
op.drop_table('tagkind')
op.drop_table('blobplacementlocation')

5
initdb.py
View file

@ -14,7 +14,7 @@ from uuid import UUID, uuid4
from threading import Event
from email.utils import formatdate
from data.database import (db, all_models, Role, TeamRole, Visibility, LoginService,
from data.database import (db, all_models, beta_classes, Role, TeamRole, Visibility, LoginService,
BuildTriggerService, AccessTokenKind, LogEntryKind, ImageStorageLocation,
ImageStorageTransformation, ImageStorageSignatureKind,
ExternalNotificationEvent, ExternalNotificationMethod, NotificationKind,
@ -824,6 +824,9 @@ def find_models_missing_data():
# As a sanity check we are going to make sure that all db tables have some data
models_missing_data = set()
for one_model in all_models:
if one_model in beta_classes:
continue
try:
one_model.select().get()
except one_model.DoesNotExist:

BIN
test/data/test.db
View file

Binary file not shown.