vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

data.database: add CNR/OCI models

Browse source
This commit is contained in:
Antoine Legrand 2016-10-26 23:19:28 +02:00 committed by Jimmy Zelinskie
parent f08e4921f2
commit c61024586d
1 changed files with 118 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
data/database.py
View file

@ -21,6 +21,7 @@ from playhouse.shortcuts import RetryOperationalError
from sqlalchemy.engine.url import make_url
import resumablehashlib
from cachetools import lru_cache
from data.fields import (ResumableSHA256Field, ResumableSHA1Field, JSONField, Base64BinaryField,
FullIndexedTextField, FullIndexedCharField)
@ -150,6 +151,7 @@ SCHEME_SPECIALIZED_FOR_UPDATE = {
'sqlite': null_for_update,
}
class CallableProxy(Proxy):
def __call__(self, *args, **kwargs):
if self.obj is None:
@ -336,10 +338,54 @@ class QuayUserField(ForeignKeyField):
self.robot_null_delete = robot_null_delete
if 'rel_model' not in kwargs:
kwargs['rel_model'] = User
super(QuayUserField, self).__init__(*args, **kwargs)
# @TODO: Generates client-side enum
class EnumField(ForeignKeyField):
""" Create a cached python Enum from an EnumTable """
def __init__(self, rel_model, enum_key_field='name', *args, **kwargs):
"""
rel_model is the EnumTable model-class (see ForeignKeyField)
enum_key_field is the field from the EnumTable to use as the enum name
"""
self.enum_key_field = enum_key_field
super(EnumField, self).__init__(rel_model, *args, **kwargs)
@property
@lru_cache(maxsize=1)
def enum(self):
""" Returns a python enun.Enum generated from the associated EnumTable """
values = []
for row in self.rel_model.select():
key = getattr(row, self.enum_key_field)
value = getattr(row, 'id')
values.append((key, value))
return Enum(self.rel_model.__name__, values)
def get_id(self, name):
""" Returns the ForeignKeyId from the name field
Example:
>>> Repository.repo_kind.get_id("application")
2
"""
try:
return self.enum[name].value
except KeyError:
raise self.rel_model.DoesNotExist
def get_name(self, value):
""" Returns the name value from the ForeignKeyId
Example:
>>> Repository.repo_kind.get_name(2)
"application"
"""
try:
return self.enum(value).name
except ValueError:
raise self.rel_model.DoesNotExist
class BaseModel(ReadSlaveModel):
class Meta:
database = db
@ -405,7 +451,8 @@ class User(BaseModel):
RepositoryTag, PermissionPrototype, DerivedStorageForImage,
TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
RepositoryNotification, OAuthAuthorizationCode,
RepositoryActionCount, TagManifestLabel}
RepositoryActionCount, TagManifestLabel, Tag,
ManifestLabel, BlobUploading}
delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
@ -502,12 +549,17 @@ class Visibility(BaseModel):
name = CharField(index=True, unique=True)
class RepositoryKind(BaseModel):
name = CharField(index=True, unique=True)
class Repository(BaseModel):
namespace_user = QuayUserField(null=True)
name = FullIndexedCharField(match_function=db_match_func)
visibility = ForeignKeyField(Visibility)
description = FullIndexedTextField(match_function=db_match_func, null=True)
badge_token = CharField(default=uuid_generator)
repo_kind = EnumField(RepositoryKind)
class Meta:
database = db
@ -1069,7 +1121,7 @@ class Label(BaseModel):
uuid = CharField(default=uuid_generator, index=True, unique=True)
key = CharField(index=True)
value = TextField()
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
source_type = ForeignKeyField(LabelSourceType)
@ -1087,25 +1139,10 @@ class TagManifestLabel(BaseModel):
)
'''
class ManifestLabel(BaseModel):
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(Manifest, index=True)
label = ForeignKeyField(Label)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'annotated', 'label'), True),
)
class Blob(BaseModel):
""" Blob represents a content-addressable object stored outside of the database. """
digest = CharField(index=True, unique=True)
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
size = BigIntegerField()
uncompressed_size = BigIntegerField(null=True)
@ -1118,13 +1155,13 @@ class BlobPlacementLocation(BaseModel):
class BlobPlacementLocationPreference(BaseModel):
""" BlobPlacementLocationPreference is a location to which a user's data will be replicated. """
user = QuayUserField(index=True, allows_robots=False)
location = ForeignKeyField(BlobPlacementLocation)
location = EnumField(BlobPlacementLocation)
class BlobPlacement(BaseModel):
""" BlobPlacement represents the location of a Blob. """
blob = ForeignKeyField(Blob)
location = ForeignKeyField(BlobPlacementLocation)
location = EnumField(BlobPlacementLocation)
class Meta:
database = db
@ -1159,10 +1196,23 @@ class BlobUploading(BaseModel):
class Manifest(BaseModel):
""" Manifest represents the metadata and collection of blobs that comprise a container image. """
digest = CharField(index=True, unique=True)
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
manifest_json = JSONField()
class ManifestLabel(BaseModel):
repository = ForeignKeyField(Repository, index=True)
annotated = ForeignKeyField(Manifest, index=True)
label = ForeignKeyField(Label)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'annotated', 'label'), True),
)
class ManifestBlob(BaseModel):
""" ManifestBlob is a many-to-many relation table linking Manifests and Blobs. """
manifest = ForeignKeyField(Manifest, index=True)
@ -1181,22 +1231,56 @@ class ManifestList(BaseModel):
digest = CharField(index=True, unique=True)
manifest_list_json = JSONField()
schema_version = CharField()
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """
manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True)
operating_system = CharField()
architecture = CharField()
platform_json = JSONField()
class TagKind(BaseModel):
""" TagKind is a enumtable of to reference tag kinds """
name = CharField(index=True, unique=True)
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList. """
name = CharField()
repository = ForeignKeyField(Repository)
manifest_list = ForeignKeyField(ManifestList, null=True)
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reverted = BooleanField(default=False)
protected = BooleanField(default=False)
tag_kind = EnumField(TagKind)
linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents')
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest_list', 'operating_system', 'architecture'), False),
(('repository', 'name', 'tag_kind'), False),
(('repository', 'name', 'hidden', 'tag_kind'), False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end', 'tag_kind'), True),
)
Channel = Tag.alias()
class ManifestListManifest(BaseModel):
""" ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """
manifest_list = ForeignKeyField(ManifestList, index=True)
manifest = ForeignKeyField(Manifest, index=True)
operating_system = CharField(null=True)
architecture = CharField(null=True)
platform_json = JSONField(null=True)
media_type = EnumField(MediaType)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('manifest_list', 'operating_system', 'architecture', 'media_type'), False),
(('manifest_list', 'media_type'), False),
)
@ -1235,7 +1319,7 @@ class DerivedImage(BaseModel):
uuid = CharField(default=uuid_generator, unique=True)
source_manifest = ForeignKeyField(Manifest)
derived_manifest_json = JSONField()
media_type = ForeignKeyField(MediaType)
media_type = EnumField(MediaType)
blob = ForeignKeyField(Blob, related_name='blob')
uniqueness_hash = CharField(index=True, unique=True)
signature_blob = ForeignKeyField(Blob, null=True, related_name='signature_blob')
@ -1249,28 +1333,6 @@ class DerivedImage(BaseModel):
)
class Tag(BaseModel):
""" Tag represents a user-facing alias for referencing a ManifestList. """
name = CharField()
repository = ForeignKeyField(Repository)
manifest_list = ForeignKeyField(ManifestList)
lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms)
lifetime_end = BigIntegerField(null=True, index=True)
hidden = BooleanField(default=False)
reverted = BooleanField(default=False)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('repository', 'name'), False),
(('repository', 'name', 'hidden') False),
# This unique index prevents deadlocks when concurrently moving and deleting tags
(('repository', 'name', 'lifetime_end'), True),
)
class BitTorrentPieces(BaseModel):
""" BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob. """
blob = ForeignKeyField(Blob)
@ -1285,11 +1347,9 @@ class BitTorrentPieces(BaseModel):
)
beta_classes = set([ManifestLayerScan, Tag, BlobPlacementLocation, ManifestLayer, ManifestList,
beta_classes = set([ManifestLayerScan, Tag, TagKind, BlobPlacementLocation, ManifestLayer, ManifestList,
BitTorrentPieces, MediaType, Label, ManifestBlob, BlobUploading, Blob,
ManifestLayerDockerV1, BlobPlacementLocationPreference, ManifestListManifest,
Manifest, DerivedImage, BlobPlacement])
'''
Manifest, DerivedImage, BlobPlacement, ManifestLabel])
is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]