diff --git a/data/database.py b/data/database.py index d51f052bf..296408080 100644 --- a/data/database.py +++ b/data/database.py @@ -21,6 +21,7 @@ from playhouse.shortcuts import RetryOperationalError from sqlalchemy.engine.url import make_url import resumablehashlib +from cachetools import lru_cache from data.fields import (ResumableSHA256Field, ResumableSHA1Field, JSONField, Base64BinaryField, FullIndexedTextField, FullIndexedCharField) @@ -150,6 +151,7 @@ SCHEME_SPECIALIZED_FOR_UPDATE = { 'sqlite': null_for_update, } + class CallableProxy(Proxy): def __call__(self, *args, **kwargs): if self.obj is None: @@ -336,10 +338,54 @@ class QuayUserField(ForeignKeyField): self.robot_null_delete = robot_null_delete if 'rel_model' not in kwargs: kwargs['rel_model'] = User - super(QuayUserField, self).__init__(*args, **kwargs) +# @TODO: Generates client-side enum +class EnumField(ForeignKeyField): + """ Create a cached python Enum from an EnumTable """ + def __init__(self, rel_model, enum_key_field='name', *args, **kwargs): + """ + rel_model is the EnumTable model-class (see ForeignKeyField) + enum_key_field is the field from the EnumTable to use as the enum name + """ + self.enum_key_field = enum_key_field + super(EnumField, self).__init__(rel_model, *args, **kwargs) + + @property + @lru_cache(maxsize=1) + def enum(self): + """ Returns a python enun.Enum generated from the associated EnumTable """ + values = [] + for row in self.rel_model.select(): + key = getattr(row, self.enum_key_field) + value = getattr(row, 'id') + values.append((key, value)) + return Enum(self.rel_model.__name__, values) + + def get_id(self, name): + """ Returns the ForeignKeyId from the name field + Example: + >>> Repository.repo_kind.get_id("application") + 2 + """ + try: + return self.enum[name].value + except KeyError: + raise self.rel_model.DoesNotExist + + def get_name(self, value): + """ Returns the name value from the ForeignKeyId + Example: + >>> Repository.repo_kind.get_name(2) + "application" + """ + try: + return self.enum(value).name + except ValueError: + raise self.rel_model.DoesNotExist + + class BaseModel(ReadSlaveModel): class Meta: database = db @@ -405,7 +451,8 @@ class User(BaseModel): RepositoryTag, PermissionPrototype, DerivedStorageForImage, TagManifest, AccessToken, OAuthAccessToken, BlobUpload, RepositoryNotification, OAuthAuthorizationCode, - RepositoryActionCount, TagManifestLabel} + RepositoryActionCount, TagManifestLabel, Tag, + ManifestLabel, BlobUploading} delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes) @@ -502,12 +549,17 @@ class Visibility(BaseModel): name = CharField(index=True, unique=True) +class RepositoryKind(BaseModel): + name = CharField(index=True, unique=True) + + class Repository(BaseModel): namespace_user = QuayUserField(null=True) name = FullIndexedCharField(match_function=db_match_func) visibility = ForeignKeyField(Visibility) description = FullIndexedTextField(match_function=db_match_func, null=True) badge_token = CharField(default=uuid_generator) + repo_kind = EnumField(RepositoryKind) class Meta: database = db @@ -1069,7 +1121,7 @@ class Label(BaseModel): uuid = CharField(default=uuid_generator, index=True, unique=True) key = CharField(index=True) value = TextField() - media_type = ForeignKeyField(MediaType) + media_type = EnumField(MediaType) source_type = ForeignKeyField(LabelSourceType) @@ -1087,25 +1139,10 @@ class TagManifestLabel(BaseModel): ) -''' - -class ManifestLabel(BaseModel): - repository = ForeignKeyField(Repository, index=True) - annotated = ForeignKeyField(Manifest, index=True) - label = ForeignKeyField(Label) - - class Meta: - database = db - read_slaves = (read_slave,) - indexes = ( - (('repository', 'annotated', 'label'), True), - ) - - class Blob(BaseModel): """ Blob represents a content-addressable object stored outside of the database. """ digest = CharField(index=True, unique=True) - media_type = ForeignKeyField(MediaType) + media_type = EnumField(MediaType) size = BigIntegerField() uncompressed_size = BigIntegerField(null=True) @@ -1118,13 +1155,13 @@ class BlobPlacementLocation(BaseModel): class BlobPlacementLocationPreference(BaseModel): """ BlobPlacementLocationPreference is a location to which a user's data will be replicated. """ user = QuayUserField(index=True, allows_robots=False) - location = ForeignKeyField(BlobPlacementLocation) + location = EnumField(BlobPlacementLocation) class BlobPlacement(BaseModel): """ BlobPlacement represents the location of a Blob. """ blob = ForeignKeyField(Blob) - location = ForeignKeyField(BlobPlacementLocation) + location = EnumField(BlobPlacementLocation) class Meta: database = db @@ -1159,10 +1196,23 @@ class BlobUploading(BaseModel): class Manifest(BaseModel): """ Manifest represents the metadata and collection of blobs that comprise a container image. """ digest = CharField(index=True, unique=True) - media_type = ForeignKeyField(MediaType) + media_type = EnumField(MediaType) manifest_json = JSONField() +class ManifestLabel(BaseModel): + repository = ForeignKeyField(Repository, index=True) + annotated = ForeignKeyField(Manifest, index=True) + label = ForeignKeyField(Label) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'annotated', 'label'), True), + ) + + class ManifestBlob(BaseModel): """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs. """ manifest = ForeignKeyField(Manifest, index=True) @@ -1181,22 +1231,56 @@ class ManifestList(BaseModel): digest = CharField(index=True, unique=True) manifest_list_json = JSONField() schema_version = CharField() - media_type = ForeignKeyField(MediaType) + media_type = EnumField(MediaType) -class ManifestListManifest(BaseModel): - """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """ - manifest_list = ForeignKeyField(ManifestList, index=True) - manifest = ForeignKeyField(Manifest, index=True) - operating_system = CharField() - architecture = CharField() - platform_json = JSONField() +class TagKind(BaseModel): + """ TagKind is a enumtable of to reference tag kinds """ + name = CharField(index=True, unique=True) + + +class Tag(BaseModel): + """ Tag represents a user-facing alias for referencing a ManifestList. """ + name = CharField() + repository = ForeignKeyField(Repository) + manifest_list = ForeignKeyField(ManifestList, null=True) + lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms) + lifetime_end = BigIntegerField(null=True, index=True) + hidden = BooleanField(default=False) + reverted = BooleanField(default=False) + protected = BooleanField(default=False) + tag_kind = EnumField(TagKind) + linked_tag = ForeignKeyField('self', null=True, related_name='tag_parents') class Meta: database = db read_slaves = (read_slave,) indexes = ( - (('manifest_list', 'operating_system', 'architecture'), False), + (('repository', 'name', 'tag_kind'), False), + (('repository', 'name', 'hidden', 'tag_kind'), False), + # This unique index prevents deadlocks when concurrently moving and deleting tags + (('repository', 'name', 'lifetime_end', 'tag_kind'), True), + ) + + + +Channel = Tag.alias() + +class ManifestListManifest(BaseModel): + """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """ + manifest_list = ForeignKeyField(ManifestList, index=True) + manifest = ForeignKeyField(Manifest, index=True) + operating_system = CharField(null=True) + architecture = CharField(null=True) + platform_json = JSONField(null=True) + media_type = EnumField(MediaType) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('manifest_list', 'operating_system', 'architecture', 'media_type'), False), + (('manifest_list', 'media_type'), False), ) @@ -1235,7 +1319,7 @@ class DerivedImage(BaseModel): uuid = CharField(default=uuid_generator, unique=True) source_manifest = ForeignKeyField(Manifest) derived_manifest_json = JSONField() - media_type = ForeignKeyField(MediaType) + media_type = EnumField(MediaType) blob = ForeignKeyField(Blob, related_name='blob') uniqueness_hash = CharField(index=True, unique=True) signature_blob = ForeignKeyField(Blob, null=True, related_name='signature_blob') @@ -1249,28 +1333,6 @@ class DerivedImage(BaseModel): ) -class Tag(BaseModel): - """ Tag represents a user-facing alias for referencing a ManifestList. """ - name = CharField() - repository = ForeignKeyField(Repository) - manifest_list = ForeignKeyField(ManifestList) - lifetime_start = BigIntegerField(default=get_epoch_timestamp_ms) - lifetime_end = BigIntegerField(null=True, index=True) - hidden = BooleanField(default=False) - reverted = BooleanField(default=False) - - class Meta: - database = db - read_slaves = (read_slave,) - indexes = ( - (('repository', 'name'), False), - (('repository', 'name', 'hidden') False), - - # This unique index prevents deadlocks when concurrently moving and deleting tags - (('repository', 'name', 'lifetime_end'), True), - ) - - class BitTorrentPieces(BaseModel): """ BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob. """ blob = ForeignKeyField(Blob) @@ -1285,11 +1347,9 @@ class BitTorrentPieces(BaseModel): ) -beta_classes = set([ManifestLayerScan, Tag, BlobPlacementLocation, ManifestLayer, ManifestList, +beta_classes = set([ManifestLayerScan, Tag, TagKind, BlobPlacementLocation, ManifestLayer, ManifestList, BitTorrentPieces, MediaType, Label, ManifestBlob, BlobUploading, Blob, ManifestLayerDockerV1, BlobPlacementLocationPreference, ManifestListManifest, - Manifest, DerivedImage, BlobPlacement]) -''' - + Manifest, DerivedImage, BlobPlacement, ManifestLabel]) is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]