diff --git a/data/database.py b/data/database.py index 935fad2a6..24598f29c 100644 --- a/data/database.py +++ b/data/database.py @@ -1,3 +1,5 @@ +# pylint: disable=old-style-class,no-init + import inspect import logging import string @@ -9,7 +11,6 @@ from collections import defaultdict from datetime import datetime from random import SystemRandom -import resumablehashlib import toposort from enum import Enum @@ -18,6 +19,8 @@ from playhouse.shortcuts import RetryOperationalError from sqlalchemy.engine.url import make_url +import resumablehashlib + from data.fields import ResumableSHA256Field, ResumableSHA1Field, JSONField, Base64BinaryField from data.read_slave import ReadSlaveModel from util.names import urn_generator @@ -115,11 +118,11 @@ def delete_instance_filtered(instance, model_class, delete_nullable, skip_transi with db_transaction(): for query, fk in filtered_ops: - model = fk.model_class + _model = fk.model_class if fk.null and not delete_nullable: - model.update(**{fk.name: None}).where(query).execute() + _model.update(**{fk.name: None}).where(query).execute() else: - model.delete().where(query).execute() + _model.delete().where(query).execute() return instance.delete().where(instance._pk_expr()).execute() @@ -144,12 +147,12 @@ class CloseForLongOperation(object): self.config_object = config_object def __enter__(self): - if self.config_object.get('TESTING') == True: + if self.config_object.get('TESTING') is True: return close_db_filter(None) - def __exit__(self, type, value, traceback): + def __exit__(self, typ, value, traceback): # Note: Nothing to do. The next SQL call will reconnect automatically. pass @@ -163,7 +166,7 @@ class UseThenDisconnect(object): def __enter__(self): configure(self.config_object) - def __exit__(self, type, value, traceback): + def __exit__(self, typ, value, traceback): close_db_filter(None) @@ -295,7 +298,7 @@ class QuayUserField(ForeignKeyField): def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs): self.allows_robots = allows_robots self.robot_null_delete = robot_null_delete - if not 'rel_model' in kwargs: + if 'rel_model' not in kwargs: kwargs['rel_model'] = User super(QuayUserField, self).__init__(*args, **kwargs) @@ -341,12 +344,12 @@ class User(BaseModel): # For all the model dependencies, only delete those that allow robots. for query, fk in reversed(list(self.dependencies(search_nullable=True))): if isinstance(fk, QuayUserField) and fk.allows_robots: - model = fk.model_class + _model = fk.model_class if fk.robot_null_delete: - model.update(**{fk.name: None}).where(query).execute() + _model.update(**{fk.name: None}).where(query).execute() else: - model.delete().where(query).execute() + _model.delete().where(query).execute() # Delete the instance itself. super(User, self).delete_instance(recursive=False, delete_nullable=False) @@ -494,7 +497,7 @@ class PermissionPrototype(BaseModel): uuid = CharField(default=uuid_generator) activating_user = QuayUserField(allows_robots=True, index=True, null=True, related_name='userpermissionproto') - delegate_user = QuayUserField(allows_robots=True,related_name='receivingpermission', + delegate_user = QuayUserField(allows_robots=True, related_name='receivingpermission', null=True) delegate_team = ForeignKeyField(Team, related_name='receivingpermission', null=True) @@ -988,6 +991,186 @@ class TagManifestLabel(BaseModel): (('annotated', 'label'), True), ) +class Blob(BaseModel): + """ Blob represents a content-addressable object stored outside of the database. """ + digest = CharField(index=True, unique=True) + media_type = ForeignKeyField(MediaType) + size = BigIntegerField() + uncompressed_size = BigIntegerField(null=True) + + +class BlobPlacementLocation(BaseModel): + """ BlobPlacementLocation is an enumeration of the possible storage locations for Blobs. """ + name = CharField(index=True, unique=True) + + +class BlobPlacementLocationPreference(BaseModel): + """ BlobPlacementLocationPreference is a location to which a user's data will be replicated. """ + user = QuayUserField(index=True, allow_robots=False) + location = ForeignKeyField(BlobPlacementLocation) + + +class BlobPlacement(BaseModel): + """ BlobPlacement represents the location of a Blob. """ + blob = ForeignKeyField(Blob) + location = ForeignKeyField(BlobPlacementLocation) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('blob', 'location'), True), + ) + + +class BlobUploading(BaseModel): + """ BlobUploading represents the state of a Blob currently being uploaded. """ + uuid = CharField(index=True, unique=True) + created = DateTimeField(default=datetime.now, index=True) + repository = ForeignKeyField(Repository, index=True) + location = ForeignKeyField(BlobPlacementLocation) + byte_count = IntegerField(default=0) + sha_state = ResumableSHA256Field(null=True, default=resumablehashlib.sha256) + storage_metadata = JSONField(null=True, default={}) + chunk_count = IntegerField(default=0) + uncompressed_byte_count = IntegerField(null=True) + piece_sha_state = ResumableSHA1Field(null=True) + piece_hashes = Base64BinaryField(null=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'uuid'), True), + ) + + +class Manifest(BaseModel): + """ Manifest represents the metadata and collection of blobs that comprise a container image. """ + digest = CharField(index=True, unique=True) + media_type = ForeignKeyField(MediaType) + manifest_json = JSONField() + + +class ManifestBlob(BaseModel): + """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs. """ + manifest = ForeignKeyField(Manifest, index=True) + blob = ForeignKeyField(Blob, index=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('manifest', 'blob'), True), + ) + + +class ManifestList(BaseModel): + """ ManifestList represents all of the various manifests that compose a Tag. """ + digest = CharField(index=True, unique=True) + manifest_list_json = JSONField() + schema_version = CharField() + media_type = ForeignKeyField(MediaType) + + +class ManifestListManifest(BaseModel): + """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """ + manifest_list = ForeignKeyField(ManifestList, index=True) + manifest = ForeignKeyField(Manifest, index=True) + operating_system = CharField() + architecture = CharField() + platform_json = JSONField() + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('manifest_list', 'operating_system', 'architecture'), False), + ) + + +class ManifestLayer(BaseModel): + """ ManifestLayer represents one of the layers that compose a Manifest. """ + blob = ForeignKeyField(Blob, index=True) + manifest = ForeignKeyField(Manifest) + manifest_index = IntegerField(index=True) # index 0 is the last command in a Dockerfile + metadata_json = JSONField() + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('manifest', 'manifest_index'), True), + ) + + +class ManifestLayerDockerV1(BaseModel): + """ ManifestLayerDockerV1 is the Docker v1 registry protocol metadata for a ManifestLayer. """ + manifest_layer = ForeignKeyField(ManifestLayer) + image_id = CharField(index=True) + checksum = CharField() + compat_json = JSONField() + + +class ManifestLayerScan(BaseModel): + """ ManifestLayerScan represents the state of security scanning for a ManifestLayer. """ + layer = ForeignKeyField(ManifestLayer, unique=True) + scannable = BooleanField() + scanned_by = CharField() + + +class DerivedImage(BaseModel): + """ DerivedImage represents a Manifest transcoded into an alternative format. """ + source_manifest = ForeignKeyField(Manifest) + derived_manifest_json = JSONField() + media_type = ForeignKeyField(MediaType) + blob = ForeignKeyField(Blob) + uniqueness_hash = CharField(index=True, unique=True) + signature_blob = ForeignKeyField(Blob, null=True) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('source_manifest', 'blob'), True), + (('source_manifest', 'media_type', 'uniqueness_hash'), True), + ) + + +class Tag(BaseModel): + """ Tag represents a user-facing alias for referencing a ManifestList. """ + name = CharField() + repository = ForeignKeyField(Repository) + manifest_list = ForeignKeyField(ManifestList) + lifetime_start = IntegerField(default=get_epoch_timestamp) + lifetime_end = IntegerField(null=True, index=True) + hidden = BooleanField(default=False) + reverted = BooleanField(default=False) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('repository', 'name'), False), + + # This unique index prevents deadlocks when concurrently moving and deleting tags + (('repository', 'name', 'lifetime_end_ts'), True), + ) + + +class BitTorrentPieces(BaseModel): + """ BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob. """ + blob = ForeignKeyField(Blob) + pieces = Base64BinaryField() + piece_length = IntegerField() + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + (('annotated', 'label'), True), + ) + is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)] diff --git a/pylintrc b/pylintrc index 7cecd0de7..123b4692d 100644 --- a/pylintrc +++ b/pylintrc @@ -9,7 +9,7 @@ # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=missing-docstring,invalid-name,too-many-locals +disable=missing-docstring,invalid-name,too-many-locals,too-few-public-methods,too-many-lines [TYPECHECK]