database: initial manifestlist schema changes

data/database.py

@@ -1,3 +1,5 @@
+# pylint: disable=old-style-class,no-init
+
 import inspect
 import logging
 import string
@@ -9,7 +11,6 @@ from collections import defaultdict
 from datetime import datetime
 from random import SystemRandom
 
-import resumablehashlib
 import toposort
 
 from enum import Enum
@@ -18,6 +19,8 @@ from playhouse.shortcuts import RetryOperationalError
 
 from sqlalchemy.engine.url import make_url
 
+import resumablehashlib
+
 from data.fields import ResumableSHA256Field, ResumableSHA1Field, JSONField, Base64BinaryField
 from data.read_slave import ReadSlaveModel
 from util.names import urn_generator
@@ -115,11 +118,11 @@ def delete_instance_filtered(instance, model_class, delete_nullable, skip_transi
 
   with db_transaction():
     for query, fk in filtered_ops:
-      model = fk.model_class
+      _model = fk.model_class
       if fk.null and not delete_nullable:
-        model.update(**{fk.name: None}).where(query).execute()
+        _model.update(**{fk.name: None}).where(query).execute()
       else:
-        model.delete().where(query).execute()
+        _model.delete().where(query).execute()
 
     return instance.delete().where(instance._pk_expr()).execute()
 
@@ -144,12 +147,12 @@ class CloseForLongOperation(object):
     self.config_object = config_object
 
   def __enter__(self):
-    if self.config_object.get('TESTING') == True:
+    if self.config_object.get('TESTING') is True:
       return
 
     close_db_filter(None)
 
-  def __exit__(self, type, value, traceback):
+  def __exit__(self, typ, value, traceback):
     # Note: Nothing to do. The next SQL call will reconnect automatically.
     pass
 
@@ -163,7 +166,7 @@ class UseThenDisconnect(object):
   def __enter__(self):
     configure(self.config_object)
 
-  def __exit__(self, type, value, traceback):
+  def __exit__(self, typ, value, traceback):
     close_db_filter(None)
 
 
@@ -295,7 +298,7 @@ class QuayUserField(ForeignKeyField):
   def __init__(self, allows_robots=False, robot_null_delete=False, *args, **kwargs):
     self.allows_robots = allows_robots
     self.robot_null_delete = robot_null_delete
-    if not 'rel_model' in kwargs:
+    if 'rel_model' not in kwargs:
       kwargs['rel_model'] = User
 
     super(QuayUserField, self).__init__(*args, **kwargs)
@@ -341,12 +344,12 @@ class User(BaseModel):
       # For all the model dependencies, only delete those that allow robots.
       for query, fk in reversed(list(self.dependencies(search_nullable=True))):
         if isinstance(fk, QuayUserField) and fk.allows_robots:
-          model = fk.model_class
+          _model = fk.model_class
 
           if fk.robot_null_delete:
-            model.update(**{fk.name: None}).where(query).execute()
+            _model.update(**{fk.name: None}).where(query).execute()
           else:
-            model.delete().where(query).execute()
+            _model.delete().where(query).execute()
 
       # Delete the instance itself.
       super(User, self).delete_instance(recursive=False, delete_nullable=False)
@@ -494,7 +497,7 @@ class PermissionPrototype(BaseModel):
   uuid = CharField(default=uuid_generator)
   activating_user = QuayUserField(allows_robots=True, index=True, null=True,
                                   related_name='userpermissionproto')
-  delegate_user = QuayUserField(allows_robots=True,related_name='receivingpermission',
+  delegate_user = QuayUserField(allows_robots=True, related_name='receivingpermission',
                                 null=True)
   delegate_team = ForeignKeyField(Team, related_name='receivingpermission',
                                   null=True)
@@ -988,6 +991,186 @@ class TagManifestLabel(BaseModel):
       (('annotated', 'label'), True),
     )
 
+class Blob(BaseModel):
+  """ Blob represents a content-addressable object stored outside of the database. """
+  digest = CharField(index=True, unique=True)
+  media_type = ForeignKeyField(MediaType)
+  size = BigIntegerField()
+  uncompressed_size = BigIntegerField(null=True)
+
+
+class BlobPlacementLocation(BaseModel):
+  """ BlobPlacementLocation is an enumeration of the possible storage locations for Blobs. """
+  name = CharField(index=True, unique=True)
+
+
+class BlobPlacementLocationPreference(BaseModel):
+  """ BlobPlacementLocationPreference is a location to which a user's data will be replicated. """
+  user = QuayUserField(index=True, allow_robots=False)
+  location = ForeignKeyField(BlobPlacementLocation)
+
+
+class BlobPlacement(BaseModel):
+  """ BlobPlacement represents the location of a Blob. """
+  blob = ForeignKeyField(Blob)
+  location = ForeignKeyField(BlobPlacementLocation)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('blob', 'location'), True),
+    )
+
+
+class BlobUploading(BaseModel):
+  """ BlobUploading represents the state of a Blob currently being uploaded. """
+  uuid = CharField(index=True, unique=True)
+  created = DateTimeField(default=datetime.now, index=True)
+  repository = ForeignKeyField(Repository, index=True)
+  location = ForeignKeyField(BlobPlacementLocation)
+  byte_count = IntegerField(default=0)
+  sha_state = ResumableSHA256Field(null=True, default=resumablehashlib.sha256)
+  storage_metadata = JSONField(null=True, default={})
+  chunk_count = IntegerField(default=0)
+  uncompressed_byte_count = IntegerField(null=True)
+  piece_sha_state = ResumableSHA1Field(null=True)
+  piece_hashes = Base64BinaryField(null=True)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('repository', 'uuid'), True),
+    )
+
+
+class Manifest(BaseModel):
+  """ Manifest represents the metadata and collection of blobs that comprise a container image. """
+  digest = CharField(index=True, unique=True)
+  media_type = ForeignKeyField(MediaType)
+  manifest_json = JSONField()
+
+
+class ManifestBlob(BaseModel):
+  """ ManifestBlob is a many-to-many relation table linking Manifests and Blobs. """
+  manifest = ForeignKeyField(Manifest, index=True)
+  blob = ForeignKeyField(Blob, index=True)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('manifest', 'blob'), True),
+    )
+
+
+class ManifestList(BaseModel):
+  """ ManifestList represents all of the various manifests that compose a Tag. """
+  digest = CharField(index=True, unique=True)
+  manifest_list_json = JSONField()
+  schema_version = CharField()
+  media_type = ForeignKeyField(MediaType)
+
+
+class ManifestListManifest(BaseModel):
+  """ ManifestListManifest is a many-to-many relation table linking ManifestLists and Manifests. """
+  manifest_list = ForeignKeyField(ManifestList, index=True)
+  manifest = ForeignKeyField(Manifest, index=True)
+  operating_system = CharField()
+  architecture = CharField()
+  platform_json = JSONField()
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('manifest_list', 'operating_system', 'architecture'), False),
+    )
+
+
+class ManifestLayer(BaseModel):
+  """ ManifestLayer represents one of the layers that compose a Manifest. """
+  blob = ForeignKeyField(Blob, index=True)
+  manifest = ForeignKeyField(Manifest)
+  manifest_index = IntegerField(index=True) # index 0 is the last command in a Dockerfile
+  metadata_json = JSONField()
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('manifest', 'manifest_index'), True),
+    )
+
+
+class ManifestLayerDockerV1(BaseModel):
+  """ ManifestLayerDockerV1 is the Docker v1 registry protocol metadata for a ManifestLayer. """
+  manifest_layer = ForeignKeyField(ManifestLayer)
+  image_id = CharField(index=True)
+  checksum = CharField()
+  compat_json = JSONField()
+
+
+class ManifestLayerScan(BaseModel):
+  """ ManifestLayerScan represents the state of security scanning for a ManifestLayer. """
+  layer = ForeignKeyField(ManifestLayer, unique=True)
+  scannable = BooleanField()
+  scanned_by = CharField()
+
+
+class DerivedImage(BaseModel):
+  """ DerivedImage represents a Manifest transcoded into an alternative format. """
+  source_manifest = ForeignKeyField(Manifest)
+  derived_manifest_json = JSONField()
+  media_type = ForeignKeyField(MediaType)
+  blob = ForeignKeyField(Blob)
+  uniqueness_hash = CharField(index=True, unique=True)
+  signature_blob = ForeignKeyField(Blob, null=True)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('source_manifest', 'blob'), True),
+      (('source_manifest', 'media_type', 'uniqueness_hash'), True),
+    )
+
+
+class Tag(BaseModel):
+  """ Tag represents a user-facing alias for referencing a ManifestList. """
+  name = CharField()
+  repository = ForeignKeyField(Repository)
+  manifest_list = ForeignKeyField(ManifestList)
+  lifetime_start = IntegerField(default=get_epoch_timestamp)
+  lifetime_end = IntegerField(null=True, index=True)
+  hidden = BooleanField(default=False)
+  reverted = BooleanField(default=False)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('repository', 'name'), False),
+
+      # This unique index prevents deadlocks when concurrently moving and deleting tags
+      (('repository', 'name', 'lifetime_end_ts'), True),
+    )
+
+
+class BitTorrentPieces(BaseModel):
+  """ BitTorrentPieces represents the BitTorrent piece metadata calculated from a Blob. """
+  blob = ForeignKeyField(Blob)
+  pieces = Base64BinaryField()
+  piece_length = IntegerField()
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      (('annotated', 'label'), True),
+    )
+
 
 is_model = lambda x: inspect.isclass(x) and issubclass(x, BaseModel) and x is not BaseModel
 all_models = [model[1] for model in inspect.getmembers(sys.modules[__name__], is_model)]

pylintrc

@@ -9,7 +9,7 @@
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=missing-docstring,invalid-name,too-many-locals
+disable=missing-docstring,invalid-name,too-many-locals,too-few-public-methods,too-many-lines
 
 [TYPECHECK]