Another huge batch of registry v2 changes

Add patch support and resumeable sha
Implement all actual registry methods
Add a simple database generation option

data/database.py

@@ -3,6 +3,9 @@ import logging
 import uuid
 import time
 import toposort
+import base64
+import resumablehashlib
+import json
 
 from random import SystemRandom
 from datetime import datetime
@@ -343,7 +346,7 @@ class Repository(BaseModel):
 
     # These models don't need to use transitive deletes, because the referenced objects
     # are cleaned up directly
-    skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger}
+    skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload}
 
     # We need to sort the ops so that models get cleaned in order of their dependencies
     ops = reversed(list(self.dependencies(delete_nullable)))
@@ -485,6 +488,7 @@ class ImageStorage(BaseModel):
   uncompressed_size = BigIntegerField(null=True)
   aggregate_size = BigIntegerField(null=True)
   uploading = BooleanField(default=True, null=True)
+  cas_path = BooleanField(default=True)
 
 
 class ImageStorageTransformation(BaseModel):
@@ -552,6 +556,12 @@ class Image(BaseModel):
 
   storage = ForeignKeyField(ImageStorage, index=True, null=True)
 
+  created = DateTimeField(null=True)
+  comment = TextField(null=True)
+  command = TextField(null=True)
+  aggregate_size = BigIntegerField(null=True)
+  v1_json_metadata = TextField(null=True)
+
   class Meta:
     database = db
     read_slaves = (read_slave,)
@@ -740,6 +750,44 @@ class RepositoryAuthorizedEmail(BaseModel):
     )
 
 
+class ResumableSHAField(TextField):
+  def db_value(self, value):
+    sha_state = value.state()
+
+    # One of the fields is a byte string, let's base64 encode it to make sure
+    # we can store and fetch it regardless of default collocation
+    sha_state[3] = base64.b64encode(sha_state[3])
+
+    return json.dumps(sha_state)
+
+  def python_value(self, value):
+    to_resume = resumablehashlib.sha256()
+    if value is None:
+      return to_resume
+
+    sha_state = json.loads(value)
+
+    # We need to base64 decode the data bytestring
+    sha_state[3] = base64.b64decode(sha_state[3])
+    to_resume.set_state(sha_state)
+    return to_resume
+
+
+class BlobUpload(BaseModel):
+  repository = ForeignKeyField(Repository, index=True)
+  uuid = CharField(index=True, unique=True)
+  byte_count = IntegerField(default=0)
+  sha_state = ResumableSHAField(null=True, default=resumablehashlib.sha256)
+  location = ForeignKeyField(ImageStorageLocation)
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      # create a unique index on email and repository
+      (('repository', 'uuid'), True),
+    )
+
 
 all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility,
               RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem,
@@ -750,4 +798,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
-              AccessTokenKind, Star, RepositoryActionCount, TagManifest]
+              AccessTokenKind, Star, RepositoryActionCount, TagManifest, BlobUpload]