More changes for registry-v2 in python.

Implement the minimal changes to the local filesystem storage driver and feed them through the distributed storage driver.
Create a digest package which contains digest_tools and checksums.
Fix the tests to use the new v1 endpoint locations.
Fix repository.delete_instance to properly filter the generated queries to avoid most subquery deletes, but still generate them when not explicitly filtered.

data/database.py

@@ -2,12 +2,14 @@ import string
 import logging
 import uuid
 import time
+import toposort
 
 from random import SystemRandom
 from datetime import datetime
 from peewee import *
 from data.read_slave import ReadSlaveModel
 from sqlalchemy.engine.url import make_url
+from collections import defaultdict
 
 from data.read_slave import ReadSlaveModel
 from util.names import urn_generator
@@ -297,23 +299,46 @@ class Repository(BaseModel):
     )
 
   def delete_instance(self, recursive=False, delete_nullable=False):
-    # Note: peewee generates extra nested deletion statements here that are slow and unnecessary.
-    # Therefore, we define our own deletion order here and use the dependency system to verify it.
-    ordered_dependencies = [RepositoryAuthorizedEmail, RepositoryTag, Image, LogEntry,
-                            RepositoryBuild, RepositoryBuildTrigger, RepositoryNotification,
-                            RepositoryPermission, AccessToken, Star, RepositoryActionCount]
+    if not recursive:
+      raise RuntimeError('Non-recursive delete on repository.')
 
-    for query, fk in self.dependencies(search_nullable=True):
+    # These models don't need to use transitive deletes, because the referenced objects
+    # are cleaned up directly
+    skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger}
+
+    # We need to sort the ops so that models get cleaned in order of their dependencies
+    ops = reversed(list(self.dependencies(delete_nullable)))
+    filtered_ops = []
+
+    dependencies = defaultdict(set)
+
+    for query, fk in ops:
+      if fk.model_class not in skip_transitive_deletes or query.op != 'in':
+        filtered_ops.append((query, fk))
+
+      if query.op == 'in':
+        dependencies[fk.model_class.__name__].add(query.rhs.model_class.__name__)
+      elif query.op == '=':
+        dependencies[fk.model_class.__name__].add(Repository.__name__)
+      else:
+        raise RuntimeError('Unknown operator in recursive repository delete query')
+
+    sorted_models = list(reversed(toposort.toposort_flatten(dependencies)))
+    def sorted_model_key(query_fk_tuple):
+      cmp_query, cmp_fk = query_fk_tuple
+      if cmp_query.op == 'in':
+        return -1
+      return sorted_models.index(cmp_fk.model_class.__name__)
+    filtered_ops.sort(key=sorted_model_key)
+
+    for query, fk in filtered_ops:
       model = fk.model_class
-      if not model in ordered_dependencies:
-        raise Exception('Missing repository deletion dependency: %s', model)
-
-    for model in ordered_dependencies:
-      model.delete().where(model.repository == self).execute()
-
-    # Delete the repository itself.
-    super(Repository, self).delete_instance(recursive=False, delete_nullable=False)
+      if fk.null and not delete_nullable:
+        model.update(**{fk.name: None}).where(query).execute()
+      else:
+        model.delete().where(query).execute()
 
+    return self.delete().where(self._pk_expr()).execute()
 
 class Star(BaseModel):
   user = ForeignKeyField(User, index=True)
@@ -679,4 +704,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               ExternalNotificationEvent, ExternalNotificationMethod, RepositoryNotification,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
-              AccessTokenKind, Star, RepositoryActionCount]
+              AccessTokenKind, Star, RepositoryActionCount, TagManifest]