Reduce database bandwidth by tracking gc candidate images.

data/database.py

@@ -34,6 +34,7 @@ _SCHEME_DRIVERS = {
   'postgresql+psycopg2': PostgresqlDatabase,
 }
 
+
 SCHEME_RANDOM_FUNCTION = {
   'mysql': fn.Rand,
   'mysql+pymysql': fn.Rand,
@@ -42,12 +43,37 @@ SCHEME_RANDOM_FUNCTION = {
   'postgresql+psycopg2': fn.Random,
 }
 
+
+def pipes_concat(arg1, arg2, *extra_args):
+  """ Concat function for sqlite, since it doesn't support fn.Concat.
+      Concatenates clauses with || characters.
+  """
+  reduced = arg1.concat(arg2)
+  for arg in extra_args:
+    reduced = reduced.concat(arg)
+  return reduced
+
+
+def function_concat(arg1, arg2, *extra_args):
+  """ Default implementation of concat which uses fn.Concat(). Used by all
+      database engines except sqlite.
+  """
+  return fn.Concat(arg1, arg2, *extra_args)
+
+
+SCHEME_SPECIALIZED_CONCAT = {
+  'sqlite': pipes_concat,
+}
+
+
 def real_for_update(query):
   return query.for_update()
 
+
 def null_for_update(query):
   return query
 
+
 def delete_instance_filtered(instance, model_class, delete_nullable, skip_transitive_deletes):
   """ Deletes the DB instance recursively, skipping any models in the skip_transitive_deletes set.
 
@@ -181,6 +207,7 @@ read_slave = Proxy()
 db_random_func = CallableProxy()
 db_for_update = CallableProxy()
 db_transaction = CallableProxy()
+db_concat_func = CallableProxy()
 
 
 def validate_database_url(url, db_kwargs, connect_timeout=5):
@@ -227,6 +254,8 @@ def configure(config_object):
   db_random_func.initialize(SCHEME_RANDOM_FUNCTION[parsed_write_uri.drivername])
   db_for_update.initialize(SCHEME_SPECIALIZED_FOR_UPDATE.get(parsed_write_uri.drivername,
                                                              real_for_update))
+  db_concat_func.initialize(SCHEME_SPECIALIZED_CONCAT.get(parsed_write_uri.drivername,
+                                                          function_concat))
 
   read_slave_uri = config_object.get('DB_READ_SLAVE_URI', None)
   if read_slave_uri is not None: