Switch to a per-namespace configurable expiration policy for time machine, and switch the tag gc to respect it.

config.py

@@ -185,8 +185,5 @@ class DefaultConfig(object):
   LOG_ARCHIVE_LOCATION = 'local_us'
   LOG_ARCHIVE_PATH = 'logarchive/'
 
-  # Number of revisions to keep expired tags
-  TIME_MACHINE_DELTA_SECONDS = 14 * 24 * 60 * 60
-
   # For enterprise:
   MAXIMUM_REPOSITORY_USAGE = 20

data/database.py

@@ -1,12 +1,14 @@
 import string
 import logging
 import uuid
+import time
 
 from random import SystemRandom
 from datetime import datetime
 from peewee import *
-from data.read_slave import ReadSlaveModel
 from sqlalchemy.engine.url import make_url
+
+from data.read_slave import ReadSlaveModel
 from util.names import urn_generator
 
 
@@ -136,6 +138,9 @@ def uuid_generator():
   return str(uuid.uuid4())
 
 
+_get_epoch_timestamp = lambda: int(time.time())
+
+
 def close_db_filter(_):
   if not db.is_closed():
     logger.debug('Disconnecting from database.')
@@ -175,6 +180,7 @@ class User(BaseModel):
   invoice_email = BooleanField(default=False)
   invalid_login_attempts = IntegerField(default=0)
   last_invalid_login = DateTimeField(default=datetime.utcnow)
+  removed_tag_expiration_s = IntegerField(default=1209600)  # Two weeks
 
   def delete_instance(self, recursive=False, delete_nullable=False):
     # If we are deleting a robot account, only execute the subset of queries necessary.
@@ -456,8 +462,8 @@ class RepositoryTag(BaseModel):
   name = CharField()
   image = ForeignKeyField(Image)
   repository = ForeignKeyField(Repository)
-  lifetime_start = DateTimeField(default=datetime.utcnow)
+  lifetime_start_ts = IntegerField(default=_get_epoch_timestamp)
-  lifetime_end = DateTimeField(null=True)
+  lifetime_end_ts = IntegerField(null=True, index=True)
 
   class Meta:
     database = db

data/model/legacy.py

@@ -2,6 +2,7 @@ import bcrypt
 import logging
 import dateutil.parser
 import json
+import time
 
 from datetime import datetime, timedelta, date
 
@@ -1531,8 +1532,8 @@ def get_repository_images(namespace_name, repository_name):
 
 
 def _tag_alive(query):
-  return query.where((RepositoryTag.lifetime_end >> None) |
+  return query.where((RepositoryTag.lifetime_end_ts >> None) |
-                     (RepositoryTag.lifetime_end > datetime.utcnow()))
+                     (RepositoryTag.lifetime_end_ts > int(time.time())))
 
 
 def list_repository_tags(namespace_name, repository_name):
@@ -1547,14 +1548,18 @@ def list_repository_tags(namespace_name, repository_name):
 
 
 def _garbage_collect_tags(namespace_name, repository_name):
-  with config.app_config['DB_TRANSACTION_FACTORY'](db):
+    to_delete = (RepositoryTag
-    repo = _get_repository(namespace_name, repository_name)
+                 .select(RepositoryTag.id)
-    collect_time = (datetime.utcnow() -
+                 .join(Repository)
-                    timedelta(seconds=config.app_config['TIME_MACHINE_DELTA_SECONDS']))
+                 .join(Namespace, on=(Repository.namespace_user == Namespace.id))
+                 .where(Repository.name == repository_name, Namespace.username == namespace_name,
+                        ~(RepositoryTag.lifetime_end_ts >> None),
+                        (RepositoryTag.lifetime_end_ts + Namespace.removed_tag_expiration_s) <=
+                         int(time.time())))
 
     (RepositoryTag
      .delete()
-     .where(RepositoryTag.repository == repo, RepositoryTag.lifetime_end < collect_time)
+     .where(RepositoryTag.id << to_delete)
      .execute())
 
 
@@ -1745,7 +1750,7 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
     except Image.DoesNotExist:
       raise DataModelException('Invalid image with id: %s' % tag_docker_image_id)
 
-    now = datetime.utcnow()
+    now_ts = int(time.time())
 
     try:
       # When we move a tag, we really end the timeline of the old one and create a new one
@@ -1753,13 +1758,14 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
                          .select()
                          .where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name))
       tag = query.get()
-      tag.lifetime_end = now
+      tag.lifetime_end_ts = now_ts
       tag.save()
     except RepositoryTag.DoesNotExist:
       # No tag that needs to be ended
       pass
 
-    tag = RepositoryTag.create(repository=repo, image=image, name=tag_name, lifetime_start=now)
+    tag = RepositoryTag.create(repository=repo, image=image, name=tag_name,
+                               lifetime_start_ts=now_ts)
 
     return tag
 
@@ -1781,7 +1787,7 @@ def delete_tag(namespace_name, repository_name, tag_name):
              (tag_name, namespace_name, repository_name))
       raise DataModelException(msg)
 
-    found.lifetime_end = datetime.utcnow()
+    found.lifetime_end_ts = int(time.time())
     found.save()