diff --git a/config.py b/config.py index 47f836587..39a257b15 100644 --- a/config.py +++ b/config.py @@ -185,8 +185,5 @@ class DefaultConfig(object): LOG_ARCHIVE_LOCATION = 'local_us' LOG_ARCHIVE_PATH = 'logarchive/' - # Number of revisions to keep expired tags - TIME_MACHINE_DELTA_SECONDS = 14 * 24 * 60 * 60 - # For enterprise: MAXIMUM_REPOSITORY_USAGE = 20 diff --git a/data/database.py b/data/database.py index 7d027aa71..ccc5900a8 100644 --- a/data/database.py +++ b/data/database.py @@ -1,12 +1,14 @@ import string import logging import uuid +import time from random import SystemRandom from datetime import datetime from peewee import * -from data.read_slave import ReadSlaveModel from sqlalchemy.engine.url import make_url + +from data.read_slave import ReadSlaveModel from util.names import urn_generator @@ -136,6 +138,9 @@ def uuid_generator(): return str(uuid.uuid4()) +_get_epoch_timestamp = lambda: int(time.time()) + + def close_db_filter(_): if not db.is_closed(): logger.debug('Disconnecting from database.') @@ -175,6 +180,7 @@ class User(BaseModel): invoice_email = BooleanField(default=False) invalid_login_attempts = IntegerField(default=0) last_invalid_login = DateTimeField(default=datetime.utcnow) + removed_tag_expiration_s = IntegerField(default=1209600) # Two weeks def delete_instance(self, recursive=False, delete_nullable=False): # If we are deleting a robot account, only execute the subset of queries necessary. @@ -456,8 +462,8 @@ class RepositoryTag(BaseModel): name = CharField() image = ForeignKeyField(Image) repository = ForeignKeyField(Repository) - lifetime_start = DateTimeField(default=datetime.utcnow) - lifetime_end = DateTimeField(null=True) + lifetime_start_ts = IntegerField(default=_get_epoch_timestamp) + lifetime_end_ts = IntegerField(null=True, index=True) class Meta: database = db diff --git a/data/model/legacy.py b/data/model/legacy.py index 7495f4fcf..b48318b28 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -2,6 +2,7 @@ import bcrypt import logging import dateutil.parser import json +import time from datetime import datetime, timedelta, date @@ -1531,8 +1532,8 @@ def get_repository_images(namespace_name, repository_name): def _tag_alive(query): - return query.where((RepositoryTag.lifetime_end >> None) | - (RepositoryTag.lifetime_end > datetime.utcnow())) + return query.where((RepositoryTag.lifetime_end_ts >> None) | + (RepositoryTag.lifetime_end_ts > int(time.time()))) def list_repository_tags(namespace_name, repository_name): @@ -1547,14 +1548,18 @@ def list_repository_tags(namespace_name, repository_name): def _garbage_collect_tags(namespace_name, repository_name): - with config.app_config['DB_TRANSACTION_FACTORY'](db): - repo = _get_repository(namespace_name, repository_name) - collect_time = (datetime.utcnow() - - timedelta(seconds=config.app_config['TIME_MACHINE_DELTA_SECONDS'])) + to_delete = (RepositoryTag + .select(RepositoryTag.id) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, Namespace.username == namespace_name, + ~(RepositoryTag.lifetime_end_ts >> None), + (RepositoryTag.lifetime_end_ts + Namespace.removed_tag_expiration_s) <= + int(time.time()))) (RepositoryTag .delete() - .where(RepositoryTag.repository == repo, RepositoryTag.lifetime_end < collect_time) + .where(RepositoryTag.id << to_delete) .execute()) @@ -1745,7 +1750,7 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, except Image.DoesNotExist: raise DataModelException('Invalid image with id: %s' % tag_docker_image_id) - now = datetime.utcnow() + now_ts = int(time.time()) try: # When we move a tag, we really end the timeline of the old one and create a new one @@ -1753,13 +1758,14 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, .select() .where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)) tag = query.get() - tag.lifetime_end = now + tag.lifetime_end_ts = now_ts tag.save() except RepositoryTag.DoesNotExist: # No tag that needs to be ended pass - tag = RepositoryTag.create(repository=repo, image=image, name=tag_name, lifetime_start=now) + tag = RepositoryTag.create(repository=repo, image=image, name=tag_name, + lifetime_start_ts=now_ts) return tag @@ -1781,7 +1787,7 @@ def delete_tag(namespace_name, repository_name, tag_name): (tag_name, namespace_name, repository_name)) raise DataModelException(msg) - found.lifetime_end = datetime.utcnow() + found.lifetime_end_ts = int(time.time()) found.save() diff --git a/test/data/test.db b/test/data/test.db index 1e810ea24..79f6011ad 100644 Binary files a/test/data/test.db and b/test/data/test.db differ