diff --git a/data/model/legacy.py b/data/model/legacy.py index 6d3809ccb..7495f4fcf 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -1530,15 +1530,20 @@ def get_repository_images(namespace_name, repository_name): return _get_repository_images_base(namespace_name, repository_name, lambda q: q) +def _tag_alive(query): + return query.where((RepositoryTag.lifetime_end >> None) | + (RepositoryTag.lifetime_end > datetime.utcnow())) + + def list_repository_tags(namespace_name, repository_name): - return (RepositoryTag - .select(RepositoryTag, Image) - .join(Repository) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .switch(RepositoryTag) - .join(Image) - .where(Repository.name == repository_name, Namespace.username == namespace_name, - RepositoryTag.lifetime_end >> None)) + return _tag_alive(RepositoryTag + .select(RepositoryTag, Image) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .switch(RepositoryTag) + .join(Image) + .where(Repository.name == repository_name, + Namespace.username == namespace_name)) def _garbage_collect_tags(namespace_name, repository_name): @@ -1688,10 +1693,10 @@ def _garbage_collect_storage(storage_id_whitelist): def get_tag_image(namespace_name, repository_name, tag_name): def limit_to_tag(query): - return (query - .switch(Image) - .join(RepositoryTag) - .where(RepositoryTag.name == tag_name)) + return _tag_alive(query + .switch(Image) + .join(RepositoryTag) + .where(RepositoryTag.name == tag_name)) images = _get_repository_images_base(namespace_name, repository_name, limit_to_tag) if not images: @@ -1744,8 +1749,10 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, try: # When we move a tag, we really end the timeline of the old one and create a new one - tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name, - RepositoryTag.lifetime_end >> None) + query = _tag_alive(RepositoryTag + .select() + .where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name)) + tag = query.get() tag.lifetime_end = now tag.save() except RepositoryTag.DoesNotExist: @@ -1760,12 +1767,13 @@ def create_or_update_tag(namespace_name, repository_name, tag_name, def delete_tag(namespace_name, repository_name, tag_name): with config.app_config['DB_TRANSACTION_FACTORY'](db): try: - query = (RepositoryTag - .select(RepositoryTag, Repository) - .join(Repository) - .join(Namespace, on=(Repository.namespace_user == Namespace.id)) - .where(Repository.name == repository_name, Namespace.username == namespace_name, - RepositoryTag.name == tag_name, RepositoryTag.lifetime_end >> None)) + query = _tag_alive(RepositoryTag + .select(RepositoryTag, Repository) + .join(Repository) + .join(Namespace, on=(Repository.namespace_user == Namespace.id)) + .where(Repository.name == repository_name, + Namespace.username == namespace_name, + RepositoryTag.name == tag_name)) found = db_for_update(query).get() except RepositoryTag.DoesNotExist: diff --git a/test/test_gc.py b/test/test_gc.py index 5a5444f2a..27c44c61c 100644 --- a/test/test_gc.py +++ b/test/test_gc.py @@ -1,8 +1,5 @@ import unittest -import json as py_json -from flask import url_for -from endpoints.api import api from app import app, storage from initdb import setup_database_for_testing, finished_database_for_testing from data import model, database @@ -14,12 +11,16 @@ REPO = 'somerepo' class TestGarbageColection(unittest.TestCase): def setUp(self): + self._old_tm_expiration = app.config['TIME_MACHINE_DELTA_SECONDS'] + app.config['TIME_MACHINE_DELTA_SECONDS'] = 0 + setup_database_for_testing(self) self.app = app.test_client() self.ctx = app.test_request_context() self.ctx.__enter__() def tearDown(self): + app.config['TIME_MACHINE_DELTA_SECONDS'] = self._old_tm_expiration finished_database_for_testing(self) self.ctx.__exit__(True, None, None) @@ -77,19 +78,21 @@ class TestGarbageColection(unittest.TestCase): model.garbage_collect_repository(repository.namespace_user.username, repository.name) def moveTag(self, repository, tag, docker_image_id): - model.create_or_update_tag(repository.namespace_user.username, repository.name, tag, docker_image_id) + model.create_or_update_tag(repository.namespace_user.username, repository.name, tag, + docker_image_id) model.garbage_collect_repository(repository.namespace_user.username, repository.name) def assertNotDeleted(self, repository, *args): for docker_image_id in args: - self.assertTrue(bool(model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id))) + self.assertTrue(bool(model.get_image_by_id(repository.namespace_user.username, + repository.name, docker_image_id))) def assertDeleted(self, repository, *args): for docker_image_id in args: try: # Verify the image is missing when accessed by the repository. model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id) - except model.DataModelException as ex: + except model.DataModelException: return self.fail('Expected image %s to be deleted' % docker_image_id) @@ -98,13 +101,13 @@ class TestGarbageColection(unittest.TestCase): def test_one_tag(self): """ Create a repository with a single tag, then remove that tag and verify that the repository is now empty. """ - repository = self.createRepository(latest = ['i1', 'i2', 'i3']) + repository = self.createRepository(latest=['i1', 'i2', 'i3']) self.deleteTag(repository, 'latest') self.assertDeleted(repository, 'i1', 'i2', 'i3') def test_two_tags_unshared_images(self): """ Repository has two tags with no shared images between them. """ - repository = self.createRepository(latest = ['i1', 'i2', 'i3'], other = ['f1', 'f2']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['f1', 'f2']) self.deleteTag(repository, 'latest') self.assertDeleted(repository, 'i1', 'i2', 'i3') self.assertNotDeleted(repository, 'f1', 'f2') @@ -113,7 +116,7 @@ class TestGarbageColection(unittest.TestCase): """ Repository has two tags with shared images. Deleting the tag should only remove the unshared images. """ - repository = self.createRepository(latest = ['i1', 'i2', 'i3'], other = ['i1', 'f1']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1']) self.deleteTag(repository, 'latest') self.assertDeleted(repository, 'i2', 'i3') self.assertNotDeleted(repository, 'i1', 'f1') @@ -122,8 +125,8 @@ class TestGarbageColection(unittest.TestCase): """ Two repositories with different images. Removing the tag from one leaves the other's images intact. """ - repository1 = self.createRepository(latest = ['i1', 'i2', 'i3'], name = 'repo1') - repository2 = self.createRepository(latest = ['j1', 'j2', 'j3'], name = 'repo2') + repository1 = self.createRepository(latest=['i1', 'i2', 'i3'], name='repo1') + repository2 = self.createRepository(latest=['j1', 'j2', 'j3'], name='repo2') self.deleteTag(repository1, 'latest') @@ -134,8 +137,8 @@ class TestGarbageColection(unittest.TestCase): """ Two repositories with shared images. Removing the tag from one leaves the other's images intact. """ - repository1 = self.createRepository(latest = ['i1', 'i2', 'i3'], name = 'repo1') - repository2 = self.createRepository(latest = ['i1', 'i2', 'j1'], name = 'repo2') + repository1 = self.createRepository(latest=['i1', 'i2', 'i3'], name='repo1') + repository2 = self.createRepository(latest=['i1', 'i2', 'j1'], name='repo2') self.deleteTag(repository1, 'latest') @@ -146,8 +149,8 @@ class TestGarbageColection(unittest.TestCase): """ Two repositories under different namespaces should result in the images being deleted but not completely removed from the database. """ - repository1 = self.createRepository(namespace = ADMIN_ACCESS_USER, latest = ['i1', 'i2', 'i3']) - repository2 = self.createRepository(namespace = PUBLIC_USER, latest = ['i1', 'i2', 'i3']) + repository1 = self.createRepository(namespace=ADMIN_ACCESS_USER, latest=['i1', 'i2', 'i3']) + repository2 = self.createRepository(namespace=PUBLIC_USER, latest=['i1', 'i2', 'i3']) self.deleteTag(repository1, 'latest') self.assertDeleted(repository1, 'i1', 'i2', 'i3') @@ -158,34 +161,33 @@ class TestGarbageColection(unittest.TestCase): """ Repository has multiple tags with shared images. Selectively deleting the tags, and verifying at each step. """ - repository = self.createRepository( - latest = ['i1', 'i2', 'i3'], - other = ['i1', 'f1', 'f2'], - third = ['t1', 't2', 't3'], - fourth = ['i1', 'f1']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], + other=['i1', 'f1', 'f2'], + third=['t1', 't2', 't3'], + fourth=['i1', 'f1']) # Delete tag other. Should delete f2, since it is not shared. self.deleteTag(repository, 'other') self.assertDeleted(repository, 'f2') - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1') # Move tag fourth to i3. This should remove f1 since it is no longer referenced. self.moveTag(repository, 'fourth', 'i3') self.assertDeleted(repository, 'f1') - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3') # Delete tag 'latest'. This should do nothing since fourth is on the same branch. self.deleteTag(repository, 'latest') - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3') # Delete tag 'third'. This should remove t1->t3. self.deleteTag(repository, 'third') self.assertDeleted(repository, 't1', 't2', 't3') - self.assertNotDeleted(repository,'i1', 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3') # Add tag to i1. self.moveTag(repository, 'newtag', 'i1') - self.assertNotDeleted(repository,'i1', 'i2', 'i3') + self.assertNotDeleted(repository, 'i1', 'i2', 'i3') # Delete tag 'fourth'. This should remove i2 and i3. self.deleteTag(repository, 'fourth') @@ -197,17 +199,11 @@ class TestGarbageColection(unittest.TestCase): self.assertDeleted(repository, 'i1') def test_empty_gc(self): - repository = self.createRepository( - latest = ['i1', 'i2', 'i3'], - other = ['i1', 'f1', 'f2'], - third = ['t1', 't2', 't3'], - fourth = ['i1', 'f1']) + repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'], + third=['t1', 't2', 't3'], fourth=['i1', 'f1']) self.gcNow(repository) - self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2') - - def test_gc_storage_empty(self): - model._garbage_collect_storage(set()) + self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2') if __name__ == '__main__': unittest.main()