vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Use a consistent concept of tag liveness everywhere. Fix the tests.

Browse source
This commit is contained in:
Jake Moshenko 2015-02-11 15:02:50 -05:00
parent 90c0a9c1e0
commit f32bd748e4
2 changed files with 58 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
data/model/legacy.py
View file

@ -1530,15 +1530,20 @@ def get_repository_images(namespace_name, repository_name):
return _get_repository_images_base(namespace_name, repository_name, lambda q: q) return _get_repository_images_base(namespace_name, repository_name, lambda q: q)
def _tag_alive(query):
return query.where((RepositoryTag.lifetime_end >> None) |
(RepositoryTag.lifetime_end > datetime.utcnow()))
def list_repository_tags(namespace_name, repository_name): def list_repository_tags(namespace_name, repository_name):
return (RepositoryTag return _tag_alive(RepositoryTag
.select(RepositoryTag, Image) .select(RepositoryTag, Image)
.join(Repository) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.switch(RepositoryTag) .switch(RepositoryTag)
.join(Image) .join(Image)
.where(Repository.name == repository_name, Namespace.username == namespace_name, .where(Repository.name == repository_name,
RepositoryTag.lifetime_end >> None)) Namespace.username == namespace_name))
def _garbage_collect_tags(namespace_name, repository_name): def _garbage_collect_tags(namespace_name, repository_name):
@ -1688,7 +1693,7 @@ def _garbage_collect_storage(storage_id_whitelist):
def get_tag_image(namespace_name, repository_name, tag_name): def get_tag_image(namespace_name, repository_name, tag_name):
def limit_to_tag(query): def limit_to_tag(query):
return (query return _tag_alive(query
.switch(Image) .switch(Image)
.join(RepositoryTag) .join(RepositoryTag)
.where(RepositoryTag.name == tag_name)) .where(RepositoryTag.name == tag_name))
@ -1744,8 +1749,10 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
try: try:
# When we move a tag, we really end the timeline of the old one and create a new one # When we move a tag, we really end the timeline of the old one and create a new one
tag = RepositoryTag.get(RepositoryTag.repository == repo, RepositoryTag.name == tag_name, query = _tag_alive(RepositoryTag
RepositoryTag.lifetime_end >> None) .select()
.where(RepositoryTag.repository == repo, RepositoryTag.name == tag_name))
tag = query.get()
tag.lifetime_end = now tag.lifetime_end = now
tag.save() tag.save()
except RepositoryTag.DoesNotExist: except RepositoryTag.DoesNotExist:
@ -1760,12 +1767,13 @@ def create_or_update_tag(namespace_name, repository_name, tag_name,
def delete_tag(namespace_name, repository_name, tag_name): def delete_tag(namespace_name, repository_name, tag_name):
with config.app_config['DB_TRANSACTION_FACTORY'](db): with config.app_config['DB_TRANSACTION_FACTORY'](db):
try: try:
query = (RepositoryTag query = _tag_alive(RepositoryTag
.select(RepositoryTag, Repository) .select(RepositoryTag, Repository)
.join(Repository) .join(Repository)
.join(Namespace, on=(Repository.namespace_user == Namespace.id)) .join(Namespace, on=(Repository.namespace_user == Namespace.id))
.where(Repository.name == repository_name, Namespace.username == namespace_name, .where(Repository.name == repository_name,
RepositoryTag.name == tag_name, RepositoryTag.lifetime_end >> None)) Namespace.username == namespace_name,
RepositoryTag.name == tag_name))
found = db_for_update(query).get() found = db_for_update(query).get()
except RepositoryTag.DoesNotExist: except RepositoryTag.DoesNotExist:

64
test/test_gc.py
View file

@ -1,8 +1,5 @@
import unittest import unittest
import json as py_json
from flask import url_for
from endpoints.api import api
from app import app, storage from app import app, storage
from initdb import setup_database_for_testing, finished_database_for_testing from initdb import setup_database_for_testing, finished_database_for_testing
from data import model, database from data import model, database
@ -14,12 +11,16 @@ REPO = 'somerepo'
class TestGarbageColection(unittest.TestCase): class TestGarbageColection(unittest.TestCase):
def setUp(self): def setUp(self):
self._old_tm_expiration = app.config['TIME_MACHINE_DELTA_SECONDS']
app.config['TIME_MACHINE_DELTA_SECONDS'] = 0
setup_database_for_testing(self) setup_database_for_testing(self)
self.app = app.test_client() self.app = app.test_client()
self.ctx = app.test_request_context() self.ctx = app.test_request_context()
self.ctx.__enter__() self.ctx.__enter__()
def tearDown(self): def tearDown(self):
app.config['TIME_MACHINE_DELTA_SECONDS'] = self._old_tm_expiration
finished_database_for_testing(self) finished_database_for_testing(self)
self.ctx.__exit__(True, None, None) self.ctx.__exit__(True, None, None)
@ -77,19 +78,21 @@ class TestGarbageColection(unittest.TestCase):
model.garbage_collect_repository(repository.namespace_user.username, repository.name) model.garbage_collect_repository(repository.namespace_user.username, repository.name)
def moveTag(self, repository, tag, docker_image_id): def moveTag(self, repository, tag, docker_image_id):
model.create_or_update_tag(repository.namespace_user.username, repository.name, tag, docker_image_id) model.create_or_update_tag(repository.namespace_user.username, repository.name, tag,
docker_image_id)
model.garbage_collect_repository(repository.namespace_user.username, repository.name) model.garbage_collect_repository(repository.namespace_user.username, repository.name)
def assertNotDeleted(self, repository, *args): def assertNotDeleted(self, repository, *args):
for docker_image_id in args: for docker_image_id in args:
self.assertTrue(bool(model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id))) self.assertTrue(bool(model.get_image_by_id(repository.namespace_user.username,
repository.name, docker_image_id)))
def assertDeleted(self, repository, *args): def assertDeleted(self, repository, *args):
for docker_image_id in args: for docker_image_id in args:
try: try:
# Verify the image is missing when accessed by the repository. # Verify the image is missing when accessed by the repository.
model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id) model.get_image_by_id(repository.namespace_user.username, repository.name, docker_image_id)
except model.DataModelException as ex: except model.DataModelException:
return return
self.fail('Expected image %s to be deleted' % docker_image_id) self.fail('Expected image %s to be deleted' % docker_image_id)
@ -98,13 +101,13 @@ class TestGarbageColection(unittest.TestCase):
def test_one_tag(self): def test_one_tag(self):
""" Create a repository with a single tag, then remove that tag and verify that the repository """ Create a repository with a single tag, then remove that tag and verify that the repository
is now empty. """ is now empty. """
repository = self.createRepository(latest = ['i1', 'i2', 'i3']) repository = self.createRepository(latest=['i1', 'i2', 'i3'])
self.deleteTag(repository, 'latest') self.deleteTag(repository, 'latest')
self.assertDeleted(repository, 'i1', 'i2', 'i3') self.assertDeleted(repository, 'i1', 'i2', 'i3')
def test_two_tags_unshared_images(self): def test_two_tags_unshared_images(self):
""" Repository has two tags with no shared images between them. """ """ Repository has two tags with no shared images between them. """
repository = self.createRepository(latest = ['i1', 'i2', 'i3'], other = ['f1', 'f2']) repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['f1', 'f2'])
self.deleteTag(repository, 'latest') self.deleteTag(repository, 'latest')
self.assertDeleted(repository, 'i1', 'i2', 'i3') self.assertDeleted(repository, 'i1', 'i2', 'i3')
self.assertNotDeleted(repository, 'f1', 'f2') self.assertNotDeleted(repository, 'f1', 'f2')
@ -113,7 +116,7 @@ class TestGarbageColection(unittest.TestCase):
""" Repository has two tags with shared images. Deleting the tag should only remove the """ Repository has two tags with shared images. Deleting the tag should only remove the
unshared images. unshared images.
""" """
repository = self.createRepository(latest = ['i1', 'i2', 'i3'], other = ['i1', 'f1']) repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1'])
self.deleteTag(repository, 'latest') self.deleteTag(repository, 'latest')
self.assertDeleted(repository, 'i2', 'i3') self.assertDeleted(repository, 'i2', 'i3')
self.assertNotDeleted(repository, 'i1', 'f1') self.assertNotDeleted(repository, 'i1', 'f1')
@ -122,8 +125,8 @@ class TestGarbageColection(unittest.TestCase):
""" Two repositories with different images. Removing the tag from one leaves the other's """ Two repositories with different images. Removing the tag from one leaves the other's
images intact. images intact.
""" """
repository1 = self.createRepository(latest = ['i1', 'i2', 'i3'], name = 'repo1') repository1 = self.createRepository(latest=['i1', 'i2', 'i3'], name='repo1')
repository2 = self.createRepository(latest = ['j1', 'j2', 'j3'], name = 'repo2') repository2 = self.createRepository(latest=['j1', 'j2', 'j3'], name='repo2')
self.deleteTag(repository1, 'latest') self.deleteTag(repository1, 'latest')
@ -134,8 +137,8 @@ class TestGarbageColection(unittest.TestCase):
""" Two repositories with shared images. Removing the tag from one leaves the other's """ Two repositories with shared images. Removing the tag from one leaves the other's
images intact. images intact.
""" """
repository1 = self.createRepository(latest = ['i1', 'i2', 'i3'], name = 'repo1') repository1 = self.createRepository(latest=['i1', 'i2', 'i3'], name='repo1')
repository2 = self.createRepository(latest = ['i1', 'i2', 'j1'], name = 'repo2') repository2 = self.createRepository(latest=['i1', 'i2', 'j1'], name='repo2')
self.deleteTag(repository1, 'latest') self.deleteTag(repository1, 'latest')
@ -146,8 +149,8 @@ class TestGarbageColection(unittest.TestCase):
""" Two repositories under different namespaces should result in the images being deleted """ Two repositories under different namespaces should result in the images being deleted
but not completely removed from the database. but not completely removed from the database.
""" """
repository1 = self.createRepository(namespace = ADMIN_ACCESS_USER, latest = ['i1', 'i2', 'i3']) repository1 = self.createRepository(namespace=ADMIN_ACCESS_USER, latest=['i1', 'i2', 'i3'])
repository2 = self.createRepository(namespace = PUBLIC_USER, latest = ['i1', 'i2', 'i3']) repository2 = self.createRepository(namespace=PUBLIC_USER, latest=['i1', 'i2', 'i3'])
self.deleteTag(repository1, 'latest') self.deleteTag(repository1, 'latest')
self.assertDeleted(repository1, 'i1', 'i2', 'i3') self.assertDeleted(repository1, 'i1', 'i2', 'i3')
@ -158,34 +161,33 @@ class TestGarbageColection(unittest.TestCase):
""" Repository has multiple tags with shared images. Selectively deleting the tags, and """ Repository has multiple tags with shared images. Selectively deleting the tags, and
verifying at each step. verifying at each step.
""" """
repository = self.createRepository( repository = self.createRepository(latest=['i1', 'i2', 'i3'],
latest = ['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'],
other = ['i1', 'f1', 'f2'], third=['t1', 't2', 't3'],
third = ['t1', 't2', 't3'], fourth=['i1', 'f1'])
fourth = ['i1', 'f1'])
# Delete tag other. Should delete f2, since it is not shared. # Delete tag other. Should delete f2, since it is not shared.
self.deleteTag(repository, 'other') self.deleteTag(repository, 'other')
self.assertDeleted(repository, 'f2') self.assertDeleted(repository, 'f2')
self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1') self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1')
# Move tag fourth to i3. This should remove f1 since it is no longer referenced. # Move tag fourth to i3. This should remove f1 since it is no longer referenced.
self.moveTag(repository, 'fourth', 'i3') self.moveTag(repository, 'fourth', 'i3')
self.assertDeleted(repository, 'f1') self.assertDeleted(repository, 'f1')
self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3') self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3')
# Delete tag 'latest'. This should do nothing since fourth is on the same branch. # Delete tag 'latest'. This should do nothing since fourth is on the same branch.
self.deleteTag(repository, 'latest') self.deleteTag(repository, 'latest')
self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3') self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3')
# Delete tag 'third'. This should remove t1->t3. # Delete tag 'third'. This should remove t1->t3.
self.deleteTag(repository, 'third') self.deleteTag(repository, 'third')
self.assertDeleted(repository, 't1', 't2', 't3') self.assertDeleted(repository, 't1', 't2', 't3')
self.assertNotDeleted(repository,'i1', 'i2', 'i3') self.assertNotDeleted(repository, 'i1', 'i2', 'i3')
# Add tag to i1. # Add tag to i1.
self.moveTag(repository, 'newtag', 'i1') self.moveTag(repository, 'newtag', 'i1')
self.assertNotDeleted(repository,'i1', 'i2', 'i3') self.assertNotDeleted(repository, 'i1', 'i2', 'i3')
# Delete tag 'fourth'. This should remove i2 and i3. # Delete tag 'fourth'. This should remove i2 and i3.
self.deleteTag(repository, 'fourth') self.deleteTag(repository, 'fourth')
@ -197,17 +199,11 @@ class TestGarbageColection(unittest.TestCase):
self.assertDeleted(repository, 'i1') self.assertDeleted(repository, 'i1')
def test_empty_gc(self): def test_empty_gc(self):
repository = self.createRepository( repository = self.createRepository(latest=['i1', 'i2', 'i3'], other=['i1', 'f1', 'f2'],
latest = ['i1', 'i2', 'i3'], third=['t1', 't2', 't3'], fourth=['i1', 'f1'])
other = ['i1', 'f1', 'f2'],
third = ['t1', 't2', 't3'],
fourth = ['i1', 'f1'])
self.gcNow(repository) self.gcNow(repository)
self.assertNotDeleted(repository,'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2') self.assertNotDeleted(repository, 'i1', 'i2', 'i3', 't1', 't2', 't3', 'f1', 'f2')
def test_gc_storage_empty(self):
model._garbage_collect_storage(set())
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()