Retarget hard links pointing to deleted files by emitting the deleted file contents under the first hard link instance. This fixes a breakage in the squashed TAR where we were pointing hard links to missing data.
Fixes https://jira.coreos.com/browse/QUAY-885
This commit is contained in:
Joseph Schorr
parent
041a7fcd36
commit
110366f656
8 changed files with 337 additions and 252 deletions
|
|
@ -2,17 +2,16 @@ import unittest
|
|||
import tarfile
|
||||
|
||||
from StringIO import StringIO
|
||||
from util.registry.streamlayerformat import StreamLayerMerger, AUFS_WHITEOUT
|
||||
from util.registry.streamlayerformat import StreamLayerMerger
|
||||
from util.registry.aufs import AUFS_WHITEOUT
|
||||
from util.registry.tarlayerformat import TarLayerReadException
|
||||
|
||||
class TestStreamLayerMerger(unittest.TestCase):
|
||||
def create_layer(self, **kwargs):
|
||||
def create_layer(self, *file_pairs):
|
||||
output = StringIO()
|
||||
with tarfile.open(fileobj=output, mode='w:gz') as tar:
|
||||
for current_contents in kwargs:
|
||||
current_filename = kwargs[current_contents]
|
||||
|
||||
if current_contents == '_':
|
||||
for current_filename, current_contents in file_pairs:
|
||||
if current_contents is None:
|
||||
# This is a deleted file.
|
||||
if current_filename.endswith('/'):
|
||||
current_filename = current_filename[:-1]
|
||||
|
|
@ -25,9 +24,15 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
current_contents = ''
|
||||
|
||||
info = tarfile.TarInfo(name=current_filename)
|
||||
info.size = len(current_contents)
|
||||
tar.addfile(info, fileobj=StringIO(current_contents))
|
||||
if current_contents.startswith('linkto:'):
|
||||
info = tarfile.TarInfo(name=current_filename)
|
||||
info.linkname = current_contents[len('linkto:'):]
|
||||
info.type = tarfile.LNKTYPE
|
||||
tar.addfile(info)
|
||||
else:
|
||||
info = tarfile.TarInfo(name=current_filename)
|
||||
info.size = len(current_contents)
|
||||
tar.addfile(info, fileobj=StringIO(current_contents))
|
||||
|
||||
return output.getvalue()
|
||||
|
||||
|
|
@ -35,10 +40,13 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
return ''
|
||||
|
||||
def squash_layers(self, layers, path_prefix=None):
|
||||
def get_layers():
|
||||
return [StringIO(layer) for layer in layers]
|
||||
def getter_for_layer(layer):
|
||||
return lambda: StringIO(layer)
|
||||
|
||||
merger = StreamLayerMerger(get_layers, path_prefix=path_prefix)
|
||||
def layer_stream_getter():
|
||||
return [getter_for_layer(layer) for layer in layers]
|
||||
|
||||
merger = StreamLayerMerger(layer_stream_getter, path_prefix=path_prefix)
|
||||
merged_data = ''.join(merger.get_generator())
|
||||
return merged_data
|
||||
|
||||
|
|
@ -58,9 +66,9 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_single_layer(self):
|
||||
tar_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
squashed = self.squash_layers([tar_layer])
|
||||
|
||||
|
|
@ -70,12 +78,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_multiple_layers(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'top_file')
|
||||
('top_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -86,12 +94,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_multiple_layers_dot(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = './some_file',
|
||||
bar = 'another_file',
|
||||
meh = './third_file')
|
||||
('./some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('./third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'top_file')
|
||||
('top_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -102,12 +110,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_multiple_layers_overwrite(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'another_file')
|
||||
('another_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -117,12 +125,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_multiple_layers_overwrite_base_dot(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = './another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('./another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'another_file')
|
||||
('another_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -133,12 +141,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_multiple_layers_overwrite_top_dot(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = './another_file')
|
||||
('./another_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -149,12 +157,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_deleted_file(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
_ = 'another_file')
|
||||
('another_file', None))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -164,15 +172,15 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_deleted_readded_file(self):
|
||||
third_layer = self.create_layer(
|
||||
bar = 'another_file')
|
||||
('another_file', 'bar'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
_ = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', None),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
newagain = 'another_file')
|
||||
('another_file', 'newagain'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer, third_layer])
|
||||
|
||||
|
|
@ -182,15 +190,15 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_deleted_in_lower_layer(self):
|
||||
third_layer = self.create_layer(
|
||||
bar = 'deleted_file')
|
||||
('deleted_file', 'bar'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
_ = 'deleted_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('deleted_file', None),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'top_file')
|
||||
('top_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer, third_layer])
|
||||
|
||||
|
|
@ -201,31 +209,31 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_deleted_in_lower_layer_with_added_dot(self):
|
||||
third_layer = self.create_layer(
|
||||
something = './deleted_file')
|
||||
('./deleted_file', 'something'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
_ = 'deleted_file')
|
||||
('deleted_file', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, third_layer])
|
||||
self.assertDoesNotHaveFile(squashed, 'deleted_file')
|
||||
|
||||
def test_deleted_in_lower_layer_with_deleted_dot(self):
|
||||
third_layer = self.create_layer(
|
||||
something = './deleted_file')
|
||||
('./deleted_file', 'something'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
_ = './deleted_file')
|
||||
('./deleted_file', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, third_layer])
|
||||
self.assertDoesNotHaveFile(squashed, 'deleted_file')
|
||||
|
||||
def test_directory(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'foo/some_file',
|
||||
bar = 'foo/another_file')
|
||||
('foo/some_file', 'foo'),
|
||||
('foo/another_file', 'bar'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'foo/some_file')
|
||||
('foo/some_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -234,11 +242,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_sub_directory(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'foo/some_file',
|
||||
bar = 'foo/bar/another_file')
|
||||
('foo/some_file', 'foo'),
|
||||
('foo/bar/another_file', 'bar'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'foo/some_file')
|
||||
('foo/some_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -247,11 +255,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_directory(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'foo/some_file',
|
||||
bar = 'foo/another_file')
|
||||
('foo/some_file', 'foo'),
|
||||
('foo/another_file', 'bar'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
_ = 'foo/')
|
||||
('foo/', None))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -260,11 +268,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_sub_directory(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'foo/some_file',
|
||||
bar = 'foo/bar/another_file')
|
||||
('foo/some_file', 'foo'),
|
||||
('foo/bar/another_file', 'bar'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
_ = 'foo/bar/')
|
||||
('foo/bar/', None))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -273,11 +281,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_sub_directory_with_dot(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'foo/some_file',
|
||||
bar = 'foo/bar/another_file')
|
||||
('foo/some_file', 'foo'),
|
||||
('foo/bar/another_file', 'bar'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
_ = './foo/bar/')
|
||||
('./foo/bar/', None))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -286,11 +294,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_sub_directory_with_subdot(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = './foo/some_file',
|
||||
bar = './foo/bar/another_file')
|
||||
('./foo/some_file', 'foo'),
|
||||
('./foo/bar/another_file', 'bar'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
_ = 'foo/bar/')
|
||||
('foo/bar/', None))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer])
|
||||
|
||||
|
|
@ -300,14 +308,14 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_directory_recreate(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'foo/some_file',
|
||||
bar = 'foo/another_file')
|
||||
('foo/some_file', 'foo'),
|
||||
('foo/another_file', 'bar'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
_ = 'foo/')
|
||||
('foo/', None))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
baz = 'foo/some_file')
|
||||
('foo/some_file', 'baz'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer, third_layer])
|
||||
|
||||
|
|
@ -316,11 +324,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_directory_prefix(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'foobar/some_file',
|
||||
bar = 'foo/another_file')
|
||||
('foobar/some_file', 'foo'),
|
||||
('foo/another_file', 'bar'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
_ = 'foo/')
|
||||
('foo/', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, third_layer])
|
||||
|
||||
|
|
@ -330,11 +338,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_directory_pre_prefix(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'foobar/baz/some_file',
|
||||
bar = 'foo/another_file')
|
||||
('foobar/baz/some_file', 'foo'),
|
||||
('foo/another_file', 'bar'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
_ = 'foo/')
|
||||
('foo/', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, third_layer])
|
||||
|
||||
|
|
@ -344,11 +352,11 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_delete_root_directory(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'build/first_file',
|
||||
bar = 'build/second_file')
|
||||
('build/first_file', 'foo'),
|
||||
('build/second_file', 'bar'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
_ = 'build')
|
||||
('build', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, third_layer])
|
||||
|
||||
|
|
@ -358,8 +366,8 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_tar_empty_layer(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'build/first_file',
|
||||
bar = 'build/second_file')
|
||||
('build/first_file', 'foo'),
|
||||
('build/second_file', 'bar'))
|
||||
|
||||
empty_layer = self.create_layer()
|
||||
|
||||
|
|
@ -371,8 +379,8 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_data_empty_layer(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'build/first_file',
|
||||
bar = 'build/second_file')
|
||||
('build/first_file', 'foo'),
|
||||
('build/second_file', 'bar'))
|
||||
|
||||
empty_layer = self.create_empty_layer()
|
||||
|
||||
|
|
@ -384,8 +392,8 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_broken_layer(self):
|
||||
third_layer = self.create_layer(
|
||||
foo = 'build/first_file',
|
||||
bar = 'build/second_file')
|
||||
('build/first_file', 'foo'),
|
||||
('build/second_file', 'bar'))
|
||||
|
||||
broken_layer = 'not valid data'
|
||||
|
||||
|
|
@ -397,9 +405,9 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_single_layer_with_prefix(self):
|
||||
tar_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
squashed = self.squash_layers([tar_layer], path_prefix='foo/')
|
||||
|
||||
|
|
@ -409,12 +417,12 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_multiple_layers_overwrite_with_prefix(self):
|
||||
second_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
first_layer = self.create_layer(
|
||||
top = 'another_file')
|
||||
('another_file', 'top'))
|
||||
|
||||
squashed = self.squash_layers([first_layer, second_layer], path_prefix='foo/')
|
||||
|
||||
|
|
@ -425,7 +433,7 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_superlong_filename(self):
|
||||
tar_layer = self.create_layer(
|
||||
meh = 'this_is_the_filename_that_never_ends_it_goes_on_and_on_my_friend_some_people_started')
|
||||
('this_is_the_filename_that_never_ends_it_goes_on_and_on_my_friend_some_people_started', 'meh'))
|
||||
|
||||
squashed = self.squash_layers([tar_layer],
|
||||
path_prefix='foo/')
|
||||
|
|
@ -435,9 +443,9 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
|
||||
def test_superlong_prefix(self):
|
||||
tar_layer = self.create_layer(
|
||||
foo = 'some_file',
|
||||
bar = 'another_file',
|
||||
meh = 'third_file')
|
||||
('some_file', 'foo'),
|
||||
('another_file', 'bar'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
squashed = self.squash_layers([tar_layer],
|
||||
path_prefix='foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/')
|
||||
|
|
@ -447,5 +455,40 @@ class TestStreamLayerMerger(unittest.TestCase):
|
|||
self.assertHasFile(squashed, 'foo/bar/baz/something/foo/bar/baz/anotherthing/whatever/this/is/a/really/long/filename/that/goes/here/third_file', 'meh')
|
||||
|
||||
|
||||
def test_hardlink_to_deleted_file(self):
|
||||
first_layer = self.create_layer(
|
||||
('tobedeletedfile', 'somecontents'),
|
||||
('link_to_deleted_file', 'linkto:tobedeletedfile'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
('tobedeletedfile', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, first_layer], path_prefix='foo/')
|
||||
|
||||
self.assertHasFile(squashed, 'foo/third_file', 'meh')
|
||||
self.assertHasFile(squashed, 'foo/link_to_deleted_file', 'somecontents')
|
||||
self.assertDoesNotHaveFile(squashed, 'foo/tobedeletedfile')
|
||||
|
||||
|
||||
def test_multiple_hardlink_to_deleted_file(self):
|
||||
first_layer = self.create_layer(
|
||||
('tobedeletedfile', 'somecontents'),
|
||||
('link_to_deleted_file', 'linkto:tobedeletedfile'),
|
||||
('another_link_to_deleted_file', 'linkto:tobedeletedfile'),
|
||||
('third_file', 'meh'))
|
||||
|
||||
second_layer = self.create_layer(
|
||||
('tobedeletedfile', None))
|
||||
|
||||
squashed = self.squash_layers([second_layer, first_layer], path_prefix='foo/')
|
||||
|
||||
self.assertHasFile(squashed, 'foo/third_file', 'meh')
|
||||
self.assertHasFile(squashed, 'foo/link_to_deleted_file', 'somecontents')
|
||||
self.assertHasFile(squashed, 'foo/another_link_to_deleted_file', 'somecontents')
|
||||
|
||||
self.assertDoesNotHaveFile(squashed, 'foo/tobedeletedfile')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Reference in a new issue