diff --git a/formats/aci.py b/formats/aci.py index 421b6bcfa..83ec9f3b9 100644 --- a/formats/aci.py +++ b/formats/aci.py @@ -1,5 +1,6 @@ import json import re +import calendar from uuid import uuid4 @@ -17,16 +18,21 @@ class ACIImage(TarImageFormatter): def stream_generator(self, namespace, repository, tag, synthetic_image_id, layer_json, get_image_iterator, get_layer_iterator, get_image_json): + image_mtime = 0 + created = next(get_image_iterator()).created + if created is not None: + image_mtime = calendar.timegm(created.utctimetuple()) + # ACI Format (.tar): # manifest - The JSON manifest # rootfs - The root file system # Yield the manifest. - yield self.tar_file('manifest', self._build_manifest(namespace, repository, tag, layer_json, - synthetic_image_id)) + manifest = self._build_manifest(namespace, repository, tag, layer_json, synthetic_image_id) + yield self.tar_file('manifest', manifest, mtime=image_mtime) # Yield the merged layer dtaa. - yield self.tar_folder('rootfs') + yield self.tar_folder('rootfs', mtime=image_mtime) layer_merger = StreamLayerMerger(get_layer_iterator, path_prefix='rootfs/') for entry in layer_merger.get_generator(): @@ -187,7 +193,6 @@ class ACIImage(TarImageFormatter): env_vars.append(pieces) - manifest = { "acKind": "ImageManifest", "acVersion": "0.6.1", diff --git a/formats/squashed.py b/formats/squashed.py index 2dc5675f2..ba0964339 100644 --- a/formats/squashed.py +++ b/formats/squashed.py @@ -6,6 +6,7 @@ from formats.tarimageformatter import TarImageFormatter import copy import json import math +import calendar class FileEstimationException(Exception): """ Exception raised by build_docker_load_stream if the estimated size of the layer TAR @@ -28,6 +29,11 @@ class SquashedDockerImage(TarImageFormatter): def stream_generator(self, namespace, repository, tag, synthetic_image_id, layer_json, get_image_iterator, get_layer_iterator, get_image_json): + image_mtime = 0 + created = next(get_image_iterator()).created + if created is not None: + image_mtime = calendar.timegm(created.utctimetuple()) + # Docker import V1 Format (.tar): # repositories - JSON file containing a repo -> tag -> image map @@ -45,17 +51,17 @@ class SquashedDockerImage(TarImageFormatter): repositories = {} repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info - yield self.tar_file('repositories', json.dumps(repositories)) + yield self.tar_file('repositories', json.dumps(repositories), mtime=image_mtime) # Yield the image ID folder. - yield self.tar_folder(synthetic_image_id) + yield self.tar_folder(synthetic_image_id, mtime=image_mtime) # Yield the JSON layer data. layer_json = SquashedDockerImage._build_layer_json(layer_json, synthetic_image_id) - yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json)) + yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json), mtime=image_mtime) # Yield the VERSION file. - yield self.tar_file(synthetic_image_id + '/VERSION', '1.0') + yield self.tar_file(synthetic_image_id + '/VERSION', '1.0', mtime=image_mtime) # Yield the merged layer data's header. estimated_file_size = 0 @@ -72,7 +78,8 @@ class SquashedDockerImage(TarImageFormatter): # Make sure the estimated file size is an integer number of bytes. estimated_file_size = int(math.ceil(estimated_file_size)) - yield self.tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size) + yield self.tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size, + mtime=image_mtime) # Yield the contents of the merged layer. yielded_size = 0 diff --git a/formats/tarimageformatter.py b/formats/tarimageformatter.py index 001c83831..2274af85e 100644 --- a/formats/tarimageformatter.py +++ b/formats/tarimageformatter.py @@ -18,10 +18,10 @@ class TarImageFormatter(object): layer_json, get_image_iterator, get_layer_iterator, get_image_json): raise NotImplementedError - def tar_file(self, name, contents): + def tar_file(self, name, contents, mtime=None): """ Returns the TAR binary representation for a file with the given name and file contents. """ length = len(contents) - tar_data = self.tar_file_header(name, length) + tar_data = self.tar_file_header(name, length, mtime=mtime) tar_data += contents tar_data += self.tar_file_padding(length) return tar_data @@ -33,17 +33,24 @@ class TarImageFormatter(object): return '' - def tar_file_header(self, name, file_size): + def tar_file_header(self, name, file_size, mtime=None): """ Returns TAR file header data for a file with the given name and size. """ info = tarfile.TarInfo(name=name) info.type = tarfile.REGTYPE info.size = file_size + + if mtime is not None: + info.mtime = mtime return info.tobuf() - def tar_folder(self, name): + def tar_folder(self, name, mtime=None): """ Returns TAR file header data for a folder with the given name. """ info = tarfile.TarInfo(name=name) info.type = tarfile.DIRTYPE + + if mtime is not None: + info.mtime = mtime + # allow the directory to be readable by non-root users info.mode = 0755 return info.tobuf() diff --git a/test/registry_tests.py b/test/registry_tests.py index a5af86700..89131c30a 100644 --- a/test/registry_tests.py +++ b/test/registry_tests.py @@ -19,7 +19,7 @@ from cryptography.x509 import load_pem_x509_certificate from cryptography.hazmat.backends import default_backend from app import app, storage -from data.database import close_db_filter, configure +from data.database import close_db_filter, configure, DerivedStorageForImage from data import model from endpoints.v1 import v1_bp from endpoints.v2 import v2_bp @@ -83,6 +83,12 @@ def set_feature(feature_name): return jsonify({'old_value': old_value}) +@testbp.route('/clearderivedcache', methods=['POST']) +def clearderivedcache(): + DerivedStorageForImage.delete().execute() + return 'OK' + + @testbp.route('/removeuncompressed/', methods=['POST']) def removeuncompressed(image_id): image = model.image.get_image_by_id('devtable', 'newrepo', image_id) @@ -1449,7 +1455,7 @@ class ACIConversionTests(RegistryTestCaseMixin, V1RegistryPushMixin, LiveServerT def get_converted_signature(self): # Give time for the signature to be written before continuing. - time.sleep(1) + time.sleep(2) response = self.conduct('GET', '/c1/aci/localhost:5000/devtable/newrepo/latest/aci.asc/linux/amd64/', auth='sig') return response.content @@ -1485,6 +1491,7 @@ class ACIConversionTests(RegistryTestCaseMixin, V1RegistryPushMixin, LiveServerT # Pull the squashed version of the tag. tar, converted = self.get_converted_image() signature = self.get_converted_signature() + first_hash = hashlib.sha256(converted).hexdigest() # Verify the manifest. self.assertEquals(['manifest', 'rootfs', 'rootfs/contents'], tar.getnames()) @@ -1523,6 +1530,24 @@ class ACIConversionTests(RegistryTestCaseMixin, V1RegistryPushMixin, LiveServerT # Verify the signature. self._verify_signature(signature, converted) + # Clear the cache and pull again, ensuring that the hash does not change even for a completely + # new generation of the image. + self.conduct('POST', '/__test/clearderivedcache') + + _, converted_again = self.get_converted_image() + second_hash = hashlib.sha256(converted_again).hexdigest() + self.assertEquals(second_hash, first_hash) + + # Ensure we have a different signature (and therefore the cache was broken). + signature_again = self.get_converted_signature() + self.assertNotEquals(signature_again, signature) + + # Ensure *both* signatures work for both images. + self._verify_signature(signature, converted_again) + self._verify_signature(signature_again, converted) + self._verify_signature(signature_again, converted_again) + + def test_multilayer_conversion(self): images = [ { diff --git a/util/registry/gzipwrap.py b/util/registry/gzipwrap.py index 685e5bb13..cf1c5d423 100644 --- a/util/registry/gzipwrap.py +++ b/util/registry/gzipwrap.py @@ -7,7 +7,7 @@ class GzipWrap(object): def __init__(self, input, filename=None, compresslevel=1): self.input = iter(input) self.buffer = '' - self.zipper = GzipFile(filename, mode='wb', fileobj=self, compresslevel=compresslevel) + self.zipper = GzipFile(filename, mode='wb', fileobj=self, compresslevel=compresslevel, mtime=0) self.is_done = False def read(self, size=-1):