Merge pull request #1490 from coreos-inc/aci-reproduce

Make ACI generation consistent across calls

formats/aci.py

@@ -1,5 +1,6 @@
 import json
 import re
+import calendar
 
 from uuid import uuid4
 
@@ -17,16 +18,21 @@ class ACIImage(TarImageFormatter):
 
   def stream_generator(self, namespace, repository, tag, synthetic_image_id,
                        layer_json, get_image_iterator, get_layer_iterator, get_image_json):
+    image_mtime = 0
+    created = next(get_image_iterator()).created
+    if created is not None:
+      image_mtime = calendar.timegm(created.utctimetuple())
+
     # ACI Format (.tar):
     #   manifest - The JSON manifest
     #   rootfs - The root file system
 
     # Yield the manifest.
-    yield self.tar_file('manifest', self._build_manifest(namespace, repository, tag, layer_json,
-                                                         synthetic_image_id))
+    manifest = self._build_manifest(namespace, repository, tag, layer_json, synthetic_image_id)
+    yield self.tar_file('manifest', manifest, mtime=image_mtime)
 
     # Yield the merged layer dtaa.
-    yield self.tar_folder('rootfs')
+    yield self.tar_folder('rootfs', mtime=image_mtime)
 
     layer_merger = StreamLayerMerger(get_layer_iterator, path_prefix='rootfs/')
     for entry in layer_merger.get_generator():
@@ -187,7 +193,6 @@ class ACIImage(TarImageFormatter):
 
       env_vars.append(pieces)
 
-
     manifest = {
       "acKind": "ImageManifest",
       "acVersion": "0.6.1",

formats/squashed.py

@@ -6,6 +6,7 @@ from formats.tarimageformatter import TarImageFormatter
 import copy
 import json
 import math
+import calendar
 
 class FileEstimationException(Exception):
   """ Exception raised by build_docker_load_stream if the estimated size of the layer TAR
@@ -28,6 +29,11 @@ class SquashedDockerImage(TarImageFormatter):
 
   def stream_generator(self, namespace, repository, tag, synthetic_image_id,
                        layer_json, get_image_iterator, get_layer_iterator, get_image_json):
+    image_mtime = 0
+    created = next(get_image_iterator()).created
+    if created is not None:
+      image_mtime = calendar.timegm(created.utctimetuple())
+
 
     # Docker import V1 Format (.tar):
     #  repositories - JSON file containing a repo -> tag -> image map
@@ -45,17 +51,17 @@ class SquashedDockerImage(TarImageFormatter):
     repositories = {}
     repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info
 
-    yield self.tar_file('repositories', json.dumps(repositories))
+    yield self.tar_file('repositories', json.dumps(repositories), mtime=image_mtime)
 
     # Yield the image ID folder.
-    yield self.tar_folder(synthetic_image_id)
+    yield self.tar_folder(synthetic_image_id, mtime=image_mtime)
 
     # Yield the JSON layer data.
     layer_json = SquashedDockerImage._build_layer_json(layer_json, synthetic_image_id)
-    yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json))
+    yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json), mtime=image_mtime)
 
     # Yield the VERSION file.
-    yield self.tar_file(synthetic_image_id + '/VERSION', '1.0')
+    yield self.tar_file(synthetic_image_id + '/VERSION', '1.0', mtime=image_mtime)
 
     # Yield the merged layer data's header.
     estimated_file_size = 0
@@ -72,7 +78,8 @@ class SquashedDockerImage(TarImageFormatter):
     # Make sure the estimated file size is an integer number of bytes.
     estimated_file_size = int(math.ceil(estimated_file_size))
 
-    yield self.tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size)
+    yield self.tar_file_header(synthetic_image_id + '/layer.tar', estimated_file_size,
+                               mtime=image_mtime)
 
     # Yield the contents of the merged layer.
     yielded_size = 0

formats/tarimageformatter.py

@@ -18,10 +18,10 @@ class TarImageFormatter(object):
                        layer_json, get_image_iterator, get_layer_iterator, get_image_json):
     raise NotImplementedError
 
-  def tar_file(self, name, contents):
+  def tar_file(self, name, contents, mtime=None):
     """ Returns the TAR binary representation for a file with the given name and file contents. """
     length = len(contents)
-    tar_data = self.tar_file_header(name, length)
+    tar_data = self.tar_file_header(name, length, mtime=mtime)
     tar_data += contents
     tar_data += self.tar_file_padding(length)
     return tar_data
@@ -33,17 +33,24 @@ class TarImageFormatter(object):
 
     return ''
 
-  def tar_file_header(self, name, file_size):
+  def tar_file_header(self, name, file_size, mtime=None):
     """ Returns TAR file header data for a file with the given name and size. """
     info = tarfile.TarInfo(name=name)
     info.type = tarfile.REGTYPE
     info.size = file_size
+
+    if mtime is not None:
+      info.mtime = mtime
     return info.tobuf()
 
-  def tar_folder(self, name):
+  def tar_folder(self, name, mtime=None):
     """ Returns TAR file header data for a folder with the given name. """
     info = tarfile.TarInfo(name=name)
     info.type = tarfile.DIRTYPE
+
+    if mtime is not None:
+      info.mtime = mtime
+
     # allow the directory to be readable by non-root users
     info.mode = 0755
     return info.tobuf()

test/registry_tests.py

@@ -19,7 +19,7 @@ from cryptography.x509 import load_pem_x509_certificate
 from cryptography.hazmat.backends import default_backend
 
 from app import app, storage
-from data.database import close_db_filter, configure
+from data.database import close_db_filter, configure, DerivedStorageForImage
 from data import model
 from endpoints.v1 import v1_bp
 from endpoints.v2 import v2_bp
@@ -83,6 +83,12 @@ def set_feature(feature_name):
   return jsonify({'old_value': old_value})
 
 
+@testbp.route('/clearderivedcache', methods=['POST'])
+def clearderivedcache():
+  DerivedStorageForImage.delete().execute()
+  return 'OK'
+
+
 @testbp.route('/removeuncompressed/<image_id>', methods=['POST'])
 def removeuncompressed(image_id):
   image = model.image.get_image_by_id('devtable', 'newrepo', image_id)
@@ -1449,7 +1455,7 @@ class ACIConversionTests(RegistryTestCaseMixin, V1RegistryPushMixin, LiveServerT
 
   def get_converted_signature(self):
     # Give time for the signature to be written before continuing.
-    time.sleep(1)
+    time.sleep(2)
     response = self.conduct('GET', '/c1/aci/localhost:5000/devtable/newrepo/latest/aci.asc/linux/amd64/', auth='sig')
     return response.content
 
@@ -1485,6 +1491,7 @@ class ACIConversionTests(RegistryTestCaseMixin, V1RegistryPushMixin, LiveServerT
     # Pull the squashed version of the tag.
     tar, converted = self.get_converted_image()
     signature = self.get_converted_signature()
+    first_hash = hashlib.sha256(converted).hexdigest()
 
     # Verify the manifest.
     self.assertEquals(['manifest', 'rootfs', 'rootfs/contents'], tar.getnames())
@@ -1523,6 +1530,24 @@ class ACIConversionTests(RegistryTestCaseMixin, V1RegistryPushMixin, LiveServerT
     # Verify the signature.
     self._verify_signature(signature, converted)
 
+    # Clear the cache and pull again, ensuring that the hash does not change even for a completely
+    # new generation of the image.
+    self.conduct('POST', '/__test/clearderivedcache')
+
+    _, converted_again = self.get_converted_image()
+    second_hash = hashlib.sha256(converted_again).hexdigest()
+    self.assertEquals(second_hash, first_hash)
+
+    # Ensure we have a different signature (and therefore the cache was broken).
+    signature_again = self.get_converted_signature()
+    self.assertNotEquals(signature_again, signature)
+
+    # Ensure *both* signatures work for both images.
+    self._verify_signature(signature, converted_again)
+    self._verify_signature(signature_again, converted)
+    self._verify_signature(signature_again, converted_again)
+
+
   def test_multilayer_conversion(self):
     images = [
       {

util/registry/gzipwrap.py

@@ -7,7 +7,7 @@ class GzipWrap(object):
   def __init__(self, input, filename=None, compresslevel=1):
     self.input = iter(input)
     self.buffer = ''
-    self.zipper = GzipFile(filename, mode='wb', fileobj=self, compresslevel=compresslevel)
+    self.zipper = GzipFile(filename, mode='wb', fileobj=self, compresslevel=compresslevel, mtime=0)
     self.is_done = False
 
   def read(self, size=-1):