Change tar stream formatters to be based on tag and manifest, instead of legacy images

image/appc/__init__.py

@@ -18,10 +18,10 @@ class AppCImageFormatter(TarImageFormatter):
   Image formatter which produces an tarball according to the AppC specification.
   """
 
-  def stream_generator(self, repo_image, tag, synthetic_image_id, get_image_iterator,
+  def stream_generator(self, tag, manifest, synthetic_image_id, layer_iterator,
                        tar_stream_getter_iterator, reporter=None):
     image_mtime = 0
-    created = next(get_image_iterator()).v1_metadata.created
+    created = manifest.created_datetime
     if created is not None:
       image_mtime = calendar.timegm(created.utctimetuple())
 
@@ -31,8 +31,8 @@ class AppCImageFormatter(TarImageFormatter):
 
     # Yield the manifest.
     manifest = json.dumps(DockerV1ToACIManifestTranslator.build_manifest(
-      repo_image,
       tag,
+      manifest,
       synthetic_image_id
     ))
     yield self.tar_file('manifest', manifest, mtime=image_mtime)
@@ -170,16 +170,16 @@ class DockerV1ToACIManifestTranslator(object):
     return volumes
 
   @staticmethod
-  def build_manifest(repo_image, tag, synthetic_image_id):
+  def build_manifest(tag, manifest, synthetic_image_id):
     """ Builds an ACI manifest of an existing repository image. """
-    docker_layer_data = JSONPathDict(repo_image.compat_metadata)
+    docker_layer_data = JSONPathDict(manifest.leaf_layer.v1_metadata)
     config = docker_layer_data['config'] or JSONPathDict({})
 
-    namespace = repo_image.repository.namespace_name
+    namespace = tag.repository.namespace_name
-    repo_name = repo_image.repository.name
+    repo_name = tag.repository.name
     source_url = "%s://%s/%s/%s:%s" % (app.config['PREFERRED_URL_SCHEME'],
                                        app.config['SERVER_HOSTNAME'],
-                                       namespace, repo_name, tag)
+                                       namespace, repo_name, tag.name)
 
     # ACI requires that the execution command be absolutely referenced. Therefore, if we find
     # a relative command, we give it as an argument to /bin/sh to resolve and execute for us.

image/appc/test/test_appc.py

@@ -1,7 +1,6 @@
 import pytest
 
 from image.appc import DockerV1ToACIManifestTranslator
-from endpoints.verbs.models_interface import RepositoryReference, ImageWithBlob
 from util.dict_wrappers import JSONPathDict
 
 
@@ -74,31 +73,6 @@ EXAMPLE_MANIFEST_OBJ = {
   "throwaway": True
 }
 
-
-@pytest.fixture
-def repo_image():
-  repo_ref = RepositoryReference(1, 'simple', 'devtable')
-  return ImageWithBlob(1, None, EXAMPLE_MANIFEST_OBJ, repo_ref, 1, None)
-
-
-def test_port_conversion(repo_image):
-  output = DockerV1ToACIManifestTranslator.build_manifest(repo_image, 'v3.0.15', 'abcdef')
-  ports = output['app']['ports']
-  ports.sort()
-  assert {'name':'port-2379', 'port':2379, 'protocol':'tcp'} == ports[0]
-  assert {'name':'port-2380', 'port':2380, 'protocol':'tcp'} == ports[1]
-
-
-def test_legacy_port_conversion(repo_image):
-  del repo_image.compat_metadata['config']['ExposedPorts']
-  repo_image.compat_metadata['config']['ports'] = ['8080', '8081']
-  output = DockerV1ToACIManifestTranslator.build_manifest(repo_image, 'v3.0.15', 'abcdef')
-  ports = output['app']['ports']
-  ports.sort()
-  assert {'name':'port-8080', 'port':8080, 'protocol':'tcp'} == ports[0]
-  assert {'name':'port-8081', 'port':8081, 'protocol':'tcp'} == ports[1]
-
-
 @pytest.mark.parametrize("vcfg,expected", [
   ({'Volumes': None}, []),
   ({'Volumes': {}}, []),

image/common.py

@@ -7,17 +7,17 @@ class TarImageFormatter(object):
   Base class for classes which produce a tar containing image and layer data.
   """
 
-  def build_stream(self, repo_image, tag, synthetic_image_id, get_image_iterator,
+  def build_stream(self, tag, manifest, synthetic_image_id, layer_iterator,
                    tar_stream_getter_iterator, reporter=None):
     """
     Builds and streams a synthetic .tar.gz that represents the formatted tar created by this class's
     implementation.
     """
-    return GzipWrap(self.stream_generator(repo_image, tag, synthetic_image_id, get_image_iterator,
+    return GzipWrap(self.stream_generator(tag, manifest, synthetic_image_id, layer_iterator,
                                           tar_stream_getter_iterator, reporter=reporter))
 
-  def stream_generator(self, repo_image, tag, synthetic_image_id, get_image_iterator,
+  def stream_generator(self, tag, manifest, synthetic_image_id, layer_iterator,
-                       tar_stream_getter_iterator):
+                       tar_stream_getter_iterator, reporter=None):
     raise NotImplementedError
 
   def tar_file(self, name, contents, mtime=None):

image/docker/squashed.py

@@ -28,10 +28,10 @@ class SquashedDockerImageFormatter(TarImageFormatter):
   # daemon dies when trying to load the entire tar into memory.
   SIZE_MULTIPLIER = 1.2
 
-  def stream_generator(self, repo_image, tag, synthetic_image_id, get_image_iterator,
+  def stream_generator(self, tag, manifest, synthetic_image_id, layer_iterator,
                        tar_stream_getter_iterator, reporter=None):
     image_mtime = 0
-    created = next(get_image_iterator()).v1_metadata.created
+    created = manifest.created_datetime
     if created is not None:
       image_mtime = calendar.timegm(created.utctimetuple())
 
@@ -50,8 +50,8 @@ class SquashedDockerImageFormatter(TarImageFormatter):
 
     hostname = app.config['SERVER_HOSTNAME']
     repositories = {}
-    namespace = repo_image.repository.namespace_name
+    namespace = tag.repository.namespace_name
-    repository = repo_image.repository.name
+    repository = tag.repository.name
     repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info
 
     yield self.tar_file('repositories', json.dumps(repositories), mtime=image_mtime)
@@ -60,7 +60,7 @@ class SquashedDockerImageFormatter(TarImageFormatter):
     yield self.tar_folder(synthetic_image_id, mtime=image_mtime)
 
     # Yield the JSON layer data.
-    layer_json = SquashedDockerImageFormatter._build_layer_json(repo_image, synthetic_image_id)
+    layer_json = SquashedDockerImageFormatter._build_layer_json(manifest, synthetic_image_id)
     yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json), mtime=image_mtime)
 
     # Yield the VERSION file.
@@ -68,16 +68,8 @@ class SquashedDockerImageFormatter(TarImageFormatter):
 
     # Yield the merged layer data's header.
     estimated_file_size = 0
-    for image in get_image_iterator():
+    for layer in layer_iterator:
-      # In V1 we have the actual uncompressed size, which is needed for back compat with
+      estimated_file_size += layer.estimated_file_size(SquashedDockerImageFormatter.SIZE_MULTIPLIER)
-      # older versions of Docker.
-      # In V2, we use the size given in the image JSON.
-      if image.blob.uncompressed_size:
-        estimated_file_size += image.blob.uncompressed_size
-      else:
-        image_json = image.compat_metadata
-        estimated_file_size += (image_json.get('Size', 0) *
-                                SquashedDockerImageFormatter.SIZE_MULTIPLIER)
 
     # Make sure the estimated file size is an integer number of bytes.
     estimated_file_size = int(math.ceil(estimated_file_size))
@@ -115,8 +107,8 @@ class SquashedDockerImageFormatter(TarImageFormatter):
 
 
   @staticmethod
-  def _build_layer_json(repo_image, synthetic_image_id):
+  def _build_layer_json(manifest, synthetic_image_id):
-    layer_json = repo_image.compat_metadata
+    layer_json = manifest.leaf_layer.v1_metadata
     updated_json = copy.deepcopy(layer_json)
     updated_json['id'] = synthetic_image_id