From bafab2e73425a028ad1fbab430ee241c01c3a03d Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 28 Aug 2018 17:30:15 -0400 Subject: [PATCH] Change tar stream formatters to be based on tag and manifest, instead of legacy images --- image/appc/__init__.py | 16 ++++++++-------- image/appc/test/test_appc.py | 26 -------------------------- image/common.py | 8 ++++---- image/docker/squashed.py | 26 +++++++++----------------- 4 files changed, 21 insertions(+), 55 deletions(-) diff --git a/image/appc/__init__.py b/image/appc/__init__.py index 91730f923..5b74309c9 100644 --- a/image/appc/__init__.py +++ b/image/appc/__init__.py @@ -18,10 +18,10 @@ class AppCImageFormatter(TarImageFormatter): Image formatter which produces an tarball according to the AppC specification. """ - def stream_generator(self, repo_image, tag, synthetic_image_id, get_image_iterator, + def stream_generator(self, tag, manifest, synthetic_image_id, layer_iterator, tar_stream_getter_iterator, reporter=None): image_mtime = 0 - created = next(get_image_iterator()).v1_metadata.created + created = manifest.created_datetime if created is not None: image_mtime = calendar.timegm(created.utctimetuple()) @@ -31,8 +31,8 @@ class AppCImageFormatter(TarImageFormatter): # Yield the manifest. manifest = json.dumps(DockerV1ToACIManifestTranslator.build_manifest( - repo_image, tag, + manifest, synthetic_image_id )) yield self.tar_file('manifest', manifest, mtime=image_mtime) @@ -170,16 +170,16 @@ class DockerV1ToACIManifestTranslator(object): return volumes @staticmethod - def build_manifest(repo_image, tag, synthetic_image_id): + def build_manifest(tag, manifest, synthetic_image_id): """ Builds an ACI manifest of an existing repository image. """ - docker_layer_data = JSONPathDict(repo_image.compat_metadata) + docker_layer_data = JSONPathDict(manifest.leaf_layer.v1_metadata) config = docker_layer_data['config'] or JSONPathDict({}) - namespace = repo_image.repository.namespace_name - repo_name = repo_image.repository.name + namespace = tag.repository.namespace_name + repo_name = tag.repository.name source_url = "%s://%s/%s/%s:%s" % (app.config['PREFERRED_URL_SCHEME'], app.config['SERVER_HOSTNAME'], - namespace, repo_name, tag) + namespace, repo_name, tag.name) # ACI requires that the execution command be absolutely referenced. Therefore, if we find # a relative command, we give it as an argument to /bin/sh to resolve and execute for us. diff --git a/image/appc/test/test_appc.py b/image/appc/test/test_appc.py index 06f1e8a8d..d078fbe9f 100644 --- a/image/appc/test/test_appc.py +++ b/image/appc/test/test_appc.py @@ -1,7 +1,6 @@ import pytest from image.appc import DockerV1ToACIManifestTranslator -from endpoints.verbs.models_interface import RepositoryReference, ImageWithBlob from util.dict_wrappers import JSONPathDict @@ -74,31 +73,6 @@ EXAMPLE_MANIFEST_OBJ = { "throwaway": True } - -@pytest.fixture -def repo_image(): - repo_ref = RepositoryReference(1, 'simple', 'devtable') - return ImageWithBlob(1, None, EXAMPLE_MANIFEST_OBJ, repo_ref, 1, None) - - -def test_port_conversion(repo_image): - output = DockerV1ToACIManifestTranslator.build_manifest(repo_image, 'v3.0.15', 'abcdef') - ports = output['app']['ports'] - ports.sort() - assert {'name':'port-2379', 'port':2379, 'protocol':'tcp'} == ports[0] - assert {'name':'port-2380', 'port':2380, 'protocol':'tcp'} == ports[1] - - -def test_legacy_port_conversion(repo_image): - del repo_image.compat_metadata['config']['ExposedPorts'] - repo_image.compat_metadata['config']['ports'] = ['8080', '8081'] - output = DockerV1ToACIManifestTranslator.build_manifest(repo_image, 'v3.0.15', 'abcdef') - ports = output['app']['ports'] - ports.sort() - assert {'name':'port-8080', 'port':8080, 'protocol':'tcp'} == ports[0] - assert {'name':'port-8081', 'port':8081, 'protocol':'tcp'} == ports[1] - - @pytest.mark.parametrize("vcfg,expected", [ ({'Volumes': None}, []), ({'Volumes': {}}, []), diff --git a/image/common.py b/image/common.py index 356288316..63b00e676 100644 --- a/image/common.py +++ b/image/common.py @@ -7,17 +7,17 @@ class TarImageFormatter(object): Base class for classes which produce a tar containing image and layer data. """ - def build_stream(self, repo_image, tag, synthetic_image_id, get_image_iterator, + def build_stream(self, tag, manifest, synthetic_image_id, layer_iterator, tar_stream_getter_iterator, reporter=None): """ Builds and streams a synthetic .tar.gz that represents the formatted tar created by this class's implementation. """ - return GzipWrap(self.stream_generator(repo_image, tag, synthetic_image_id, get_image_iterator, + return GzipWrap(self.stream_generator(tag, manifest, synthetic_image_id, layer_iterator, tar_stream_getter_iterator, reporter=reporter)) - def stream_generator(self, repo_image, tag, synthetic_image_id, get_image_iterator, - tar_stream_getter_iterator): + def stream_generator(self, tag, manifest, synthetic_image_id, layer_iterator, + tar_stream_getter_iterator, reporter=None): raise NotImplementedError def tar_file(self, name, contents, mtime=None): diff --git a/image/docker/squashed.py b/image/docker/squashed.py index 5a52c7109..57b680dff 100644 --- a/image/docker/squashed.py +++ b/image/docker/squashed.py @@ -28,10 +28,10 @@ class SquashedDockerImageFormatter(TarImageFormatter): # daemon dies when trying to load the entire tar into memory. SIZE_MULTIPLIER = 1.2 - def stream_generator(self, repo_image, tag, synthetic_image_id, get_image_iterator, + def stream_generator(self, tag, manifest, synthetic_image_id, layer_iterator, tar_stream_getter_iterator, reporter=None): image_mtime = 0 - created = next(get_image_iterator()).v1_metadata.created + created = manifest.created_datetime if created is not None: image_mtime = calendar.timegm(created.utctimetuple()) @@ -50,8 +50,8 @@ class SquashedDockerImageFormatter(TarImageFormatter): hostname = app.config['SERVER_HOSTNAME'] repositories = {} - namespace = repo_image.repository.namespace_name - repository = repo_image.repository.name + namespace = tag.repository.namespace_name + repository = tag.repository.name repositories[hostname + '/' + namespace + '/' + repository] = synthetic_layer_info yield self.tar_file('repositories', json.dumps(repositories), mtime=image_mtime) @@ -60,7 +60,7 @@ class SquashedDockerImageFormatter(TarImageFormatter): yield self.tar_folder(synthetic_image_id, mtime=image_mtime) # Yield the JSON layer data. - layer_json = SquashedDockerImageFormatter._build_layer_json(repo_image, synthetic_image_id) + layer_json = SquashedDockerImageFormatter._build_layer_json(manifest, synthetic_image_id) yield self.tar_file(synthetic_image_id + '/json', json.dumps(layer_json), mtime=image_mtime) # Yield the VERSION file. @@ -68,16 +68,8 @@ class SquashedDockerImageFormatter(TarImageFormatter): # Yield the merged layer data's header. estimated_file_size = 0 - for image in get_image_iterator(): - # In V1 we have the actual uncompressed size, which is needed for back compat with - # older versions of Docker. - # In V2, we use the size given in the image JSON. - if image.blob.uncompressed_size: - estimated_file_size += image.blob.uncompressed_size - else: - image_json = image.compat_metadata - estimated_file_size += (image_json.get('Size', 0) * - SquashedDockerImageFormatter.SIZE_MULTIPLIER) + for layer in layer_iterator: + estimated_file_size += layer.estimated_file_size(SquashedDockerImageFormatter.SIZE_MULTIPLIER) # Make sure the estimated file size is an integer number of bytes. estimated_file_size = int(math.ceil(estimated_file_size)) @@ -115,8 +107,8 @@ class SquashedDockerImageFormatter(TarImageFormatter): @staticmethod - def _build_layer_json(repo_image, synthetic_image_id): - layer_json = repo_image.compat_metadata + def _build_layer_json(manifest, synthetic_image_id): + layer_json = manifest.leaf_layer.v1_metadata updated_json = copy.deepcopy(layer_json) updated_json['id'] = synthetic_image_id