From fa58f3b1d24a5a71fc5d6a0f29efa0cd63a1deb3 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 15 Aug 2018 11:41:15 -0400 Subject: [PATCH] Fix handling of manifests with unicode in the backfill Also adds a bunch of tests around manifests to ensure we get the same information in and out --- image/docker/schema1.py | 10 ++- image/docker/test/manifest_unicode_row.json | 1 + image/docker/test/test_schema1.py | 33 +++++++++- image/docker/test/validated_manifest.json | 62 +++++++++++++++++++ .../test/validated_manifest_with_unicode.json | 50 +++++++++++++++ test/registry/protocol_v2.py | 3 +- test/registry/protocols.py | 1 + test/registry/registry_tests.py | 19 ++++++ workers/manifestbackfillworker.py | 4 +- 9 files changed, 176 insertions(+), 7 deletions(-) create mode 100644 image/docker/test/manifest_unicode_row.json create mode 100644 image/docker/test/validated_manifest.json create mode 100644 image/docker/test/validated_manifest_with_unicode.json diff --git a/image/docker/schema1.py b/image/docker/schema1.py index 47134e178..b11d6f2ac 100644 --- a/image/docker/schema1.py +++ b/image/docker/schema1.py @@ -183,10 +183,14 @@ class DockerSchema1Manifest(ManifestInterface): if validate: self._validate() + @classmethod + def for_latin1_bytes(cls, encoded_bytes, validate=True): + return DockerSchema1Manifest(encoded_bytes.encode('utf-8'), validate) + def _validate(self): for signature in self._signatures: bytes_to_verify = '{0}.{1}'.format(signature['protected'], - base64url_encode(self.payload)) + base64url_encode(self._payload)) signer = SIGNER_ALGS[signature['header']['alg']] key = keyrep(signature['header']['jwk']) gk = key.get_key() @@ -241,7 +245,7 @@ class DockerSchema1Manifest(ManifestInterface): @property def digest(self): - return digest_tools.sha256_digest(self.payload) + return digest_tools.sha256_digest(self._payload) @property def image_ids(self): @@ -304,7 +308,7 @@ class DockerSchema1Manifest(ManifestInterface): yield Schema1Layer(image_digest, extracted, metadata_string) @property - def payload(self): + def _payload(self): protected = str(self._signatures[0][DOCKER_SCHEMA1_PROTECTED_KEY]) parsed_protected = json.loads(base64url_decode(protected)) signed_content_head = self._bytes[:parsed_protected[DOCKER_SCHEMA1_FORMAT_LENGTH_KEY]] diff --git a/image/docker/test/manifest_unicode_row.json b/image/docker/test/manifest_unicode_row.json new file mode 100644 index 000000000..9d6b663c0 --- /dev/null +++ b/image/docker/test/manifest_unicode_row.json @@ -0,0 +1 @@ +[{"id":"13080314","tag_id":"93362429","digest":"sha256:dde3714ce7e23edc6413aa85c0b42792e4f2f79e9ea36afc154d63ff3d04e86c","json_data":"{\n \"schemaVersion\": 1,\n \"name\": \"josephschorr\/buildtest2\",\n \"tag\": \"unicode\",\n \"architecture\": \"amd64\",\n \"fsLayers\": [\n {\n \"blobSum\": \"sha256:9dcda8e13dc6f3aa30ce7867d8a9e3941dc3a54cfefb5e76cbdfa90d2b56ed2f\"\n },\n {\n \"blobSum\": \"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\"\n },\n {\n \"blobSum\": \"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\"\n },\n {\n \"blobSum\": \"sha256:8c5a7da1afbc602695fcb2cd6445743cec5ff32053ea589ea9bd8773b7068185\"\n }\n ],\n \"history\": [\n {\n \"v1Compatibility\": \"{\\\"architecture\\\":\\\"amd64\\\",\\\"config\\\":{\\\"Hostname\\\":\\\"\\\",\\\"Domainname\\\":\\\"\\\",\\\"User\\\":\\\"\\\",\\\"AttachStdin\\\":false,\\\"AttachStdout\\\":false,\\\"AttachStderr\\\":false,\\\"Tty\\\":false,\\\"OpenStdin\\\":false,\\\"StdinOnce\\\":false,\\\"Env\\\":[\\\"PATH=\/usr\/local\/sbin:\/usr\/local\/bin:\/usr\/sbin:\/usr\/bin:\/sbin:\/bin\\\"],\\\"Cmd\\\":[\\\"sh\\\"],\\\"ArgsEscaped\\\":true,\\\"Image\\\":\\\"sha256:746d49e88c1eac6e3d3384d73db788f166a51b5a2eb9da49671586f62baf6c0c\\\",\\\"Volumes\\\":null,\\\"WorkingDir\\\":\\\"\\\",\\\"Entrypoint\\\":null,\\\"OnBuild\\\":[],\\\"Labels\\\":{\\\"maintainer\\\":\\\"Ge\u00e9 L\u00e9fleur\\\"}},\\\"container\\\":\\\"654ee2461cf64a54484624d8b7efbb76c5e197ba6f3322538b6810dad097c11f\\\",\\\"container_config\\\":{\\\"Hostname\\\":\\\"\\\",\\\"Domainname\\\":\\\"\\\",\\\"User\\\":\\\"\\\",\\\"AttachStdin\\\":false,\\\"AttachStdout\\\":false,\\\"AttachStderr\\\":false,\\\"Tty\\\":false,\\\"OpenStdin\\\":false,\\\"StdinOnce\\\":false,\\\"Env\\\":[\\\"PATH=\/usr\/local\/sbin:\/usr\/local\/bin:\/usr\/sbin:\/usr\/bin:\/sbin:\/bin\\\"],\\\"Cmd\\\":[\\\"\/bin\/sh\\\",\\\"-c\\\",\\\"echo foo \\\\u003e bar\\\"],\\\"ArgsEscaped\\\":true,\\\"Image\\\":\\\"sha256:746d49e88c1eac6e3d3384d73db788f166a51b5a2eb9da49671586f62baf6c0c\\\",\\\"Volumes\\\":null,\\\"WorkingDir\\\":\\\"\\\",\\\"Entrypoint\\\":null,\\\"OnBuild\\\":[],\\\"Labels\\\":{\\\"maintainer\\\":\\\"Ge\u00e9 L\u00e9fleur\\\"}},\\\"created\\\":\\\"2018-08-14T22:17:55.7294283Z\\\",\\\"docker_version\\\":\\\"17.09.0-ce\\\",\\\"id\\\":\\\"db077d203993a3a1cfeaf4bbaedb34ff1a706452cb598c62d2873ba78dd0d2fe\\\",\\\"os\\\":\\\"linux\\\",\\\"parent\\\":\\\"539016dae3ce29f825af4d27a60b8d42306a86727f7406371682612124bc6db3\\\"}\"\n },\n {\n \"v1Compatibility\": \"{\\\"id\\\":\\\"539016dae3ce29f825af4d27a60b8d42306a86727f7406371682612124bc6db3\\\",\\\"parent\\\":\\\"5a1738daa8064e42d79a0b1f3d1b75ca4406c6695969860ff8e814999bda9470\\\",\\\"created\\\":\\\"2018-08-14T22:17:54.5902216Z\\\",\\\"container_config\\\":{\\\"Cmd\\\":[\\\"\/bin\/sh -c #(nop) LABEL maintainer=Ge\u00e9 L\u00e9fleur\\\"]},\\\"throwaway\\\":true}\"\n },\n {\n \"v1Compatibility\": \"{\\\"id\\\":\\\"5a1738daa8064e42d79a0b1f3d1b75ca4406c6695969860ff8e814999bda9470\\\",\\\"parent\\\":\\\"97d7c933c31fa951536cacfdfe3f862ce589020fa58bdf2fccc66204191a4273\\\",\\\"created\\\":\\\"2018-07-31T22:20:07.617575594Z\\\",\\\"container_config\\\":{\\\"Cmd\\\":[\\\"\/bin\/sh -c #(nop) CMD [\\\\\\\"sh\\\\\\\"]\\\"]},\\\"throwaway\\\":true}\"\n },\n {\n \"v1Compatibility\": \"{\\\"id\\\":\\\"97d7c933c31fa951536cacfdfe3f862ce589020fa58bdf2fccc66204191a4273\\\",\\\"created\\\":\\\"2018-07-31T22:20:07.361628468Z\\\",\\\"container_config\\\":{\\\"Cmd\\\":[\\\"\/bin\/sh -c #(nop) ADD file:96fda64a6b725d4df5249c12e32245e2f02469ff637c38077740f4984cd883dd in \/ \\\"]}}\"\n }\n ],\n \"signatures\": [\n {\n \"header\": {\n \"jwk\": {\n \"crv\": \"P-256\",\n \"kid\": \"AARA:PFUD:3V54:7F2S:2P7E:WMCU:WRE7:KUYD:CFKH:UHZ7:AZ4I:UQEX\",\n \"kty\": \"EC\",\n \"x\": \"34N4h_uM7FedPw4k3_VabKlt7qoBWpHgpko7zE0RkeY\",\n \"y\": \"LhxxtCYh_b1EwUbl3-tQFTbg1mTu34vMxj4UaKjWZk8\"\n },\n \"alg\": \"ES256\"\n },\n \"signature\": \"XvA_yxSPZwnln-pl_VyT5HgfC_NRnVj2IDZjnPy4NRm99Ik82jjliZmoNL4g54AGe3CUD4i6eJiDdCgSCqjxQw\",\n \"protected\": \"eyJmb3JtYXRMZW5ndGgiOjMwODAsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxOC0wOC0xNFQyMjoyNTo0M1oifQ\"\n }\n ]\n}"}] diff --git a/image/docker/test/test_schema1.py b/image/docker/test/test_schema1.py index 96ae2d12b..e584ec9ed 100644 --- a/image/docker/test/test_schema1.py +++ b/image/docker/test/test_schema1.py @@ -1,4 +1,7 @@ +import os +import hashlib import json + import pytest from image.docker.schema1 import MalformedSchema1Manifest, DockerSchema1Manifest @@ -76,4 +79,32 @@ def test_valid_manifest(): assert manifest.leaf_layer == manifest.layers[1] - assert manifest.digest + +def test_validate_manifest(): + test_dir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(test_dir, 'validated_manifest.json'), 'r') as f: + manifest_bytes = f.read() + + manifest = DockerSchema1Manifest(manifest_bytes, validate=True) + digest = manifest.digest + assert digest == 'sha256:b5dc4f63fdbd64f34f2314c0747ef81008f9fcddce4edfc3fd0e8ec8b358d571' + + +def test_validate_manifest_with_unicode(): + test_dir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(test_dir, 'validated_manifest_with_unicode.json'), 'r') as f: + manifest_bytes = f.read() + + manifest = DockerSchema1Manifest(manifest_bytes, validate=True) + digest = manifest.digest + assert digest == 'sha256:815ecf45716a96b19d54d911e6ace91f78bab26ca0dd299645d9995dacd9f1ef' + + +def test_validate_manifest_with_unicode_encoded(): + test_dir = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(test_dir, 'manifest_unicode_row.json'), 'r') as f: + manifest_bytes = json.loads(f.read())[0]['json_data'] + + manifest = DockerSchema1Manifest.for_latin1_bytes(manifest_bytes, validate=True) + digest = manifest.digest + assert digest == 'sha256:dde3714ce7e23edc6413aa85c0b42792e4f2f79e9ea36afc154d63ff3d04e86c' diff --git a/image/docker/test/validated_manifest.json b/image/docker/test/validated_manifest.json new file mode 100644 index 000000000..8462f516e --- /dev/null +++ b/image/docker/test/validated_manifest.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "name": "josephschorr/buildtest2", + "tag": "latest", + "architecture": "amd64", + "fsLayers": [ + { + "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + }, + { + "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + }, + { + "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + }, + { + "blobSum": "sha256:f0880d1639d2e72499fe0cfb218a98ca7aa3bffda6e0b808861505a1536cca10" + }, + { + "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + }, + { + "blobSum": "sha256:8e3ba11ec2a2b39ab372c60c16b421536e50e5ce64a0bc81765c2e38381bcff6" + } + ], + "history": [ + { + "v1Compatibility": "{\"architecture\":\"amd64\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\"],\"ArgsEscaped\":true,\"Image\":\"sha256:ebd938adb98827e85616f288beb990fd9f07335305c3d77ff783253b97d84b99\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{\"foo\":\"bar\",\"this.is.cool\":\"{\\\"some\\\": \\\"json\\\"}\"}},\"container\":\"a06cd9c29efac778d1e670a2d26971cf21360f9c59eb250e771f5852ff9f49ca\",\"container_config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"cat baz\"],\"ArgsEscaped\":true,\"Image\":\"sha256:ebd938adb98827e85616f288beb990fd9f07335305c3d77ff783253b97d84b99\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{\"foo\":\"bar\",\"this.is.cool\":\"{\\\"some\\\": \\\"json\\\"}\"}},\"created\":\"2018-08-13T19:19:53.358734989Z\",\"docker_version\":\"18.02.0-ce\",\"id\":\"08b0a1239a30dc9c12585c415538a3a43fab399a07cb341881b46e2fb69ae8f7\",\"os\":\"linux\",\"parent\":\"bc560233cb7ec4158c1858fd24fb093dc70a6fb7ad80b25f2a6f36a2138dd724\",\"throwaway\":true}" + }, + { + "v1Compatibility": "{\"id\":\"bc560233cb7ec4158c1858fd24fb093dc70a6fb7ad80b25f2a6f36a2138dd724\",\"parent\":\"cefdff8f1be4330d2e0414f598b0f38def3fb5c6a383d3b709162d51efe859b9\",\"created\":\"2018-08-13T19:19:52.919686159Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) LABEL this.is.cool={\\\"some\\\": \\\"json\\\"}\"]},\"throwaway\":true}" + }, + { + "v1Compatibility": "{\"id\":\"cefdff8f1be4330d2e0414f598b0f38def3fb5c6a383d3b709162d51efe859b9\",\"parent\":\"a86432a6eeb137d2342ee5ddcbc0dd32b5e58dfe3301dd09991147bb458ad6a9\",\"created\":\"2018-08-13T19:19:52.834827335Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) LABEL foo=bar\"]},\"throwaway\":true}" + }, + { + "v1Compatibility": "{\"id\":\"a86432a6eeb137d2342ee5ddcbc0dd32b5e58dfe3301dd09991147bb458ad6a9\",\"parent\":\"8b5fc1032bbcc570c28adc9b13525051c83bbf37ce305735f9c7be6e36ebff7d\",\"created\":\"2018-08-13T19:19:52.766315533Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) COPY file:9990e969595bc050f081c07b0bdf71524f3c46e6ffe8537c1778516c123f9f55 in baz \"]}}" + }, + { + "v1Compatibility": "{\"id\":\"8b5fc1032bbcc570c28adc9b13525051c83bbf37ce305735f9c7be6e36ebff7d\",\"parent\":\"f18ee96f0b1656cab52554b270f19e8df5046d307296d2146539c04565d67747\",\"created\":\"2018-07-06T14:14:06.393355914Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) CMD [\\\"/bin/sh\\\"]\"]},\"throwaway\":true}" + }, + { + "v1Compatibility": "{\"id\":\"f18ee96f0b1656cab52554b270f19e8df5046d307296d2146539c04565d67747\",\"created\":\"2018-07-06T14:14:06.165546783Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) ADD file:25f61d70254b9807a40cd3e8d820f6a5ec0e1e596de04e325f6a33810393e95a in / \"]}}" + } + ], + "signatures": [ + { + "header": { + "jwk": { + "crv": "P-256", + "kid": "H4QD:5X6G:2G7T:QXGN:EH3X:3UQU:REXP:7LAH:SGCZ:4FBI:EUSI:3P7Z", + "kty": "EC", + "x": "FowcV0YK1Dsn8FldhFJQJnxE247QUH43EchdZSmWrsQ", + "y": "4uUZBA9U1jC-AxmNzrwb1r9Oh2SXNXE3yqSpz7pwoiI" + }, + "alg": "ES256" + }, + "signature": "rJNUkqKUZ2_d2JTWTLu4XWFcNpNIMDEH6qoiOie9o_BlD_Ifhrw31OIUT23eKa-HyVm5sYOfx4DY3N5Xy1kr9A", + "protected": "eyJmb3JtYXRMZW5ndGgiOjQxMzksImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxOC0wOC0xM1QxOToyMDowMVoifQ" + } + ] +} \ No newline at end of file diff --git a/image/docker/test/validated_manifest_with_unicode.json b/image/docker/test/validated_manifest_with_unicode.json new file mode 100644 index 000000000..adc378492 --- /dev/null +++ b/image/docker/test/validated_manifest_with_unicode.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 1, + "name": "devtable/simple", + "tag": "unicode", + "architecture": "amd64", + "fsLayers": [ + { + "blobSum": "sha256:9dcda8e13dc6f3aa30ce7867d8a9e3941dc3a54cfefb5e76cbdfa90d2b56ed2f" + }, + { + "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + }, + { + "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" + }, + { + "blobSum": "sha256:8c5a7da1afbc602695fcb2cd6445743cec5ff32053ea589ea9bd8773b7068185" + } + ], + "history": [ + { + "v1Compatibility": "{\"architecture\":\"amd64\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"sh\"],\"ArgsEscaped\":true,\"Image\":\"sha256:746d49e88c1eac6e3d3384d73db788f166a51b5a2eb9da49671586f62baf6c0c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":[],\"Labels\":{\"maintainer\":\"Geé Léfleur\"}},\"container\":\"654ee2461cf64a54484624d8b7efbb76c5e197ba6f3322538b6810dad097c11f\",\"container_config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"echo foo \\u003e bar\"],\"ArgsEscaped\":true,\"Image\":\"sha256:746d49e88c1eac6e3d3384d73db788f166a51b5a2eb9da49671586f62baf6c0c\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":[],\"Labels\":{\"maintainer\":\"Geé Léfleur\"}},\"created\":\"2018-08-14T22:17:55.7294283Z\",\"docker_version\":\"17.09.0-ce\",\"id\":\"db077d203993a3a1cfeaf4bbaedb34ff1a706452cb598c62d2873ba78dd0d2fe\",\"os\":\"linux\",\"parent\":\"539016dae3ce29f825af4d27a60b8d42306a86727f7406371682612124bc6db3\"}" + }, + { + "v1Compatibility": "{\"id\":\"539016dae3ce29f825af4d27a60b8d42306a86727f7406371682612124bc6db3\",\"parent\":\"5a1738daa8064e42d79a0b1f3d1b75ca4406c6695969860ff8e814999bda9470\",\"created\":\"2018-08-14T22:17:54.5902216Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) LABEL maintainer=Geé Léfleur\"]},\"throwaway\":true}" + }, + { + "v1Compatibility": "{\"id\":\"5a1738daa8064e42d79a0b1f3d1b75ca4406c6695969860ff8e814999bda9470\",\"parent\":\"97d7c933c31fa951536cacfdfe3f862ce589020fa58bdf2fccc66204191a4273\",\"created\":\"2018-07-31T22:20:07.617575594Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) CMD [\\\"sh\\\"]\"]},\"throwaway\":true}" + }, + { + "v1Compatibility": "{\"id\":\"97d7c933c31fa951536cacfdfe3f862ce589020fa58bdf2fccc66204191a4273\",\"created\":\"2018-07-31T22:20:07.361628468Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) ADD file:96fda64a6b725d4df5249c12e32245e2f02469ff637c38077740f4984cd883dd in / \"]}}" + } + ], + "signatures": [ + { + "header": { + "jwk": { + "crv": "P-256", + "kid": "AARA:PFUD:3V54:7F2S:2P7E:WMCU:WRE7:KUYD:CFKH:UHZ7:AZ4I:UQEX", + "kty": "EC", + "x": "34N4h_uM7FedPw4k3_VabKlt7qoBWpHgpko7zE0RkeY", + "y": "LhxxtCYh_b1EwUbl3-tQFTbg1mTu34vMxj4UaKjWZk8" + }, + "alg": "ES256" + }, + "signature": "WCTPkAwHteVVjQCbY4GWRtoFJewKnZ9b0syTm72hi3n3Z_G30Gn5EDTU3adyXQx24aMzTFI_vryexeuypHv2Rw", + "protected": "eyJmb3JtYXRMZW5ndGgiOjMwNzIsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxOC0wOC0xNFQyMjoxOTozOFoifQ" + } + ] +} diff --git a/test/registry/protocol_v2.py b/test/registry/protocol_v2.py index 4ae3399cc..052c7a403 100644 --- a/test/registry/protocol_v2.py +++ b/test/registry/protocol_v2.py @@ -279,8 +279,9 @@ class V2Protocol(RegistryProtocol): if options.manifest_content_type is not None: manifest_headers['Content-Type'] = options.manifest_content_type + tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest self.conduct(session, 'PUT', - '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_name), + '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest), data=manifest.bytes, expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST), headers=manifest_headers) diff --git a/test/registry/protocols.py b/test/registry/protocols.py index 2cd073441..05471f0ef 100644 --- a/test/registry/protocols.py +++ b/test/registry/protocols.py @@ -65,6 +65,7 @@ class ProtocolOptions(object): self.skip_head_checks = False self.manifest_content_type = None self.mount_blobs = None + self.push_by_manifest_digest = False @add_metaclass(ABCMeta) diff --git a/test/registry/registry_tests.py b/test/registry/registry_tests.py index bd1da71ee..ed7ef2096 100644 --- a/test/registry/registry_tests.py +++ b/test/registry/registry_tests.py @@ -66,6 +66,25 @@ def test_basic_push_pull_by_manifest(manifest_protocol, basic_images, liveserver credentials=credentials) +def test_basic_push_by_manifest_digest(manifest_protocol, basic_images, liveserver_session, + app_reloader): + """ Test: Basic push-by-manifest and pull-by-manifest of an image to a new repository. """ + credentials = ('devtable', 'password') + + # Push a new repository. + options = ProtocolOptions() + options.push_by_manifest_digest = True + + result = manifest_protocol.push(liveserver_session, 'devtable', 'newrepo', 'latest', basic_images, + credentials=credentials, options=options) + + # Pull the repository by digests to verify. + digests = [str(manifest.digest) for manifest in result.manifests.values()] + manifest_protocol.pull(liveserver_session, 'devtable', 'newrepo', digests, basic_images, + credentials=credentials) + + + def test_push_invalid_credentials(pusher, basic_images, liveserver_session, app_reloader): """ Test: Ensure we get auth errors when trying to push with invalid credentials. """ invalid_credentials = ('devtable', 'notcorrectpassword') diff --git a/workers/manifestbackfillworker.py b/workers/manifestbackfillworker.py index 78dda070e..3a6aab4c6 100644 --- a/workers/manifestbackfillworker.py +++ b/workers/manifestbackfillworker.py @@ -114,11 +114,11 @@ def backfill_manifest(tag_manifest): if lookup_map_row(tag_manifest): return False - # Parse the manifest. If we cannot parse, then we treat the manifest as broken and just emit it + # Parse the manifest. If we cannot parse, then we treat the manifest as broken and just emit it # without additional rows or data, as it will eventually not be useful. is_broken = False try: - manifest = DockerSchema1Manifest(tag_manifest.json_data, validate=False) + manifest = DockerSchema1Manifest.for_latin1_bytes(tag_manifest.json_data, validate=False) except ManifestException: logger.exception('Exception when trying to parse manifest %s', tag_manifest.id) manifest = BrokenManifest(tag_manifest.digest, tag_manifest.json_data)