Further fixes for unicode handling in manifests
We were occasionally trying to compute schema 2 version 1 signatures on the *unicode* representation, which was failing the signature check. This PR adds a new wrapper type called `Bytes`, which all manifests must take in, and which handles the unicodes vs encoded utf-8 stuff in a central location. This PR also adds a test for the manifest that was breaking in production.
This commit is contained in:
Joseph Schorr
parent
05fa2bcbe0
commit
171c7e5238
28 changed files with 275 additions and 106 deletions
|
|
@ -8,6 +8,8 @@ import pytest
|
|||
from app import docker_v2_signing_key
|
||||
from image.docker.schema1 import (MalformedSchema1Manifest, DockerSchema1Manifest,
|
||||
DockerSchema1ManifestBuilder)
|
||||
from util.bytes import Bytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize('json_data', [
|
||||
'',
|
||||
|
|
@ -20,7 +22,7 @@ from image.docker.schema1 import (MalformedSchema1Manifest, DockerSchema1Manifes
|
|||
])
|
||||
def test_malformed_manifests(json_data):
|
||||
with pytest.raises(MalformedSchema1Manifest):
|
||||
DockerSchema1Manifest(json_data)
|
||||
DockerSchema1Manifest(Bytes.for_string_or_unicode(json_data))
|
||||
|
||||
|
||||
MANIFEST_BYTES = json.dumps({
|
||||
|
|
@ -64,7 +66,7 @@ MANIFEST_BYTES = json.dumps({
|
|||
|
||||
|
||||
def test_valid_manifest():
|
||||
manifest = DockerSchema1Manifest(MANIFEST_BYTES, validate=False)
|
||||
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(MANIFEST_BYTES), validate=False)
|
||||
assert len(manifest.signatures) == 1
|
||||
assert manifest.namespace == ''
|
||||
assert manifest.repo_name == 'hello-world'
|
||||
|
|
@ -107,7 +109,7 @@ def test_validate_manifest():
|
|||
with open(os.path.join(test_dir, 'validated_manifest.json'), 'r') as f:
|
||||
manifest_bytes = f.read()
|
||||
|
||||
manifest = DockerSchema1Manifest(manifest_bytes, validate=True)
|
||||
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes), validate=True)
|
||||
digest = manifest.digest
|
||||
assert digest == 'sha256:b5dc4f63fdbd64f34f2314c0747ef81008f9fcddce4edfc3fd0e8ec8b358d571'
|
||||
assert manifest.created_datetime
|
||||
|
|
@ -118,7 +120,7 @@ def test_validate_manifest_with_unicode():
|
|||
with open(os.path.join(test_dir, 'validated_manifest_with_unicode.json'), 'r') as f:
|
||||
manifest_bytes = f.read()
|
||||
|
||||
manifest = DockerSchema1Manifest(manifest_bytes, validate=True)
|
||||
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes), validate=True)
|
||||
digest = manifest.digest
|
||||
assert digest == 'sha256:815ecf45716a96b19d54d911e6ace91f78bab26ca0dd299645d9995dacd9f1ef'
|
||||
assert manifest.created_datetime
|
||||
|
|
@ -140,7 +142,7 @@ def test_validate_manifest_with_unencoded_unicode():
|
|||
with open(os.path.join(test_dir, 'manifest_unencoded_unicode.json'), 'r') as f:
|
||||
manifest_bytes = f.read()
|
||||
|
||||
manifest = DockerSchema1Manifest(manifest_bytes)
|
||||
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes))
|
||||
digest = manifest.digest
|
||||
assert digest == 'sha256:5d8a0f34744a39bf566ba430251adc0cc86587f86aed3ac2acfb897f349777bc'
|
||||
assert manifest.created_datetime
|
||||
|
|
@ -162,3 +164,17 @@ def test_build_unencoded_unicode_manifest(with_key):
|
|||
|
||||
built = builder.build(with_key, ensure_ascii=False)
|
||||
built._validate()
|
||||
|
||||
|
||||
def test_validate_manifest_known_issue():
|
||||
test_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
with open(os.path.join(test_dir, 'validate_manifest_known_issue.json'), 'r') as f:
|
||||
manifest_bytes = f.read()
|
||||
|
||||
manifest = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest_bytes))
|
||||
digest = manifest.digest
|
||||
assert digest == 'sha256:44518f5a4d1cb5b7a6347763116fb6e10f6a8563b6c40bb389a0a982f0a9f47a'
|
||||
assert manifest.created_datetime
|
||||
|
||||
layers = list(manifest.get_layers(None))
|
||||
assert layers[-1].author is None
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from image.docker.schema2 import DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE
|
|||
from image.docker.test.test_schema1 import MANIFEST_BYTES as SCHEMA1_BYTES
|
||||
from image.docker.schema2.test.test_list import MANIFESTLIST_BYTES
|
||||
from image.docker.schema2.test.test_manifest import MANIFEST_BYTES as SCHEMA2_BYTES
|
||||
from util.bytes import Bytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize('media_type, manifest_bytes', [
|
||||
|
|
@ -15,4 +16,5 @@ from image.docker.schema2.test.test_manifest import MANIFEST_BYTES as SCHEMA2_BY
|
|||
(DOCKER_SCHEMA2_MANIFESTLIST_CONTENT_TYPE, MANIFESTLIST_BYTES),
|
||||
])
|
||||
def test_parse_manifest_from_bytes(media_type, manifest_bytes):
|
||||
assert parse_manifest_from_bytes(manifest_bytes, media_type, validate=False)
|
||||
assert parse_manifest_from_bytes(Bytes.for_string_or_unicode(manifest_bytes), media_type,
|
||||
validate=False)
|
||||
|
|
|
|||
56
image/docker/test/validate_manifest_known_issue.json
Normal file
56
image/docker/test/validate_manifest_known_issue.json
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"name": "quaymonitor/monitortest2",
|
||||
"tag": "latest",
|
||||
"architecture": "x86_64",
|
||||
"fsLayers": [
|
||||
{
|
||||
"blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:184dc3db39b5e19dc39547f43db46ea48cd6cc779e806a3c8a5e5396acd20206"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:db80bcab0e8b69656505332fcdff3ef2b9f664a2029d1b2f97224cffcf689afc"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:184dc3db39b5e19dc39547f43db46ea48cd6cc779e806a3c8a5e5396acd20206"
|
||||
},
|
||||
{
|
||||
"blobSum": "sha256:f0a98344d604e54694fc6118cf7a0cbd10dc7b2e9be8607ba8c5bfd7ba3c1067"
|
||||
}
|
||||
],
|
||||
"history": [
|
||||
{
|
||||
"v1Compatibility": "{\"architecture\":\"x86_64\",\"config\":{\"Hostname\":\"4c9181ab6b87\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"HOME=/\",\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"sh\",\"echo\",\"\\\"2019-01-08 19:13:20 +0000\\\" \\u003e foo\"],\"Image\":\"quay.io/quay/busybox\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"container\":\"4c9181ab6b87fe75b5c0955c6c78983dec337914b05e65fb0073cce0ad076106\",\"container_config\":{\"Hostname\":\"4c9181ab6b87\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"HOME=/\",\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"sh\",\"echo\",\"\\\"2019-01-08 19:13:20 +0000\\\" \\u003e foo\"],\"Image\":\"quay.io/quay/busybox\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"created\":\"2019-01-08T19:13:20.674196032Z\",\"docker_version\":\"18.06.1-ce\",\"id\":\"7da7c4e4bcb121915fb33eb5c76ffef194cdcc14608010692cfce5734bd84751\",\"os\":\"linux\",\"parent\":\"ec75e623647b299585bdb0991293bd446e5545e9a4dabf9d37922d5671d9d860\",\"throwaway\":true}"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"ec75e623647b299585bdb0991293bd446e5545e9a4dabf9d37922d5671d9d860\",\"parent\":\"f32bc6daa02c76f0b1773688684bf3bee719a69db06192432e6c28a238f4cf4a\",\"created\":\"2014-02-03T15:58:08.872585903Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) CMD [/bin/sh -c /bin/sh]\"]},\"author\":\"Jérôme Petazzoni \\u003cjerome@docker.com\\u003e\"}"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"f32bc6daa02c76f0b1773688684bf3bee719a69db06192432e6c28a238f4cf4a\",\"parent\":\"02feaf4fdc57dba2b142dae9d8dd0c90e710be710bea25ce63269e65d8f32872\",\"created\":\"2014-02-03T15:58:08.72383042Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) ADD rootfs.tar in /\"]},\"author\":\"Jérôme Petazzoni \\u003cjerome@docker.com\\u003e\"}"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"02feaf4fdc57dba2b142dae9d8dd0c90e710be710bea25ce63269e65d8f32872\",\"parent\":\"f9a6e54178f312aa3686d7305b970e7d908d58b32e3f4554731b647e07b48fd2\",\"created\":\"2014-02-03T15:58:08.52236968Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) MAINTAINER Jérôme Petazzoni \\u003cjerome@docker.com\\u003e\"]},\"author\":\"Jérôme Petazzoni \\u003cjerome@docker.com\\u003e\"}"
|
||||
},
|
||||
{
|
||||
"v1Compatibility": "{\"id\":\"f9a6e54178f312aa3686d7305b970e7d908d58b32e3f4554731b647e07b48fd2\",\"comment\":\"Imported from -\",\"created\":\"2013-06-13T14:03:50.821769-07:00\",\"container_config\":{\"Cmd\":[\"\"]}}"
|
||||
}
|
||||
],
|
||||
"signatures": [
|
||||
{
|
||||
"header": {
|
||||
"jwk": {
|
||||
"crv": "P-256",
|
||||
"kid": "XPAM:RVQE:4LWW:ABXI:QLLK:O2LK:XJ4V:UAOJ:WM24:ZG6J:UIJ3:JAYM",
|
||||
"kty": "EC",
|
||||
"x": "ijnW3d93SINE1y3GjNsCMYghAb7NT21vSiYK8pWdBkM",
|
||||
"y": "7t-mGjoYOhEIGVaCSEclLLkMgHz2S9WXkReZJEBx-_U"
|
||||
},
|
||||
"alg": "ES256"
|
||||
},
|
||||
"signature": "N9m-NNL8CdGwxEHHHaJDhbT5_FFKBSdyy-7lP4jnWG3AQmOWbPEXTFANTeH2CNPvAbaM9ZqQm0dQFQVnOe5GNQ",
|
||||
"protected": "eyJmb3JtYXRMZW5ndGgiOjM1OTgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxOS0wMS0wOFQxOToxMzoyM1oifQ"
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in a new issue