Merge pull request #3313 from quay/fix-unicode-error

Make sure to decode manifests into utf-8 when necessary

test/registry/protocol_fixtures.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import random
 import string
 
@@ -24,6 +26,20 @@ def basic_images():
   ]
 
 
+@pytest.fixture(scope="session")
+def unicode_images():
+  """ Returns basic images for push and pull testing that contain unicode in the image metadata. """
+  # Note: order is from base layer down to leaf.
+  parent_bytes = layer_bytes_for_contents('parent contents')
+  image_bytes = layer_bytes_for_contents('some contents')
+  return [
+    Image(id='parentid', bytes=parent_bytes, parent_id=None),
+    Image(id='someid', bytes=image_bytes, parent_id='parentid',
+          config={'comment': u'the Pawe\xc5\x82 Kami\xc5\x84ski image',
+                  'author': u'Sômé guy'}),
+  ]
+
+
 @pytest.fixture(scope="session")
 def different_images():
   """ Returns different basic images for push and pull testing. """

test/registry/protocol_v2.py

@@ -281,12 +281,12 @@ class V2Protocol(RegistryProtocol):
     if images[-1].config:
       config['config'] = images[-1].config
 
-    config_json = json.dumps(config)
+    config_json = json.dumps(config, ensure_ascii=options.ensure_ascii)
     schema2_config = DockerSchema2Config(config_json)
     builder.set_config(schema2_config)
 
-    blobs[schema2_config.digest] = schema2_config.bytes
-    return builder.build()
+    blobs[schema2_config.digest] = schema2_config.bytes.encode('utf-8')
+    return builder.build(ensure_ascii=options.ensure_ascii)
 
   def build_schema1(self, namespace, repo_name, tag_name, images, blobs, options):
     builder = DockerSchema1ManifestBuilder(namespace, repo_name, tag_name)
@@ -311,10 +311,14 @@ class V2Protocol(RegistryProtocol):
       if image.created is not None:
         layer_dict['created'] = image.created
 
-      builder.add_layer(checksum, json.dumps(layer_dict))
+      builder.add_layer(checksum, json.dumps(layer_dict, ensure_ascii=options.ensure_ascii))
 
     # Build the manifest.
-    return builder.build(self.jwk)
+    built = builder.build(self.jwk, ensure_ascii=options.ensure_ascii)
+
+    # Validate it before we send it.
+    DockerSchema1Manifest(built.bytes)
+    return built
 
   def push(self, session, namespace, repo_name, tag_names, images, credentials=None,
            expected_failure=None, options=None):
@@ -368,7 +372,7 @@ class V2Protocol(RegistryProtocol):
       tag_or_digest = tag_name if not options.push_by_manifest_digest else manifest.digest
       self.conduct(session, 'PUT',
                    '/v2/%s/manifests/%s' % (self.repo_name(namespace, repo_name), tag_or_digest),
-                   data=manifest.bytes,
+                   data=manifest.bytes.encode('utf-8'),
                    expected_status=(put_code, expected_failure, V2ProtocolSteps.PUT_MANIFEST),
                    headers=manifest_headers)
 

test/registry/protocols.py

@@ -81,6 +81,7 @@ class ProtocolOptions(object):
     self.push_by_manifest_digest = False
     self.request_addr = None
     self.skip_blob_push_checks = False
+    self.ensure_ascii = True
 
 
 @add_metaclass(ABCMeta)
@@ -120,7 +121,7 @@ class RegistryProtocol(object):
   def conduct(self, session, method, url, expected_status=200, params=None, data=None,
               json_data=None, headers=None, auth=None, options=None):
     if json_data is not None:
-      data = json.dumps(json_data)
+      data = json.dumps(json_data).encode('utf-8')
       headers = headers or {}
       headers['Content-Type'] = 'application/json'
 

test/registry/registry_tests.py

@@ -1773,3 +1773,33 @@ def test_pull_manifest_list_schema2_only(v22_protocol, basic_images, different_i
 
   if has_amd64_linux:
     assert result.manifests['latest'].media_type == DOCKER_SCHEMA2_MANIFEST_CONTENT_TYPE
+
+
+def test_push_pull_unicode(pusher, puller, unicode_images, liveserver_session, app_reloader):
+  """ Test: Push an image with unicode inside and then pull it. """
+  credentials = ('devtable', 'password')
+
+  # Push a new repository.
+  pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', unicode_images,
+              credentials=credentials)
+
+  # Pull the repository to verify.
+  puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', unicode_images,
+              credentials=credentials)
+
+
+def test_push_pull_unicode_direct(pusher, puller, unicode_images, liveserver_session, app_reloader):
+  """ Test: Push an image with *unescaped* unicode inside and then pull it. """
+  credentials = ('devtable', 'password')
+
+  # Turn off automatic unicode encoding when building the manifests.
+  options = ProtocolOptions()
+  options.ensure_ascii = False
+
+  # Push a new repository.
+  pusher.push(liveserver_session, 'devtable', 'newrepo', 'latest', unicode_images,
+              credentials=credentials, options=options)
+
+  # Pull the repository to verify.
+  puller.pull(liveserver_session, 'devtable', 'newrepo', 'latest', unicode_images,
+              credentials=credentials, options=options)