Further fixes for unicode handling in manifests

We were occasionally trying to compute schema 2 version 1 signatures on the *unicode* representation, which was failing the signature check. This PR adds a new wrapper type called `Bytes`, which all manifests must take in, and which handles the unicodes vs encoded utf-8 stuff in a central location. This PR also adds a test for the manifest that was breaking in production.

image/docker/schema1.py

@@ -23,7 +23,8 @@ from image.docker import ManifestException
 from image.docker.types import ManifestImageLayer
 from image.docker.interfaces import ManifestInterface
 from image.docker.v1 import DockerV1Metadata
-from image.docker.schemautil import ensure_utf8, to_canonical_json
+from image.docker.schemautil import to_canonical_json
+from util.bytes import Bytes
 
 logger = logging.getLogger(__name__)
 
@@ -160,11 +161,13 @@ class DockerSchema1Manifest(ManifestInterface):
   }
 
   def __init__(self, manifest_bytes, validate=True):
+    assert isinstance(manifest_bytes, Bytes)
+
     self._layers = None
     self._bytes = manifest_bytes
 
     try:
-      self._parsed = json.loads(manifest_bytes)
+      self._parsed = json.loads(manifest_bytes.as_encoded_str())
     except ValueError as ve:
       raise MalformedSchema1Manifest('malformed manifest data: %s' % ve)
 
@@ -193,13 +196,13 @@ class DockerSchema1Manifest(ManifestInterface):
 
   @classmethod
   def for_latin1_bytes(cls, encoded_bytes, validate=True):
-    return DockerSchema1Manifest(encoded_bytes.encode('utf-8'), validate)
+    return DockerSchema1Manifest(Bytes.for_string_or_unicode(encoded_bytes), validate)
 
   def _validate(self):
     if not self._signatures:
       return
 
-    payload_str = ensure_utf8(self._payload)
+    payload_str = self._payload
     for signature in self._signatures:
       bytes_to_verify = '{0}.{1}'.format(signature['protected'], base64url_encode(payload_str))
       signer = SIGNER_ALGS[signature['header']['alg']]
@@ -248,10 +251,6 @@ class DockerSchema1Manifest(ManifestInterface):
   def tag(self):
     return self._tag
 
-  @property
-  def json(self):
-    return self._bytes
-
   @property
   def bytes(self):
     return self._bytes
@@ -270,7 +269,7 @@ class DockerSchema1Manifest(ManifestInterface):
 
   @property
   def digest(self):
-    return digest_tools.sha256_digest(ensure_utf8(self._payload))
+    return digest_tools.sha256_digest(self._payload)
 
   @property
   def image_ids(self):
@@ -395,11 +394,12 @@ class DockerSchema1Manifest(ManifestInterface):
   @property
   def _payload(self):
     if self._signatures is None:
-      return self._bytes
+      return self._bytes.as_encoded_str()
 
+    byte_data = self._bytes.as_encoded_str()
     protected = str(self._signatures[0][DOCKER_SCHEMA1_PROTECTED_KEY])
     parsed_protected = json.loads(base64url_decode(protected))
-    signed_content_head = self._bytes[:parsed_protected[DOCKER_SCHEMA1_FORMAT_LENGTH_KEY]]
+    signed_content_head = byte_data[:parsed_protected[DOCKER_SCHEMA1_FORMAT_LENGTH_KEY]]
     signed_content_tail = base64url_decode(str(parsed_protected[DOCKER_SCHEMA1_FORMAT_TAIL_KEY]))
     return signed_content_head + signed_content_tail
 
@@ -548,8 +548,9 @@ class DockerSchema1ManifestBuilder(object):
 
     payload_str = json.dumps(payload, indent=3, ensure_ascii=ensure_ascii)
     if json_web_key is None:
-      return DockerSchema1Manifest(payload_str)
+      return DockerSchema1Manifest(Bytes.for_string_or_unicode(payload_str))
 
+    payload_str = Bytes.for_string_or_unicode(payload_str).as_encoded_str()
     split_point = payload_str.rfind('\n}')
 
     protected_payload = {
@@ -560,7 +561,6 @@ class DockerSchema1ManifestBuilder(object):
     protected = base64url_encode(json.dumps(protected_payload, ensure_ascii=ensure_ascii))
     logger.debug('Generated protected block: %s', protected)
 
-    payload_str = ensure_utf8(payload_str)
     bytes_to_sign = '{0}.{1}'.format(protected, base64url_encode(payload_str))
 
     signer = SIGNER_ALGS[_JWS_SIGNING_ALGORITHM]
@@ -579,7 +579,9 @@ class DockerSchema1ManifestBuilder(object):
 
     logger.debug('Encoded signature block: %s', json.dumps(signature_block))
     payload.update({DOCKER_SCHEMA1_SIGNATURES_KEY: [signature_block]})
-    return DockerSchema1Manifest(json.dumps(payload, indent=3, ensure_ascii=ensure_ascii))
+
+    json_str = json.dumps(payload, indent=3, ensure_ascii=ensure_ascii)
+    return DockerSchema1Manifest(Bytes.for_string_or_unicode(json_str))
 
 
 def _updated_v1_metadata(v1_metadata_json, updated_id_map):