Further fixes for unicode handling in manifests

We were occasionally trying to compute schema 2 version 1 signatures on the *unicode* representation, which was failing the signature check. This PR adds a new wrapper type called `Bytes`, which all manifests must take in, and which handles the unicodes vs encoded utf-8 stuff in a central location. This PR also adds a test for the manifest that was breaking in production.

data/model/oci/manifest.py

@@ -149,7 +149,7 @@ def _create_manifest(repository_id, manifest_interface_instance, storage):
       manifest = Manifest.create(repository=repository_id,
                                  digest=manifest_interface_instance.digest,
                                  media_type=media_type,
-                                 manifest_bytes=manifest_interface_instance.bytes)
+                                 manifest_bytes=manifest_interface_instance.bytes.as_encoded_str())
     except IntegrityError:
       manifest = Manifest.get(repository=repository_id, digest=manifest_interface_instance.digest)
       return CreatedManifest(manifest=manifest, newly_created=False, labels_to_apply=None)

data/model/oci/tag.py

@@ -11,6 +11,7 @@ from data.model.oci.shared import get_legacy_image_for_manifest
 from data.model import config
 from image.docker.schema1 import (DOCKER_SCHEMA1_CONTENT_TYPES, DockerSchema1Manifest,
                                   MalformedSchema1Manifest)
+from util.bytes import Bytes
 from util.timedeltastring import convert_to_timedelta
 
 logger = logging.getLogger(__name__)
@@ -215,7 +216,8 @@ def retarget_tag(tag_name, manifest_id, is_reversion=False, now_ms=None):
   # name.
   if manifest.media_type.name in DOCKER_SCHEMA1_CONTENT_TYPES:
     try:
-      parsed = DockerSchema1Manifest(manifest.manifest_bytes, validate=False)
+      parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(manifest.manifest_bytes),
+                                     validate=False)
       if parsed.tag != tag_name:
         logger.error('Tried to re-target schema1 manifest with tag `%s` to tag `%s', parsed.tag,
                      tag_name)

data/model/oci/test/test_oci_manifest.py

@@ -18,6 +18,7 @@ from data.model.storage import get_layer_path
 from image.docker.schema1 import DockerSchema1ManifestBuilder, DockerSchema1Manifest
 from image.docker.schema2.manifest import DockerSchema2ManifestBuilder
 from image.docker.schema2.list import DockerSchema2ManifestListBuilder
+from util.bytes import Bytes
 
 from test.fixtures import *
 
@@ -163,7 +164,7 @@ def test_get_or_create_manifest(schema_version, initialized_db):
   assert created is not None
   assert created.media_type.name == sample_manifest_instance.media_type
   assert created.digest == sample_manifest_instance.digest
-  assert created.manifest_bytes == sample_manifest_instance.bytes
+  assert created.manifest_bytes == sample_manifest_instance.bytes.as_encoded_str()
   assert created_manifest.labels_to_apply == expected_labels
 
   # Verify the legacy image.
@@ -199,7 +200,8 @@ def test_get_or_create_manifest_invalid_image(initialized_db):
   repository = get_repository('devtable', 'simple')
 
   latest_tag = get_tag(repository, 'latest')
-  parsed = DockerSchema1Manifest(latest_tag.manifest.manifest_bytes, validate=False)
+  parsed = DockerSchema1Manifest(Bytes.for_string_or_unicode(latest_tag.manifest.manifest_bytes),
+                                 validate=False)
 
   builder = DockerSchema1ManifestBuilder('devtable', 'simple', 'anothertag')
   builder.add_layer(parsed.blob_digests[0], '{"id": "foo", "parent": "someinvalidimageid"}')

data/model/tag.py

@@ -793,7 +793,8 @@ def populate_manifest(repository, manifest, legacy_image, storage_ids):
   with db_transaction():
     try:
       manifest_row = Manifest.create(digest=manifest.digest, repository=repository,
-                                     manifest_bytes=manifest.bytes, media_type=media_type)
+                                     manifest_bytes=manifest.bytes.as_encoded_str(),
+                                     media_type=media_type)
     except IntegrityError:
       return Manifest.get(repository=repository, digest=manifest.digest)
 

data/model/test/test_tag.py

@@ -325,7 +325,7 @@ def test_store_tag_manifest(get_storages, initialized_db):
   mapping_row = TagManifestToManifest.get(tag_manifest=tag_manifest)
 
   assert mapping_row.manifest is not None
-  assert mapping_row.manifest.manifest_bytes == manifest.bytes
+  assert mapping_row.manifest.manifest_bytes == manifest.bytes.as_encoded_str()
   assert mapping_row.manifest.digest == str(manifest.digest)
 
   blob_rows = {m.blob_id for m in