V1 Docker ID <-> V2 layer SHA mismatch fix

Fix handling of V1 Docker ID <-> V2 layer SHA mismatch by dynamically rewriting the manifest to use new synthesized IDs for all layers above the mismatch. Also adds a bunch of tests for this and other use cases, fixes a bug around manifest digest uniqueness and fixes the 5.5 migration for MySQL.

data/database.py

@@ -642,7 +642,7 @@ class RepositoryTag(BaseModel):
 
 class TagManifest(BaseModel):
   tag = ForeignKeyField(RepositoryTag, index=True, unique=True)
-  digest = CharField(index=True, unique=True)
+  digest = CharField(index=True)
   json_data = TextField()
 
 

data/migrations/versions/88e0f440a2f_add_created_field_to_the_blobupload_.py

@@ -16,7 +16,7 @@ from datetime import datetime
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
-    now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    now = datetime.now().strftime("'%Y-%m-%d %H:%M:%S'")
     op.add_column('blobupload', sa.Column('created', sa.DateTime(), nullable=False, server_default=sa.text(now)))
     op.create_index('blobupload_created', 'blobupload', ['created'], unique=False)
     ### end Alembic commands ###

data/migrations/versions/e4129c93e477_remove_uniqueness_constraint_on_the_.py

@@ -0,0 +1,28 @@
+"""Remove uniqueness constraint on the TagManifest digest column
+
+Revision ID: e4129c93e477
+Revises: 956a0833223
+Create Date: 2016-02-12 17:22:48.039791
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e4129c93e477'
+down_revision = '956a0833223'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('tagmanifest_digest', table_name='tagmanifest')
+    op.create_index('tagmanifest_digest', 'tagmanifest', ['digest'], unique=False)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('tagmanifest_digest', table_name='tagmanifest')
+    op.create_index('tagmanifest_digest', 'tagmanifest', ['digest'], unique=True)
+    ### end Alembic commands ###

data/model/image.py

@@ -2,7 +2,7 @@ import logging
 import dateutil.parser
 import random
 
-from peewee import JOIN_LEFT_OUTER, SQL
+from peewee import JOIN_LEFT_OUTER, IntegrityError
 from datetime import datetime
 
 from data.model import (DataModelException, db_transaction, _basequery, storage,
@@ -405,10 +405,13 @@ def synthesize_v1_image(repo, image_storage, docker_image_id, created_date_str,
   aggregate_size = _basequery.calculate_image_aggregate_size(ancestors, image_storage.image_size,
                                                              parent_image)
 
-  return Image.create(docker_image_id=docker_image_id, ancestors=ancestors, comment=comment,
-                      command=command, v1_json_metadata=v1_json_metadata, created=created,
-                      storage=image_storage, repository=repo, parent=parent_image,
-                      aggregate_size=aggregate_size)
+  try:
+    return Image.create(docker_image_id=docker_image_id, ancestors=ancestors, comment=comment,
+                        command=command, v1_json_metadata=v1_json_metadata, created=created,
+                        storage=image_storage, repository=repo, parent=parent_image,
+                        aggregate_size=aggregate_size)
+  except IntegrityError:
+    return Image.get(docker_image_id=docker_image_id, repository=repo)
 
 
 def ensure_image_locations(*names):