Merge pull request #488 from coreos-inc/migrateback

Migrate data back to Image in preparation for v2

data/database.py

@@ -561,6 +561,12 @@ class Image(BaseModel):
 
   storage = ForeignKeyField(ImageStorage, index=True, null=True)
 
+  created = DateTimeField(null=True)
+  comment = TextField(null=True)
+  command = TextField(null=True)
+  aggregate_size = BigIntegerField(null=True)
+  v1_json_metadata = TextField(null=True)
+
   class Meta:
     database = db
     read_slaves = (read_slave,)

data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py

@@ -0,0 +1,34 @@
+"""Migrate image data back to image table
+
+Revision ID: 545794454f49
+Revises: 3a3bb77e17d5
+Create Date: 2015-09-15 11:48:47.554255
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '545794454f49'
+down_revision = '3a3bb77e17d5'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('image', sa.Column('aggregate_size', sa.BigInteger(), nullable=True))
+    op.add_column('image', sa.Column('command', sa.Text(), nullable=True))
+    op.add_column('image', sa.Column('comment', sa.Text(), nullable=True))
+    op.add_column('image', sa.Column('created', sa.DateTime(), nullable=True))
+    op.add_column('image', sa.Column('v1_json_metadata', sa.Text(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('image', 'v1_json_metadata')
+    op.drop_column('image', 'created')
+    op.drop_column('image', 'comment')
+    op.drop_column('image', 'command')
+    op.drop_column('image', 'aggregate_size')
+    ### end Alembic commands ###

data/model/image.py

@@ -255,7 +255,7 @@ def find_create_or_link_image(docker_image_id, repo_obj, username, translations,
 
 
 def set_image_metadata(docker_image_id, namespace_name, repository_name, created_date_str, comment,
-                       command, parent=None):
+                       command, v1_json_metadata, parent=None):
   with db_transaction():
     query = (Image
              .select(Image, ImageStorage)
@@ -273,7 +273,10 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
 
     # We cleanup any old checksum in case it's a retry after a fail
     fetched.storage.checksum = None
-    fetched.storage.created = datetime.now()
+    now = datetime.now()
+    # TODO stop writing to storage when all readers are removed
+    fetched.storage.created = now
+    fetched.created = now
 
     if created_date_str is not None:
       try:
@@ -282,8 +285,12 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created
         # parse raises different exceptions, so we cannot use a specific kind of handler here.
         pass
 
+    # TODO stop writing to storage fields when all readers are removed
     fetched.storage.comment = comment
     fetched.storage.command = command
+    fetched.comment = comment
+    fetched.command = command
+    fetched.v1_json_metadata = v1_json_metadata
 
     if parent:
       fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id)
@@ -323,13 +330,18 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size,
                     .where(Image.id << ancestors)
                     .scalar()) + image_size
 
+      # TODO stop writing to storage when all readers are removed
       image.storage.aggregate_size = total_size
+      image.aggregate_size = total_size
     except Image.DoesNotExist:
       pass
   else:
+      # TODO stop writing to storage when all readers are removed
     image.storage.aggregate_size = image_size
+    image.aggregate_size = image_size
 
   image.storage.save()
+  image.save()
 
   return image
 

endpoints/v1/registry.py

@@ -456,7 +456,8 @@ def put_image_json(namespace, repository, image_id):
 
   logger.debug('Parsing image JSON')
   try:
-    data = json.loads(request.data.decode('utf8'))
+    v1_metadata = request.data
+    data = json.loads(v1_metadata.decode('utf8'))
   except ValueError:
     pass
 
@@ -530,7 +531,7 @@ def put_image_json(namespace, repository, image_id):
 
   logger.debug('Setting image metadata')
   model.image.set_image_metadata(image_id, namespace, repository, data.get('created'),
-                                 data.get('comment'), command, parent_image)
+                                 data.get('comment'), command, v1_metadata, parent_image)
 
   logger.debug('Putting json path')
   store.put_content(repo_image.storage.locations, json_path, request.data)

initdb.py

@@ -99,9 +99,16 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map):
     creation_time = REFERENCE_DATE + timedelta(weeks=image_num) + timedelta(days=model_num)
     command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)]
     command = json.dumps(command_list) if command_list else None
+
+    v1_metadata = {
+      'id': docker_image_id,
+    }
+    if parent is not None:
+      v1_metadata['parent'] = parent.docker_image_id
+
     new_image = model.image.set_image_metadata(docker_image_id, repo.namespace_user.username,
                                                repo.name, str(creation_time), 'no comment', command,
-                                               parent)
+                                               v1_metadata, parent)
 
     compressed_size = random.randrange(1, 1024 * 1024 * 1024)
     model.image.set_image_size(docker_image_id, repo.namespace_user.username, repo.name,

test/test_gc.py

@@ -68,8 +68,14 @@ class TestGarbageCollection(unittest.TestCase):
         if not image_id in image_map:
           image_map[image_id] = self.createImage(image_id, repo, namespace)
 
+        v1_metadata = {
+          'id': image_id,
+        }
+        if parent is not None:
+          v1_metadata['parent'] = parent.docker_image_id
+
         # Set the ancestors for the image.
-        parent = model.image.set_image_metadata(image_id, namespace, name, '', '', '',
+        parent = model.image.set_image_metadata(image_id, namespace, name, '', '', '', v1_metadata,
                                                 parent=parent)
 
       # Set the tag for the image.