diff --git a/data/database.py b/data/database.py index cc802ac59..660976bcf 100644 --- a/data/database.py +++ b/data/database.py @@ -561,6 +561,12 @@ class Image(BaseModel): storage = ForeignKeyField(ImageStorage, index=True, null=True) + created = DateTimeField(null=True) + comment = TextField(null=True) + command = TextField(null=True) + aggregate_size = BigIntegerField(null=True) + v1_json_metadata = TextField(null=True) + class Meta: database = db read_slaves = (read_slave,) diff --git a/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py b/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py new file mode 100644 index 000000000..17af21eb3 --- /dev/null +++ b/data/migrations/versions/545794454f49_migrate_image_data_back_to_image_table.py @@ -0,0 +1,34 @@ +"""Migrate image data back to image table + +Revision ID: 545794454f49 +Revises: 3a3bb77e17d5 +Create Date: 2015-09-15 11:48:47.554255 + +""" + +# revision identifiers, used by Alembic. +revision = '545794454f49' +down_revision = '3a3bb77e17d5' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('image', sa.Column('aggregate_size', sa.BigInteger(), nullable=True)) + op.add_column('image', sa.Column('command', sa.Text(), nullable=True)) + op.add_column('image', sa.Column('comment', sa.Text(), nullable=True)) + op.add_column('image', sa.Column('created', sa.DateTime(), nullable=True)) + op.add_column('image', sa.Column('v1_json_metadata', sa.Text(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('image', 'v1_json_metadata') + op.drop_column('image', 'created') + op.drop_column('image', 'comment') + op.drop_column('image', 'command') + op.drop_column('image', 'aggregate_size') + ### end Alembic commands ### diff --git a/data/model/image.py b/data/model/image.py index 0da208b46..aef076336 100644 --- a/data/model/image.py +++ b/data/model/image.py @@ -255,7 +255,7 @@ def find_create_or_link_image(docker_image_id, repo_obj, username, translations, def set_image_metadata(docker_image_id, namespace_name, repository_name, created_date_str, comment, - command, parent=None): + command, v1_json_metadata, parent=None): with db_transaction(): query = (Image .select(Image, ImageStorage) @@ -273,7 +273,10 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created # We cleanup any old checksum in case it's a retry after a fail fetched.storage.checksum = None - fetched.storage.created = datetime.now() + now = datetime.now() + # TODO stop writing to storage when all readers are removed + fetched.storage.created = now + fetched.created = now if created_date_str is not None: try: @@ -282,8 +285,12 @@ def set_image_metadata(docker_image_id, namespace_name, repository_name, created # parse raises different exceptions, so we cannot use a specific kind of handler here. pass + # TODO stop writing to storage fields when all readers are removed fetched.storage.comment = comment fetched.storage.command = command + fetched.comment = comment + fetched.command = command + fetched.v1_json_metadata = v1_json_metadata if parent: fetched.ancestors = '%s%s/' % (parent.ancestors, parent.id) @@ -323,13 +330,18 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size, .where(Image.id << ancestors) .scalar()) + image_size + # TODO stop writing to storage when all readers are removed image.storage.aggregate_size = total_size + image.aggregate_size = total_size except Image.DoesNotExist: pass else: + # TODO stop writing to storage when all readers are removed image.storage.aggregate_size = image_size + image.aggregate_size = image_size image.storage.save() + image.save() return image diff --git a/endpoints/v1/registry.py b/endpoints/v1/registry.py index e1838730d..5b94c48b5 100644 --- a/endpoints/v1/registry.py +++ b/endpoints/v1/registry.py @@ -456,7 +456,8 @@ def put_image_json(namespace, repository, image_id): logger.debug('Parsing image JSON') try: - data = json.loads(request.data.decode('utf8')) + v1_metadata = request.data + data = json.loads(v1_metadata.decode('utf8')) except ValueError: pass @@ -530,7 +531,7 @@ def put_image_json(namespace, repository, image_id): logger.debug('Setting image metadata') model.image.set_image_metadata(image_id, namespace, repository, data.get('created'), - data.get('comment'), command, parent_image) + data.get('comment'), command, v1_metadata, parent_image) logger.debug('Putting json path') store.put_content(repo_image.storage.locations, json_path, request.data) diff --git a/initdb.py b/initdb.py index 57024f00d..29b151601 100644 --- a/initdb.py +++ b/initdb.py @@ -99,9 +99,16 @@ def __create_subtree(repo, structure, creator_username, parent, tag_map): creation_time = REFERENCE_DATE + timedelta(weeks=image_num) + timedelta(days=model_num) command_list = SAMPLE_CMDS[image_num % len(SAMPLE_CMDS)] command = json.dumps(command_list) if command_list else None + + v1_metadata = { + 'id': docker_image_id, + } + if parent is not None: + v1_metadata['parent'] = parent.docker_image_id + new_image = model.image.set_image_metadata(docker_image_id, repo.namespace_user.username, repo.name, str(creation_time), 'no comment', command, - parent) + v1_metadata, parent) compressed_size = random.randrange(1, 1024 * 1024 * 1024) model.image.set_image_size(docker_image_id, repo.namespace_user.username, repo.name, diff --git a/test/data/test.db b/test/data/test.db index ec9242221..762ad4378 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_gc.py b/test/test_gc.py index 27f67f82a..a5992188b 100644 --- a/test/test_gc.py +++ b/test/test_gc.py @@ -68,8 +68,14 @@ class TestGarbageCollection(unittest.TestCase): if not image_id in image_map: image_map[image_id] = self.createImage(image_id, repo, namespace) + v1_metadata = { + 'id': image_id, + } + if parent is not None: + v1_metadata['parent'] = parent.docker_image_id + # Set the ancestors for the image. - parent = model.image.set_image_metadata(image_id, namespace, name, '', '', '', + parent = model.image.set_image_metadata(image_id, namespace, name, '', '', '', v1_metadata, parent=parent) # Set the tag for the image.