Revert "Merge pull request #557 from coreos-inc/revert-migration"

This reverts commit c4f938898a, reversing
changes made to 7ad2522dbe.

util/migrate/backfill_aggregate_sizes.py

@@ -1,44 +1,50 @@
 import logging
 
-from data.database import ImageStorage, Image, db
+from data.database import ImageStorage, Image, db, db_for_update
 from app import app
 
-LOGGER = logging.getLogger(__name__)
+
+logger = logging.getLogger(__name__)
+
 
 def backfill_aggregate_sizes():
   """ Generates aggregate sizes for any image storage entries without them """
-  LOGGER.setLevel(logging.DEBUG)
-  LOGGER.debug('Aggregate sizes backfill: Began execution')
+  logger.debug('Aggregate sizes backfill: Began execution')
   while True:
-    batch_storage_ids = list(ImageStorage
-                          .select(ImageStorage.id)
-                          .where(ImageStorage.aggregate_size >> None)
-                          .limit(10))
+    batch_image_ids = list(Image
+                           .select(Image.id)
+                           .where(Image.aggregate_size >> None)
+                           .limit(100))
 
-    if len(batch_storage_ids) == 0:
+    if len(batch_image_ids) == 0:
       # There are no storages left to backfill. We're done!
-      LOGGER.debug('Aggregate sizes backfill: Backfill completed')
+      logger.debug('Aggregate sizes backfill: Backfill completed')
       return
 
-    LOGGER.debug('Aggregate sizes backfill: Found %s records to update', len(batch_storage_ids))
-    for image_storage_id in batch_storage_ids:
-      LOGGER.debug('Updating image storage: %s', image_storage_id.id)
+    logger.debug('Aggregate sizes backfill: Found %s records to update', len(batch_image_ids))
+    for image_id in batch_image_ids:
+      logger.debug('Updating image : %s', image_id.id)
 
       with app.config['DB_TRANSACTION_FACTORY'](db):
         try:
-          storage = ImageStorage.select().where(ImageStorage.id == image_storage_id.id).get()
-          image  = Image.select().where(Image.storage == storage).get()
+          image = (Image
+                   .select(Image, ImageStorage)
+                   .join(ImageStorage)
+                   .where(Image.id == image_id)
+                   .get())
+
+          aggregate_size = image.storage.image_size
 
           image_ids = image.ancestors.split('/')[1:-1]
-          aggregate_size = storage.image_size
           for image_id in image_ids:
-            current_image = Image.select().where(Image.id == image_id).join(ImageStorage)
-            aggregate_size += image.storage.image_size
+            to_add = db_for_update(Image
+                                   .select(Image, ImageStorage)
+                                   .join(ImageStorage)
+                                   .where(Image.id == image_id)).get()
+            aggregate_size += to_add.storage.image_size
 
-          storage.aggregate_size = aggregate_size
-          storage.save()
-        except ImageStorage.DoesNotExist:
-          pass
+          image.aggregate_size = aggregate_size
+          image.save()
         except Image.DoesNotExist:
           pass
 

util/migrate/backfill_image_fields.py

@@ -0,0 +1,87 @@
+import logging
+
+from peewee import (CharField, BigIntegerField, BooleanField, ForeignKeyField, DateTimeField,
+                    TextField)
+from data.database import BaseModel, db, db_for_update
+from app import app
+
+
+logger = logging.getLogger(__name__)
+
+
+class Repository(BaseModel):
+  pass
+
+
+# Vendor the information from tables we will be writing to at the time of this migration
+class ImageStorage(BaseModel):
+  created = DateTimeField(null=True)
+  comment = TextField(null=True)
+  command = TextField(null=True)
+  aggregate_size = BigIntegerField(null=True)
+  uploading = BooleanField(default=True, null=True)
+
+
+class Image(BaseModel):
+  # This class is intentionally denormalized. Even though images are supposed
+  # to be globally unique we can't treat them as such for permissions and
+  # security reasons. So rather than Repository <-> Image being many to many
+  # each image now belongs to exactly one repository.
+  docker_image_id = CharField(index=True)
+  repository = ForeignKeyField(Repository)
+
+  # '/' separated list of ancestory ids, e.g. /1/2/6/7/10/
+  ancestors = CharField(index=True, default='/', max_length=64535, null=True)
+
+  storage = ForeignKeyField(ImageStorage, index=True, null=True)
+
+  created = DateTimeField(null=True)
+  comment = TextField(null=True)
+  command = TextField(null=True)
+  aggregate_size = BigIntegerField(null=True)
+  v1_json_metadata = TextField(null=True)
+
+
+def backfill_image_fields():
+  """ Copies metadata from image storages to their images. """
+  logger.debug('Image metadata backfill: Began execution')
+  while True:
+    batch_image_ids = list(Image
+                           .select(Image.id)
+                           .join(ImageStorage)
+                           .where(Image.created >> None, Image.comment >> None,
+                                  Image.command >> None, Image.aggregate_size >> None,
+                                  ImageStorage.uploading == False,
+                                  ~((ImageStorage.created >> None) &
+                                    (ImageStorage.comment >> None) &
+                                    (ImageStorage.command >> None) &
+                                    (ImageStorage.aggregate_size >> None)))
+                           .limit(100))
+
+    if len(batch_image_ids) == 0:
+      logger.debug('Image metadata backfill: Backfill completed')
+      return
+
+    logger.debug('Image metadata backfill: Found %s records to update', len(batch_image_ids))
+    for image_id in batch_image_ids:
+      logger.debug('Updating image: %s', image_id.id)
+
+      with app.config['DB_TRANSACTION_FACTORY'](db):
+        try:
+          image = db_for_update(Image
+                                .select(Image, ImageStorage)
+                                .join(ImageStorage)
+                                .where(Image.id == image_id.id)).get()
+
+          image.created = image.storage.created
+          image.comment = image.storage.comment
+          image.command = image.storage.command
+          image.aggregate_size = image.storage.aggregate_size
+          image.save()
+        except Image.DoesNotExist:
+          pass
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.DEBUG)
+  logging.getLogger('peewee').setLevel(logging.CRITICAL)
+  backfill_image_fields()

util/migrate/backfill_v1_metadata.py

@@ -0,0 +1,63 @@
+import logging
+
+from peewee import JOIN_LEFT_OUTER
+
+from data.database import (Image, ImageStorage, ImageStoragePlacement, ImageStorageLocation, db,
+                           db_for_update)
+from app import app, storage
+from data import model
+
+
+logger = logging.getLogger(__name__)
+
+
+def backfill_v1_metadata():
+  """ Copies metadata from image storages to their images. """
+  logger.debug('Image v1 metadata backfill: Began execution')
+  while True:
+    batch_image_ids = list(Image
+                           .select(Image.id)
+                           .join(ImageStorage)
+                           .where(Image.v1_json_metadata >> None, ImageStorage.uploading == False)
+                           .limit(100))
+
+    if len(batch_image_ids) == 0:
+      logger.debug('Image v1 metadata backfill: Backfill completed')
+      return
+
+    logger.debug('Image v1 metadata backfill: Found %s records to update', len(batch_image_ids))
+    for one_id in batch_image_ids:
+      with app.config['DB_TRANSACTION_FACTORY'](db):
+        try:
+          logger.debug('Loading image: %s', one_id.id)
+
+          raw_query = (ImageStoragePlacement
+                       .select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation)
+                       .join(ImageStorageLocation)
+                       .switch(ImageStoragePlacement)
+                       .join(ImageStorage, JOIN_LEFT_OUTER)
+                       .join(Image)
+                       .where(Image.id == one_id.id))
+
+          placement_query = db_for_update(raw_query)
+
+          repo_image_list = model.image.invert_placement_query_results(placement_query)
+          if len(repo_image_list) > 1:
+            logger.error('Found more images than we requested, something is wrong with the query')
+            return
+
+          repo_image = repo_image_list[0]
+          uuid = repo_image.storage.uuid
+          json_path = storage.image_json_path(uuid)
+
+          logger.debug('Updating image: %s from: %s', repo_image.id, json_path)
+          data = storage.get_content(repo_image.storage.locations, json_path)
+          repo_image.v1_json_metadata = data
+          repo_image.save()
+        except ImageStoragePlacement.DoesNotExist:
+          pass
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.DEBUG)
+  # logging.getLogger('peewee').setLevel(logging.CRITICAL)
+  backfill_v1_metadata()