quay/util/migrate/backfill_v1_checksums.py

import logging

from peewee import (CharField, BigIntegerField, BooleanField, ForeignKeyField, DateTimeField,
                    TextField, fn)
from data.database import BaseModel, db, db_for_update
from util.migrate.allocator import yield_random_entries
from app import app


logger = logging.getLogger(__name__)


class Repository(BaseModel):
  pass


# Vendor the information from tables we will be writing to at the time of this migration
class ImageStorage(BaseModel):
  uuid = CharField(index=True, unique=True)
  checksum = CharField(null=True)
  image_size = BigIntegerField(null=True)
  uncompressed_size = BigIntegerField(null=True)
  uploading = BooleanField(default=True, null=True)
  cas_path = BooleanField(default=True)
  content_checksum = CharField(null=True, index=True)


class Image(BaseModel):
  docker_image_id = CharField(index=True)
  repository = ForeignKeyField(Repository)
  ancestors = CharField(index=True, default='/', max_length=64535, null=True)
  storage = ForeignKeyField(ImageStorage, index=True, null=True)
  created = DateTimeField(null=True)
  comment = TextField(null=True)
  command = TextField(null=True)
  aggregate_size = BigIntegerField(null=True)
  v1_json_metadata = TextField(null=True)
  v1_checksum = CharField(null=True)


def backfill_checksums():
  """ Copies checksums from image storages to their images. """
  logger.debug('Image v1 checksum backfill: Began execution')
  def batch_query():
    return (Image
            .select(Image.id)
            .join(ImageStorage)
            .where(Image.v1_checksum >> None, ImageStorage.uploading == False,
                   ~(ImageStorage.checksum >> None)))

  max_id = Image.select(fn.Max(Image.id)).scalar()

  for candidate_image, abort in yield_random_entries(batch_query, Image.id, 1000, max_id):
    with app.config['DB_TRANSACTION_FACTORY'](db):
      try:
        image = db_for_update(Image
                              .select(Image, ImageStorage)
                              .join(ImageStorage)
                              .where(Image.id == candidate_image.id,
                                     Image.v1_checksum >> None)).get()

        image.v1_checksum = image.storage.checksum
        image.save()
      except Image.DoesNotExist:
        logger.info('Collision with another worker, aborting batch')
        abort.set()


if __name__ == "__main__":
  logging.basicConfig(level=logging.DEBUG)
  logging.getLogger('peewee').setLevel(logging.CRITICAL)
  backfill_checksums()