Revert "Merge pull request #557 from coreos-inc/revert-migration"
This reverts commitc4f938898a, reversing changes made to7ad2522dbe.
This commit is contained in:
Jimmy Zelinskie
parent
5c063bd285
commit
39cfe77d42
11 changed files with 232 additions and 65 deletions
|
|
@ -1,44 +1,50 @@
|
|||
import logging
|
||||
|
||||
from data.database import ImageStorage, Image, db
|
||||
from data.database import ImageStorage, Image, db, db_for_update
|
||||
from app import app
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def backfill_aggregate_sizes():
|
||||
""" Generates aggregate sizes for any image storage entries without them """
|
||||
LOGGER.setLevel(logging.DEBUG)
|
||||
LOGGER.debug('Aggregate sizes backfill: Began execution')
|
||||
logger.debug('Aggregate sizes backfill: Began execution')
|
||||
while True:
|
||||
batch_storage_ids = list(ImageStorage
|
||||
.select(ImageStorage.id)
|
||||
.where(ImageStorage.aggregate_size >> None)
|
||||
.limit(10))
|
||||
batch_image_ids = list(Image
|
||||
.select(Image.id)
|
||||
.where(Image.aggregate_size >> None)
|
||||
.limit(100))
|
||||
|
||||
if len(batch_storage_ids) == 0:
|
||||
if len(batch_image_ids) == 0:
|
||||
# There are no storages left to backfill. We're done!
|
||||
LOGGER.debug('Aggregate sizes backfill: Backfill completed')
|
||||
logger.debug('Aggregate sizes backfill: Backfill completed')
|
||||
return
|
||||
|
||||
LOGGER.debug('Aggregate sizes backfill: Found %s records to update', len(batch_storage_ids))
|
||||
for image_storage_id in batch_storage_ids:
|
||||
LOGGER.debug('Updating image storage: %s', image_storage_id.id)
|
||||
logger.debug('Aggregate sizes backfill: Found %s records to update', len(batch_image_ids))
|
||||
for image_id in batch_image_ids:
|
||||
logger.debug('Updating image : %s', image_id.id)
|
||||
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
try:
|
||||
storage = ImageStorage.select().where(ImageStorage.id == image_storage_id.id).get()
|
||||
image = Image.select().where(Image.storage == storage).get()
|
||||
image = (Image
|
||||
.select(Image, ImageStorage)
|
||||
.join(ImageStorage)
|
||||
.where(Image.id == image_id)
|
||||
.get())
|
||||
|
||||
aggregate_size = image.storage.image_size
|
||||
|
||||
image_ids = image.ancestors.split('/')[1:-1]
|
||||
aggregate_size = storage.image_size
|
||||
for image_id in image_ids:
|
||||
current_image = Image.select().where(Image.id == image_id).join(ImageStorage)
|
||||
aggregate_size += image.storage.image_size
|
||||
to_add = db_for_update(Image
|
||||
.select(Image, ImageStorage)
|
||||
.join(ImageStorage)
|
||||
.where(Image.id == image_id)).get()
|
||||
aggregate_size += to_add.storage.image_size
|
||||
|
||||
storage.aggregate_size = aggregate_size
|
||||
storage.save()
|
||||
except ImageStorage.DoesNotExist:
|
||||
pass
|
||||
image.aggregate_size = aggregate_size
|
||||
image.save()
|
||||
except Image.DoesNotExist:
|
||||
pass
|
||||
|
||||
|
|
|
|||
87
util/migrate/backfill_image_fields.py
Normal file
87
util/migrate/backfill_image_fields.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import logging
|
||||
|
||||
from peewee import (CharField, BigIntegerField, BooleanField, ForeignKeyField, DateTimeField,
|
||||
TextField)
|
||||
from data.database import BaseModel, db, db_for_update
|
||||
from app import app
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Repository(BaseModel):
|
||||
pass
|
||||
|
||||
|
||||
# Vendor the information from tables we will be writing to at the time of this migration
|
||||
class ImageStorage(BaseModel):
|
||||
created = DateTimeField(null=True)
|
||||
comment = TextField(null=True)
|
||||
command = TextField(null=True)
|
||||
aggregate_size = BigIntegerField(null=True)
|
||||
uploading = BooleanField(default=True, null=True)
|
||||
|
||||
|
||||
class Image(BaseModel):
|
||||
# This class is intentionally denormalized. Even though images are supposed
|
||||
# to be globally unique we can't treat them as such for permissions and
|
||||
# security reasons. So rather than Repository <-> Image being many to many
|
||||
# each image now belongs to exactly one repository.
|
||||
docker_image_id = CharField(index=True)
|
||||
repository = ForeignKeyField(Repository)
|
||||
|
||||
# '/' separated list of ancestory ids, e.g. /1/2/6/7/10/
|
||||
ancestors = CharField(index=True, default='/', max_length=64535, null=True)
|
||||
|
||||
storage = ForeignKeyField(ImageStorage, index=True, null=True)
|
||||
|
||||
created = DateTimeField(null=True)
|
||||
comment = TextField(null=True)
|
||||
command = TextField(null=True)
|
||||
aggregate_size = BigIntegerField(null=True)
|
||||
v1_json_metadata = TextField(null=True)
|
||||
|
||||
|
||||
def backfill_image_fields():
|
||||
""" Copies metadata from image storages to their images. """
|
||||
logger.debug('Image metadata backfill: Began execution')
|
||||
while True:
|
||||
batch_image_ids = list(Image
|
||||
.select(Image.id)
|
||||
.join(ImageStorage)
|
||||
.where(Image.created >> None, Image.comment >> None,
|
||||
Image.command >> None, Image.aggregate_size >> None,
|
||||
ImageStorage.uploading == False,
|
||||
~((ImageStorage.created >> None) &
|
||||
(ImageStorage.comment >> None) &
|
||||
(ImageStorage.command >> None) &
|
||||
(ImageStorage.aggregate_size >> None)))
|
||||
.limit(100))
|
||||
|
||||
if len(batch_image_ids) == 0:
|
||||
logger.debug('Image metadata backfill: Backfill completed')
|
||||
return
|
||||
|
||||
logger.debug('Image metadata backfill: Found %s records to update', len(batch_image_ids))
|
||||
for image_id in batch_image_ids:
|
||||
logger.debug('Updating image: %s', image_id.id)
|
||||
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
try:
|
||||
image = db_for_update(Image
|
||||
.select(Image, ImageStorage)
|
||||
.join(ImageStorage)
|
||||
.where(Image.id == image_id.id)).get()
|
||||
|
||||
image.created = image.storage.created
|
||||
image.comment = image.storage.comment
|
||||
image.command = image.storage.command
|
||||
image.aggregate_size = image.storage.aggregate_size
|
||||
image.save()
|
||||
except Image.DoesNotExist:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
logging.getLogger('peewee').setLevel(logging.CRITICAL)
|
||||
backfill_image_fields()
|
||||
63
util/migrate/backfill_v1_metadata.py
Normal file
63
util/migrate/backfill_v1_metadata.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import logging
|
||||
|
||||
from peewee import JOIN_LEFT_OUTER
|
||||
|
||||
from data.database import (Image, ImageStorage, ImageStoragePlacement, ImageStorageLocation, db,
|
||||
db_for_update)
|
||||
from app import app, storage
|
||||
from data import model
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def backfill_v1_metadata():
|
||||
""" Copies metadata from image storages to their images. """
|
||||
logger.debug('Image v1 metadata backfill: Began execution')
|
||||
while True:
|
||||
batch_image_ids = list(Image
|
||||
.select(Image.id)
|
||||
.join(ImageStorage)
|
||||
.where(Image.v1_json_metadata >> None, ImageStorage.uploading == False)
|
||||
.limit(100))
|
||||
|
||||
if len(batch_image_ids) == 0:
|
||||
logger.debug('Image v1 metadata backfill: Backfill completed')
|
||||
return
|
||||
|
||||
logger.debug('Image v1 metadata backfill: Found %s records to update', len(batch_image_ids))
|
||||
for one_id in batch_image_ids:
|
||||
with app.config['DB_TRANSACTION_FACTORY'](db):
|
||||
try:
|
||||
logger.debug('Loading image: %s', one_id.id)
|
||||
|
||||
raw_query = (ImageStoragePlacement
|
||||
.select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation)
|
||||
.join(ImageStorageLocation)
|
||||
.switch(ImageStoragePlacement)
|
||||
.join(ImageStorage, JOIN_LEFT_OUTER)
|
||||
.join(Image)
|
||||
.where(Image.id == one_id.id))
|
||||
|
||||
placement_query = db_for_update(raw_query)
|
||||
|
||||
repo_image_list = model.image.invert_placement_query_results(placement_query)
|
||||
if len(repo_image_list) > 1:
|
||||
logger.error('Found more images than we requested, something is wrong with the query')
|
||||
return
|
||||
|
||||
repo_image = repo_image_list[0]
|
||||
uuid = repo_image.storage.uuid
|
||||
json_path = storage.image_json_path(uuid)
|
||||
|
||||
logger.debug('Updating image: %s from: %s', repo_image.id, json_path)
|
||||
data = storage.get_content(repo_image.storage.locations, json_path)
|
||||
repo_image.v1_json_metadata = data
|
||||
repo_image.save()
|
||||
except ImageStoragePlacement.DoesNotExist:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
# logging.getLogger('peewee').setLevel(logging.CRITICAL)
|
||||
backfill_v1_metadata()
|
||||
Reference in a new issue