vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Migrate old data to new locations, read only new.

Browse source
This commit is contained in:
Jake Moshenko 2015-09-15 19:07:35 -04:00
parent 0316f6e471
commit 8baacd2741
10 changed files with 204 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
buildman/jobutil/buildjob.py
View file

@ -128,10 +128,9 @@ class BuildJob(object):
return False
full_command = '["/bin/sh", "-c", "%s"]' % cache_commands[step]
logger.debug('Checking step #%s: %s, %s == %s', step, image.id,
image.storage.command, full_command)
logger.debug('Checking step #%s: %s, %s == %s', step, image.id, image.command, full_command)
return image.storage.command == full_command
return image.command == full_command
path = tree.find_longest_path(base_image.id, checker)
if not path:

24
data/migrations/versions/2e0380215d01_backfill_image_fields_from_image_.py Normal file
View file

@ -0,0 +1,24 @@
"""Backfill image fields from image storages
Revision ID: 2e0380215d01
Revises: 3ff4fbc94644
Create Date: 2015-09-15 16:57:42.850246
"""
# revision identifiers, used by Alembic.
revision = '2e0380215d01'
down_revision = '3ff4fbc94644'
from alembic import op
import sqlalchemy as sa
from util.migrate.backfill_image_fields import backfill_image_fields
from util.migrate.backfill_v1_metadata import backfill_v1_metadata
def upgrade(tables):
backfill_image_fields()
backfill_v1_metadata()
def downgrade(tables):
pass

11
data/model/image.py
View file

@ -79,7 +79,14 @@ def get_repository_images_base(namespace_name, repository_name, query_modifier):
query = query_modifier(query)
location_list = list(query)
return invert_placement_query_results(query)
def invert_placement_query_results(placement_query):
""" This method will take a query which returns placements, storages, and images, and have it
return images and their storages, along with the placement set on each storage.
"""
location_list = list(placement_query)
images = {}
for location in location_list:
@ -326,7 +333,7 @@ def set_image_size(docker_image_id, namespace_name, repository_name, image_size,
try:
# TODO(jschorr): Switch to this faster route once we have full ancestor aggregate_size
# parent_image = Image.get(Image.id == ancestors[-1])
# total_size = image_size + parent_image.storage.aggregate_size
# total_size = image_size + parent_image.aggregate_size
total_size = (ImageStorage
.select(fn.Sum(ImageStorage.image_size))
.join(Image)

2
digest/checksums.py
View file

@ -68,7 +68,7 @@ def compute_tarsum(fp, json_data):
def simple_checksum_handler(json_data):
h = hashlib.sha256(json_data + '\n')
h = hashlib.sha256(json_data.encode('utf8') + '\n')
def fn(buf):
h.update(buf)

12
endpoints/api/image.py
View file

@ -12,11 +12,7 @@ from util.cache import cache_control_flask_restful
def image_view(image, image_map, include_ancestors=True):
extended_props = image
if image.storage and image.storage.id:
extended_props = image.storage
command = extended_props.command
command = image.command
def docker_id(aid):
if not aid or not aid in image_map:
@ -26,10 +22,10 @@ def image_view(image, image_map, include_ancestors=True):
image_data = {
'id': image.docker_image_id,
'created': format_date(extended_props.created),
'comment': extended_props.comment,
'created': format_date(image.created),
'comment': image.comment,
'command': json.loads(command) if command else None,
'size': extended_props.image_size,
'size': image.storage.image_size,
'uploading': image.storage.uploading,
'sort_index': len(image.ancestors),
}

2
endpoints/api/repository.py
View file

@ -243,7 +243,7 @@ class Repository(RepositoryParamResource):
tag_info = {
'name': tag.name,
'image_id': tag.image.docker_image_id,
'size': tag.image.storage.aggregate_size
'size': tag.image.aggregate_size
}
if tag.lifetime_start_ts > 0:

33
endpoints/v1/registry.py
View file

@ -211,11 +211,10 @@ def put_image_layer(namespace, repository, image_id):
try:
logger.debug('Retrieving image data')
uuid = repo_image.storage.uuid
json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
except (IOError, AttributeError):
json_data = repo_image.v1_json_metadata
except (AttributeError):
logger.exception('Exception when retrieving image data')
abort(404, 'Image %(image_id)s not found', issue='unknown-image',
image_id=image_id)
abort(404, 'Image %(image_id)s not found', issue='unknown-image', image_id=image_id)
logger.debug('Retrieving image path info')
layer_path = store.image_layer_path(uuid)
@ -331,10 +330,8 @@ def put_image_checksum(namespace, repository, image_id):
if not repo_image or not repo_image.storage:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
uuid = repo_image.storage.uuid
logger.debug('Looking up repo layer data')
if not store.exists(repo_image.storage.locations, store.image_json_path(uuid)):
if not repo_image.v1_json_metadata:
abort(404, 'Image not found: %(image_id)s', issue='unknown-image', image_id=image_id)
logger.debug('Marking image path')
@ -374,19 +371,14 @@ def get_image_json(namespace, repository, image_id, headers):
logger.debug('Looking up repo image')
repo_image = model.image.get_repo_image_extended(namespace, repository, image_id)
logger.debug('Looking up repo layer data')
try:
uuid = repo_image.storage.uuid
data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
except (IOError, AttributeError):
if repo_image is None:
flask_abort(404)
logger.debug('Looking up repo layer size')
size = repo_image.storage.image_size
headers['X-Docker-Size'] = str(size)
response = make_response(data, 200)
response = make_response(repo_image.v1_json_metadata, 200)
response.headers.extend(headers)
return response
@ -489,8 +481,6 @@ def put_image_json(namespace, repository, image_id):
model.tag.create_temporary_hidden_tag(repo, repo_image,
app.config['PUSH_TEMP_TAG_EXPIRATION_SEC'])
uuid = repo_image.storage.uuid
if image_id != data['id']:
abort(400, 'JSON data contains invalid id for image: %(image_id)s',
issue='invalid-request', image_id=image_id)
@ -508,17 +498,12 @@ def put_image_json(namespace, repository, image_id):
if parent_id:
logger.debug('Looking up parent image data')
if (parent_id and not
store.exists(parent_locations, store.image_json_path(parent_uuid))):
if parent_id and not parent_image.v1_json_metadata:
abort(400, 'Image %(image_id)s depends on non existing parent image %(parent_id)s',
issue='invalid-request', image_id=image_id, parent_id=parent_id)
logger.debug('Looking up image storage paths')
json_path = store.image_json_path(uuid)
logger.debug('Checking if image already exists')
if (store.exists(repo_image.storage.locations, json_path) and not
image_is_uploading(repo_image)):
if repo_image.v1_json_metadata and not image_is_uploading(repo_image):
exact_abort(409, 'Image already exists')
set_uploading_flag(repo_image, True)
@ -534,6 +519,8 @@ def put_image_json(namespace, repository, image_id):
data.get('comment'), command, v1_metadata, parent_image)
logger.debug('Putting json path')
uuid = repo_image.storage.uuid
json_path = store.image_json_path(uuid)
store.put_content(repo_image.storage.locations, json_path, request.data)
logger.debug('Generating image ancestry')

9
endpoints/verbs.py
View file

@ -114,17 +114,15 @@ def _verify_repo_verb(store, namespace, repository, tag, verb, checker=None):
abort(404)
# Lookup the tag's image and storage.
repo_image = model.image.get_repo_image_extended(namespace, repository, tag_image.docker_image_id)
repo_image = model.image.get_repo_image(namespace, repository, tag_image.docker_image_id)
if not repo_image:
abort(404)
# If there is a data checker, call it first.
uuid = repo_image.storage.uuid
image_json = None
if checker is not None:
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
image_json = json.loads(image_json_data)
image_json = json.loads(repo_image.v1_json_metadata)
if not checker(image_json):
logger.debug('Check mismatch on %s/%s:%s, verb %s', namespace, repository, tag, verb)
@ -193,8 +191,7 @@ def _repo_verb(namespace, repository, tag, verb, formatter, sign=False, checker=
# Load the image's JSON layer.
if not image_json:
image_json_data = store.get_content(repo_image.storage.locations, store.image_json_path(uuid))
image_json = json.loads(image_json_data)
image_json = json.loads(repo_image.v1_json_metadata)
# Calculate a synthetic image ID.
synthetic_image_id = hashlib.sha256(tag_image.docker_image_id + ':' + verb).hexdigest()

87
util/migrate/backfill_image_fields.py Normal file
View file

@ -0,0 +1,87 @@
import logging
from peewee import (CharField, BigIntegerField, BooleanField, ForeignKeyField, DateTimeField,
TextField)
from data.database import BaseModel, db, db_for_update
from app import app
logger = logging.getLogger(__name__)
class Repository(BaseModel):
pass
# Vendor the information from tables we will be writing to at the time of this migration
class ImageStorage(BaseModel):
created = DateTimeField(null=True)
comment = TextField(null=True)
command = TextField(null=True)
aggregate_size = BigIntegerField(null=True)
uploading = BooleanField(default=True, null=True)
class Image(BaseModel):
# This class is intentionally denormalized. Even though images are supposed
# to be globally unique we can't treat them as such for permissions and
# security reasons. So rather than Repository <-> Image being many to many
# each image now belongs to exactly one repository.
docker_image_id = CharField(index=True)
repository = ForeignKeyField(Repository)
# '/' separated list of ancestory ids, e.g. /1/2/6/7/10/
ancestors = CharField(index=True, default='/', max_length=64535, null=True)
storage = ForeignKeyField(ImageStorage, index=True, null=True)
created = DateTimeField(null=True)
comment = TextField(null=True)
command = TextField(null=True)
aggregate_size = BigIntegerField(null=True)
v1_json_metadata = TextField(null=True)
def backfill_image_fields():
""" Copies metadata from image storages to their images. """
logger.debug('Image metadata backfill: Began execution')
while True:
batch_image_ids = list(Image
.select(Image.id)
.join(ImageStorage)
.where(Image.created >> None, Image.comment >> None,
Image.command >> None, Image.aggregate_size >> None,
ImageStorage.uploading == False,
~((ImageStorage.created >> None) &
(ImageStorage.comment >> None) &
(ImageStorage.command >> None) &
(ImageStorage.aggregate_size >> None)))
.limit(100))
if len(batch_image_ids) == 0:
logger.debug('Image metadata backfill: Backfill completed')
return
logger.debug('Image metadata backfill: Found %s records to update', len(batch_image_ids))
for image_id in batch_image_ids:
logger.debug('Updating image: %s', image_id.id)
with app.config['DB_TRANSACTION_FACTORY'](db):
try:
image = db_for_update(Image
.select(Image, ImageStorage)
.join(ImageStorage)
.where(Image.id == image_id.id)).get()
image.created = image.storage.created
image.comment = image.storage.comment
image.command = image.storage.command
image.aggregate_size = image.storage.aggregate_size
image.save()
except Image.DoesNotExist:
pass
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
logging.getLogger('peewee').setLevel(logging.CRITICAL)
backfill_image_fields()

63
util/migrate/backfill_v1_metadata.py Normal file
View file

@ -0,0 +1,63 @@
import logging
from peewee import JOIN_LEFT_OUTER
from data.database import (Image, ImageStorage, ImageStoragePlacement, ImageStorageLocation, db,
db_for_update)
from app import app, storage
from data import model
logger = logging.getLogger(__name__)
def backfill_v1_metadata():
""" Copies metadata from image storages to their images. """
logger.debug('Image v1 metadata backfill: Began execution')
while True:
batch_image_ids = list(Image
.select(Image.id)
.join(ImageStorage)
.where(Image.v1_json_metadata >> None, ImageStorage.uploading == False)
.limit(100))
if len(batch_image_ids) == 0:
logger.debug('Image v1 metadata backfill: Backfill completed')
return
logger.debug('Image v1 metadata backfill: Found %s records to update', len(batch_image_ids))
for one_id in batch_image_ids:
with app.config['DB_TRANSACTION_FACTORY'](db):
try:
logger.debug('Loading image: %s', one_id.id)
raw_query = (ImageStoragePlacement
.select(ImageStoragePlacement, Image, ImageStorage, ImageStorageLocation)
.join(ImageStorageLocation)
.switch(ImageStoragePlacement)
.join(ImageStorage, JOIN_LEFT_OUTER)
.join(Image)
.where(Image.id == one_id.id))
placement_query = db_for_update(raw_query)
repo_image_list = model.image.invert_placement_query_results(placement_query)
if len(repo_image_list) > 1:
logger.error('Found more images than we requested, something is wrong with the query')
return
repo_image = repo_image_list[0]
uuid = repo_image.storage.uuid
json_path = storage.image_json_path(uuid)
logger.debug('Updating image: %s from: %s', repo_image.id, json_path)
data = storage.get_content(repo_image.storage.locations, json_path)
repo_image.v1_json_metadata = data
repo_image.save()
except ImageStoragePlacement.DoesNotExist:
pass
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
# logging.getLogger('peewee').setLevel(logging.CRITICAL)
backfill_v1_metadata()