vbatts/quay
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Change derived storage to be based on image

Browse source 
Fixes #971
This commit is contained in:
Joseph Schorr 2015-11-24 12:44:07 -05:00
parent 8d05d40cf7
commit 762cd56e64
10 changed files with 155 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

16
data/database.py
View file

@ -514,6 +514,7 @@ class ImageStorageSignature(BaseModel):
)
# NOTE: This table is *deprecated* and will be removed in the next PR.
class DerivedImageStorage(BaseModel):
source = ForeignKeyField(ImageStorage, null=True, related_name='source')
derivative = ForeignKeyField(ImageStorage, related_name='derivative')
@ -596,6 +597,19 @@ class Image(BaseModel):
_ImageProxy.initialize(Image)
class DerivedStorageForImage(BaseModel):
source_image = ForeignKeyField(Image)
derivative = ForeignKeyField(ImageStorage)
transformation = ForeignKeyField(ImageStorageTransformation)
class Meta:
database = db
read_slaves = (read_slave,)
indexes = (
(('source_image', 'transformation'), True),
)
class RepositoryTag(BaseModel):
name = CharField()
image = ForeignKeyField(Image)
@ -831,4 +845,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion,
QuayService, QuayRegion, QuayRelease, BlobUpload]
QuayService, QuayRegion, QuayRelease, BlobUpload, DerivedStorageForImage]

39
data/migrations/versions/5a2445ffe21b_add_new_derivedstorageforimage_table.py Normal file
View file

@ -0,0 +1,39 @@
"""Add new DerivedStorageForImage table
Revision ID: 5a2445ffe21b
Revises: 1b2bb93ceb82
Create Date: 2015-11-24 11:58:02.956687
"""
# revision identifiers, used by Alembic.
revision = '5a2445ffe21b'
down_revision = '1b2bb93ceb82'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
def upgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.create_table('derivedstorageforimage',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('source_image_id', sa.Integer(), nullable=False),
sa.Column('derivative_id', sa.Integer(), nullable=False),
sa.Column('transformation_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['derivative_id'], ['imagestorage.id'], name=op.f('fk_derivedstorageforimage_derivative_id_imagestorage')),
sa.ForeignKeyConstraint(['source_image_id'], ['image.id'], name=op.f('fk_derivedstorageforimage_source_image_id_image')),
sa.ForeignKeyConstraint(['transformation_id'], ['imagestoragetransformation.id'], name=op.f('fk_derivedstorageforimage_transformation_constraint')),
sa.PrimaryKeyConstraint('id', name=op.f('pk_derivedstorageforimage'))
)
op.create_index('derivedstorageforimage_derivative_id', 'derivedstorageforimage', ['derivative_id'], unique=False)
op.create_index('derivedstorageforimage_source_image_id', 'derivedstorageforimage', ['source_image_id'], unique=False)
op.create_index('derivedstorageforimage_source_image_id_transformation_id', 'derivedstorageforimage', ['source_image_id', 'transformation_id'], unique=True)
op.create_index('derivedstorageforimage_transformation_id', 'derivedstorageforimage', ['transformation_id'], unique=False)
### end Alembic commands ###
def downgrade(tables):
### commands auto generated by Alembic - please adjust! ###
op.drop_table('derivedstorageforimage')
### end Alembic commands ###

49
data/model/image.py
View file

@ -8,8 +8,8 @@ from datetime import datetime
from data.model import (DataModelException, db_transaction, _basequery, storage,
InvalidImageException, config)
from data.database import (Image, Repository, ImageStoragePlacement, Namespace, ImageStorage,
ImageStorageLocation, RepositoryPermission, db_for_update,
db_random_func)
ImageStorageLocation, RepositoryPermission, DerivedStorageForImage,
ImageStorageTransformation, db_random_func, db_for_update)
logger = logging.getLogger(__name__)
@ -542,3 +542,48 @@ def set_secscan_status(image, indexed, version):
.update(security_indexed=indexed, security_indexed_engine=version)
.where(Image.id << ids_to_update)
.execute())
def find_or_create_derived_storage(source_image, transformation_name, preferred_location):
existing = find_derived_storage_for_image(source_image, transformation_name)
if existing is not None:
return existing
logger.debug('Creating storage dervied from source image: %s', source_image.id)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = storage.create_v1_storage(preferred_location)
DerivedStorageForImage.create(source_image=source_image, derivative=new_storage,
transformation=trans)
return new_storage
def find_derived_storage_for_image(source_image, transformation_name):
try:
found = (ImageStorage
.select(ImageStorage, DerivedStorageForImage)
.join(DerivedStorageForImage)
.join(ImageStorageTransformation)
.where(DerivedStorageForImage.source_image == source_image,
ImageStorageTransformation.name == transformation_name)
.get())
found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
return found
except ImageStorage.DoesNotExist:
return None
def delete_derived_storage_by_uuid(storage_uuid):
try:
image_storage = storage.get_storage_by_uuid(storage_uuid)
except InvalidImageException:
return
try:
DerivedStorageForImage.get(derivative=image_storage)
except DerivedStorageForImage.DoesNotExist:
return
image_storage.delete_instance(recursive=True)

21
data/model/repository.py
View file

@ -7,8 +7,8 @@ from data.model import (DataModelException, tag, db_transaction, storage, image,
_basequery, config)
from data.database import (Repository, Namespace, RepositoryTag, Star, Image, ImageStorage, User,
Visibility, RepositoryPermission, TupleSelector, RepositoryActionCount,
Role, RepositoryAuthorizedEmail, TagManifest, db_for_update,
get_epoch_timestamp, db_random_func)
Role, RepositoryAuthorizedEmail, TagManifest, DerivedStorageForImage,
db_for_update, get_epoch_timestamp, db_random_func)
logger = logging.getLogger(__name__)
@ -108,7 +108,7 @@ def garbage_collect_repository(namespace_name, repository_name):
def garbage_collect_repo(repo):
logger.debug('Garbage collecting repository %s', repo.id)
storage_id_whitelist = {}
storage_id_whitelist = set()
tag.garbage_collect_tags(repo)
with db_transaction():
@ -142,6 +142,21 @@ def garbage_collect_repo(repo):
if len(to_remove) > 0:
logger.info('Cleaning up unreferenced images: %s', to_remove)
storage_id_whitelist = {images_to_storages[to_remove_id] for to_remove_id in to_remove}
# Lookup any derived images for the images to remove.
derived = DerivedStorageForImage.select().where(
DerivedStorageForImage.source_image << to_remove)
has_derived = False
for derived_image in derived:
has_derived = True
storage_id_whitelist.add(derived_image.derivative_id)
# Delete any derived images and the images themselves.
if has_derived:
DerivedStorageForImage.delete().where(
DerivedStorageForImage.source_image << to_remove).execute()
Image.delete().where(Image.id << to_remove).execute()
if len(to_remove) > 0:

76
data/model/storage.py
View file

@ -3,7 +3,7 @@ import logging
from peewee import JOIN_LEFT_OUTER, fn, SQL
from data.model import config, db_transaction, InvalidImageException
from data.database import (ImageStorage, Image, DerivedImageStorage, ImageStoragePlacement,
from data.database import (ImageStorage, Image, DerivedStorageForImage, ImageStoragePlacement,
ImageStorageLocation, ImageStorageTransformation, ImageStorageSignature,
ImageStorageSignatureKind, Repository, Namespace)
@ -17,18 +17,6 @@ def add_storage_placement(storage, location_name):
ImageStoragePlacement.create(location=location, storage=storage)
def find_or_create_derived_storage(source, transformation_name, preferred_location):
existing = find_derived_storage(source, transformation_name)
if existing is not None:
return existing
logger.debug('Creating storage dervied from source: %s', source.uuid)
trans = ImageStorageTransformation.get(name=transformation_name)
new_storage = create_v1_storage(preferred_location)
DerivedImageStorage.create(source=source, derivative=new_storage, transformation=trans)
return new_storage
def garbage_collect_storage(storage_id_whitelist):
if len(storage_id_whitelist) == 0:
return
@ -42,37 +30,11 @@ def garbage_collect_storage(storage_id_whitelist):
.switch(ImageStorage)
.join(Image, JOIN_LEFT_OUTER)
.switch(ImageStorage)
.join(DerivedImageStorage, JOIN_LEFT_OUTER,
on=(ImageStorage.id == DerivedImageStorage.derivative))
.join(DerivedStorageForImage, JOIN_LEFT_OUTER,
on=(ImageStorage.id == DerivedStorageForImage.derivative))
.where(ImageStorage.id << list(candidates))
.group_by(*group_by)
.having((fn.Count(Image.id) == 0) & (fn.Count(DerivedImageStorage.id) == 0)))
# Note: We remove the derived image storage in its own transaction as a way to reduce the
# time that the transaction holds on the database indicies. This could result in a derived
# image storage being deleted for an image storage which is later reused during this time,
# but since these are caches anyway, it isn't terrible and worth the tradeoff (for now).
logger.debug('Garbage collecting derived storage from candidates: %s', storage_id_whitelist)
with db_transaction():
# Find out which derived storages will be removed, and add them to the whitelist
# The comma after ImageStorage.id is VERY important, it makes it a tuple, which is a sequence
orphaned_from_candidates = list(orphaned_storage_query(ImageStorage.select(ImageStorage.id),
storage_id_whitelist,
(ImageStorage.id,)))
if len(orphaned_from_candidates) > 0:
derived_to_remove = (ImageStorage
.select(ImageStorage.id)
.join(DerivedImageStorage,
on=(ImageStorage.id == DerivedImageStorage.derivative))
.where(DerivedImageStorage.source << orphaned_from_candidates))
storage_id_whitelist.update({derived.id for derived in derived_to_remove})
# Remove the dervived image storages with sources of orphaned storages
(DerivedImageStorage
.delete()
.where(DerivedImageStorage.source << orphaned_from_candidates)
.execute())
.having((fn.Count(Image.id) == 0) & (fn.Count(DerivedStorageForImage.id) == 0)))
# Note: Both of these deletes must occur in the same transaction (unfortunately) because a
# storage without any placement is invalid, and a placement cannot exist without a storage.
@ -151,36 +113,6 @@ def lookup_storage_signature(storage, signature_kind):
return None
def find_derived_storage(source, transformation_name):
try:
found = (ImageStorage
.select(ImageStorage, DerivedImageStorage)
.join(DerivedImageStorage, on=(ImageStorage.id == DerivedImageStorage.derivative))
.join(ImageStorageTransformation)
.where(DerivedImageStorage.source == source,
ImageStorageTransformation.name == transformation_name)
.get())
found.locations = {placement.location.name for placement in found.imagestorageplacement_set}
return found
except ImageStorage.DoesNotExist:
return None
def delete_derived_storage_by_uuid(storage_uuid):
try:
image_storage = get_storage_by_uuid(storage_uuid)
except InvalidImageException:
return
try:
DerivedImageStorage.get(derivative=image_storage)
except DerivedImageStorage.DoesNotExist:
return
image_storage.delete_instance(recursive=True)
def _get_storage(query_modifier):
query = (ImageStoragePlacement
.select(ImageStoragePlacement, ImageStorage, ImageStorageLocation)