From cb7ec2f2394d2b381ceca67f1980eddeea5b6a04 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Fri, 23 Oct 2015 13:49:23 -0400 Subject: [PATCH] Backport remaining v2 changes to phase4 --- data/database.py | 26 +++++++++-- data/fields.py | 38 ++++++++++++++++ .../33bd39ef5ed6_backport_v2_db_changes.py | 43 +++++++++++++++++++ data/model/storage.py | 2 +- 4 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 data/fields.py create mode 100644 data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py diff --git a/data/database.py b/data/database.py index 765a4948e..cc0beafa6 100644 --- a/data/database.py +++ b/data/database.py @@ -3,15 +3,16 @@ import logging import uuid import time import toposort +import resumablehashlib from random import SystemRandom from datetime import datetime from peewee import * from data.read_slave import ReadSlaveModel +from data.fields import ResumableSHAField, JSONField from sqlalchemy.engine.url import make_url from collections import defaultdict -from data.read_slave import ReadSlaveModel from util.names import urn_generator @@ -348,7 +349,7 @@ class Repository(BaseModel): # These models don't need to use transitive deletes, because the referenced objects # are cleaned up directly - skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger} + skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload} # We need to sort the ops so that models get cleaned in order of their dependencies ops = reversed(list(self.dependencies(delete_nullable))) @@ -490,6 +491,7 @@ class ImageStorage(BaseModel): image_size = BigIntegerField(null=True) uncompressed_size = BigIntegerField(null=True) uploading = BooleanField(default=True, null=True) + cas_path = BooleanField(default=True) class ImageStorageTransformation(BaseModel): @@ -761,6 +763,23 @@ class RepositoryAuthorizedEmail(BaseModel): ) +class BlobUpload(BaseModel): + repository = ForeignKeyField(Repository, index=True) + uuid = CharField(index=True, unique=True) + byte_count = IntegerField(default=0) + sha_state = ResumableSHAField(null=True, default=resumablehashlib.sha256) + location = ForeignKeyField(ImageStorageLocation) + storage_metadata = JSONField(null=True, default={}) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + # create a unique index on email and repository + (('repository', 'uuid'), True), + ) + + class QuayService(BaseModel): name = CharField(index=True, unique=True) @@ -788,7 +807,6 @@ class QuayRelease(BaseModel): ) - all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem, RepositoryBuild, Team, TeamMember, TeamRole, LogEntryKind, LogEntry, @@ -799,4 +817,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion, - QuayService, QuayRegion, QuayRelease] + QuayService, QuayRegion, QuayRelease, BlobUpload] diff --git a/data/fields.py b/data/fields.py new file mode 100644 index 000000000..123811ccd --- /dev/null +++ b/data/fields.py @@ -0,0 +1,38 @@ +import base64 +import resumablehashlib +import json + +from peewee import TextField + + +class ResumableSHAField(TextField): + def db_value(self, value): + sha_state = value.state() + + # One of the fields is a byte string, let's base64 encode it to make sure + # we can store and fetch it regardless of default collocation. + sha_state[3] = base64.b64encode(sha_state[3]) + + return json.dumps(sha_state) + + def python_value(self, value): + to_resume = resumablehashlib.sha256() + if value is None: + return to_resume + + sha_state = json.loads(value) + + # We need to base64 decode the data bytestring. + sha_state[3] = base64.b64decode(sha_state[3]) + to_resume.set_state(sha_state) + return to_resume + + +class JSONField(TextField): + def db_value(self, value): + return json.dumps(value) + + def python_value(self, value): + if value is None or value == "": + return {} + return json.loads(value) diff --git a/data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py b/data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py new file mode 100644 index 000000000..c63924c93 --- /dev/null +++ b/data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py @@ -0,0 +1,43 @@ +"""Backport v2 db changes. + +Revision ID: 33bd39ef5ed6 +Revises: 127905a52fdd +Create Date: 2015-10-23 12:34:22.776542 + +""" + +# revision identifiers, used by Alembic. +revision = '33bd39ef5ed6' +down_revision = '127905a52fdd' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('blobupload', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=255), nullable=False), + sa.Column('byte_count', sa.Integer(), nullable=False), + sa.Column('sha_state', sa.Text(), nullable=True), + sa.Column('location_id', sa.Integer(), nullable=False), + sa.Column('storage_metadata', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_blobupload_location_id_imagestoragelocation')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_blobupload_repository_id_repository')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_blobupload')) + ) + op.create_index('blobupload_location_id', 'blobupload', ['location_id'], unique=False) + op.create_index('blobupload_repository_id', 'blobupload', ['repository_id'], unique=False) + op.create_index('blobupload_repository_id_uuid', 'blobupload', ['repository_id', 'uuid'], unique=True) + op.create_index('blobupload_uuid', 'blobupload', ['uuid'], unique=True) + op.add_column(u'imagestorage', sa.Column('cas_path', sa.Boolean(), nullable=False, server_default="0")) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column(u'imagestorage', 'cas_path') + op.drop_table('blobupload') + ### end Alembic commands ### diff --git a/data/model/storage.py b/data/model/storage.py index 97b94ed4e..0ddbc8ac8 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -124,7 +124,7 @@ def garbage_collect_storage(storage_id_whitelist): def create_storage(location_name): - storage = ImageStorage.create() + storage = ImageStorage.create(cas_path=False) location = ImageStorageLocation.get(name=location_name) ImageStoragePlacement.create(location=location, storage=storage) storage.locations = {location_name}