diff --git a/data/database.py b/data/database.py index 765a4948e..cc0beafa6 100644 --- a/data/database.py +++ b/data/database.py @@ -3,15 +3,16 @@ import logging import uuid import time import toposort +import resumablehashlib from random import SystemRandom from datetime import datetime from peewee import * from data.read_slave import ReadSlaveModel +from data.fields import ResumableSHAField, JSONField from sqlalchemy.engine.url import make_url from collections import defaultdict -from data.read_slave import ReadSlaveModel from util.names import urn_generator @@ -348,7 +349,7 @@ class Repository(BaseModel): # These models don't need to use transitive deletes, because the referenced objects # are cleaned up directly - skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger} + skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload} # We need to sort the ops so that models get cleaned in order of their dependencies ops = reversed(list(self.dependencies(delete_nullable))) @@ -490,6 +491,7 @@ class ImageStorage(BaseModel): image_size = BigIntegerField(null=True) uncompressed_size = BigIntegerField(null=True) uploading = BooleanField(default=True, null=True) + cas_path = BooleanField(default=True) class ImageStorageTransformation(BaseModel): @@ -761,6 +763,23 @@ class RepositoryAuthorizedEmail(BaseModel): ) +class BlobUpload(BaseModel): + repository = ForeignKeyField(Repository, index=True) + uuid = CharField(index=True, unique=True) + byte_count = IntegerField(default=0) + sha_state = ResumableSHAField(null=True, default=resumablehashlib.sha256) + location = ForeignKeyField(ImageStorageLocation) + storage_metadata = JSONField(null=True, default={}) + + class Meta: + database = db + read_slaves = (read_slave,) + indexes = ( + # create a unique index on email and repository + (('repository', 'uuid'), True), + ) + + class QuayService(BaseModel): name = CharField(index=True, unique=True) @@ -788,7 +807,6 @@ class QuayRelease(BaseModel): ) - all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem, RepositoryBuild, Team, TeamMember, TeamRole, LogEntryKind, LogEntry, @@ -799,4 +817,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage, TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind, AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion, - QuayService, QuayRegion, QuayRelease] + QuayService, QuayRegion, QuayRelease, BlobUpload] diff --git a/data/fields.py b/data/fields.py new file mode 100644 index 000000000..123811ccd --- /dev/null +++ b/data/fields.py @@ -0,0 +1,38 @@ +import base64 +import resumablehashlib +import json + +from peewee import TextField + + +class ResumableSHAField(TextField): + def db_value(self, value): + sha_state = value.state() + + # One of the fields is a byte string, let's base64 encode it to make sure + # we can store and fetch it regardless of default collocation. + sha_state[3] = base64.b64encode(sha_state[3]) + + return json.dumps(sha_state) + + def python_value(self, value): + to_resume = resumablehashlib.sha256() + if value is None: + return to_resume + + sha_state = json.loads(value) + + # We need to base64 decode the data bytestring. + sha_state[3] = base64.b64decode(sha_state[3]) + to_resume.set_state(sha_state) + return to_resume + + +class JSONField(TextField): + def db_value(self, value): + return json.dumps(value) + + def python_value(self, value): + if value is None or value == "": + return {} + return json.loads(value) diff --git a/data/migrations/migration.sh b/data/migrations/migration.sh index 9d20c5a6a..65521f6a6 100755 --- a/data/migrations/migration.sh +++ b/data/migrations/migration.sh @@ -108,6 +108,16 @@ test_migrate $MYSQL_CONFIG_OVERRIDE set -e down_mysql +# Test via Postgres. +echo '> Starting Postgres' +up_postgres + +echo '> Testing Migration (postgres)' +set +e +test_migrate $PGSQL_CONFIG_OVERRIDE +set -e +down_postgres + # Test via MariaDB. echo '> Starting MariaDB' up_mariadb @@ -127,13 +137,3 @@ set +e test_migrate $PERCONA_CONFIG_OVERRIDE set -e down_percona - -# Test via Postgres. -echo '> Starting Postgres' -up_postgres - -echo '> Testing Migration (postgres)' -set +e -test_migrate $PGSQL_CONFIG_OVERRIDE -set -e -down_postgres diff --git a/data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py b/data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py new file mode 100644 index 000000000..6ab6d79b7 --- /dev/null +++ b/data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py @@ -0,0 +1,31 @@ +"""Remove the deprecated imagestorage columns. + +Revision ID: 127905a52fdd +Revises: 2e0380215d01 +Create Date: 2015-09-17 15:48:56.667823 + +""" + +# revision identifiers, used by Alembic. +revision = '127905a52fdd' +down_revision = '2e0380215d01' + +from alembic import op +import sqlalchemy as sa + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('imagestorage', 'comment') + op.drop_column('imagestorage', 'aggregate_size') + op.drop_column('imagestorage', 'command') + op.drop_column('imagestorage', 'created') + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('imagestorage', sa.Column('created', sa.DateTime(), nullable=True)) + op.add_column('imagestorage', sa.Column('command', sa.Text(), nullable=True)) + op.add_column('imagestorage', sa.Column('aggregate_size', sa.BigInteger(), nullable=True)) + op.add_column('imagestorage', sa.Column('comment', sa.Text(), nullable=True)) + ### end Alembic commands ### diff --git a/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py b/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py index 5e8d21211..561a32dca 100644 --- a/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py +++ b/data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py @@ -12,7 +12,6 @@ down_revision = '5ad999136045' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py index 2f6c60706..e4276effe 100644 --- a/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py +++ b/data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py @@ -12,7 +12,6 @@ down_revision = 'f42b0ea7a4d' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py index 9371941f8..3bf692fe6 100644 --- a/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py +++ b/data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py @@ -12,7 +12,6 @@ down_revision = '313d297811c4' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): diff --git a/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py b/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py index 44ea6f5ec..baa78465b 100644 --- a/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py +++ b/data/migrations/versions/1c5b738283a5_backfill_user_uuids.py @@ -12,7 +12,6 @@ down_revision = '2fb36d4be80d' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql from util.migrate.backfill_user_uuids import backfill_user_uuids def upgrade(tables): diff --git a/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py b/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py index a7942b7d4..460296f17 100644 --- a/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py +++ b/data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py @@ -12,7 +12,6 @@ down_revision = '14fe12ade3df' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql from sqlalchemy.exc import InternalError def upgrade(tables): @@ -29,7 +28,7 @@ def upgrade(tables): def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### try: - op.add_column('logentry', sa.Column('access_token_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + op.add_column('logentry', sa.Column('access_token_id', sa.Integer(), nullable=True)) op.create_foreign_key(u'fk_logentry_access_token_id_accesstoken', 'logentry', 'accesstoken', ['access_token_id'], ['id']) op.create_index('logentry_access_token_id', 'logentry', ['access_token_id'], unique=False) except InternalError: diff --git a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py index 8185c1118..02a119074 100644 --- a/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py +++ b/data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py @@ -12,7 +12,6 @@ down_revision = '5a07499ce53f' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/2088f2b81010_add_stars.py b/data/migrations/versions/2088f2b81010_add_stars.py index ad4ccdf2b..af27da83e 100644 --- a/data/migrations/versions/2088f2b81010_add_stars.py +++ b/data/migrations/versions/2088f2b81010_add_stars.py @@ -12,7 +12,6 @@ down_revision = '707d5191eda' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): op.create_table('star', diff --git a/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py index 2f6ff722b..ed7fdc8be 100644 --- a/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py +++ b/data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py @@ -12,7 +12,6 @@ down_revision = '5b84373e5db' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): op.drop_index('queueitem_queue_name', table_name='queueitem') diff --git a/data/migrations/versions/31288f79df53_make_resource_key_nullable.py b/data/migrations/versions/31288f79df53_make_resource_key_nullable.py index e52795ce6..e14dfaca1 100644 --- a/data/migrations/versions/31288f79df53_make_resource_key_nullable.py +++ b/data/migrations/versions/31288f79df53_make_resource_key_nullable.py @@ -12,12 +12,11 @@ down_revision = '214350b6a8b1' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.alter_column('repositorybuild', 'resource_key', - existing_type=mysql.VARCHAR(length=255), + existing_type=sa.String(length=255), nullable=True) ### end Alembic commands ### @@ -25,6 +24,6 @@ def upgrade(tables): def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.alter_column('repositorybuild', 'resource_key', - existing_type=mysql.VARCHAR(length=255), + existing_type=sa.String(length=255), nullable=False) ### end Alembic commands ### diff --git a/data/migrations/versions/313d297811c4_add_an_index_to_the_docker_image_id_.py b/data/migrations/versions/313d297811c4_add_an_index_to_the_docker_image_id_.py index 2ed6bd2f5..3987fe2cc 100644 --- a/data/migrations/versions/313d297811c4_add_an_index_to_the_docker_image_id_.py +++ b/data/migrations/versions/313d297811c4_add_an_index_to_the_docker_image_id_.py @@ -12,7 +12,6 @@ down_revision = '204abf14783d' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py index d6bdcb35e..c11199a67 100644 --- a/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py +++ b/data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py @@ -12,7 +12,6 @@ down_revision = '4b7ef0c7bdb2' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py b/data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py new file mode 100644 index 000000000..c63924c93 --- /dev/null +++ b/data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py @@ -0,0 +1,43 @@ +"""Backport v2 db changes. + +Revision ID: 33bd39ef5ed6 +Revises: 127905a52fdd +Create Date: 2015-10-23 12:34:22.776542 + +""" + +# revision identifiers, used by Alembic. +revision = '33bd39ef5ed6' +down_revision = '127905a52fdd' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('blobupload', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('repository_id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=255), nullable=False), + sa.Column('byte_count', sa.Integer(), nullable=False), + sa.Column('sha_state', sa.Text(), nullable=True), + sa.Column('location_id', sa.Integer(), nullable=False), + sa.Column('storage_metadata', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_blobupload_location_id_imagestoragelocation')), + sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_blobupload_repository_id_repository')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_blobupload')) + ) + op.create_index('blobupload_location_id', 'blobupload', ['location_id'], unique=False) + op.create_index('blobupload_repository_id', 'blobupload', ['repository_id'], unique=False) + op.create_index('blobupload_repository_id_uuid', 'blobupload', ['repository_id', 'uuid'], unique=True) + op.create_index('blobupload_uuid', 'blobupload', ['uuid'], unique=True) + op.add_column(u'imagestorage', sa.Column('cas_path', sa.Boolean(), nullable=False, server_default="0")) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column(u'imagestorage', 'cas_path') + op.drop_table('blobupload') + ### end Alembic commands ### diff --git a/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py b/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py new file mode 100644 index 000000000..f11b5336a --- /dev/null +++ b/data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py @@ -0,0 +1,47 @@ +"""Switch manifest text to a longtext. + +Revision ID: 35f538da62 +Revises: 33bd39ef5ed6 +Create Date: 2015-10-23 15:31:27.353995 + +""" + +# revision identifiers, used by Alembic. +revision = '35f538da62' +down_revision = '33bd39ef5ed6' + +from alembic import op +import sqlalchemy as sa + +from sqlalchemy.types import TypeDecorator, Text +from sqlalchemy.dialects.mysql import LONGTEXT +import uuid + +class EngineLongText(TypeDecorator): + """Platform-independent LongText type. + + Uses MySQL's LONGTEXT type, otherwise uses + Text, because other engines are not as limited + as MySQL. + + """ + impl = Text + + def load_dialect_impl(self, dialect): + if dialect.name == 'mysql': + return dialect.type_descriptor(LONGTEXT()) + else: + return dialect.type_descriptor(Text()) + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column(u'tagmanifest', 'json_data') + op.add_column(u'tagmanifest', sa.Column('json_data', EngineLongText(), nullable=False)) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column(u'tagmanifest', 'json_data') + op.add_column(u'tagmanifest', sa.Column('json_data', sa.Text(), nullable=False)) + ### end Alembic commands ### diff --git a/data/migrations/versions/3fee6f979c2a_make_auth_token_nullable.py b/data/migrations/versions/3fee6f979c2a_make_auth_token_nullable.py index 2574271ef..04379eb60 100644 --- a/data/migrations/versions/3fee6f979c2a_make_auth_token_nullable.py +++ b/data/migrations/versions/3fee6f979c2a_make_auth_token_nullable.py @@ -12,12 +12,11 @@ down_revision = '31288f79df53' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.alter_column('repositorybuildtrigger', 'auth_token', - existing_type=mysql.VARCHAR(length=255), + existing_type=sa.String(length=255), nullable=True) ### end Alembic commands ### @@ -25,6 +24,6 @@ def upgrade(tables): def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### op.alter_column('repositorybuildtrigger', 'auth_token', - existing_type=mysql.VARCHAR(length=255), + existing_type=sa.String(length=255), nullable=False) ### end Alembic commands ### diff --git a/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py b/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py index 6b4160b19..efd3d1c60 100644 --- a/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py +++ b/data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py @@ -12,7 +12,6 @@ down_revision = '1594a74a74ca' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): op.bulk_insert(tables.externalnotificationmethod, diff --git a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py index bc8373655..42afef28f 100644 --- a/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py +++ b/data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py @@ -12,7 +12,6 @@ down_revision = '43e943c0639f' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/5ad999136045_add_signature_storage.py b/data/migrations/versions/5ad999136045_add_signature_storage.py index f306c58b8..210b91175 100644 --- a/data/migrations/versions/5ad999136045_add_signature_storage.py +++ b/data/migrations/versions/5ad999136045_add_signature_storage.py @@ -12,7 +12,6 @@ down_revision = '228d1af6af1c' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/migrations/versions/707d5191eda_change_build_queue_reference_from_.py b/data/migrations/versions/707d5191eda_change_build_queue_reference_from_.py index 9b2110df7..dc8f88087 100644 --- a/data/migrations/versions/707d5191eda_change_build_queue_reference_from_.py +++ b/data/migrations/versions/707d5191eda_change_build_queue_reference_from_.py @@ -12,7 +12,6 @@ down_revision = '4ef04c61fcf9' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### @@ -26,7 +25,7 @@ def upgrade(tables): def downgrade(tables): ### commands auto generated by Alembic - please adjust! ### - op.add_column('repositorybuild', sa.Column('queue_item_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + op.add_column('repositorybuild', sa.Column('queue_item_id', sa.Integer(), autoincrement=False, nullable=True)) op.create_foreign_key(u'fk_repositorybuild_queue_item_id_queueitem', 'repositorybuild', 'queueitem', ['queue_item_id'], ['id']) op.create_index('repositorybuild_queue_item_id', 'repositorybuild', ['queue_item_id'], unique=False) op.drop_index('repositorybuild_queue_id', table_name='repositorybuild') diff --git a/data/migrations/versions/82297d834ad_add_us_west_location.py b/data/migrations/versions/82297d834ad_add_us_west_location.py index b939a939e..33a543062 100644 --- a/data/migrations/versions/82297d834ad_add_us_west_location.py +++ b/data/migrations/versions/82297d834ad_add_us_west_location.py @@ -12,7 +12,6 @@ down_revision = '47670cbeced' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): op.bulk_insert(tables.imagestoragelocation, diff --git a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py index 5b3f6c812..e36586a09 100644 --- a/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py +++ b/data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py @@ -12,7 +12,6 @@ down_revision = '4fdb65816b8d' from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects import mysql def upgrade(tables): ### commands auto generated by Alembic - please adjust! ### diff --git a/data/model/storage.py b/data/model/storage.py index 97b94ed4e..0ddbc8ac8 100644 --- a/data/model/storage.py +++ b/data/model/storage.py @@ -124,7 +124,7 @@ def garbage_collect_storage(storage_id_whitelist): def create_storage(location_name): - storage = ImageStorage.create() + storage = ImageStorage.create(cas_path=False) location = ImageStorageLocation.get(name=location_name) ImageStoragePlacement.create(location=location, storage=storage) storage.locations = {location_name}