Merge pull request #717 from coreos-inc/v2-phase4

V2 phase4

data/database.py

@@ -3,15 +3,16 @@ import logging
 import uuid
 import time
 import toposort
+import resumablehashlib
 
 from random import SystemRandom
 from datetime import datetime
 from peewee import *
 from data.read_slave import ReadSlaveModel
+from data.fields import ResumableSHAField, JSONField
 from sqlalchemy.engine.url import make_url
 from collections import defaultdict
 
-from data.read_slave import ReadSlaveModel
 from util.names import urn_generator
 
 
@@ -348,7 +349,7 @@ class Repository(BaseModel):
 
     # These models don't need to use transitive deletes, because the referenced objects
     # are cleaned up directly
-    skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger}
+    skip_transitive_deletes = {RepositoryTag, RepositoryBuild, RepositoryBuildTrigger, BlobUpload}
 
     # We need to sort the ops so that models get cleaned in order of their dependencies
     ops = reversed(list(self.dependencies(delete_nullable)))
@@ -490,6 +491,7 @@ class ImageStorage(BaseModel):
   image_size = BigIntegerField(null=True)
   uncompressed_size = BigIntegerField(null=True)
   uploading = BooleanField(default=True, null=True)
+  cas_path = BooleanField(default=True)
 
 
 class ImageStorageTransformation(BaseModel):
@@ -761,6 +763,23 @@ class RepositoryAuthorizedEmail(BaseModel):
     )
 
 
+class BlobUpload(BaseModel):
+  repository = ForeignKeyField(Repository, index=True)
+  uuid = CharField(index=True, unique=True)
+  byte_count = IntegerField(default=0)
+  sha_state = ResumableSHAField(null=True, default=resumablehashlib.sha256)
+  location = ForeignKeyField(ImageStorageLocation)
+  storage_metadata = JSONField(null=True, default={})
+
+  class Meta:
+    database = db
+    read_slaves = (read_slave,)
+    indexes = (
+      # create a unique index on email and repository
+      (('repository', 'uuid'), True),
+    )
+
+
 class QuayService(BaseModel):
   name = CharField(index=True, unique=True)
 
@@ -788,7 +807,6 @@ class QuayRelease(BaseModel):
     )
 
 
-
 all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission, Visibility,
               RepositoryTag, EmailConfirmation, FederatedLogin, LoginService, QueueItem,
               RepositoryBuild, Team, TeamMember, TeamRole, LogEntryKind, LogEntry,
@@ -799,4 +817,4 @@ all_models = [User, Repository, Image, AccessToken, Role, RepositoryPermission,
               RepositoryAuthorizedEmail, ImageStorageTransformation, DerivedImageStorage,
               TeamMemberInvite, ImageStorageSignature, ImageStorageSignatureKind,
               AccessTokenKind, Star, RepositoryActionCount, TagManifest, UserRegion,
-              QuayService, QuayRegion, QuayRelease]
+              QuayService, QuayRegion, QuayRelease, BlobUpload]

data/fields.py

@@ -0,0 +1,38 @@
+import base64
+import resumablehashlib
+import json
+
+from peewee import TextField
+
+
+class ResumableSHAField(TextField):
+  def db_value(self, value):
+    sha_state = value.state()
+
+    # One of the fields is a byte string, let's base64 encode it to make sure
+    # we can store and fetch it regardless of default collocation.
+    sha_state[3] = base64.b64encode(sha_state[3])
+
+    return json.dumps(sha_state)
+
+  def python_value(self, value):
+    to_resume = resumablehashlib.sha256()
+    if value is None:
+      return to_resume
+
+    sha_state = json.loads(value)
+
+    # We need to base64 decode the data bytestring.
+    sha_state[3] = base64.b64decode(sha_state[3])
+    to_resume.set_state(sha_state)
+    return to_resume
+
+
+class JSONField(TextField):
+  def db_value(self, value):
+    return json.dumps(value)
+
+  def python_value(self, value):
+    if value is None or value == "":
+      return {}
+    return json.loads(value)

data/migrations/migration.sh

@@ -108,6 +108,16 @@ test_migrate $MYSQL_CONFIG_OVERRIDE
 set -e
 down_mysql
 
+# Test via Postgres.
+echo '> Starting Postgres'
+up_postgres
+
+echo '> Testing Migration (postgres)'
+set +e
+test_migrate $PGSQL_CONFIG_OVERRIDE
+set -e
+down_postgres
+
 # Test via MariaDB.
 echo '> Starting MariaDB'
 up_mariadb
@@ -127,13 +137,3 @@ set +e
 test_migrate $PERCONA_CONFIG_OVERRIDE
 set -e
 down_percona
-
-# Test via Postgres.
-echo '> Starting Postgres'
-up_postgres
-
-echo '> Testing Migration (postgres)'
-set +e
-test_migrate $PGSQL_CONFIG_OVERRIDE
-set -e
-down_postgres

data/migrations/versions/127905a52fdd_remove_the_deprecated_imagestorage_.py

@@ -0,0 +1,31 @@
+"""Remove the deprecated imagestorage columns.
+
+Revision ID: 127905a52fdd
+Revises: 2e0380215d01
+Create Date: 2015-09-17 15:48:56.667823
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '127905a52fdd'
+down_revision = '2e0380215d01'
+
+from alembic import op
+import sqlalchemy as sa
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('imagestorage', 'comment')
+    op.drop_column('imagestorage', 'aggregate_size')
+    op.drop_column('imagestorage', 'command')
+    op.drop_column('imagestorage', 'created')
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('imagestorage', sa.Column('created', sa.DateTime(), nullable=True))
+    op.add_column('imagestorage', sa.Column('command', sa.Text(), nullable=True))
+    op.add_column('imagestorage', sa.Column('aggregate_size', sa.BigInteger(), nullable=True))
+    op.add_column('imagestorage', sa.Column('comment', sa.Text(), nullable=True))
+    ### end Alembic commands ###

data/migrations/versions/14fe12ade3df_add_build_queue_item_reference_to_the_.py

@@ -12,7 +12,6 @@ down_revision = '5ad999136045'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/1594a74a74ca_add_metadata_field_to_external_logins.py

@@ -12,7 +12,6 @@ down_revision = 'f42b0ea7a4d'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/17f11e265e13_add_uuid_field_to_user.py

@@ -12,7 +12,6 @@ down_revision = '313d297811c4'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 
 def upgrade(tables):

data/migrations/versions/1c5b738283a5_backfill_user_uuids.py

@@ -12,7 +12,6 @@ down_revision = '2fb36d4be80d'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 from util.migrate.backfill_user_uuids import backfill_user_uuids
 
 def upgrade(tables):

data/migrations/versions/1d2d86d09fcd_actually_remove_the_column.py

@@ -12,7 +12,6 @@ down_revision = '14fe12ade3df'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 from sqlalchemy.exc import InternalError
 
 def upgrade(tables):
@@ -29,7 +28,7 @@ def upgrade(tables):
 def downgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     try:
-        op.add_column('logentry', sa.Column('access_token_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True))
+        op.add_column('logentry', sa.Column('access_token_id', sa.Integer(), nullable=True))
         op.create_foreign_key(u'fk_logentry_access_token_id_accesstoken', 'logentry', 'accesstoken', ['access_token_id'], ['id'])
         op.create_index('logentry_access_token_id', 'logentry', ['access_token_id'], unique=False)
     except InternalError:

data/migrations/versions/201d55b38649_remove_fields_from_image_table_that_.py

@@ -12,7 +12,6 @@ down_revision = '5a07499ce53f'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/2088f2b81010_add_stars.py

@@ -12,7 +12,6 @@ down_revision = '707d5191eda'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     op.create_table('star',

data/migrations/versions/228d1af6af1c_mysql_max_index_lengths.py

@@ -12,7 +12,6 @@ down_revision = '5b84373e5db'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
   op.drop_index('queueitem_queue_name', table_name='queueitem')

data/migrations/versions/31288f79df53_make_resource_key_nullable.py

@@ -12,12 +12,11 @@ down_revision = '214350b6a8b1'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('repositorybuild', 'resource_key',
-               existing_type=mysql.VARCHAR(length=255),
+               existing_type=sa.String(length=255),
                nullable=True)
     ### end Alembic commands ###
 
@@ -25,6 +24,6 @@ def upgrade(tables):
 def downgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('repositorybuild', 'resource_key',
-               existing_type=mysql.VARCHAR(length=255),
+               existing_type=sa.String(length=255),
                nullable=False)
     ### end Alembic commands ###

data/migrations/versions/313d297811c4_add_an_index_to_the_docker_image_id_.py

@@ -12,7 +12,6 @@ down_revision = '204abf14783d'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/325a4d7c79d9_prepare_the_database_for_the_new_.py

@@ -12,7 +12,6 @@ down_revision = '4b7ef0c7bdb2'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/33bd39ef5ed6_backport_v2_db_changes.py

@@ -0,0 +1,43 @@
+"""Backport v2 db changes.
+
+Revision ID: 33bd39ef5ed6
+Revises: 127905a52fdd
+Create Date: 2015-10-23 12:34:22.776542
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '33bd39ef5ed6'
+down_revision = '127905a52fdd'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('blobupload',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('repository_id', sa.Integer(), nullable=False),
+    sa.Column('uuid', sa.String(length=255), nullable=False),
+    sa.Column('byte_count', sa.Integer(), nullable=False),
+    sa.Column('sha_state', sa.Text(), nullable=True),
+    sa.Column('location_id', sa.Integer(), nullable=False),
+    sa.Column('storage_metadata', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['location_id'], ['imagestoragelocation.id'], name=op.f('fk_blobupload_location_id_imagestoragelocation')),
+    sa.ForeignKeyConstraint(['repository_id'], ['repository.id'], name=op.f('fk_blobupload_repository_id_repository')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_blobupload'))
+    )
+    op.create_index('blobupload_location_id', 'blobupload', ['location_id'], unique=False)
+    op.create_index('blobupload_repository_id', 'blobupload', ['repository_id'], unique=False)
+    op.create_index('blobupload_repository_id_uuid', 'blobupload', ['repository_id', 'uuid'], unique=True)
+    op.create_index('blobupload_uuid', 'blobupload', ['uuid'], unique=True)
+    op.add_column(u'imagestorage', sa.Column('cas_path', sa.Boolean(), nullable=False, server_default="0"))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column(u'imagestorage', 'cas_path')
+    op.drop_table('blobupload')
+    ### end Alembic commands ###

data/migrations/versions/35f538da62_switch_manifest_text_to_a_longtext.py

@@ -0,0 +1,47 @@
+"""Switch manifest text to a longtext.
+
+Revision ID: 35f538da62
+Revises: 33bd39ef5ed6
+Create Date: 2015-10-23 15:31:27.353995
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '35f538da62'
+down_revision = '33bd39ef5ed6'
+
+from alembic import op
+import sqlalchemy as sa
+
+from sqlalchemy.types import TypeDecorator, Text
+from sqlalchemy.dialects.mysql import LONGTEXT
+import uuid
+
+class EngineLongText(TypeDecorator):
+    """Platform-independent LongText type.
+
+    Uses MySQL's LONGTEXT type, otherwise uses
+    Text, because other engines are not as limited
+    as MySQL.
+
+    """
+    impl = Text
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == 'mysql':
+            return dialect.type_descriptor(LONGTEXT())
+        else:
+            return dialect.type_descriptor(Text())
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column(u'tagmanifest', 'json_data')
+    op.add_column(u'tagmanifest', sa.Column('json_data', EngineLongText(), nullable=False))
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column(u'tagmanifest', 'json_data')
+    op.add_column(u'tagmanifest', sa.Column('json_data', sa.Text(), nullable=False))
+    ### end Alembic commands ###

data/migrations/versions/3fee6f979c2a_make_auth_token_nullable.py

@@ -12,12 +12,11 @@ down_revision = '31288f79df53'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('repositorybuildtrigger', 'auth_token',
-               existing_type=mysql.VARCHAR(length=255),
+               existing_type=sa.String(length=255),
                nullable=True)
     ### end Alembic commands ###
 
@@ -25,6 +24,6 @@ def upgrade(tables):
 def downgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('repositorybuildtrigger', 'auth_token',
-               existing_type=mysql.VARCHAR(length=255),
+               existing_type=sa.String(length=255),
                nullable=False)
     ### end Alembic commands ###

data/migrations/versions/4a0c94399f38_add_new_notification_kinds.py

@@ -12,7 +12,6 @@ down_revision = '1594a74a74ca'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     op.bulk_insert(tables.externalnotificationmethod,

data/migrations/versions/4fdb65816b8d_add_brute_force_prevention_metadata_to_.py

@@ -12,7 +12,6 @@ down_revision = '43e943c0639f'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/5ad999136045_add_signature_storage.py

@@ -12,7 +12,6 @@ down_revision = '228d1af6af1c'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/migrations/versions/707d5191eda_change_build_queue_reference_from_.py

@@ -12,7 +12,6 @@ down_revision = '4ef04c61fcf9'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
@@ -26,7 +25,7 @@ def upgrade(tables):
 
 def downgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###
-    op.add_column('repositorybuild', sa.Column('queue_item_id', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True))
+    op.add_column('repositorybuild', sa.Column('queue_item_id', sa.Integer(), autoincrement=False, nullable=True))
     op.create_foreign_key(u'fk_repositorybuild_queue_item_id_queueitem', 'repositorybuild', 'queueitem', ['queue_item_id'], ['id'])
     op.create_index('repositorybuild_queue_item_id', 'repositorybuild', ['queue_item_id'], unique=False)
     op.drop_index('repositorybuild_queue_id', table_name='repositorybuild')

data/migrations/versions/82297d834ad_add_us_west_location.py

@@ -12,7 +12,6 @@ down_revision = '47670cbeced'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     op.bulk_insert(tables.imagestoragelocation,

data/migrations/versions/f42b0ea7a4d_remove_the_old_webhooks_table.py

@@ -12,7 +12,6 @@ down_revision = '4fdb65816b8d'
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.dialects import mysql
 
 def upgrade(tables):
     ### commands auto generated by Alembic - please adjust! ###

data/model/storage.py

@@ -124,7 +124,7 @@ def garbage_collect_storage(storage_id_whitelist):
 
 
 def create_storage(location_name):
-  storage = ImageStorage.create()
+  storage = ImageStorage.create(cas_path=False)
   location = ImageStorageLocation.get(name=location_name)
   ImageStoragePlacement.create(location=location, storage=storage)
   storage.locations = {location_name}