Fix the encrypted token migration issue encountered on HEAD
This change ensures there is better messaging around the encrypted token migration, including a new phase to use for new installations, and fixes an issue encountered when running database migrations for new installations
This commit is contained in:
Joseph Schorr
parent
a0f7c4f396
commit
a54fb1b23a
7 changed files with 194 additions and 163 deletions
|
|
@ -2,7 +2,7 @@ import json
|
|||
import logging
|
||||
import uuid
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from abc import ABCMeta, abstractmethod, abstractproperty
|
||||
from datetime import datetime
|
||||
from six import add_metaclass
|
||||
|
||||
|
|
@ -92,6 +92,10 @@ class MigrationTester(object):
|
|||
"""
|
||||
TestDataType = DataTypes
|
||||
|
||||
@abstractproperty
|
||||
def is_testing(self):
|
||||
""" Returns whether we are currently under a migration test. """
|
||||
|
||||
@abstractmethod
|
||||
def populate_table(self, table_name, fields):
|
||||
""" Called to populate a table with the given fields filled in with testing data. """
|
||||
|
|
@ -107,6 +111,10 @@ class NoopTester(MigrationTester):
|
|||
|
||||
|
||||
class PopulateTestDataTester(MigrationTester):
|
||||
@property
|
||||
def is_testing(self):
|
||||
return True
|
||||
|
||||
def populate_table(self, table_name, fields):
|
||||
columns = {field_name: field_type() for field_name, field_type in fields}
|
||||
field_name_vars = [':' + field_name for field_name, _ in fields]
|
||||
|
|
|
|||
|
|
@ -80,11 +80,13 @@ def upgrade(tables, tester, progress_reporter):
|
|||
|
||||
op.add_column('repomirrorconfig', sa.Column('external_reference', sa.Text(), nullable=True))
|
||||
|
||||
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_reference >> None)):
|
||||
repo = '%s/%s/%s' % (repo_mirror.external_registry, repo_mirror.external_namespace, repo_mirror.external_repository)
|
||||
logger.info('migrating %s' % repo)
|
||||
repo_mirror.external_reference = repo
|
||||
repo_mirror.save()
|
||||
from app import app
|
||||
if app.config.get('SETUP_COMPLETE', False) or tester.is_testing:
|
||||
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_reference >> None)):
|
||||
repo = '%s/%s/%s' % (repo_mirror.external_registry, repo_mirror.external_namespace, repo_mirror.external_repository)
|
||||
logger.info('migrating %s' % repo)
|
||||
repo_mirror.external_reference = repo
|
||||
repo_mirror.save()
|
||||
|
||||
op.drop_column('repomirrorconfig', 'external_registry')
|
||||
op.drop_column('repomirrorconfig', 'external_namespace')
|
||||
|
|
@ -109,14 +111,16 @@ def downgrade(tables, tester, progress_reporter):
|
|||
op.add_column('repomirrorconfig', sa.Column('external_namespace', sa.String(length=255), nullable=True))
|
||||
op.add_column('repomirrorconfig', sa.Column('external_repository', sa.String(length=255), nullable=True))
|
||||
|
||||
logger.info('Restoring columns from external_reference')
|
||||
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_registry >> None)):
|
||||
logger.info('Restoring %s' % repo_mirror.external_reference)
|
||||
parts = repo_mirror.external_reference.split('/', 2)
|
||||
repo_mirror.external_registry = parts[0] if len(parts) >= 1 else 'DOWNGRADE-FAILED'
|
||||
repo_mirror.external_namespace = parts[1] if len(parts) >= 2 else 'DOWNGRADE-FAILED'
|
||||
repo_mirror.external_repository = parts[2] if len(parts) >= 3 else 'DOWNGRADE-FAILED'
|
||||
repo_mirror.save()
|
||||
from app import app
|
||||
if app.config.get('SETUP_COMPLETE', False):
|
||||
logger.info('Restoring columns from external_reference')
|
||||
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_registry >> None)):
|
||||
logger.info('Restoring %s' % repo_mirror.external_reference)
|
||||
parts = repo_mirror.external_reference.split('/', 2)
|
||||
repo_mirror.external_registry = parts[0] if len(parts) >= 1 else 'DOWNGRADE-FAILED'
|
||||
repo_mirror.external_namespace = parts[1] if len(parts) >= 2 else 'DOWNGRADE-FAILED'
|
||||
repo_mirror.external_repository = parts[2] if len(parts) >= 3 else 'DOWNGRADE-FAILED'
|
||||
repo_mirror.save()
|
||||
|
||||
op.drop_column('repomirrorconfig', 'external_reference')
|
||||
|
||||
|
|
|
|||
|
|
@ -98,157 +98,159 @@ class OAuthApplication(BaseModel):
|
|||
def upgrade(tables, tester, progress_reporter):
|
||||
op = ProgressWrapper(original_op, progress_reporter)
|
||||
|
||||
# Empty all access token names to fix the bug where we put the wrong name and code
|
||||
# in for some tokens.
|
||||
AccessToken.update(token_name=None).where(AccessToken.token_name >> None).execute()
|
||||
from app import app
|
||||
if app.config.get('SETUP_COMPLETE', False) or tester.is_testing:
|
||||
# Empty all access token names to fix the bug where we put the wrong name and code
|
||||
# in for some tokens.
|
||||
AccessToken.update(token_name=None).where(AccessToken.token_name >> None).execute()
|
||||
|
||||
# AccessToken.
|
||||
logger.info('Backfilling encrypted credentials for access tokens')
|
||||
for access_token in _iterate(AccessToken, ((AccessToken.token_name >> None) |
|
||||
(AccessToken.token_name == ''))):
|
||||
logger.info('Backfilling encrypted credentials for access token %s', access_token.id)
|
||||
assert access_token.code is not None
|
||||
assert access_token.code[:ACCESS_TOKEN_NAME_PREFIX_LENGTH]
|
||||
assert access_token.code[ACCESS_TOKEN_NAME_PREFIX_LENGTH:]
|
||||
# AccessToken.
|
||||
logger.info('Backfilling encrypted credentials for access tokens')
|
||||
for access_token in _iterate(AccessToken, ((AccessToken.token_name >> None) |
|
||||
(AccessToken.token_name == ''))):
|
||||
logger.info('Backfilling encrypted credentials for access token %s', access_token.id)
|
||||
assert access_token.code is not None
|
||||
assert access_token.code[:ACCESS_TOKEN_NAME_PREFIX_LENGTH]
|
||||
assert access_token.code[ACCESS_TOKEN_NAME_PREFIX_LENGTH:]
|
||||
|
||||
token_name = access_token.code[:ACCESS_TOKEN_NAME_PREFIX_LENGTH]
|
||||
token_code = _decrypted(access_token.code[ACCESS_TOKEN_NAME_PREFIX_LENGTH:])
|
||||
token_name = access_token.code[:ACCESS_TOKEN_NAME_PREFIX_LENGTH]
|
||||
token_code = _decrypted(access_token.code[ACCESS_TOKEN_NAME_PREFIX_LENGTH:])
|
||||
|
||||
(AccessToken
|
||||
.update(token_name=token_name, token_code=token_code)
|
||||
.where(AccessToken.id == access_token.id, AccessToken.code == access_token.code)
|
||||
.execute())
|
||||
(AccessToken
|
||||
.update(token_name=token_name, token_code=token_code)
|
||||
.where(AccessToken.id == access_token.id, AccessToken.code == access_token.code)
|
||||
.execute())
|
||||
|
||||
assert AccessToken.select().where(AccessToken.token_name >> None).count() == 0
|
||||
assert AccessToken.select().where(AccessToken.token_name >> None).count() == 0
|
||||
|
||||
# Robots.
|
||||
logger.info('Backfilling encrypted credentials for robots')
|
||||
while True:
|
||||
has_row = False
|
||||
query = (User
|
||||
.select()
|
||||
.join(RobotAccountToken, JOIN.LEFT_OUTER)
|
||||
.where(User.robot == True, RobotAccountToken.id >> None)
|
||||
.limit(BATCH_SIZE))
|
||||
# Robots.
|
||||
logger.info('Backfilling encrypted credentials for robots')
|
||||
while True:
|
||||
has_row = False
|
||||
query = (User
|
||||
.select()
|
||||
.join(RobotAccountToken, JOIN.LEFT_OUTER)
|
||||
.where(User.robot == True, RobotAccountToken.id >> None)
|
||||
.limit(BATCH_SIZE))
|
||||
|
||||
for robot_user in query:
|
||||
logger.info('Backfilling encrypted credentials for robot %s', robot_user.id)
|
||||
has_row = True
|
||||
try:
|
||||
RobotAccountToken.create(robot_account=robot_user,
|
||||
token=_decrypted(robot_user.email),
|
||||
fully_migrated=False)
|
||||
except IntegrityError:
|
||||
for robot_user in query:
|
||||
logger.info('Backfilling encrypted credentials for robot %s', robot_user.id)
|
||||
has_row = True
|
||||
try:
|
||||
RobotAccountToken.create(robot_account=robot_user,
|
||||
token=_decrypted(robot_user.email),
|
||||
fully_migrated=False)
|
||||
except IntegrityError:
|
||||
break
|
||||
|
||||
if not has_row:
|
||||
break
|
||||
|
||||
if not has_row:
|
||||
break
|
||||
# RepositoryBuildTrigger
|
||||
logger.info('Backfilling encrypted credentials for repo build triggers')
|
||||
for repo_build_trigger in _iterate(RepositoryBuildTrigger,
|
||||
(RepositoryBuildTrigger.fully_migrated == False)):
|
||||
logger.info('Backfilling encrypted credentials for repo build trigger %s',
|
||||
repo_build_trigger.id)
|
||||
|
||||
# RepositoryBuildTrigger
|
||||
logger.info('Backfilling encrypted credentials for repo build triggers')
|
||||
for repo_build_trigger in _iterate(RepositoryBuildTrigger,
|
||||
(RepositoryBuildTrigger.fully_migrated == False)):
|
||||
logger.info('Backfilling encrypted credentials for repo build trigger %s',
|
||||
repo_build_trigger.id)
|
||||
(RepositoryBuildTrigger
|
||||
.update(secure_auth_token=_decrypted(repo_build_trigger.auth_token),
|
||||
secure_private_key=_decrypted(repo_build_trigger.private_key),
|
||||
fully_migrated=True)
|
||||
.where(RepositoryBuildTrigger.id == repo_build_trigger.id,
|
||||
RepositoryBuildTrigger.uuid == repo_build_trigger.uuid)
|
||||
.execute())
|
||||
|
||||
(RepositoryBuildTrigger
|
||||
.update(secure_auth_token=_decrypted(repo_build_trigger.auth_token),
|
||||
secure_private_key=_decrypted(repo_build_trigger.private_key),
|
||||
fully_migrated=True)
|
||||
.where(RepositoryBuildTrigger.id == repo_build_trigger.id,
|
||||
RepositoryBuildTrigger.uuid == repo_build_trigger.uuid)
|
||||
.execute())
|
||||
assert (RepositoryBuildTrigger
|
||||
.select()
|
||||
.where(RepositoryBuildTrigger.fully_migrated == False)
|
||||
.count()) == 0
|
||||
|
||||
assert (RepositoryBuildTrigger
|
||||
.select()
|
||||
.where(RepositoryBuildTrigger.fully_migrated == False)
|
||||
.count()) == 0
|
||||
# AppSpecificAuthToken
|
||||
logger.info('Backfilling encrypted credentials for app specific auth tokens')
|
||||
for token in _iterate(AppSpecificAuthToken, ((AppSpecificAuthToken.token_name >> None) |
|
||||
(AppSpecificAuthToken.token_name == '') |
|
||||
(AppSpecificAuthToken.token_secret >> None))):
|
||||
logger.info('Backfilling encrypted credentials for app specific auth %s',
|
||||
token.id)
|
||||
assert token.token_code[AST_TOKEN_NAME_PREFIX_LENGTH:]
|
||||
|
||||
# AppSpecificAuthToken
|
||||
logger.info('Backfilling encrypted credentials for app specific auth tokens')
|
||||
for token in _iterate(AppSpecificAuthToken, ((AppSpecificAuthToken.token_name >> None) |
|
||||
(AppSpecificAuthToken.token_name == '') |
|
||||
(AppSpecificAuthToken.token_secret >> None))):
|
||||
logger.info('Backfilling encrypted credentials for app specific auth %s',
|
||||
token.id)
|
||||
assert token.token_code[AST_TOKEN_NAME_PREFIX_LENGTH:]
|
||||
token_name = token.token_code[:AST_TOKEN_NAME_PREFIX_LENGTH]
|
||||
token_secret = _decrypted(token.token_code[AST_TOKEN_NAME_PREFIX_LENGTH:])
|
||||
assert token_name
|
||||
assert token_secret
|
||||
|
||||
token_name = token.token_code[:AST_TOKEN_NAME_PREFIX_LENGTH]
|
||||
token_secret = _decrypted(token.token_code[AST_TOKEN_NAME_PREFIX_LENGTH:])
|
||||
assert token_name
|
||||
assert token_secret
|
||||
(AppSpecificAuthToken
|
||||
.update(token_name=token_name,
|
||||
token_secret=token_secret)
|
||||
.where(AppSpecificAuthToken.id == token.id,
|
||||
AppSpecificAuthToken.token_code == token.token_code)
|
||||
.execute())
|
||||
|
||||
(AppSpecificAuthToken
|
||||
.update(token_name=token_name,
|
||||
token_secret=token_secret)
|
||||
.where(AppSpecificAuthToken.id == token.id,
|
||||
AppSpecificAuthToken.token_code == token.token_code)
|
||||
.execute())
|
||||
assert (AppSpecificAuthToken
|
||||
.select()
|
||||
.where(AppSpecificAuthToken.token_name >> None)
|
||||
.count()) == 0
|
||||
|
||||
assert (AppSpecificAuthToken
|
||||
.select()
|
||||
.where(AppSpecificAuthToken.token_name >> None)
|
||||
.count()) == 0
|
||||
# OAuthAccessToken
|
||||
logger.info('Backfilling credentials for OAuth access tokens')
|
||||
for token in _iterate(OAuthAccessToken, ((OAuthAccessToken.token_name >> None) |
|
||||
(OAuthAccessToken.token_name == ''))):
|
||||
logger.info('Backfilling credentials for OAuth access token %s', token.id)
|
||||
token_name = token.access_token[:OAUTH_ACCESS_TOKEN_PREFIX_LENGTH]
|
||||
token_code = Credential.from_string(token.access_token[OAUTH_ACCESS_TOKEN_PREFIX_LENGTH:])
|
||||
assert token_name
|
||||
assert token.access_token[OAUTH_ACCESS_TOKEN_PREFIX_LENGTH:]
|
||||
|
||||
# OAuthAccessToken
|
||||
logger.info('Backfilling credentials for OAuth access tokens')
|
||||
for token in _iterate(OAuthAccessToken, ((OAuthAccessToken.token_name >> None) |
|
||||
(OAuthAccessToken.token_name == ''))):
|
||||
logger.info('Backfilling credentials for OAuth access token %s', token.id)
|
||||
token_name = token.access_token[:OAUTH_ACCESS_TOKEN_PREFIX_LENGTH]
|
||||
token_code = Credential.from_string(token.access_token[OAUTH_ACCESS_TOKEN_PREFIX_LENGTH:])
|
||||
assert token_name
|
||||
assert token.access_token[OAUTH_ACCESS_TOKEN_PREFIX_LENGTH:]
|
||||
(OAuthAccessToken
|
||||
.update(token_name=token_name,
|
||||
token_code=token_code)
|
||||
.where(OAuthAccessToken.id == token.id,
|
||||
OAuthAccessToken.access_token == token.access_token)
|
||||
.execute())
|
||||
|
||||
(OAuthAccessToken
|
||||
.update(token_name=token_name,
|
||||
token_code=token_code)
|
||||
.where(OAuthAccessToken.id == token.id,
|
||||
OAuthAccessToken.access_token == token.access_token)
|
||||
.execute())
|
||||
assert (OAuthAccessToken
|
||||
.select()
|
||||
.where(OAuthAccessToken.token_name >> None)
|
||||
.count()) == 0
|
||||
|
||||
assert (OAuthAccessToken
|
||||
.select()
|
||||
.where(OAuthAccessToken.token_name >> None)
|
||||
.count()) == 0
|
||||
# OAuthAuthorizationCode
|
||||
logger.info('Backfilling credentials for OAuth auth code')
|
||||
for code in _iterate(OAuthAuthorizationCode, ((OAuthAuthorizationCode.code_name >> None) |
|
||||
(OAuthAuthorizationCode.code_name == ''))):
|
||||
logger.info('Backfilling credentials for OAuth auth code %s', code.id)
|
||||
user_code = code.code or random_string_generator(AUTHORIZATION_CODE_PREFIX_LENGTH * 2)()
|
||||
code_name = user_code[:AUTHORIZATION_CODE_PREFIX_LENGTH]
|
||||
code_credential = Credential.from_string(user_code[AUTHORIZATION_CODE_PREFIX_LENGTH:])
|
||||
assert code_name
|
||||
assert user_code[AUTHORIZATION_CODE_PREFIX_LENGTH:]
|
||||
|
||||
# OAuthAuthorizationCode
|
||||
logger.info('Backfilling credentials for OAuth auth code')
|
||||
for code in _iterate(OAuthAuthorizationCode, ((OAuthAuthorizationCode.code_name >> None) |
|
||||
(OAuthAuthorizationCode.code_name == ''))):
|
||||
logger.info('Backfilling credentials for OAuth auth code %s', code.id)
|
||||
user_code = code.code or random_string_generator(AUTHORIZATION_CODE_PREFIX_LENGTH * 2)()
|
||||
code_name = user_code[:AUTHORIZATION_CODE_PREFIX_LENGTH]
|
||||
code_credential = Credential.from_string(user_code[AUTHORIZATION_CODE_PREFIX_LENGTH:])
|
||||
assert code_name
|
||||
assert user_code[AUTHORIZATION_CODE_PREFIX_LENGTH:]
|
||||
(OAuthAuthorizationCode
|
||||
.update(code_name=code_name, code_credential=code_credential)
|
||||
.where(OAuthAuthorizationCode.id == code.id)
|
||||
.execute())
|
||||
|
||||
(OAuthAuthorizationCode
|
||||
.update(code_name=code_name, code_credential=code_credential)
|
||||
.where(OAuthAuthorizationCode.id == code.id)
|
||||
.execute())
|
||||
assert (OAuthAuthorizationCode
|
||||
.select()
|
||||
.where(OAuthAuthorizationCode.code_name >> None)
|
||||
.count()) == 0
|
||||
|
||||
assert (OAuthAuthorizationCode
|
||||
.select()
|
||||
.where(OAuthAuthorizationCode.code_name >> None)
|
||||
.count()) == 0
|
||||
# OAuthApplication
|
||||
logger.info('Backfilling secret for OAuth applications')
|
||||
for app in _iterate(OAuthApplication, OAuthApplication.fully_migrated == False):
|
||||
logger.info('Backfilling secret for OAuth application %s', app.id)
|
||||
client_secret = app.client_secret or str(uuid.uuid4())
|
||||
secure_client_secret = _decrypted(client_secret)
|
||||
|
||||
# OAuthApplication
|
||||
logger.info('Backfilling secret for OAuth applications')
|
||||
for app in _iterate(OAuthApplication, OAuthApplication.fully_migrated == False):
|
||||
logger.info('Backfilling secret for OAuth application %s', app.id)
|
||||
client_secret = app.client_secret or str(uuid.uuid4())
|
||||
secure_client_secret = _decrypted(client_secret)
|
||||
(OAuthApplication
|
||||
.update(secure_client_secret=secure_client_secret, fully_migrated=True)
|
||||
.where(OAuthApplication.id == app.id, OAuthApplication.fully_migrated == False)
|
||||
.execute())
|
||||
|
||||
(OAuthApplication
|
||||
.update(secure_client_secret=secure_client_secret, fully_migrated=True)
|
||||
.where(OAuthApplication.id == app.id, OAuthApplication.fully_migrated == False)
|
||||
.execute())
|
||||
|
||||
assert (OAuthApplication
|
||||
.select()
|
||||
.where(OAuthApplication.fully_migrated == False)
|
||||
.count()) == 0
|
||||
assert (OAuthApplication
|
||||
.select()
|
||||
.where(OAuthApplication.fully_migrated == False)
|
||||
.count()) == 0
|
||||
|
||||
# Adjust existing fields to be nullable.
|
||||
op.alter_column('accesstoken', 'code', nullable=True, existing_type=sa.String(length=255))
|
||||
|
|
@ -271,10 +273,6 @@ def upgrade(tables, tester, progress_reporter):
|
|||
|
||||
def downgrade(tables, tester, progress_reporter):
|
||||
op = ProgressWrapper(original_op, progress_reporter)
|
||||
op.alter_column('accesstoken', 'code', nullable=False, existing_type=sa.String(length=255))
|
||||
op.alter_column('oauthaccesstoken', 'access_token', nullable=False, existing_type=sa.String(length=255))
|
||||
op.alter_column('oauthauthorizationcode', 'code', nullable=False, existing_type=sa.String(length=255))
|
||||
op.alter_column('appspecificauthtoken', 'token_code', nullable=False, existing_type=sa.String(length=255))
|
||||
|
||||
op.alter_column('accesstoken', 'token_name', nullable=True, existing_type=sa.String(length=255))
|
||||
op.alter_column('accesstoken', 'token_code', nullable=True, existing_type=sa.String(length=255))
|
||||
|
|
|
|||
|
|
@ -39,22 +39,24 @@ def upgrade(tables, tester, progress_reporter):
|
|||
# ### end Alembic commands ###
|
||||
|
||||
# Overwrite all plaintext robot credentials.
|
||||
while True:
|
||||
try:
|
||||
robot_account_token = RobotAccountToken.get(fully_migrated=False)
|
||||
robot_account = robot_account_token.robot_account
|
||||
from app import app
|
||||
if app.config.get('SETUP_COMPLETE', False) or tester.is_testing:
|
||||
while True:
|
||||
try:
|
||||
robot_account_token = RobotAccountToken.get(fully_migrated=False)
|
||||
robot_account = robot_account_token.robot_account
|
||||
|
||||
robot_account.email = str(uuid.uuid4())
|
||||
robot_account.save()
|
||||
robot_account.email = str(uuid.uuid4())
|
||||
robot_account.save()
|
||||
|
||||
federated_login = FederatedLogin.get(user=robot_account)
|
||||
federated_login.service_ident = 'robot:%s' % robot_account.id
|
||||
federated_login.save()
|
||||
federated_login = FederatedLogin.get(user=robot_account)
|
||||
federated_login.service_ident = 'robot:%s' % robot_account.id
|
||||
federated_login.save()
|
||||
|
||||
robot_account_token.fully_migrated = True
|
||||
robot_account_token.save()
|
||||
except RobotAccountToken.DoesNotExist:
|
||||
break
|
||||
robot_account_token.fully_migrated = True
|
||||
robot_account_token.save()
|
||||
except RobotAccountToken.DoesNotExist:
|
||||
break
|
||||
|
||||
|
||||
def downgrade(tables, tester, progress_reporter):
|
||||
|
|
|
|||
Reference in a new issue