a54fb1b23a
This change ensures there is better messaging around the encrypted token migration, including a new phase to use for new installations, and fixes an issue encountered when running database migrations for new installations
129 lines
5.1 KiB
Python
129 lines
5.1 KiB
Python
"""repo mirror columns
|
|
|
|
Revision ID: 34c8ef052ec9
|
|
Revises: c059b952ed76
|
|
Create Date: 2019-10-07 13:11:20.424715
|
|
|
|
"""
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision = '34c8ef052ec9'
|
|
down_revision = 'cc6778199cdb'
|
|
|
|
from alembic import op
|
|
from alembic import op as original_op
|
|
from data.migrations.progress import ProgressWrapper
|
|
from datetime import datetime
|
|
import sqlalchemy as sa
|
|
from sqlalchemy.dialects import mysql
|
|
from peewee import ForeignKeyField, DateTimeField, BooleanField
|
|
from data.database import (BaseModel, RepoMirrorType, RepoMirrorStatus, RepoMirrorRule, uuid_generator,
|
|
QuayUserField, Repository, IntegerField, JSONField)
|
|
from data.fields import EnumField as ClientEnumField, CharField, EncryptedCharField
|
|
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BATCH_SIZE = 10
|
|
|
|
|
|
# Original model
|
|
class RepoMirrorConfig(BaseModel):
|
|
"""
|
|
Represents a repository to be mirrored and any additional configuration
|
|
required to perform the mirroring.
|
|
"""
|
|
repository = ForeignKeyField(Repository, index=True, unique=True, backref='mirror')
|
|
creation_date = DateTimeField(default=datetime.utcnow)
|
|
is_enabled = BooleanField(default=True)
|
|
|
|
# Mirror Configuration
|
|
mirror_type = ClientEnumField(RepoMirrorType, default=RepoMirrorType.PULL)
|
|
internal_robot = QuayUserField(allows_robots=True, null=True, backref='mirrorpullrobot',
|
|
robot_null_delete=True)
|
|
external_reference = CharField()
|
|
external_registry = CharField()
|
|
external_namespace = CharField()
|
|
external_repository = CharField()
|
|
external_registry_username = EncryptedCharField(max_length=2048, null=True)
|
|
external_registry_password = EncryptedCharField(max_length=2048, null=True)
|
|
external_registry_config = JSONField(default={})
|
|
|
|
# Worker Queuing
|
|
sync_interval = IntegerField() # seconds between syncs
|
|
sync_start_date = DateTimeField(null=True) # next start time
|
|
sync_expiration_date = DateTimeField(null=True) # max duration
|
|
sync_retries_remaining = IntegerField(default=3)
|
|
sync_status = ClientEnumField(RepoMirrorStatus, default=RepoMirrorStatus.NEVER_RUN)
|
|
sync_transaction_id = CharField(default=uuid_generator, max_length=36)
|
|
|
|
# Tag-Matching Rules
|
|
root_rule = ForeignKeyField(RepoMirrorRule)
|
|
|
|
|
|
def _iterate(model_class, clause):
|
|
while True:
|
|
has_rows = False
|
|
for row in list(model_class.select().where(clause).limit(BATCH_SIZE)):
|
|
has_rows = True
|
|
yield row
|
|
|
|
if not has_rows:
|
|
break
|
|
|
|
|
|
def upgrade(tables, tester, progress_reporter):
|
|
op = ProgressWrapper(original_op, progress_reporter)
|
|
|
|
logger.info('Migrating to external_reference from existing columns')
|
|
|
|
op.add_column('repomirrorconfig', sa.Column('external_reference', sa.Text(), nullable=True))
|
|
|
|
from app import app
|
|
if app.config.get('SETUP_COMPLETE', False) or tester.is_testing:
|
|
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_reference >> None)):
|
|
repo = '%s/%s/%s' % (repo_mirror.external_registry, repo_mirror.external_namespace, repo_mirror.external_repository)
|
|
logger.info('migrating %s' % repo)
|
|
repo_mirror.external_reference = repo
|
|
repo_mirror.save()
|
|
|
|
op.drop_column('repomirrorconfig', 'external_registry')
|
|
op.drop_column('repomirrorconfig', 'external_namespace')
|
|
op.drop_column('repomirrorconfig', 'external_repository')
|
|
|
|
op.alter_column('repomirrorconfig', 'external_reference', nullable=False, existing_type=sa.Text())
|
|
|
|
|
|
tester.populate_column('repomirrorconfig', 'external_reference', tester.TestDataType.String)
|
|
|
|
|
|
def downgrade(tables, tester, progress_reporter):
|
|
op = ProgressWrapper(original_op, progress_reporter)
|
|
|
|
'''
|
|
This will downgrade existing data but may not exactly match previous data structure. If the
|
|
external_reference does not have three parts (registry, namespace, repository) then a failed
|
|
value is inserted.
|
|
'''
|
|
|
|
op.add_column('repomirrorconfig', sa.Column('external_registry', sa.String(length=255), nullable=True))
|
|
op.add_column('repomirrorconfig', sa.Column('external_namespace', sa.String(length=255), nullable=True))
|
|
op.add_column('repomirrorconfig', sa.Column('external_repository', sa.String(length=255), nullable=True))
|
|
|
|
from app import app
|
|
if app.config.get('SETUP_COMPLETE', False):
|
|
logger.info('Restoring columns from external_reference')
|
|
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_registry >> None)):
|
|
logger.info('Restoring %s' % repo_mirror.external_reference)
|
|
parts = repo_mirror.external_reference.split('/', 2)
|
|
repo_mirror.external_registry = parts[0] if len(parts) >= 1 else 'DOWNGRADE-FAILED'
|
|
repo_mirror.external_namespace = parts[1] if len(parts) >= 2 else 'DOWNGRADE-FAILED'
|
|
repo_mirror.external_repository = parts[2] if len(parts) >= 3 else 'DOWNGRADE-FAILED'
|
|
repo_mirror.save()
|
|
|
|
op.drop_column('repomirrorconfig', 'external_reference')
|
|
|
|
op.alter_column('repomirrorconfig', 'external_registry', nullable=False, existing_type=sa.String(length=255))
|
|
op.alter_column('repomirrorconfig', 'external_namespace', nullable=False, existing_type=sa.String(length=255))
|
|
op.alter_column('repomirrorconfig', 'external_repository', nullable=False, existing_type=sa.String(length=255))
|