130 lines
5.1 KiB
Python
130 lines
5.1 KiB
Python
"""repo mirror columns
|
|
|
|
Revision ID: 34c8ef052ec9
|
|
Revises: c059b952ed76
|
|
Create Date: 2019-10-07 13:11:20.424715
|
|
|
|
"""
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision = '34c8ef052ec9'
|
|
down_revision = 'cc6778199cdb'
|
|
|
|
from alembic import op
|
|
from alembic import op as original_op
|
|
from data.migrations.progress import ProgressWrapper
|
|
from datetime import datetime
|
|
import sqlalchemy as sa
|
|
from sqlalchemy.dialects import mysql
|
|
from peewee import ForeignKeyField, DateTimeField, BooleanField
|
|
from data.database import (BaseModel, RepoMirrorType, RepoMirrorStatus, RepoMirrorRule, uuid_generator,
|
|
QuayUserField, Repository, IntegerField, JSONField)
|
|
from data.fields import EnumField as ClientEnumField, CharField, EncryptedCharField
|
|
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
BATCH_SIZE = 10
|
|
|
|
|
|
# Original model
|
|
class RepoMirrorConfig(BaseModel):
|
|
"""
|
|
Represents a repository to be mirrored and any additional configuration
|
|
required to perform the mirroring.
|
|
"""
|
|
repository = ForeignKeyField(Repository, index=True, unique=True, backref='mirror')
|
|
creation_date = DateTimeField(default=datetime.utcnow)
|
|
is_enabled = BooleanField(default=True)
|
|
|
|
# Mirror Configuration
|
|
mirror_type = ClientEnumField(RepoMirrorType, default=RepoMirrorType.PULL)
|
|
internal_robot = QuayUserField(allows_robots=True, null=True, backref='mirrorpullrobot',
|
|
robot_null_delete=True)
|
|
external_reference = CharField()
|
|
external_registry = CharField()
|
|
external_namespace = CharField()
|
|
external_repository = CharField()
|
|
external_registry_username = EncryptedCharField(max_length=2048, null=True)
|
|
external_registry_password = EncryptedCharField(max_length=2048, null=True)
|
|
external_registry_config = JSONField(default={})
|
|
|
|
# Worker Queuing
|
|
sync_interval = IntegerField() # seconds between syncs
|
|
sync_start_date = DateTimeField(null=True) # next start time
|
|
sync_expiration_date = DateTimeField(null=True) # max duration
|
|
sync_retries_remaining = IntegerField(default=3)
|
|
sync_status = ClientEnumField(RepoMirrorStatus, default=RepoMirrorStatus.NEVER_RUN)
|
|
sync_transaction_id = CharField(default=uuid_generator, max_length=36)
|
|
|
|
# Tag-Matching Rules
|
|
root_rule = ForeignKeyField(RepoMirrorRule)
|
|
|
|
|
|
def _iterate(model_class, clause):
|
|
while True:
|
|
has_rows = False
|
|
for row in list(model_class.select().where(clause).limit(BATCH_SIZE)):
|
|
has_rows = True
|
|
yield row
|
|
|
|
if not has_rows:
|
|
break
|
|
|
|
|
|
def upgrade(tables, tester, progress_reporter):
|
|
op = ProgressWrapper(original_op, progress_reporter)
|
|
|
|
logger.info('Migrating to external_reference from existing columns')
|
|
|
|
op.add_column('repomirrorconfig', sa.Column('external_reference', sa.Text(), nullable=True))
|
|
|
|
from app import app
|
|
if app.config.get('SETUP_COMPLETE', False) or tester.is_testing:
|
|
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_reference >> None)):
|
|
repo = '%s/%s/%s' % (repo_mirror.external_registry, repo_mirror.external_namespace, repo_mirror.external_repository)
|
|
logger.info('migrating %s' % repo)
|
|
repo_mirror.external_reference = repo
|
|
repo_mirror.save()
|
|
|
|
op.drop_column('repomirrorconfig', 'external_registry')
|
|
op.drop_column('repomirrorconfig', 'external_namespace')
|
|
op.drop_column('repomirrorconfig', 'external_repository')
|
|
|
|
op.alter_column('repomirrorconfig', 'external_reference', nullable=False, existing_type=sa.Text())
|
|
|
|
|
|
tester.populate_column('repomirrorconfig', 'external_reference', tester.TestDataType.String)
|
|
|
|
|
|
def downgrade(tables, tester, progress_reporter):
|
|
op = ProgressWrapper(original_op, progress_reporter)
|
|
|
|
'''
|
|
This will downgrade existing data but may not exactly match previous data structure. If the
|
|
external_reference does not have three parts (registry, namespace, repository) then a failed
|
|
value is inserted.
|
|
'''
|
|
|
|
op.add_column('repomirrorconfig', sa.Column('external_registry', sa.String(length=255), nullable=True))
|
|
op.add_column('repomirrorconfig', sa.Column('external_namespace', sa.String(length=255), nullable=True))
|
|
op.add_column('repomirrorconfig', sa.Column('external_repository', sa.String(length=255), nullable=True))
|
|
|
|
from app import app
|
|
if app.config.get('SETUP_COMPLETE', False):
|
|
logger.info('Restoring columns from external_reference')
|
|
for repo_mirror in _iterate(RepoMirrorConfig, (RepoMirrorConfig.external_registry >> None)):
|
|
logger.info('Restoring %s' % repo_mirror.external_reference)
|
|
parts = repo_mirror.external_reference.split('/', 2)
|
|
repo_mirror.external_registry = parts[0] if len(parts) >= 1 else 'DOWNGRADE-FAILED'
|
|
repo_mirror.external_namespace = parts[1] if len(parts) >= 2 else 'DOWNGRADE-FAILED'
|
|
repo_mirror.external_repository = parts[2] if len(parts) >= 3 else 'DOWNGRADE-FAILED'
|
|
repo_mirror.save()
|
|
|
|
op.drop_column('repomirrorconfig', 'external_reference')
|
|
|
|
op.alter_column('repomirrorconfig', 'external_registry', nullable=False, existing_type=sa.String(length=255))
|
|
op.alter_column('repomirrorconfig', 'external_namespace', nullable=False, existing_type=sa.String(length=255))
|
|
op.alter_column('repomirrorconfig', 'external_repository', nullable=False, existing_type=sa.String(length=255))
|