The new strategy is to do a three phase migration. This is the first phase: getting the namespace user in the db and written for all new repositories.
This commit is contained in:
parent
8626d1cd70
commit
3259cda000
7 changed files with 62 additions and 74 deletions
|
@ -7,7 +7,7 @@ from peewee import Proxy
|
|||
from app import app as application
|
||||
from flask import request, Request
|
||||
from util.names import urn_generator
|
||||
from data.model import db as model_db, read_slave
|
||||
from data.database import db as model_db, read_slave
|
||||
|
||||
# Turn off debug logging for boto
|
||||
logging.getLogger('boto').setLevel(logging.CRITICAL)
|
||||
|
|
|
@ -181,6 +181,7 @@ class Repository(BaseModel):
|
|||
indexes = (
|
||||
# create a unique index on namespace and name
|
||||
(('namespace', 'name'), True),
|
||||
(('namespace_user', 'name'), False),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
@ -19,36 +19,6 @@ def upgrade(tables):
|
|||
# Add the namespace_user column, allowing it to be nullable
|
||||
op.add_column('repository', sa.Column('namespace_user', sa.Integer(), sa.ForeignKey('user.id')))
|
||||
|
||||
# backfill the namespace_user column
|
||||
namespace_to_user = {}
|
||||
for user in User.select(User.name, User.id).where(User.robot == False):
|
||||
namespace_to_user[user.username] = user
|
||||
|
||||
for repo in Repository.select():
|
||||
repo.namespace_user = namespace_to_user[repo.namespace]
|
||||
repo.save()
|
||||
|
||||
# Ensure the backfill was a success, then remove the namespace column
|
||||
op.alter_column('repository', 'namespace_user', nullable=False)
|
||||
op.create_index('repository_namespace_user_name', 'repository', ['namespace_user', 'name'],
|
||||
unique=True)
|
||||
|
||||
op.drop_index('repository_namespace_user', table_name='repository')
|
||||
op.drop_column('repository', 'namespace')
|
||||
|
||||
|
||||
def downgrade(tables):
|
||||
# Add the namespace column, allowing it to be nullable
|
||||
op.add_column('repository', sa.Column('namespace', sa.String(length=255)))
|
||||
|
||||
# backfill the namespace column
|
||||
for repo in Repository.select(Repository, User.username).join(User):
|
||||
repo.namespace = repo.namespace_user.username
|
||||
repo.save()
|
||||
|
||||
# Ensure the backfill was a success, then remove the namespace column
|
||||
op.alter_column('repository', 'namespace', nullable=False)
|
||||
op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True)
|
||||
|
||||
op.drop_index('repository_namespace_user', table_name='repository')
|
||||
op.drop_column('repository', 'namespace_user')
|
||||
|
|
|
@ -5,8 +5,18 @@ import json
|
|||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from data.database import *
|
||||
from util.validation import *
|
||||
from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
|
||||
Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
|
||||
LoginService, RepositoryBuild, Team, TeamMember, TeamRole,
|
||||
LogEntryKind, LogEntry, PermissionPrototype, ImageStorage,
|
||||
BuildTriggerService, RepositoryBuildTrigger, NotificationKind,
|
||||
Notification, ImageStorageLocation, ImageStoragePlacement,
|
||||
ExternalNotificationEvent, ExternalNotificationMethod,
|
||||
RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
|
||||
random_string_generator, db, BUILD_PHASE)
|
||||
from peewee import JOIN_LEFT_OUTER, fn
|
||||
from util.validation import (validate_username, validate_email, validate_password,
|
||||
INVALID_PASSWORD_MESSAGE)
|
||||
from util.names import format_robot_username
|
||||
from util.backoff import exponential_backoff
|
||||
|
||||
|
@ -560,9 +570,9 @@ def get_user_or_org(username):
|
|||
return None
|
||||
|
||||
|
||||
def get_user_by_id(user_dbid):
|
||||
def get_user_by_id(user_db_id):
|
||||
try:
|
||||
return User.get(User.id == user_dbid, User.organization == False)
|
||||
return User.get(User.id == user_db_id, User.organization == False)
|
||||
except User.DoesNotExist:
|
||||
return None
|
||||
|
||||
|
@ -632,7 +642,8 @@ def verify_user(username_or_email, password):
|
|||
retry_after = can_retry_at - now
|
||||
raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds())
|
||||
|
||||
if (fetched.password_hash and hash_password(password, fetched.password_hash) == fetched.password_hash):
|
||||
if (fetched.password_hash and
|
||||
hash_password(password, fetched.password_hash) == fetched.password_hash):
|
||||
if fetched.invalid_login_attempts > 0:
|
||||
fetched.invalid_login_attempts = 0
|
||||
fetched.save()
|
||||
|
@ -765,23 +776,23 @@ def _filter_to_repos_for_user(query, username=None, namespace=None,
|
|||
AdminUser = User.alias()
|
||||
|
||||
query = (query
|
||||
.switch(RepositoryPermission)
|
||||
.join(User, JOIN_LEFT_OUTER)
|
||||
.switch(RepositoryPermission)
|
||||
.join(Team, JOIN_LEFT_OUTER)
|
||||
.join(TeamMember, JOIN_LEFT_OUTER)
|
||||
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
|
||||
TeamMember.user))
|
||||
.switch(Repository)
|
||||
.join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
|
||||
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
|
||||
AdminTeam.organization))
|
||||
.join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
|
||||
.switch(AdminTeam)
|
||||
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
|
||||
AdminTeamMember.team))
|
||||
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
|
||||
AdminUser.id)))
|
||||
.switch(RepositoryPermission)
|
||||
.join(User, JOIN_LEFT_OUTER)
|
||||
.switch(RepositoryPermission)
|
||||
.join(Team, JOIN_LEFT_OUTER)
|
||||
.join(TeamMember, JOIN_LEFT_OUTER)
|
||||
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
|
||||
TeamMember.user))
|
||||
.switch(Repository)
|
||||
.join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
|
||||
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
|
||||
AdminTeam.organization))
|
||||
.join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
|
||||
.switch(AdminTeam)
|
||||
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
|
||||
AdminTeamMember.team))
|
||||
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
|
||||
AdminUser.id)))
|
||||
|
||||
where_clause = ((User.username == username) |
|
||||
(UserThroughTeam.username == username) |
|
||||
|
@ -1048,8 +1059,9 @@ def __apply_default_permissions(repo, proto_query, name_property,
|
|||
|
||||
def create_repository(namespace, name, creating_user, visibility='private'):
|
||||
private = Visibility.get(name=visibility)
|
||||
repo = Repository.create(namespace=namespace, name=name,
|
||||
visibility=private)
|
||||
namespace_user = User.get(username=namespace)
|
||||
repo = Repository.create(namespace=namespace, name=name, visibility=private,
|
||||
namespace_user=namespace_user)
|
||||
admin = Role.get(name='admin')
|
||||
|
||||
if creating_user and not creating_user.organization:
|
||||
|
@ -1121,26 +1133,26 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
|
|||
return repo_image
|
||||
|
||||
query = (Image
|
||||
.select(Image, ImageStorage)
|
||||
.distinct()
|
||||
.join(ImageStorage)
|
||||
.switch(Image)
|
||||
.join(Repository)
|
||||
.join(Visibility)
|
||||
.switch(Repository)
|
||||
.join(RepositoryPermission, JOIN_LEFT_OUTER)
|
||||
.where(ImageStorage.uploading == False))
|
||||
.select(Image, ImageStorage)
|
||||
.distinct()
|
||||
.join(ImageStorage)
|
||||
.switch(Image)
|
||||
.join(Repository)
|
||||
.join(Visibility)
|
||||
.switch(Repository)
|
||||
.join(RepositoryPermission, JOIN_LEFT_OUTER)
|
||||
.where(ImageStorage.uploading == False))
|
||||
|
||||
query = (_filter_to_repos_for_user(query, username)
|
||||
.where(Image.docker_image_id == docker_image_id))
|
||||
|
||||
.where(Image.docker_image_id == docker_image_id))
|
||||
|
||||
new_image_ancestry = '/'
|
||||
origin_image_id = None
|
||||
try:
|
||||
to_copy = query.get()
|
||||
msg = 'Linking image to existing storage with docker id: %s and uuid: %s'
|
||||
logger.debug(msg, docker_image_id, to_copy.storage.uuid)
|
||||
|
||||
|
||||
new_image_ancestry = __translate_ancestry(to_copy.ancestors, translations, repository,
|
||||
username, preferred_location)
|
||||
|
||||
|
|
|
@ -82,15 +82,18 @@ def param_required(param_name):
|
|||
|
||||
|
||||
@login_manager.user_loader
|
||||
def load_user(user_dbid):
|
||||
logger.debug('User loader loading deferred user id: %s' % user_dbid)
|
||||
return _LoginWrappedDBUser(user_dbid)
|
||||
def load_user(user_db_id):
|
||||
logger.debug('User loader loading deferred user id: %s' % user_db_id)
|
||||
try:
|
||||
user_db_id_int = int(user_db_id)
|
||||
return _LoginWrappedDBUser(user_db_id_int)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class _LoginWrappedDBUser(UserMixin):
|
||||
def __init__(self, user_dbid, db_user=None):
|
||||
|
||||
self._db_id = int(user_dbid)
|
||||
def __init__(self, user_db_id, db_user=None):
|
||||
self._db_id = user_db_id
|
||||
self._db_user = db_user
|
||||
|
||||
def db_user(self):
|
||||
|
|
Binary file not shown.
|
@ -5,6 +5,7 @@ from urllib import urlencode
|
|||
from urlparse import urlparse, urlunparse, parse_qs
|
||||
|
||||
from app import app
|
||||
from data import model
|
||||
from initdb import setup_database_for_testing, finished_database_for_testing
|
||||
from endpoints.api import api_bp, api
|
||||
|
||||
|
@ -75,7 +76,8 @@ class ApiTestCase(unittest.TestCase):
|
|||
|
||||
with client.session_transaction() as sess:
|
||||
if auth_username:
|
||||
sess['user_id'] = auth_username
|
||||
loaded = model.get_user(auth_username)
|
||||
sess['user_id'] = loaded.id
|
||||
sess[CSRF_TOKEN_KEY] = CSRF_TOKEN
|
||||
|
||||
# Restore the teardown functions
|
||||
|
|
Reference in a new issue