The new strategy is to do a three phase migration. This is the first phase: getting the namespace user in the db and written for all new repositories.

This commit is contained in:
Jake Moshenko 2014-09-22 17:27:02 -04:00
parent 8626d1cd70
commit 3259cda000
7 changed files with 62 additions and 74 deletions

View file

@ -7,7 +7,7 @@ from peewee import Proxy
from app import app as application
from flask import request, Request
from util.names import urn_generator
from data.model import db as model_db, read_slave
from data.database import db as model_db, read_slave
# Turn off debug logging for boto
logging.getLogger('boto').setLevel(logging.CRITICAL)

View file

@ -181,6 +181,7 @@ class Repository(BaseModel):
indexes = (
# create a unique index on namespace and name
(('namespace', 'name'), True),
(('namespace_user', 'name'), False),
)

View file

@ -19,36 +19,6 @@ def upgrade(tables):
# Add the namespace_user column, allowing it to be nullable
op.add_column('repository', sa.Column('namespace_user', sa.Integer(), sa.ForeignKey('user.id')))
# backfill the namespace_user column
namespace_to_user = {}
for user in User.select(User.name, User.id).where(User.robot == False):
namespace_to_user[user.username] = user
for repo in Repository.select():
repo.namespace_user = namespace_to_user[repo.namespace]
repo.save()
# Ensure the backfill was a success, then remove the namespace column
op.alter_column('repository', 'namespace_user', nullable=False)
op.create_index('repository_namespace_user_name', 'repository', ['namespace_user', 'name'],
unique=True)
op.drop_index('repository_namespace_user', table_name='repository')
op.drop_column('repository', 'namespace')
def downgrade(tables):
# Add the namespace column, allowing it to be nullable
op.add_column('repository', sa.Column('namespace', sa.String(length=255)))
# backfill the namespace column
for repo in Repository.select(Repository, User.username).join(User):
repo.namespace = repo.namespace_user.username
repo.save()
# Ensure the backfill was a success, then remove the namespace column
op.alter_column('repository', 'namespace', nullable=False)
op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True)
op.drop_index('repository_namespace_user', table_name='repository')
op.drop_column('repository', 'namespace_user')

View file

@ -5,8 +5,18 @@ import json
from datetime import datetime, timedelta
from data.database import *
from util.validation import *
from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
LoginService, RepositoryBuild, Team, TeamMember, TeamRole,
LogEntryKind, LogEntry, PermissionPrototype, ImageStorage,
BuildTriggerService, RepositoryBuildTrigger, NotificationKind,
Notification, ImageStorageLocation, ImageStoragePlacement,
ExternalNotificationEvent, ExternalNotificationMethod,
RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
random_string_generator, db, BUILD_PHASE)
from peewee import JOIN_LEFT_OUTER, fn
from util.validation import (validate_username, validate_email, validate_password,
INVALID_PASSWORD_MESSAGE)
from util.names import format_robot_username
from util.backoff import exponential_backoff
@ -560,9 +570,9 @@ def get_user_or_org(username):
return None
def get_user_by_id(user_dbid):
def get_user_by_id(user_db_id):
try:
return User.get(User.id == user_dbid, User.organization == False)
return User.get(User.id == user_db_id, User.organization == False)
except User.DoesNotExist:
return None
@ -632,7 +642,8 @@ def verify_user(username_or_email, password):
retry_after = can_retry_at - now
raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds())
if (fetched.password_hash and hash_password(password, fetched.password_hash) == fetched.password_hash):
if (fetched.password_hash and
hash_password(password, fetched.password_hash) == fetched.password_hash):
if fetched.invalid_login_attempts > 0:
fetched.invalid_login_attempts = 0
fetched.save()
@ -765,23 +776,23 @@ def _filter_to_repos_for_user(query, username=None, namespace=None,
AdminUser = User.alias()
query = (query
.switch(RepositoryPermission)
.join(User, JOIN_LEFT_OUTER)
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER)
.join(TeamMember, JOIN_LEFT_OUTER)
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
TeamMember.user))
.switch(Repository)
.join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
AdminTeam.organization))
.join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
.switch(AdminTeam)
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
AdminTeamMember.team))
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
AdminUser.id)))
.switch(RepositoryPermission)
.join(User, JOIN_LEFT_OUTER)
.switch(RepositoryPermission)
.join(Team, JOIN_LEFT_OUTER)
.join(TeamMember, JOIN_LEFT_OUTER)
.join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
TeamMember.user))
.switch(Repository)
.join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
.join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
AdminTeam.organization))
.join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
.switch(AdminTeam)
.join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
AdminTeamMember.team))
.join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
AdminUser.id)))
where_clause = ((User.username == username) |
(UserThroughTeam.username == username) |
@ -1048,8 +1059,9 @@ def __apply_default_permissions(repo, proto_query, name_property,
def create_repository(namespace, name, creating_user, visibility='private'):
private = Visibility.get(name=visibility)
repo = Repository.create(namespace=namespace, name=name,
visibility=private)
namespace_user = User.get(username=namespace)
repo = Repository.create(namespace=namespace, name=name, visibility=private,
namespace_user=namespace_user)
admin = Role.get(name='admin')
if creating_user and not creating_user.organization:
@ -1121,26 +1133,26 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
return repo_image
query = (Image
.select(Image, ImageStorage)
.distinct()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.where(ImageStorage.uploading == False))
.select(Image, ImageStorage)
.distinct()
.join(ImageStorage)
.switch(Image)
.join(Repository)
.join(Visibility)
.switch(Repository)
.join(RepositoryPermission, JOIN_LEFT_OUTER)
.where(ImageStorage.uploading == False))
query = (_filter_to_repos_for_user(query, username)
.where(Image.docker_image_id == docker_image_id))
.where(Image.docker_image_id == docker_image_id))
new_image_ancestry = '/'
origin_image_id = None
try:
to_copy = query.get()
msg = 'Linking image to existing storage with docker id: %s and uuid: %s'
logger.debug(msg, docker_image_id, to_copy.storage.uuid)
new_image_ancestry = __translate_ancestry(to_copy.ancestors, translations, repository,
username, preferred_location)

View file

@ -82,15 +82,18 @@ def param_required(param_name):
@login_manager.user_loader
def load_user(user_dbid):
logger.debug('User loader loading deferred user id: %s' % user_dbid)
return _LoginWrappedDBUser(user_dbid)
def load_user(user_db_id):
logger.debug('User loader loading deferred user id: %s' % user_db_id)
try:
user_db_id_int = int(user_db_id)
return _LoginWrappedDBUser(user_db_id_int)
except ValueError:
return None
class _LoginWrappedDBUser(UserMixin):
def __init__(self, user_dbid, db_user=None):
self._db_id = int(user_dbid)
def __init__(self, user_db_id, db_user=None):
self._db_id = user_db_id
self._db_user = db_user
def db_user(self):

Binary file not shown.

View file

@ -5,6 +5,7 @@ from urllib import urlencode
from urlparse import urlparse, urlunparse, parse_qs
from app import app
from data import model
from initdb import setup_database_for_testing, finished_database_for_testing
from endpoints.api import api_bp, api
@ -75,7 +76,8 @@ class ApiTestCase(unittest.TestCase):
with client.session_transaction() as sess:
if auth_username:
sess['user_id'] = auth_username
loaded = model.get_user(auth_username)
sess['user_id'] = loaded.id
sess[CSRF_TOKEN_KEY] = CSRF_TOKEN
# Restore the teardown functions