The new strategy is to do a three phase migration. This is the first phase: getting the namespace user in the db and written for all new repositories.

application.py

@@ -7,7 +7,7 @@ from peewee import Proxy
 from app import app as application
 from flask import request, Request
 from util.names import urn_generator
-from data.model import db as model_db, read_slave
+from data.database import db as model_db, read_slave
 
 # Turn off debug logging for boto
 logging.getLogger('boto').setLevel(logging.CRITICAL)

data/database.py

@@ -181,6 +181,7 @@ class Repository(BaseModel):
     indexes = (
       # create a unique index on namespace and name
       (('namespace', 'name'), True),
+      (('namespace_user', 'name'), False),
     )
 
 

data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py

@@ -19,36 +19,6 @@ def upgrade(tables):
     # Add the namespace_user column, allowing it to be nullable
     op.add_column('repository', sa.Column('namespace_user', sa.Integer(), sa.ForeignKey('user.id')))
 
-    # backfill the namespace_user column
-    namespace_to_user = {}
-    for user in User.select(User.name, User.id).where(User.robot == False):
-        namespace_to_user[user.username] = user
-
-    for repo in Repository.select():
-        repo.namespace_user = namespace_to_user[repo.namespace]
-        repo.save()
-
-    # Ensure the backfill was a success, then remove the namespace column
-    op.alter_column('repository', 'namespace_user', nullable=False)
-    op.create_index('repository_namespace_user_name', 'repository', ['namespace_user', 'name'],
-                    unique=True)
-
-    op.drop_index('repository_namespace_user', table_name='repository')
-    op.drop_column('repository', 'namespace')
-
 
 def downgrade(tables):
-    # Add the namespace column, allowing it to be nullable
-    op.add_column('repository', sa.Column('namespace', sa.String(length=255)))
-
-    # backfill the namespace column
-    for repo in Repository.select(Repository, User.username).join(User):
-        repo.namespace = repo.namespace_user.username
-        repo.save()
-
-    # Ensure the backfill was a success, then remove the namespace column
-    op.alter_column('repository', 'namespace', nullable=False)
-    op.create_index('repository_namespace_name', 'repository', ['namespace', 'name'], unique=True)
-
-    op.drop_index('repository_namespace_user', table_name='repository')
     op.drop_column('repository', 'namespace_user')

data/model/legacy.py

@@ -5,8 +5,18 @@ import json
 
 from datetime import datetime, timedelta
 
-from data.database import *
-from util.validation import *
+from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
+                           Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
+                           LoginService, RepositoryBuild, Team, TeamMember, TeamRole,
+                           LogEntryKind, LogEntry, PermissionPrototype, ImageStorage,
+                           BuildTriggerService, RepositoryBuildTrigger, NotificationKind,
+                           Notification, ImageStorageLocation, ImageStoragePlacement,
+                           ExternalNotificationEvent, ExternalNotificationMethod,
+                           RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
+                           random_string_generator, db, BUILD_PHASE)
+from peewee import JOIN_LEFT_OUTER, fn
+from util.validation import (validate_username, validate_email, validate_password,
+                             INVALID_PASSWORD_MESSAGE)
 from util.names import format_robot_username
 from util.backoff import exponential_backoff
 
@@ -560,9 +570,9 @@ def get_user_or_org(username):
     return None
 
 
-def get_user_by_id(user_dbid):
+def get_user_by_id(user_db_id):
   try:
-    return User.get(User.id == user_dbid, User.organization == False)
+    return User.get(User.id == user_db_id, User.organization == False)
   except User.DoesNotExist:
     return None
 
@@ -632,7 +642,8 @@ def verify_user(username_or_email, password):
       retry_after = can_retry_at - now
       raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds())
 
-  if (fetched.password_hash and hash_password(password, fetched.password_hash) == fetched.password_hash):
+  if (fetched.password_hash and
+      hash_password(password, fetched.password_hash) == fetched.password_hash):
     if fetched.invalid_login_attempts > 0:
       fetched.invalid_login_attempts = 0
       fetched.save()
@@ -765,23 +776,23 @@ def _filter_to_repos_for_user(query, username=None, namespace=None,
     AdminUser = User.alias()
 
     query = (query
-      .switch(RepositoryPermission)
-      .join(User, JOIN_LEFT_OUTER)
-      .switch(RepositoryPermission)
-      .join(Team, JOIN_LEFT_OUTER)
-      .join(TeamMember, JOIN_LEFT_OUTER)
-      .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
-                                                  TeamMember.user))
-      .switch(Repository)
-      .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
-      .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
-                                            AdminTeam.organization))
-      .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
-      .switch(AdminTeam)
-      .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
-                                                  AdminTeamMember.team))
-      .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
-                                            AdminUser.id)))
+             .switch(RepositoryPermission)
+             .join(User, JOIN_LEFT_OUTER)
+             .switch(RepositoryPermission)
+             .join(Team, JOIN_LEFT_OUTER)
+             .join(TeamMember, JOIN_LEFT_OUTER)
+             .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
+                                                        TeamMember.user))
+             .switch(Repository)
+             .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
+             .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
+                                                  AdminTeam.organization))
+             .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
+             .switch(AdminTeam)
+             .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
+                                                        AdminTeamMember.team))
+             .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
+                                                  AdminUser.id)))
 
     where_clause = ((User.username == username) |
                     (UserThroughTeam.username == username) |
@@ -1048,8 +1059,9 @@ def __apply_default_permissions(repo, proto_query, name_property,
 
 def create_repository(namespace, name, creating_user, visibility='private'):
   private = Visibility.get(name=visibility)
-  repo = Repository.create(namespace=namespace, name=name,
-                           visibility=private)
+  namespace_user = User.get(username=namespace)
+  repo = Repository.create(namespace=namespace, name=name, visibility=private,
+                           namespace_user=namespace_user)
   admin = Role.get(name='admin')
 
   if creating_user and not creating_user.organization:
@@ -1121,18 +1133,18 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
       return repo_image
 
     query = (Image
-      .select(Image, ImageStorage)
-      .distinct()
-      .join(ImageStorage)
-      .switch(Image)
-      .join(Repository)
-      .join(Visibility)
-      .switch(Repository)
-      .join(RepositoryPermission, JOIN_LEFT_OUTER)
-      .where(ImageStorage.uploading == False))
+             .select(Image, ImageStorage)
+             .distinct()
+             .join(ImageStorage)
+             .switch(Image)
+             .join(Repository)
+             .join(Visibility)
+             .switch(Repository)
+             .join(RepositoryPermission, JOIN_LEFT_OUTER)
+             .where(ImageStorage.uploading == False))
 
     query = (_filter_to_repos_for_user(query, username)
-      .where(Image.docker_image_id == docker_image_id))
+             .where(Image.docker_image_id == docker_image_id))
 
     new_image_ancestry = '/'
     origin_image_id = None

endpoints/common.py

@@ -82,15 +82,18 @@ def param_required(param_name):
 
 
 @login_manager.user_loader
-def load_user(user_dbid):
-  logger.debug('User loader loading deferred user id: %s' % user_dbid)
-  return _LoginWrappedDBUser(user_dbid)
+def load_user(user_db_id):
+  logger.debug('User loader loading deferred user id: %s' % user_db_id)
+  try:
+    user_db_id_int = int(user_db_id)
+    return _LoginWrappedDBUser(user_db_id_int)
+  except ValueError:
+    return None
 
 
 class _LoginWrappedDBUser(UserMixin):
-  def __init__(self, user_dbid, db_user=None):
-
-    self._db_id = int(user_dbid)
+  def __init__(self, user_db_id, db_user=None):
+    self._db_id = user_db_id
     self._db_user = db_user
 
   def db_user(self):

test/test_api_security.py

@@ -5,6 +5,7 @@ from urllib import urlencode
 from urlparse import urlparse, urlunparse, parse_qs
 
 from app import app
+from data import model
 from initdb import setup_database_for_testing, finished_database_for_testing
 from endpoints.api import api_bp, api
 
@@ -75,7 +76,8 @@ class ApiTestCase(unittest.TestCase):
 
       with client.session_transaction() as sess:
         if auth_username:
-          sess['user_id'] = auth_username
+          loaded = model.get_user(auth_username)
+          sess['user_id'] = loaded.id
         sess[CSRF_TOKEN_KEY] = CSRF_TOKEN
 
       # Restore the teardown functions