Merge remote-tracking branch 'origin/nomenclature'

application.py

@@ -7,7 +7,7 @@ from peewee import Proxy
 from app import app as application
 from flask import request, Request
 from util.names import urn_generator
-from data.model import db as model_db, read_slave
+from data.database import db as model_db, read_slave
 
 # Turn off debug logging for boto
 logging.getLogger('boto').setLevel(logging.CRITICAL)

auth/auth.py

@@ -25,7 +25,7 @@ def _load_user_from_cookie():
   if not current_user.is_anonymous():
     logger.debug('Loading user from cookie: %s', current_user.get_id())
     set_authenticated_user_deferred(current_user.get_id())
-    loaded = QuayDeferredPermissionUser(current_user.get_id(), 'username', {scopes.DIRECT_LOGIN})
+    loaded = QuayDeferredPermissionUser(current_user.get_id(), 'user_db_id', {scopes.DIRECT_LOGIN})
     identity_changed.send(app, identity=loaded)
     return current_user.db_user()
   return None
@@ -58,12 +58,10 @@ def _validate_and_apply_oauth_token(token):
   set_authenticated_user(validated.authorized_user)
   set_validated_oauth_token(validated)
 
-  new_identity = QuayDeferredPermissionUser(validated.authorized_user.username, 'username',
+  new_identity = QuayDeferredPermissionUser(validated.authorized_user.id, 'user_db_id', scope_set)
-                                            scope_set)
   identity_changed.send(app, identity=new_identity)
 
 
-
 def process_basic_auth(auth):
   normalized = [part.strip() for part in auth.split(' ') if part]
   if normalized[0].lower() != 'basic' or len(normalized) != 2:
@@ -100,8 +98,7 @@ def process_basic_auth(auth):
       logger.debug('Successfully validated robot: %s' % credentials[0])
       set_authenticated_user(robot)
       
-      deferred_robot = QuayDeferredPermissionUser(robot.username, 'username',
+      deferred_robot = QuayDeferredPermissionUser(robot.id, 'user_db_id', {scopes.DIRECT_LOGIN})
-                                                  {scopes.DIRECT_LOGIN})
       identity_changed.send(app, identity=deferred_robot)
       return
     except model.InvalidRobotException:
@@ -114,7 +111,7 @@ def process_basic_auth(auth):
       logger.debug('Successfully validated user: %s' % authenticated.username)
       set_authenticated_user(authenticated)
 
-      new_identity = QuayDeferredPermissionUser(authenticated.username, 'username',
+      new_identity = QuayDeferredPermissionUser(authenticated.id, 'user_db_id',
                                                 {scopes.DIRECT_LOGIN})
       identity_changed.send(app, identity=new_identity)
       return

auth/auth_context.py

@@ -10,13 +10,13 @@ logger = logging.getLogger(__name__)
 def get_authenticated_user():
   user = getattr(_request_ctx_stack.top, 'authenticated_user', None)
   if not user:
-    username = getattr(_request_ctx_stack.top, 'authenticated_username', None)
+    db_id = getattr(_request_ctx_stack.top, 'authenticated_db_id', None)
-    if not username:
+    if not db_id:
-      logger.debug('No authenticated user or deferred username.')
+      logger.debug('No authenticated user or deferred database id.')
       return None
 
     logger.debug('Loading deferred authenticated user.')
-    loaded = model.get_user(username)
+    loaded = model.get_user_by_id(db_id)
     set_authenticated_user(loaded)
     user = loaded
 
@@ -30,10 +30,10 @@ def set_authenticated_user(user_or_robot):
   ctx.authenticated_user = user_or_robot
 
 
-def set_authenticated_user_deferred(username_or_robotname):
+def set_authenticated_user_deferred(user_or_robot_db_id):
-  logger.debug('Deferring loading of authenticated user object: %s', username_or_robotname)
+  logger.debug('Deferring loading of authenticated user object: %s', user_or_robot_db_id)
   ctx = _request_ctx_stack.top
-  ctx.authenticated_username = username_or_robotname
+  ctx.authenticated_db_id = user_or_robot_db_id
 
 
 def get_validated_oauth_token():

auth/permissions.py

@@ -58,8 +58,8 @@ SCOPE_MAX_USER_ROLES.update({
 
 
 class QuayDeferredPermissionUser(Identity):
-  def __init__(self, id, auth_type, scopes):
+  def __init__(self, db_id, auth_type, scopes):
-    super(QuayDeferredPermissionUser, self).__init__(id, auth_type)
+    super(QuayDeferredPermissionUser, self).__init__(db_id, auth_type)
 
     self._permissions_loaded = False
     self._scope_set = scopes
@@ -88,7 +88,7 @@ class QuayDeferredPermissionUser(Identity):
   def can(self, permission):
     if not self._permissions_loaded:
       logger.debug('Loading user permissions after deferring.')
-      user_object = model.get_user(self.id)
+      user_object = model.get_user_by_id(self.id)
 
       # Add the superuser need, if applicable.
       if (user_object.username is not None and
@@ -230,9 +230,9 @@ def on_identity_loaded(sender, identity):
   if isinstance(identity, QuayDeferredPermissionUser):
     logger.debug('Deferring permissions for user: %s', identity.id)
 
-  elif identity.auth_type == 'username':
+  elif identity.auth_type == 'user_db_id':
     logger.debug('Switching username permission to deferred object: %s', identity.id)
-    switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'username', {scopes.DIRECT_LOGIN})
+    switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'user_db_id', {scopes.DIRECT_LOGIN})
     identity_changed.send(app, identity=switch_to_deferred)
 
   elif identity.auth_type == 'token':

data/buildlogs.py

@@ -17,7 +17,7 @@ class RedisBuildLogs(object):
   PHASE = 'phase'
 
   def __init__(self, redis_host):
-    self._redis = redis.StrictRedis(host=redis_host)
+    self._redis = redis.StrictRedis(host=redis_host, socket_timeout=5)
 
   @staticmethod
   def _logs_key(build_id):

data/database.py

@@ -169,6 +169,7 @@ class Visibility(BaseModel):
 
 class Repository(BaseModel):
   namespace = CharField()
+  namespace_user = ForeignKeyField(User, null=True)
   name = CharField()
   visibility = ForeignKeyField(Visibility)
   description = TextField(null=True)
@@ -180,6 +181,7 @@ class Repository(BaseModel):
     indexes = (
       # create a unique index on namespace and name
       (('namespace', 'name'), True),
+      (('namespace_user', 'name'), False),
     )
 
 

data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py

@@ -0,0 +1,24 @@
+"""Migrate registry namespaces to reference a user.
+
+Revision ID: 13da56878560
+Revises: 51d04d0e7e6f
+Create Date: 2014-09-18 13:56:45.130455
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '13da56878560'
+down_revision = '51d04d0e7e6f'
+
+from alembic import op
+import sqlalchemy as sa
+
+from data.database import Repository, User
+
+def upgrade(tables):
+    # Add the namespace_user column, allowing it to be nullable
+    op.add_column('repository', sa.Column('namespace_user', sa.Integer(), sa.ForeignKey('user.id')))
+
+
+def downgrade(tables):
+    op.drop_column('repository', 'namespace_user')

data/model/legacy.py

@@ -5,8 +5,18 @@ import json
 
 from datetime import datetime, timedelta
 
-from data.database import *
+from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission,
-from util.validation import *
+                           Visibility, RepositoryTag, EmailConfirmation, FederatedLogin,
+                           LoginService, RepositoryBuild, Team, TeamMember, TeamRole,
+                           LogEntryKind, LogEntry, PermissionPrototype, ImageStorage,
+                           BuildTriggerService, RepositoryBuildTrigger, NotificationKind,
+                           Notification, ImageStorageLocation, ImageStoragePlacement,
+                           ExternalNotificationEvent, ExternalNotificationMethod,
+                           RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite,
+                           random_string_generator, db, BUILD_PHASE)
+from peewee import JOIN_LEFT_OUTER, fn
+from util.validation import (validate_username, validate_email, validate_password,
+                             INVALID_PASSWORD_MESSAGE)
 from util.names import format_robot_username
 from util.backoff import exponential_backoff
 
@@ -560,6 +570,13 @@ def get_user_or_org(username):
     return None
 
 
+def get_user_by_id(user_db_id):
+  try:
+    return User.get(User.id == user_db_id, User.organization == False)
+  except User.DoesNotExist:
+    return None
+
+
 def get_user_or_org_by_customer_id(customer_id):
   try:
     return User.get(User.stripe_id == customer_id)
@@ -584,12 +601,13 @@ def get_matching_users(username_prefix, robot_namespace=None,
                           (User.robot == True)))
 
   query = (User
-    .select(User.username, fn.Sum(Team.id), User.robot)
+    .select(User.username, User.robot)
     .group_by(User.username)
     .where(direct_user_query))
 
   if organization:
     query = (query
+      .select(User.username, User.robot, fn.Sum(Team.id))
       .join(TeamMember, JOIN_LEFT_OUTER)
       .join(Team, JOIN_LEFT_OUTER, on=((Team.id == TeamMember.team) &
                                        (Team.organization == organization))))
@@ -598,9 +616,9 @@ def get_matching_users(username_prefix, robot_namespace=None,
   class MatchingUserResult(object):
     def __init__(self, *args):
       self.username = args[0]
-      self.is_robot = args[2]
+      self.is_robot = args[1]
       if organization:
-        self.is_org_member = (args[1] != None)
+        self.is_org_member = (args[2] != None)
       else:
         self.is_org_member = None
 
@@ -625,7 +643,8 @@ def verify_user(username_or_email, password):
       retry_after = can_retry_at - now
       raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds())
 
-  if (fetched.password_hash and hash_password(password, fetched.password_hash) == fetched.password_hash):
+  if (fetched.password_hash and
+      hash_password(password, fetched.password_hash) == fetched.password_hash):
     if fetched.invalid_login_attempts > 0:
       fetched.invalid_login_attempts = 0
       fetched.save()
@@ -758,23 +777,23 @@ def _filter_to_repos_for_user(query, username=None, namespace=None,
     AdminUser = User.alias()
 
     query = (query
-      .switch(RepositoryPermission)
+             .switch(RepositoryPermission)
-      .join(User, JOIN_LEFT_OUTER)
+             .join(User, JOIN_LEFT_OUTER)
-      .switch(RepositoryPermission)
+             .switch(RepositoryPermission)
-      .join(Team, JOIN_LEFT_OUTER)
+             .join(Team, JOIN_LEFT_OUTER)
-      .join(TeamMember, JOIN_LEFT_OUTER)
+             .join(TeamMember, JOIN_LEFT_OUTER)
-      .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
+             .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id ==
-                                                  TeamMember.user))
+                                                        TeamMember.user))
-      .switch(Repository)
+             .switch(Repository)
-      .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
+             .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace))
-      .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
+             .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id ==
-                                            AdminTeam.organization))
+                                                  AdminTeam.organization))
-      .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
+             .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id))
-      .switch(AdminTeam)
+             .switch(AdminTeam)
-      .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
+             .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id ==
-                                                  AdminTeamMember.team))
+                                                        AdminTeamMember.team))
-      .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
+             .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user ==
-                                            AdminUser.id)))
+                                                  AdminUser.id)))
 
     where_clause = ((User.username == username) |
                     (UserThroughTeam.username == username) |
@@ -1041,8 +1060,9 @@ def __apply_default_permissions(repo, proto_query, name_property,
 
 def create_repository(namespace, name, creating_user, visibility='private'):
   private = Visibility.get(name=visibility)
-  repo = Repository.create(namespace=namespace, name=name,
+  namespace_user = User.get(username=namespace)
-                           visibility=private)
+  repo = Repository.create(namespace=namespace, name=name, visibility=private,
+                           namespace_user=namespace_user)
   admin = Role.get(name='admin')
 
   if creating_user and not creating_user.organization:
@@ -1114,26 +1134,26 @@ def find_create_or_link_image(docker_image_id, repository, username, translation
       return repo_image
 
     query = (Image
-      .select(Image, ImageStorage)
+             .select(Image, ImageStorage)
-      .distinct()
+             .distinct()
-      .join(ImageStorage)
+             .join(ImageStorage)
-      .switch(Image)
+             .switch(Image)
-      .join(Repository)
+             .join(Repository)
-      .join(Visibility)
+             .join(Visibility)
-      .switch(Repository)
+             .switch(Repository)
-      .join(RepositoryPermission, JOIN_LEFT_OUTER)
+             .join(RepositoryPermission, JOIN_LEFT_OUTER)
-      .where(ImageStorage.uploading == False))
+             .where(ImageStorage.uploading == False))
 
     query = (_filter_to_repos_for_user(query, username)
-      .where(Image.docker_image_id == docker_image_id))
+             .where(Image.docker_image_id == docker_image_id))
-    
+
     new_image_ancestry = '/'
     origin_image_id = None
     try:
       to_copy = query.get()
       msg = 'Linking image to existing storage with docker id: %s and uuid: %s'
       logger.debug(msg, docker_image_id, to_copy.storage.uuid)
-      
+
       new_image_ancestry = __translate_ancestry(to_copy.ancestors, translations, repository,
                                                 username, preferred_location)
 

data/userevent.py

@@ -44,7 +44,7 @@ class UserEvent(object):
   as backed by Redis.
   """
   def __init__(self, redis_host, username):
-    self._redis = redis.StrictRedis(host=redis_host)
+    self._redis = redis.StrictRedis(host=redis_host, socket_timeout=5)
     self._username = username
 
   @staticmethod
@@ -77,7 +77,7 @@ class UserEventListener(object):
   def __init__(self, redis_host, username, events=set([])):
     channels = [self._user_event_key(username, e) for e in events]
 
-    self._redis = redis.StrictRedis(host=redis_host)
+    self._redis = redis.StrictRedis(host=redis_host, socket_timeout=5)
     self._pubsub = self._redis.pubsub()
     self._pubsub.subscribe(channels)
 

endpoints/common.py

@@ -82,20 +82,23 @@ def param_required(param_name):
 
 
 @login_manager.user_loader
-def load_user(username):
+def load_user(user_db_id):
-  logger.debug('User loader loading deferred user: %s' % username)
+  logger.debug('User loader loading deferred user id: %s' % user_db_id)
-  return _LoginWrappedDBUser(username)
+  try:
+    user_db_id_int = int(user_db_id)
+    return _LoginWrappedDBUser(user_db_id_int)
+  except ValueError:
+    return None
 
 
 class _LoginWrappedDBUser(UserMixin):
-  def __init__(self, db_username, db_user=None):
+  def __init__(self, user_db_id, db_user=None):
-
+    self._db_id = user_db_id
-    self._db_username = db_username
     self._db_user = db_user
 
   def db_user(self):
     if not self._db_user:
-      self._db_user = model.get_user(self._db_username)
+      self._db_user = model.get_user_by_id(self._db_id)
     return self._db_user
 
   def is_authenticated(self):
@@ -105,13 +108,13 @@ class _LoginWrappedDBUser(UserMixin):
     return self.db_user().verified
 
   def get_id(self):
-    return unicode(self._db_username)
+    return unicode(self._db_id)
 
 
 def common_login(db_user):
-  if login_user(_LoginWrappedDBUser(db_user.username, db_user)):
+  if login_user(_LoginWrappedDBUser(db_user.id, db_user)):
     logger.debug('Successfully signed in as: %s' % db_user.username)
-    new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN})
+    new_identity = QuayDeferredPermissionUser(db_user.id, 'user_db_id', {scopes.DIRECT_LOGIN})
     identity_changed.send(app, identity=new_identity)
     session['login_time'] = datetime.datetime.now()
     return True

test/test_api_security.py

@@ -5,6 +5,7 @@ from urllib import urlencode
 from urlparse import urlparse, urlunparse, parse_qs
 
 from app import app
+from data import model
 from initdb import setup_database_for_testing, finished_database_for_testing
 from endpoints.api import api_bp, api
 
@@ -75,7 +76,8 @@ class ApiTestCase(unittest.TestCase):
 
       with client.session_transaction() as sess:
         if auth_username:
-          sess['user_id'] = auth_username
+          loaded = model.get_user(auth_username)
+          sess['user_id'] = loaded.id
         sess[CSRF_TOKEN_KEY] = CSRF_TOKEN
 
       # Restore the teardown functions

test/test_api_usage.py

@@ -342,8 +342,8 @@ class TestChangeUserDetails(ApiTestCase):
   def test_changepassword_unicode(self):
     self.login(READ_ACCESS_USER)
     self.putJsonResponse(User,
-                         data=dict(password='someunicode北京市pass'))
+                         data=dict(password=u'someunicode北京市pass'))
-    self.login(READ_ACCESS_USER, password='someunicode北京市pass')
+    self.login(READ_ACCESS_USER, password=u'someunicode北京市pass')
 
   def test_changeeemail(self):
     self.login(READ_ACCESS_USER)