diff --git a/application.py b/application.py index 2fb79835b..493b34fed 100644 --- a/application.py +++ b/application.py @@ -7,7 +7,7 @@ from peewee import Proxy from app import app as application from flask import request, Request from util.names import urn_generator -from data.model import db as model_db, read_slave +from data.database import db as model_db, read_slave # Turn off debug logging for boto logging.getLogger('boto').setLevel(logging.CRITICAL) diff --git a/auth/auth.py b/auth/auth.py index a81876e54..ed0c8d82a 100644 --- a/auth/auth.py +++ b/auth/auth.py @@ -25,7 +25,7 @@ def _load_user_from_cookie(): if not current_user.is_anonymous(): logger.debug('Loading user from cookie: %s', current_user.get_id()) set_authenticated_user_deferred(current_user.get_id()) - loaded = QuayDeferredPermissionUser(current_user.get_id(), 'username', {scopes.DIRECT_LOGIN}) + loaded = QuayDeferredPermissionUser(current_user.get_id(), 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=loaded) return current_user.db_user() return None @@ -58,12 +58,10 @@ def _validate_and_apply_oauth_token(token): set_authenticated_user(validated.authorized_user) set_validated_oauth_token(validated) - new_identity = QuayDeferredPermissionUser(validated.authorized_user.username, 'username', - scope_set) + new_identity = QuayDeferredPermissionUser(validated.authorized_user.id, 'user_db_id', scope_set) identity_changed.send(app, identity=new_identity) - def process_basic_auth(auth): normalized = [part.strip() for part in auth.split(' ') if part] if normalized[0].lower() != 'basic' or len(normalized) != 2: @@ -100,8 +98,7 @@ def process_basic_auth(auth): logger.debug('Successfully validated robot: %s' % credentials[0]) set_authenticated_user(robot) - deferred_robot = QuayDeferredPermissionUser(robot.username, 'username', - {scopes.DIRECT_LOGIN}) + deferred_robot = QuayDeferredPermissionUser(robot.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=deferred_robot) return except model.InvalidRobotException: @@ -114,7 +111,7 @@ def process_basic_auth(auth): logger.debug('Successfully validated user: %s' % authenticated.username) set_authenticated_user(authenticated) - new_identity = QuayDeferredPermissionUser(authenticated.username, 'username', + new_identity = QuayDeferredPermissionUser(authenticated.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) return diff --git a/auth/auth_context.py b/auth/auth_context.py index b97ffa02d..6c587f901 100644 --- a/auth/auth_context.py +++ b/auth/auth_context.py @@ -10,13 +10,13 @@ logger = logging.getLogger(__name__) def get_authenticated_user(): user = getattr(_request_ctx_stack.top, 'authenticated_user', None) if not user: - username = getattr(_request_ctx_stack.top, 'authenticated_username', None) - if not username: - logger.debug('No authenticated user or deferred username.') + db_id = getattr(_request_ctx_stack.top, 'authenticated_db_id', None) + if not db_id: + logger.debug('No authenticated user or deferred database id.') return None logger.debug('Loading deferred authenticated user.') - loaded = model.get_user(username) + loaded = model.get_user_by_id(db_id) set_authenticated_user(loaded) user = loaded @@ -30,10 +30,10 @@ def set_authenticated_user(user_or_robot): ctx.authenticated_user = user_or_robot -def set_authenticated_user_deferred(username_or_robotname): - logger.debug('Deferring loading of authenticated user object: %s', username_or_robotname) +def set_authenticated_user_deferred(user_or_robot_db_id): + logger.debug('Deferring loading of authenticated user object: %s', user_or_robot_db_id) ctx = _request_ctx_stack.top - ctx.authenticated_username = username_or_robotname + ctx.authenticated_db_id = user_or_robot_db_id def get_validated_oauth_token(): diff --git a/auth/permissions.py b/auth/permissions.py index 2b27f9583..59b190b3c 100644 --- a/auth/permissions.py +++ b/auth/permissions.py @@ -58,8 +58,8 @@ SCOPE_MAX_USER_ROLES.update({ class QuayDeferredPermissionUser(Identity): - def __init__(self, id, auth_type, scopes): - super(QuayDeferredPermissionUser, self).__init__(id, auth_type) + def __init__(self, db_id, auth_type, scopes): + super(QuayDeferredPermissionUser, self).__init__(db_id, auth_type) self._permissions_loaded = False self._scope_set = scopes @@ -88,7 +88,7 @@ class QuayDeferredPermissionUser(Identity): def can(self, permission): if not self._permissions_loaded: logger.debug('Loading user permissions after deferring.') - user_object = model.get_user(self.id) + user_object = model.get_user_by_id(self.id) # Add the superuser need, if applicable. if (user_object.username is not None and @@ -230,9 +230,9 @@ def on_identity_loaded(sender, identity): if isinstance(identity, QuayDeferredPermissionUser): logger.debug('Deferring permissions for user: %s', identity.id) - elif identity.auth_type == 'username': + elif identity.auth_type == 'user_db_id': logger.debug('Switching username permission to deferred object: %s', identity.id) - switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'username', {scopes.DIRECT_LOGIN}) + switch_to_deferred = QuayDeferredPermissionUser(identity.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=switch_to_deferred) elif identity.auth_type == 'token': diff --git a/data/buildlogs.py b/data/buildlogs.py index 9128390af..b33f25559 100644 --- a/data/buildlogs.py +++ b/data/buildlogs.py @@ -17,7 +17,7 @@ class RedisBuildLogs(object): PHASE = 'phase' def __init__(self, redis_host): - self._redis = redis.StrictRedis(host=redis_host) + self._redis = redis.StrictRedis(host=redis_host, socket_timeout=5) @staticmethod def _logs_key(build_id): diff --git a/data/database.py b/data/database.py index 740743df5..fea62286a 100644 --- a/data/database.py +++ b/data/database.py @@ -169,6 +169,7 @@ class Visibility(BaseModel): class Repository(BaseModel): namespace = CharField() + namespace_user = ForeignKeyField(User, null=True) name = CharField() visibility = ForeignKeyField(Visibility) description = TextField(null=True) @@ -180,6 +181,7 @@ class Repository(BaseModel): indexes = ( # create a unique index on namespace and name (('namespace', 'name'), True), + (('namespace_user', 'name'), False), ) diff --git a/data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py b/data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py new file mode 100644 index 000000000..8d2810af7 --- /dev/null +++ b/data/migrations/versions/13da56878560_migrate_registry_namespaces_to_.py @@ -0,0 +1,24 @@ +"""Migrate registry namespaces to reference a user. + +Revision ID: 13da56878560 +Revises: 51d04d0e7e6f +Create Date: 2014-09-18 13:56:45.130455 + +""" + +# revision identifiers, used by Alembic. +revision = '13da56878560' +down_revision = '51d04d0e7e6f' + +from alembic import op +import sqlalchemy as sa + +from data.database import Repository, User + +def upgrade(tables): + # Add the namespace_user column, allowing it to be nullable + op.add_column('repository', sa.Column('namespace_user', sa.Integer(), sa.ForeignKey('user.id'))) + + +def downgrade(tables): + op.drop_column('repository', 'namespace_user') diff --git a/data/model/legacy.py b/data/model/legacy.py index 28f73dafe..8a5a3b9fa 100644 --- a/data/model/legacy.py +++ b/data/model/legacy.py @@ -5,8 +5,18 @@ import json from datetime import datetime, timedelta -from data.database import * -from util.validation import * +from data.database import (User, Repository, Image, AccessToken, Role, RepositoryPermission, + Visibility, RepositoryTag, EmailConfirmation, FederatedLogin, + LoginService, RepositoryBuild, Team, TeamMember, TeamRole, + LogEntryKind, LogEntry, PermissionPrototype, ImageStorage, + BuildTriggerService, RepositoryBuildTrigger, NotificationKind, + Notification, ImageStorageLocation, ImageStoragePlacement, + ExternalNotificationEvent, ExternalNotificationMethod, + RepositoryNotification, RepositoryAuthorizedEmail, TeamMemberInvite, + random_string_generator, db, BUILD_PHASE) +from peewee import JOIN_LEFT_OUTER, fn +from util.validation import (validate_username, validate_email, validate_password, + INVALID_PASSWORD_MESSAGE) from util.names import format_robot_username from util.backoff import exponential_backoff @@ -560,6 +570,13 @@ def get_user_or_org(username): return None +def get_user_by_id(user_db_id): + try: + return User.get(User.id == user_db_id, User.organization == False) + except User.DoesNotExist: + return None + + def get_user_or_org_by_customer_id(customer_id): try: return User.get(User.stripe_id == customer_id) @@ -584,12 +601,13 @@ def get_matching_users(username_prefix, robot_namespace=None, (User.robot == True))) query = (User - .select(User.username, fn.Sum(Team.id), User.robot) + .select(User.username, User.robot) .group_by(User.username) .where(direct_user_query)) if organization: query = (query + .select(User.username, User.robot, fn.Sum(Team.id)) .join(TeamMember, JOIN_LEFT_OUTER) .join(Team, JOIN_LEFT_OUTER, on=((Team.id == TeamMember.team) & (Team.organization == organization)))) @@ -598,9 +616,9 @@ def get_matching_users(username_prefix, robot_namespace=None, class MatchingUserResult(object): def __init__(self, *args): self.username = args[0] - self.is_robot = args[2] + self.is_robot = args[1] if organization: - self.is_org_member = (args[1] != None) + self.is_org_member = (args[2] != None) else: self.is_org_member = None @@ -625,7 +643,8 @@ def verify_user(username_or_email, password): retry_after = can_retry_at - now raise TooManyLoginAttemptsException('Too many login attempts.', retry_after.total_seconds()) - if (fetched.password_hash and hash_password(password, fetched.password_hash) == fetched.password_hash): + if (fetched.password_hash and + hash_password(password, fetched.password_hash) == fetched.password_hash): if fetched.invalid_login_attempts > 0: fetched.invalid_login_attempts = 0 fetched.save() @@ -758,23 +777,23 @@ def _filter_to_repos_for_user(query, username=None, namespace=None, AdminUser = User.alias() query = (query - .switch(RepositoryPermission) - .join(User, JOIN_LEFT_OUTER) - .switch(RepositoryPermission) - .join(Team, JOIN_LEFT_OUTER) - .join(TeamMember, JOIN_LEFT_OUTER) - .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == - TeamMember.user)) - .switch(Repository) - .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace)) - .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == - AdminTeam.organization)) - .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) - .switch(AdminTeam) - .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == - AdminTeamMember.team)) - .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == - AdminUser.id))) + .switch(RepositoryPermission) + .join(User, JOIN_LEFT_OUTER) + .switch(RepositoryPermission) + .join(Team, JOIN_LEFT_OUTER) + .join(TeamMember, JOIN_LEFT_OUTER) + .join(UserThroughTeam, JOIN_LEFT_OUTER, on=(UserThroughTeam.id == + TeamMember.user)) + .switch(Repository) + .join(Org, JOIN_LEFT_OUTER, on=(Org.username == Repository.namespace)) + .join(AdminTeam, JOIN_LEFT_OUTER, on=(Org.id == + AdminTeam.organization)) + .join(TeamRole, JOIN_LEFT_OUTER, on=(AdminTeam.role == TeamRole.id)) + .switch(AdminTeam) + .join(AdminTeamMember, JOIN_LEFT_OUTER, on=(AdminTeam.id == + AdminTeamMember.team)) + .join(AdminUser, JOIN_LEFT_OUTER, on=(AdminTeamMember.user == + AdminUser.id))) where_clause = ((User.username == username) | (UserThroughTeam.username == username) | @@ -1041,8 +1060,9 @@ def __apply_default_permissions(repo, proto_query, name_property, def create_repository(namespace, name, creating_user, visibility='private'): private = Visibility.get(name=visibility) - repo = Repository.create(namespace=namespace, name=name, - visibility=private) + namespace_user = User.get(username=namespace) + repo = Repository.create(namespace=namespace, name=name, visibility=private, + namespace_user=namespace_user) admin = Role.get(name='admin') if creating_user and not creating_user.organization: @@ -1114,26 +1134,26 @@ def find_create_or_link_image(docker_image_id, repository, username, translation return repo_image query = (Image - .select(Image, ImageStorage) - .distinct() - .join(ImageStorage) - .switch(Image) - .join(Repository) - .join(Visibility) - .switch(Repository) - .join(RepositoryPermission, JOIN_LEFT_OUTER) - .where(ImageStorage.uploading == False)) + .select(Image, ImageStorage) + .distinct() + .join(ImageStorage) + .switch(Image) + .join(Repository) + .join(Visibility) + .switch(Repository) + .join(RepositoryPermission, JOIN_LEFT_OUTER) + .where(ImageStorage.uploading == False)) query = (_filter_to_repos_for_user(query, username) - .where(Image.docker_image_id == docker_image_id)) - + .where(Image.docker_image_id == docker_image_id)) + new_image_ancestry = '/' origin_image_id = None try: to_copy = query.get() msg = 'Linking image to existing storage with docker id: %s and uuid: %s' logger.debug(msg, docker_image_id, to_copy.storage.uuid) - + new_image_ancestry = __translate_ancestry(to_copy.ancestors, translations, repository, username, preferred_location) diff --git a/data/userevent.py b/data/userevent.py index bcdafd078..d137eac04 100644 --- a/data/userevent.py +++ b/data/userevent.py @@ -44,7 +44,7 @@ class UserEvent(object): as backed by Redis. """ def __init__(self, redis_host, username): - self._redis = redis.StrictRedis(host=redis_host) + self._redis = redis.StrictRedis(host=redis_host, socket_timeout=5) self._username = username @staticmethod @@ -77,7 +77,7 @@ class UserEventListener(object): def __init__(self, redis_host, username, events=set([])): channels = [self._user_event_key(username, e) for e in events] - self._redis = redis.StrictRedis(host=redis_host) + self._redis = redis.StrictRedis(host=redis_host, socket_timeout=5) self._pubsub = self._redis.pubsub() self._pubsub.subscribe(channels) diff --git a/endpoints/common.py b/endpoints/common.py index 52715a1d1..1c6439371 100644 --- a/endpoints/common.py +++ b/endpoints/common.py @@ -82,20 +82,23 @@ def param_required(param_name): @login_manager.user_loader -def load_user(username): - logger.debug('User loader loading deferred user: %s' % username) - return _LoginWrappedDBUser(username) +def load_user(user_db_id): + logger.debug('User loader loading deferred user id: %s' % user_db_id) + try: + user_db_id_int = int(user_db_id) + return _LoginWrappedDBUser(user_db_id_int) + except ValueError: + return None class _LoginWrappedDBUser(UserMixin): - def __init__(self, db_username, db_user=None): - - self._db_username = db_username + def __init__(self, user_db_id, db_user=None): + self._db_id = user_db_id self._db_user = db_user def db_user(self): if not self._db_user: - self._db_user = model.get_user(self._db_username) + self._db_user = model.get_user_by_id(self._db_id) return self._db_user def is_authenticated(self): @@ -105,13 +108,13 @@ class _LoginWrappedDBUser(UserMixin): return self.db_user().verified def get_id(self): - return unicode(self._db_username) + return unicode(self._db_id) def common_login(db_user): - if login_user(_LoginWrappedDBUser(db_user.username, db_user)): + if login_user(_LoginWrappedDBUser(db_user.id, db_user)): logger.debug('Successfully signed in as: %s' % db_user.username) - new_identity = QuayDeferredPermissionUser(db_user.username, 'username', {scopes.DIRECT_LOGIN}) + new_identity = QuayDeferredPermissionUser(db_user.id, 'user_db_id', {scopes.DIRECT_LOGIN}) identity_changed.send(app, identity=new_identity) session['login_time'] = datetime.datetime.now() return True diff --git a/test/data/test.db b/test/data/test.db index 68f838e57..f6e5f83d9 100644 Binary files a/test/data/test.db and b/test/data/test.db differ diff --git a/test/test_api_security.py b/test/test_api_security.py index 04498ad2a..d9e4f223b 100644 --- a/test/test_api_security.py +++ b/test/test_api_security.py @@ -5,6 +5,7 @@ from urllib import urlencode from urlparse import urlparse, urlunparse, parse_qs from app import app +from data import model from initdb import setup_database_for_testing, finished_database_for_testing from endpoints.api import api_bp, api @@ -75,7 +76,8 @@ class ApiTestCase(unittest.TestCase): with client.session_transaction() as sess: if auth_username: - sess['user_id'] = auth_username + loaded = model.get_user(auth_username) + sess['user_id'] = loaded.id sess[CSRF_TOKEN_KEY] = CSRF_TOKEN # Restore the teardown functions diff --git a/test/test_api_usage.py b/test/test_api_usage.py index a38b761c8..b85388cdc 100644 --- a/test/test_api_usage.py +++ b/test/test_api_usage.py @@ -342,8 +342,8 @@ class TestChangeUserDetails(ApiTestCase): def test_changepassword_unicode(self): self.login(READ_ACCESS_USER) self.putJsonResponse(User, - data=dict(password='someunicode北京市pass')) - self.login(READ_ACCESS_USER, password='someunicode北京市pass') + data=dict(password=u'someunicode北京市pass')) + self.login(READ_ACCESS_USER, password=u'someunicode北京市pass') def test_changeeemail(self): self.login(READ_ACCESS_USER)