diff --git a/config.py b/config.py index 57dc1342b..7220800c9 100644 --- a/config.py +++ b/config.py @@ -432,3 +432,8 @@ class DefaultConfig(object): # Maximum size allowed for layers in the registry. MAXIMUM_LAYER_SIZE = '20G' + + # Feature Flag: Whether team syncing from the backing auth is enabled. + FEATURE_TEAM_SYNCING = False + TEAM_RESYNC_STALE_TIME = '30m' + TEAM_SYNC_WORKER_FREQUENCY = 60 # seconds diff --git a/data/database.py b/data/database.py index 6d7b7e3b2..8a7a87004 100644 --- a/data/database.py +++ b/data/database.py @@ -451,7 +451,7 @@ class User(BaseModel): TagManifest, AccessToken, OAuthAccessToken, BlobUpload, RepositoryNotification, OAuthAuthorizationCode, RepositoryActionCount, TagManifestLabel, Tag, - ManifestLabel, BlobUploading} | beta_classes + ManifestLabel, BlobUploading, TeamSync} | beta_classes delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes) @@ -526,6 +526,15 @@ class LoginService(BaseModel): name = CharField(unique=True, index=True) +class TeamSync(BaseModel): + team = ForeignKeyField(Team, unique=True) + + transaction_id = CharField() + last_updated = DateTimeField(null=True, index=True) + service = ForeignKeyField(LoginService) + config = JSONField() + + class FederatedLogin(BaseModel): user = QuayUserField(allows_robots=True, index=True) service = ForeignKeyField(LoginService) diff --git a/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py b/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py new file mode 100644 index 000000000..e13d0a344 --- /dev/null +++ b/data/migrations/versions/be8d1c402ce0_add_teamsync_table.py @@ -0,0 +1,39 @@ +"""Add TeamSync table + +Revision ID: be8d1c402ce0 +Revises: a6c463dfb9fe +Create Date: 2017-02-23 13:34:52.356812 + +""" + +# revision identifiers, used by Alembic. +revision = 'be8d1c402ce0' +down_revision = 'a6c463dfb9fe' + +from alembic import op +import sqlalchemy as sa +from util.migrate import UTF8LongText + +def upgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('teamsync', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('team_id', sa.Integer(), nullable=False), + sa.Column('transaction_id', sa.String(length=255), nullable=False), + sa.Column('last_updated', sa.DateTime(), nullable=True), + sa.Column('service_id', sa.Integer(), nullable=False), + sa.Column('config', UTF8LongText(), nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['loginservice.id'], name=op.f('fk_teamsync_service_id_loginservice')), + sa.ForeignKeyConstraint(['team_id'], ['team.id'], name=op.f('fk_teamsync_team_id_team')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_teamsync')) + ) + op.create_index('teamsync_last_updated', 'teamsync', ['last_updated'], unique=False) + op.create_index('teamsync_service_id', 'teamsync', ['service_id'], unique=False) + op.create_index('teamsync_team_id', 'teamsync', ['team_id'], unique=True) + ### end Alembic commands ### + + +def downgrade(tables): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('teamsync') + ### end Alembic commands ### diff --git a/data/model/team.py b/data/model/team.py index 753d00e2f..923951299 100644 --- a/data/model/team.py +++ b/data/model/team.py @@ -1,9 +1,15 @@ -from data.database import Team, TeamMember, TeamRole, User, TeamMemberInvite, RepositoryPermission +import json +import uuid + +from datetime import datetime +from peewee import fn + +from data.database import (Team, TeamMember, TeamRole, User, TeamMemberInvite, RepositoryPermission, + TeamSync, LoginService, FederatedLogin, db_random_func, db_transaction) from data.model import (DataModelException, InvalidTeamException, UserAlreadyInTeam, - InvalidTeamMemberException, user, _basequery) + InvalidTeamMemberException, _basequery) from data.text import prefix_search from util.validation import validate_username -from peewee import fn, JOIN_LEFT_OUTER from util.morecollections import AttrDict @@ -188,7 +194,7 @@ def get_matching_teams(team_prefix, organization): return query.limit(10) -def get_teams_within_org(organization): +def get_teams_within_org(organization, has_external_auth=False): """ Returns a AttrDict of team info (id, name, description), its role under the org, the number of repositories on which it has permission, and the number of members. """ @@ -205,6 +211,8 @@ def get_teams_within_org(organization): 'repo_count': 0, 'member_count': 0, + + 'is_synced': False, } teams = {team.id: _team_view(team) for team in query} @@ -232,6 +240,12 @@ def get_teams_within_org(organization): for member_tuple in members_tuples: teams[member_tuple[0]]['member_count'] = member_tuple[1] + # Add syncing information. + if has_external_auth: + sync_query = TeamSync.select(TeamSync.team).where(TeamSync.team << teams.keys()) + for team_sync in sync_query: + teams[team_sync.team_id]['is_synced'] = True + return [AttrDict(team_info) for team_info in teams.values()] @@ -369,3 +383,121 @@ def confirm_team_invite(code, user_obj): team = found.team inviter = found.inviter return (team, inviter) + + +def get_federated_team_member_mapping(team, login_service_name): + """ Returns a dict of all federated IDs for all team members in the team whose users are + bound to the login service within the given name. The dictionary is from federated service + identifier (username) to their Quay User table ID. + """ + login_service = LoginService.get(name=login_service_name) + + query = (FederatedLogin + .select(FederatedLogin.service_ident, User.id) + .join(User) + .join(TeamMember) + .join(Team) + .where(Team.id == team, User.robot == False, FederatedLogin.service == login_service)) + return dict(query.tuples()) + + +def list_team_users(team): + """ Returns an iterator of all the *users* found in a team. Does not include robots. """ + return (User + .select() + .join(TeamMember) + .join(Team) + .where(Team.id == team, User.robot == False)) + + +def list_team_robots(team): + """ Returns an iterator of all the *robots* found in a team. Does not include users. """ + return (User + .select() + .join(TeamMember) + .join(Team) + .where(Team.id == team, User.robot == True)) + + +def set_team_syncing(team, login_service_name, config): + """ Sets the given team to sync to the given service using the given config. """ + login_service = LoginService.get(name=login_service_name) + return TeamSync.create(team=team, transaction_id='', service=login_service, + config=json.dumps(config)) + + +def remove_team_syncing(orgname, teamname): + """ Removes syncing on the team matching the given organization name and team name. """ + existing = get_team_sync_information(orgname, teamname) + if existing: + existing.delete_instance() + + +def get_stale_team(stale_timespan): + """ Returns a team that is setup to sync to an external group, and who has not been synced in + now - stale_timespan. Returns None if none found. + """ + stale_at = datetime.now() - stale_timespan + + try: + candidates = (TeamSync + .select(TeamSync.id) + .where((TeamSync.last_updated <= stale_at) | (TeamSync.last_updated >> None)) + .limit(500) + .alias('candidates')) + + found = (TeamSync + .select(candidates.c.id) + .from_(candidates) + .order_by(db_random_func()) + .get()) + + if found is None: + return + + return TeamSync.select(TeamSync, Team).join(Team).where(TeamSync.id == found.id).get() + except TeamSync.DoesNotExist: + return None + + +def get_team_sync_information(orgname, teamname): + """ Returns the team syncing information for the team with the given name under the organization + with the given name or None if none. + """ + query = (TeamSync + .select(TeamSync, LoginService) + .join(Team) + .join(User) + .switch(TeamSync) + .join(LoginService) + .where(Team.name == teamname, User.organization == True, User.username == orgname)) + + try: + return query.get() + except TeamSync.DoesNotExist: + return None + + +def update_sync_status(team_sync_info): + """ Attempts to update the transaction ID and last updated time on a TeamSync object. If the + transaction ID on the entry in the DB does not match that found on the object, this method + returns False, which indicates another caller updated it first. + """ + new_transaction_id = str(uuid.uuid4()) + query = (TeamSync + .update(transaction_id=new_transaction_id, last_updated=datetime.now()) + .where(TeamSync.id == team_sync_info.id, + TeamSync.transaction_id == team_sync_info.transaction_id)) + return query.execute() == 1 + + +def delete_members_not_present(team, member_id_set): + """ Deletes all members of the given team that are not found in the member ID set. """ + with db_transaction(): + user_ids = set([u.id for u in list_team_users(team)]) + to_delete = list(user_ids - member_id_set) + if to_delete: + query = TeamMember.delete().where(TeamMember.team == team, TeamMember.user << to_delete) + return query.execute() + + return 0 diff --git a/data/users/__init__.py b/data/users/__init__.py index 1f083e8ff..f8e47ba16 100644 --- a/data/users/__init__.py +++ b/data/users/__init__.py @@ -175,6 +175,12 @@ class UserAuthentication(object): """ return self.state.link_user(username_or_email) + def get_and_link_federated_user_info(self, user_info): + """ Returns a tuple containing the database user record linked to the given UserInformation + pair and any error that occurred when trying to link the user. + """ + return self.state.get_and_link_federated_user_info(user_info) + def confirm_existing_user(self, username, password): """ Verifies that the given password matches to the given DB username. Unlike verify_credentials, this call first translates the DB user via the FederatedLogin table @@ -186,6 +192,28 @@ class UserAuthentication(object): """ Verifies that the given username and password credentials are valid. """ return self.state.verify_credentials(username_or_email, password) + def check_group_lookup_args(self, group_lookup_args): + """ Verifies that the given group lookup args point to a valid group. Returns a tuple consisting + of a boolean status and an error message (if any). + """ + return self.state.check_group_lookup_args(group_lookup_args) + + def service_metadata(self): + """ Returns a dictionary of extra metadata to present to *superusers* about this auth engine. + For example, LDAP returns the base DN so we can display to the user during sync setup. + """ + return self.state.service_metadata() + + def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False): + """ Returns a tuple of an iterator over all the members of the group matching the given lookup + args dictionary, or the error that occurred if the initial call failed or is unsupported. + The format of the lookup args dictionary is specific to the implementation. + Each result in the iterator is a tuple of (UserInformation, error_message), and only + one will be not-None. + """ + return self.state.iterate_group_members(group_lookup_args, page_size=page_size, + disable_pagination=disable_pagination) + def verify_and_link_user(self, username_or_email, password, basic_auth=False): """ Verifies that the given username and password credentials are valid and, if so, creates or links the database user to the federated identity. """ diff --git a/data/users/database.py b/data/users/database.py index 4edb0f442..862065723 100644 --- a/data/users/database.py +++ b/data/users/database.py @@ -24,7 +24,22 @@ class DatabaseUsers(object): """ Never used since all users being added are already, by definition, in the database. """ return (None, 'Unsupported for this authentication system') + def get_and_link_federated_user_info(self, user_info): + """ Never used since all users being added are already, by definition, in the database. """ + return (None, 'Unsupported for this authentication system') + def query_users(self, query, limit): """ No need to implement, as we already query for users directly in the database. """ return (None, '', '') + def check_group_lookup_args(self, group_lookup_args): + """ Never used since all groups, by definition, are in the database. """ + return (False, 'Not supported') + + def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False): + """ Never used since all groups, by definition, are in the database. """ + return (None, 'Not supported') + + def service_metadata(self): + """ Never used since database has no metadata """ + return {} diff --git a/data/users/externalldap.py b/data/users/externalldap.py index 8cd82ac28..1cfb61385 100644 --- a/data/users/externalldap.py +++ b/data/users/externalldap.py @@ -2,6 +2,8 @@ import ldap import logging import os +from ldap.controls import SimplePagedResultsControl + from collections import namedtuple from data.users.federated import FederatedUsers, UserInformation from util.itertoolrecipes import take @@ -10,6 +12,7 @@ logger = logging.getLogger(__name__) _DEFAULT_NETWORK_TIMEOUT = 10.0 # seconds _DEFAULT_TIMEOUT = 10.0 # seconds +_DEFAULT_PAGE_SIZE = 1000 class LDAPConnectionBuilder(object): @@ -63,7 +66,7 @@ class LDAPUsers(FederatedUsers): def __init__(self, ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr, allow_tls_fallback=False, secondary_user_rdns=None, requires_email=True, - timeout=None, network_timeout=None): + timeout=None, network_timeout=None, force_no_pagination=False): super(LDAPUsers, self).__init__('ldap', requires_email) self._ldap = LDAPConnectionBuilder(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback, @@ -73,6 +76,7 @@ class LDAPUsers(FederatedUsers): self._email_attr = email_attr self._allow_tls_fallback = allow_tls_fallback self._requires_email = requires_email + self._force_no_pagination = force_no_pagination # Note: user_rdn is a list of RDN pieces (for historical reasons), and secondary_user_rds # is a list of RDN strings. @@ -84,6 +88,7 @@ class LDAPUsers(FederatedUsers): # Create the set of full DN paths. self._user_dns = [get_full_rdn(relative_dn) for relative_dn in relative_user_dns] + self._base_dn = ','.join(base_dn) def _get_ldap_referral_dn(self, referral_exception): logger.debug('Got referral: %s', referral_exception.args[0]) @@ -174,7 +179,7 @@ class LDAPUsers(FederatedUsers): with_mail = [result for result in with_dns if result.attrs.get(self._email_attr)] return (with_mail[0] if with_mail else with_dns[0], None) - def _credential_for_user(self, response): + def _build_user_information(self, response): if not response.get(self._uid_attr): return (None, 'Missing uid field "%s" in user record' % self._uid_attr) @@ -194,7 +199,7 @@ class LDAPUsers(FederatedUsers): logger.debug('Found user for LDAP username or email %s', username_or_email) _, found_response = found_user - return self._credential_for_user(found_response) + return self._build_user_information(found_response) def query_users(self, query, limit=20): """ Queries LDAP for matching users. """ @@ -208,7 +213,7 @@ class LDAPUsers(FederatedUsers): final_results = [] for result in results[0:limit]: - credentials, err_msg = self._credential_for_user(result.attrs) + credentials, err_msg = self._build_user_information(result.attrs) if err_msg is not None: continue @@ -253,4 +258,87 @@ class LDAPUsers(FederatedUsers): logger.debug('Invalid LDAP credentials') return (None, 'Invalid password') - return self._credential_for_user(found_response) + return self._build_user_information(found_response) + + def service_metadata(self): + return { + 'base_dn': self._base_dn, + } + + def check_group_lookup_args(self, group_lookup_args, disable_pagination=False): + if not group_lookup_args.get('group_dn'): + return (False, 'Missing group_dn') + + (it, err) = self.iterate_group_members(group_lookup_args, page_size=1, + disable_pagination=disable_pagination) + if err is not None: + return (False, err) + + if not list(it): + return (False, 'Group does not exist or is empty') + + return (True, None) + + def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False): + try: + with self._ldap.get_connection(): + pass + except ldap.INVALID_CREDENTIALS: + return (None, 'LDAP Admin dn or password is invalid') + + group_dn = group_lookup_args['group_dn'] + page_size = page_size or _DEFAULT_PAGE_SIZE + return (self._iterate_members(group_dn, page_size, disable_pagination), None) + + def _iterate_members(self, group_dn, page_size, disable_pagination): + has_pagination = not(self._force_no_pagination or disable_pagination) + with self._ldap.get_connection() as conn: + lc = ldap.controls.libldap.SimplePagedResultsControl(criticality=True, size=page_size, + cookie='') + + search_flt = '(memberOf=%s,%s)' % (group_dn, self._base_dn) + attributes = [self._uid_attr, self._email_attr] + + for user_search_dn in self._user_dns: + # Conduct the initial search for users that are a member of the group. + if has_pagination: + msgid = conn.search_ext(user_search_dn, ldap.SCOPE_SUBTREE, search_flt, serverctrls=[lc], + attrlist=attributes) + else: + msgid = conn.search(user_search_dn, ldap.SCOPE_SUBTREE, search_flt, attrlist=attributes) + + while True: + if has_pagination: + _, rdata, _, serverctrls = conn.result3(msgid) + else: + _, rdata = conn.result(msgid) + + # Yield any users found. + for userdata in rdata: + yield self._build_user_information(userdata[1]) + + # If pagination is disabled, nothing more to do. + if not has_pagination: + break + + # Filter down the controls with which the server responded, looking for the paging + # control type. If not found, then the server does not support pagination and we already + # got all of the results. + pctrls = [control for control in serverctrls + if control.controlType == ldap.controls.SimplePagedResultsControl.controlType] + + if pctrls: + # Server supports pagination. Update the cookie so the next search finds the next page, + # then conduct the next search. + cookie = lc.cookie = pctrls[0].cookie + if cookie: + msgid = conn.search_ext(user_search_dn, ldap.SCOPE_SUBTREE, search_flt, + serverctrls=[lc], attrlist=attributes) + continue + else: + # No additional results. + break + else: + # Pagination is not supported. + logger.debug('Pagination is not supported for this LDAP server') + break diff --git a/data/users/federated.py b/data/users/federated.py index 8b6e570cb..cdf6a28ce 100644 --- a/data/users/federated.py +++ b/data/users/federated.py @@ -37,7 +37,63 @@ class FederatedUsers(object): """ If implemented, get_user must be implemented as well. """ return (None, 'Not supported') - def _get_federated_user(self, username, email): + def link_user(self, username_or_email): + (user_info, err_msg) = self.get_user(username_or_email) + if user_info is None: + return (None, err_msg) + + return self.get_and_link_federated_user_info(user_info) + + def get_and_link_federated_user_info(self, user_info): + return self._get_and_link_federated_user_info(user_info.username, user_info.email) + + def verify_and_link_user(self, username_or_email, password): + """ Verifies the given credentials and, if valid, creates/links a database user to the + associated federated service. + """ + (credentials, err_msg) = self.verify_credentials(username_or_email, password) + if credentials is None: + return (None, err_msg) + + return self._get_and_link_federated_user_info(credentials.username, credentials.email) + + def confirm_existing_user(self, username, password): + """ Confirms that the given *database* username and service password are valid for the linked + service. This method is used when the federated service's username is not known. + """ + db_user = model.user.get_user(username) + if not db_user: + return (None, 'Invalid user') + + federated_login = model.user.lookup_federated_login(db_user, self._federated_service) + if not federated_login: + return (None, 'Invalid user') + + (credentials, err_msg) = self.verify_credentials(federated_login.service_ident, password) + if credentials is None: + return (None, err_msg) + + return (db_user, None) + + def service_metadata(self): + """ Returns a dictionary of extra metadata to present to *superusers* about this auth engine. + For example, LDAP returns the base DN so we can display to the user during sync setup. + """ + return {} + + def check_group_lookup_args(self, group_lookup_args): + """ Verifies that the given group lookup args point to a valid group. Returns a tuple consisting + of a boolean status and an error message (if any). + """ + return (False, 'Not supported') + + def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False): + """ Returns an iterator over all the members of the group matching the given lookup args + dictionary. The format of the lookup args dictionary is specific to the implementation. + """ + return (None, 'Not supported') + + def _get_and_link_federated_user_info(self, username, email): db_user = model.user.verify_federated_login(self._federated_service, username) if not db_user: # We must create the user in our db @@ -58,43 +114,8 @@ class FederatedUsers(object): prompts=prompts) else: # Update the db attributes from the federated service. - if email: + if email and db_user.email != email: db_user.email = email db_user.save() return (db_user, None) - - def link_user(self, username_or_email): - (credentials, err_msg) = self.get_user(username_or_email) - if credentials is None: - return (None, err_msg) - - return self._get_federated_user(credentials.username, credentials.email) - - def verify_and_link_user(self, username_or_email, password): - """ Verifies the given credentials and, if valid, creates/links a database user to the - associated federated service. - """ - (credentials, err_msg) = self.verify_credentials(username_or_email, password) - if credentials is None: - return (None, err_msg) - - return self._get_federated_user(credentials.username, credentials.email) - - def confirm_existing_user(self, username, password): - """ Confirms that the given *database* username and service password are valid for the linked - service. This method is used when the federated service's username is not known. - """ - db_user = model.user.get_user(username) - if not db_user: - return (None, 'Invalid user') - - federated_login = model.user.lookup_federated_login(db_user, self._federated_service) - if not federated_login: - return (None, 'Invalid user') - - (credentials, err_msg) = self.verify_credentials(federated_login.service_ident, password) - if credentials is None: - return (None, err_msg) - - return (db_user, None) diff --git a/data/users/keystone.py b/data/users/keystone.py index b3e7ad442..8c46a7869 100644 --- a/data/users/keystone.py +++ b/data/users/keystone.py @@ -5,6 +5,7 @@ from keystoneclient.v2_0 import client as kclient from keystoneclient.v3 import client as kv3client from keystoneclient.exceptions import AuthorizationFailure as KeystoneAuthorizationFailure from keystoneclient.exceptions import Unauthorized as KeystoneUnauthorized +from keystoneclient.exceptions import NotFound as KeystoneNotFound from data.users.federated import FederatedUsers, UserInformation from util.itertoolrecipes import take @@ -83,6 +84,11 @@ class KeystoneV3Users(FederatedUsers): self.debug = os.environ.get('USERS_DEBUG') == '1' self.requires_email = requires_email + def _get_admin_client(self): + return kv3client.Client(username=self.admin_username, password=self.admin_password, + tenant_name=self.admin_tenant, auth_url=self.auth_url, + timeout=self.timeout, debug=self.debug) + def verify_credentials(self, username_or_email, password): try: keystone_client = kv3client.Client(username=username_or_email, password=password, @@ -116,6 +122,46 @@ class KeystoneV3Users(FederatedUsers): return (user, None) + def check_group_lookup_args(self, group_lookup_args): + if not group_lookup_args.get('group_id'): + return (False, 'Missing group_id') + + group_id = group_lookup_args['group_id'] + return self._check_group(group_id) + + def _check_group(self, group_id): + try: + return (bool(self._get_admin_client().groups.get(group_id)), None) + except KeystoneNotFound: + return (False, 'Group not found') + except KeystoneAuthorizationFailure as kaf: + logger.exception('Keystone auth failure for admin user for group lookup %s', group_id) + return (False, kaf.message or 'Invalid admin username or password') + except KeystoneUnauthorized as kut: + logger.exception('Keystone unauthorized for admin user for group lookup %s', group_id) + return (False, kut.message or 'Invalid admin username or password') + + def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False): + group_id = group_lookup_args['group_id'] + + (status, err) = self._check_group(group_id) + if not status: + return (None, err) + + try: + user_info_iterator = self._get_admin_client().users.list(group=group_id) + def iterator(): + for user in user_info_iterator: + yield (self._user_info(user), None) + + return (iterator(), None) + except KeystoneAuthorizationFailure as kaf: + logger.exception('Keystone auth failure for admin user for group lookup %s', group_id) + return (False, kaf.message or 'Invalid admin username or password') + except KeystoneUnauthorized as kut: + logger.exception('Keystone unauthorized for admin user for group lookup %s', group_id) + return (False, kut.message or 'Invalid admin username or password') + @staticmethod def _user_info(user): email = user.email if hasattr(user, 'email') else None @@ -126,10 +172,7 @@ class KeystoneV3Users(FederatedUsers): return ([], self.federated_service, None) try: - keystone_client = kv3client.Client(username=self.admin_username, password=self.admin_password, - tenant_name=self.admin_tenant, auth_url=self.auth_url, - timeout=self.timeout, debug=self.debug) - found_users = list(take(limit, keystone_client.users.list(name=query))) + found_users = list(take(limit, self._get_admin_client().users.list(name=query))) logger.debug('For Keystone query %s found users: %s', query, found_users) if not found_users: return ([], self.federated_service, None) diff --git a/data/users/teamsync.py b/data/users/teamsync.py new file mode 100644 index 000000000..44dc52032 --- /dev/null +++ b/data/users/teamsync.py @@ -0,0 +1,135 @@ +import logging +import json + +from data import model + +logger = logging.getLogger(__name__) + + +MAX_TEAMS_PER_ITERATION = 500 + + +def sync_teams_to_groups(authentication, stale_cutoff): + """ Performs team syncing by looking up any stale team(s) found, and performing the sync + operation on them. + """ + logger.debug('Looking up teams to sync to groups') + + sync_team_tried = set() + while len(sync_team_tried) < MAX_TEAMS_PER_ITERATION: + # Find a stale team. + stale_team_sync = model.team.get_stale_team(stale_cutoff) + if not stale_team_sync: + logger.debug('No additional stale team found; sleeping') + return + + # Make sure we don't try to reprocess a team on this iteration. + if stale_team_sync.id in sync_team_tried: + break + + sync_team_tried.add(stale_team_sync.id) + + # Sync the team. + sync_successful = sync_team(authentication, stale_team_sync) + if not sync_successful: + return + + +def sync_team(authentication, stale_team_sync): + """ Performs synchronization of a team (as referenced by the TeamSync stale_team_sync). + Returns True on success and False otherwise. + """ + sync_config = json.loads(stale_team_sync.config) + logger.info('Syncing team `%s` under organization %s via %s (#%s)', stale_team_sync.team.name, + stale_team_sync.team.organization.username, sync_config, stale_team_sync.team_id, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config}) + + # Load all the existing members of the team in Quay that are bound to the auth service. + existing_users = model.team.get_federated_team_member_mapping(stale_team_sync.team, + authentication.federated_service) + + logger.debug('Existing membership of %s for team `%s` under organization %s via %s (#%s)', + len(existing_users), stale_team_sync.team.name, + stale_team_sync.team.organization.username, sync_config, stale_team_sync.team_id, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config, + 'existing_member_count': len(existing_users)}) + + # Load all the members of the team from the authenication system. + (member_iterator, err) = authentication.iterate_group_members(sync_config) + if err is not None: + logger.error('Got error when trying to iterate group members with config %s: %s', + sync_config, err) + return False + + # Collect all the members currently found in the group, adding them to the team as we go + # along. + group_membership = set() + for (member_info, err) in member_iterator: + if err is not None: + logger.error('Got error when trying to construct a member: %s', err) + continue + + # If the member is already in the team, nothing more to do. + if member_info.username in existing_users: + logger.debug('Member %s already in team `%s` under organization %s via %s (#%s)', + member_info.username, stale_team_sync.team.name, + stale_team_sync.team.organization.username, sync_config, + stale_team_sync.team_id, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config, + 'member': member_info.username}) + + group_membership.add(existing_users[member_info.username]) + continue + + # Retrieve the Quay user associated with the member info. + (quay_user, err) = authentication.get_and_link_federated_user_info(member_info) + if err is not None: + logger.error('Could not link external user %s to an internal user: %s', + member_info.username, err, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config, + 'member': member_info.username, 'error': err}) + continue + + # Add the user to the membership set. + group_membership.add(quay_user.id) + + # Add the user to the team. + try: + logger.info('Adding member %s to team `%s` under organization %s via %s (#%s)', + quay_user.username, stale_team_sync.team.name, + stale_team_sync.team.organization.username, sync_config, + stale_team_sync.team_id, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config, + 'member': quay_user.username}) + + model.team.add_user_to_team(quay_user, stale_team_sync.team) + except model.UserAlreadyInTeam: + # If the user is already present, nothing more to do for them. + pass + + # Update the transaction and last_updated time of the team sync. Only if it matches + # the current value will we then perform the deletion step. + got_transaction_handle = model.team.update_sync_status(stale_team_sync) + if not got_transaction_handle: + # Another worker updated this team. Nothing more to do. + logger.debug('Another worker synced team `%s` under organization %s via %s (#%s)', + stale_team_sync.team.name, + stale_team_sync.team.organization.username, sync_config, + stale_team_sync.team_id, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config}) + return True + + # Delete any team members not found in the backing auth system. + logger.debug('Deleting stale members for team `%s` under organization %s via %s (#%s)', + stale_team_sync.team.name, stale_team_sync.team.organization.username, + sync_config, stale_team_sync.team_id, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config}) + + deleted = model.team.delete_members_not_present(stale_team_sync.team, group_membership) + + # Done! + logger.info('Finishing sync for team `%s` under organization %s via %s (#%s): %s deleted', + stale_team_sync.team.name, stale_team_sync.team.organization.username, + sync_config, stale_team_sync.team_id, deleted, + extra={'team': stale_team_sync.team_id, 'sync_config': sync_config}) + return True diff --git a/data/users/test/test_teamsync.py b/data/users/test/test_teamsync.py new file mode 100644 index 000000000..51ab82e6f --- /dev/null +++ b/data/users/test/test_teamsync.py @@ -0,0 +1,260 @@ +from datetime import datetime, timedelta + +import pytest + +from data import model, database +from data.users.federated import FederatedUsers, UserInformation +from data.users.teamsync import sync_team, sync_teams_to_groups +from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file +from test.test_ldap import mock_ldap +from test.test_keystone_auth import fake_keystone +from util.names import parse_robot_username + +_FAKE_AUTH = 'fake' + +class FakeUsers(FederatedUsers): + def __init__(self, group_members): + super(FakeUsers, self).__init__(_FAKE_AUTH, False) + self.group_tuples = [(m, None) for m in group_members] + + def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False): + return (self.group_tuples, None) + + +@pytest.mark.parametrize('starting_membership,group_membership,expected_membership', [ + # Empty team + single member in group => Single member in team. + ([], + [ + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + ], + ['someuser']), + + # Team with a Quay user + empty group => empty team. + ([('someuser', None)], + [], + []), + + # Team with an existing external user + user is in the group => no changes. + ([ + ('someuser', 'someuser'), + ], + [ + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + ], + ['someuser']), + + # Team with an existing external user (with a different Quay username) + user is in the group. + # => no changes + ([ + ('anotherquayname', 'someuser'), + ], + [ + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + ], + ['someuser']), + + # Team missing a few members that are in the group => members added. + ([('someuser', 'someuser')], + [ + UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'), + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + UserInformation('thirduser', 'thirduser', 'thirduser@devtable.com'), + ], + ['anotheruser', 'someuser', 'thirduser']), + + # Team has a few extra members no longer in the group => members removed. + ([ + ('anotheruser', 'anotheruser'), + ('someuser', 'someuser'), + ('thirduser', 'thirduser'), + ('nontestuser', None), + ], + [ + UserInformation('thirduser', 'thirduser', 'thirduser@devtable.com'), + ], + ['thirduser']), + + # Team has different membership than the group => members added and removed. + ([ + ('anotheruser', 'anotheruser'), + ('someuser', 'someuser'), + ('nontestuser', None), + ], + [ + UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'), + UserInformation('missinguser', 'missinguser', 'missinguser@devtable.com'), + ], + ['anotheruser', 'missinguser']), + + # Team has same membership but some robots => robots remain and no other changes. + ([ + ('someuser', 'someuser'), + ('buynlarge+anotherbot', None), + ('buynlarge+somerobot', None), + ], + [ + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + ], + ['someuser', 'buynlarge+somerobot', 'buynlarge+anotherbot']), + + # Team has an extra member and some robots => member removed and robots remain. + ([ + ('someuser', 'someuser'), + ('buynlarge+anotherbot', None), + ('buynlarge+somerobot', None), + ], + [ + # No members. + ], + ['buynlarge+somerobot', 'buynlarge+anotherbot']), + + # Team has a different member and some robots => member changed and robots remain. + ([ + ('someuser', 'someuser'), + ('buynlarge+anotherbot', None), + ('buynlarge+somerobot', None), + ], + [ + UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'), + ], + ['anotheruser', 'buynlarge+somerobot', 'buynlarge+anotherbot']), + + # Team with an existing external user (with a different Quay username) + user is in the group. + # => no changes and robots remain. + ([ + ('anotherquayname', 'someuser'), + ('buynlarge+anotherbot', None), + ], + [ + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + ], + ['someuser', 'buynlarge+anotherbot']), + + # Team which returns the same member twice, as pagination in some engines (like LDAP) is not + # stable. + ([], + [ + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'), + UserInformation('someuser', 'someuser', 'someuser@devtable.com'), + ], + ['anotheruser', 'someuser']), +]) +def test_syncing(starting_membership, group_membership, expected_membership, app): + org = model.organization.get_organization('buynlarge') + + # Necessary for the fake auth entries to be created in FederatedLogin. + database.LoginService.create(name=_FAKE_AUTH) + + # Assert the team is empty, so we have a clean slate. + sync_team_info = model.team.get_team_sync_information('buynlarge', 'synced') + assert len(list(model.team.list_team_users(sync_team_info.team))) == 0 + + # Add the existing starting members to the team. + for starting_member in starting_membership: + (quay_username, fakeauth_username) = starting_member + if '+' in quay_username: + # Add a robot. + (_, shortname) = parse_robot_username(quay_username) + robot, _ = model.user.create_robot(shortname, org) + model.team.add_user_to_team(robot, sync_team_info.team) + else: + email = quay_username + '@devtable.com' + + if fakeauth_username is None: + quay_user = model.user.create_user_noverify(quay_username, email) + else: + quay_user = model.user.create_federated_user(quay_username, email, _FAKE_AUTH, + fakeauth_username, False) + + model.team.add_user_to_team(quay_user, sync_team_info.team) + + # Call syncing on the team. + fake_auth = FakeUsers(group_membership) + assert sync_team(fake_auth, sync_team_info) + + # Ensure the last updated time and transaction_id's have changed. + updated_sync_info = model.team.get_team_sync_information('buynlarge', 'synced') + assert updated_sync_info.last_updated is not None + assert updated_sync_info.transaction_id != sync_team_info.transaction_id + + users_expected = set([name for name in expected_membership if '+' not in name]) + robots_expected = set([name for name in expected_membership if '+' in name]) + assert len(users_expected) + len(robots_expected) == len(expected_membership) + + # Check that the team's users match those expected. + service_user_map = model.team.get_federated_team_member_mapping(sync_team_info.team, _FAKE_AUTH) + assert set(service_user_map.keys()) == users_expected + + quay_users = model.team.list_team_users(sync_team_info.team) + assert len(quay_users) == len(users_expected) + + for quay_user in quay_users: + fakeauth_record = model.user.lookup_federated_login(quay_user, _FAKE_AUTH) + assert fakeauth_record is not None + assert fakeauth_record.service_ident in users_expected + assert service_user_map[fakeauth_record.service_ident] == quay_user.id + + # Check that the team's robots match those expected. + robots_found = set([r.username for r in model.team.list_team_robots(sync_team_info.team)]) + assert robots_expected == robots_found + + +def test_sync_teams_to_groups(app): + # Necessary for the fake auth entries to be created in FederatedLogin. + database.LoginService.create(name=_FAKE_AUTH) + + # Assert the team has not yet been updated. + sync_team_info = model.team.get_team_sync_information('buynlarge', 'synced') + assert sync_team_info.last_updated is None + + # Call to sync all teams. + fake_auth = FakeUsers([]) + sync_teams_to_groups(fake_auth, timedelta(seconds=1)) + + # Ensure the team was synced. + updated_sync_info = model.team.get_team_sync_information('buynlarge', 'synced') + assert updated_sync_info.last_updated is not None + assert updated_sync_info.transaction_id != sync_team_info.transaction_id + + # Set the stale threshold to a high amount and ensure the team is not resynced. + current_info = model.team.get_team_sync_information('buynlarge', 'synced') + current_info.last_updated = datetime.now() - timedelta(seconds=2) + current_info.save() + + sync_teams_to_groups(fake_auth, timedelta(seconds=120)) + + third_sync_info = model.team.get_team_sync_information('buynlarge', 'synced') + assert third_sync_info.last_updated == current_info.last_updated + assert third_sync_info.transaction_id == updated_sync_info.transaction_id + + # Set the stale threshold to 10 seconds, and ensure the team is resynced, after making it + # "updated" 20s ago. + current_info = model.team.get_team_sync_information('buynlarge', 'synced') + current_info.last_updated = datetime.now() - timedelta(seconds=20) + current_info.save() + + sync_teams_to_groups(fake_auth, timedelta(seconds=10)) + + fourth_sync_info = model.team.get_team_sync_information('buynlarge', 'synced') + assert fourth_sync_info.transaction_id != updated_sync_info.transaction_id + + +@pytest.mark.parametrize('auth_system_builder,config', [ + (mock_ldap, {'group_dn': 'cn=AwesomeFolk'}), + (fake_keystone, {'group_id': 'somegroupid'}), +]) +def test_teamsync_end_to_end(auth_system_builder, config, app): + with auth_system_builder() as auth: + # Create an new team to sync. + org = model.organization.get_organization('buynlarge') + new_synced_team = model.team.create_team('synced2', org, 'member', 'Some synced team.') + sync_team_info = model.team.set_team_syncing(new_synced_team, auth.federated_service, config) + + # Sync the team. + assert sync_team(auth, sync_team_info) + + # Ensure we now have members. + msg = 'Auth system: %s' % auth.federated_service + sync_team_info = model.team.get_team_sync_information('buynlarge', 'synced2') + assert len(list(model.team.list_team_users(sync_team_info.team))) > 0, msg diff --git a/endpoints/api/organization.py b/endpoints/api/organization.py index 854451454..9411d8edd 100644 --- a/endpoints/api/organization.py +++ b/endpoints/api/organization.py @@ -6,7 +6,7 @@ from flask import request import features -from app import billing as stripe, avatar, all_queues +from app import billing as stripe, avatar, all_queues, authentication from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error, related_user_resource, internal_only, require_user_admin, log_action, show_if, path_param, require_scope, require_fresh_login) @@ -33,6 +33,8 @@ def team_view(orgname, team): 'repo_count': team.repo_count, 'member_count': team.member_count, + + 'is_synced': team.is_synced, } @@ -157,7 +159,8 @@ class Organization(ApiResource): teams = None if OrganizationMemberPermission(orgname).can(): - teams = model.team.get_teams_within_org(org) + has_syncing = features.TEAM_SYNCING and bool(authentication.federated_service) + teams = model.team.get_teams_within_org(org, has_syncing) return org_view(org, teams) diff --git a/endpoints/api/team.py b/endpoints/api/team.py index 81b779b58..981e0f1cc 100644 --- a/endpoints/api/team.py +++ b/endpoints/api/team.py @@ -1,19 +1,27 @@ """ Create, list and manage an organization's teams. """ +import json + +from functools import wraps + from flask import request import features -from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error, - log_action, internal_only, require_scope, path_param, query_param, - truthy_bool, parse_args, require_user_admin, show_if) -from endpoints.exception import Unauthorized, NotFound -from auth.permissions import AdministerOrganizationPermission, ViewTeamPermission +from app import avatar, authentication +from auth.permissions import (AdministerOrganizationPermission, ViewTeamPermission, + SuperUserPermission) + from auth.auth_context import get_authenticated_user from auth import scopes from data import model +from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error, + log_action, internal_only, require_scope, path_param, query_param, + truthy_bool, parse_args, require_user_admin, show_if, format_date, + verify_not_prod, require_fresh_login) +from endpoints.exception import Unauthorized, NotFound, InvalidRequest from util.useremails import send_org_invite_email -from app import avatar +from util.names import parse_robot_username def permission_view(permission): return { @@ -24,7 +32,6 @@ def permission_view(permission): 'role': permission.role.name } - def try_accept_invite(code, user): (team, inviter) = model.team.confirm_team_invite(code, user) @@ -40,7 +47,6 @@ def try_accept_invite(code, user): return team - def handle_addinvite_team(inviter, team, user=None, email=None): requires_invite = features.MAILING and features.REQUIRE_TEAM_INVITE invite = model.team.add_or_invite_to_team(inviter, team, user, email, @@ -82,7 +88,6 @@ def member_view(member, invited=False): 'invited': invited, } - def invite_view(invite): if invite.user: return member_view(invite.user, invited=True) @@ -94,6 +99,30 @@ def invite_view(invite): 'invited': True } +def disallow_for_synced_team(except_robots=False): + """ Disallows the decorated operation for a team that is marked as being synced from an internal + auth provider such as LDAP. If except_robots is True, then the operation is allowed if the + member specified on the operation is a robot account. + """ + def inner(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + # Team syncing can only be enabled if we have a federated service. + if features.TEAM_SYNCING and authentication.federated_service: + orgname = kwargs['orgname'] + teamname = kwargs['teamname'] + if model.team.get_team_sync_information(orgname, teamname): + if not except_robots or not parse_robot_username(kwargs.get('membername', '')): + raise InvalidRequest('Cannot call this method on an auth-synced team') + + return func(self, *args, **kwargs) + return wrapper + return inner + + +disallow_nonrobots_for_synced_team = disallow_for_synced_team(except_robots=True) +disallow_all_for_synced_team = disallow_for_synced_team(except_robots=False) + @resource('/v1/organization//team/') @path_param('orgname', 'The name of the organization') @@ -180,6 +209,58 @@ class OrganizationTeam(ApiResource): raise Unauthorized() +@resource('/v1/organization//team//syncing') +@path_param('orgname', 'The name of the organization') +@path_param('teamname', 'The name of the team') +@show_if(features.TEAM_SYNCING) +class OrganizationTeamSyncing(ApiResource): + """ Resource for managing syncing of a team by a backing group. """ + @require_scope(scopes.ORG_ADMIN) + @require_scope(scopes.SUPERUSER) + @nickname('enableOrganizationTeamSync') + @verify_not_prod + @require_fresh_login + def post(self, orgname, teamname): + # User must be both the org admin AND a superuser. + if SuperUserPermission().can() and AdministerOrganizationPermission(orgname).can(): + try: + team = model.team.get_organization_team(orgname, teamname) + except model.InvalidTeamException: + raise NotFound() + + config = request.get_json() + + # Ensure that the specified config points to a valid group. + status, err = authentication.check_group_lookup_args(config) + if not status: + raise InvalidRequest('Could not sync to group: %s' % err) + + # Set the team's syncing config. + model.team.set_team_syncing(team, authentication.federated_service, config) + + return team_view(orgname, team) + + raise Unauthorized() + + @require_scope(scopes.ORG_ADMIN) + @require_scope(scopes.SUPERUSER) + @nickname('disableOrganizationTeamSync') + @verify_not_prod + @require_fresh_login + def delete(self, orgname, teamname): + # User must be both the org admin AND a superuser. + if SuperUserPermission().can() and AdministerOrganizationPermission(orgname).can(): + try: + team = model.team.get_organization_team(orgname, teamname) + except model.InvalidTeamException: + raise NotFound() + + model.team.remove_team_syncing(orgname, teamname) + return team_view(orgname, team) + + raise Unauthorized() + + @resource('/v1/organization//team//members') @path_param('orgname', 'The name of the organization') @path_param('teamname', 'The name of the team') @@ -211,9 +292,29 @@ class TeamMemberList(ApiResource): data = { 'name': teamname, 'members': [member_view(m) for m in members] + [invite_view(i) for i in invites], - 'can_edit': edit_permission.can() + 'can_edit': edit_permission.can(), } + if features.TEAM_SYNCING and authentication.federated_service: + if SuperUserPermission().can() and AdministerOrganizationPermission(orgname).can(): + data['can_sync'] = { + 'service': authentication.federated_service, + } + + data['can_sync'].update(authentication.service_metadata()) + + sync_info = model.team.get_team_sync_information(orgname, teamname) + if sync_info is not None: + data['synced'] = { + 'service': sync_info.service.name, + } + + if SuperUserPermission().can(): + data['synced'].update({ + 'last_updated': format_date(sync_info.last_updated), + 'config': json.loads(sync_info.config), + }) + return data raise Unauthorized() @@ -228,6 +329,7 @@ class TeamMember(ApiResource): @require_scope(scopes.ORG_ADMIN) @nickname('updateOrganizationTeamMember') + @disallow_nonrobots_for_synced_team def put(self, orgname, teamname, membername): """ Adds or invites a member to an existing team. """ permission = AdministerOrganizationPermission(orgname) @@ -265,6 +367,7 @@ class TeamMember(ApiResource): @require_scope(scopes.ORG_ADMIN) @nickname('deleteOrganizationTeamMember') + @disallow_nonrobots_for_synced_team def delete(self, orgname, teamname, membername): """ Delete a member of a team. If the user is merely invited to join the team, then the invite is removed instead. @@ -308,6 +411,7 @@ class InviteTeamMember(ApiResource): """ Resource for inviting a team member via email address. """ @require_scope(scopes.ORG_ADMIN) @nickname('inviteTeamMemberEmail') + @disallow_all_for_synced_team def put(self, orgname, teamname, email): """ Invites an email address to an existing team. """ permission = AdministerOrganizationPermission(orgname) @@ -407,7 +511,7 @@ class TeamMemberInvite(ApiResource): @nickname('declineOrganizationTeamInvite') @require_user_admin def delete(self, code): - """ Delete an existing member of a team. """ + """ Delete an existing invitation to join a team. """ (team, inviter) = model.team.delete_team_invite(code, user_obj=get_authenticated_user()) model.notification.delete_matching_notifications(get_authenticated_user(), 'org_team_invite', diff --git a/endpoints/api/test/shared.py b/endpoints/api/test/shared.py index 5b9c2f090..3d1f0cffa 100644 --- a/endpoints/api/test/shared.py +++ b/endpoints/api/test/shared.py @@ -2,7 +2,6 @@ import datetime import json from contextlib import contextmanager - from data import model from endpoints.api import api diff --git a/endpoints/api/test/test_security.py b/endpoints/api/test/test_security.py index 7d094f44c..1ae2c31cc 100644 --- a/endpoints/api/test/test_security.py +++ b/endpoints/api/test/test_security.py @@ -1,5 +1,7 @@ import pytest +from endpoints.api import api +from endpoints.api.team import OrganizationTeamSyncing from endpoints.api.test.shared import client_with_identity, conduct_api_call from endpoints.api.superuser import SuperUserRepositoryBuildLogs, SuperUserRepositoryBuildResource from endpoints.api.superuser import SuperUserRepositoryBuildStatus @@ -9,6 +11,16 @@ TEAM_PARAMS = {'orgname': 'buynlarge', 'teamname': 'owners'} BUILD_PARAMS = {'build_uuid': 'test-1234'} @pytest.mark.parametrize('resource,method,params,body,identity,expected', [ + (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, None, 403), + (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, 'freshuser', 403), + (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, 'reader', 403), + (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, 'devtable', 400), + + (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, None, 403), + (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, 'freshuser', 403), + (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, 'reader', 403), + (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, 'devtable', 200), + (SuperUserRepositoryBuildLogs, 'GET', BUILD_PARAMS, None, None, 401), (SuperUserRepositoryBuildLogs, 'GET', BUILD_PARAMS, None, 'freshuser', 403), (SuperUserRepositoryBuildLogs, 'GET', BUILD_PARAMS, None, 'reader', 403), diff --git a/endpoints/api/test/test_team.py b/endpoints/api/test/test_team.py new file mode 100644 index 000000000..4dfa98b7f --- /dev/null +++ b/endpoints/api/test/test_team.py @@ -0,0 +1,87 @@ +import json + +from mock import patch + +from data import model +from endpoints.api import api +from endpoints.api.test.shared import client_with_identity, conduct_api_call +from endpoints.api.team import OrganizationTeamSyncing, TeamMemberList +from endpoints.api.organization import Organization +from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file +from test.test_ldap import mock_ldap + +SYNCED_TEAM_PARAMS = {'orgname': 'sellnsmall', 'teamname': 'synced'} +UNSYNCED_TEAM_PARAMS = {'orgname': 'sellnsmall', 'teamname': 'owners'} + +def test_team_syncing(client): + with mock_ldap() as ldap: + with patch('endpoints.api.team.authentication', ldap): + with client_with_identity('devtable', client) as cl: + config = { + 'group_dn': 'cn=AwesomeFolk', + } + + conduct_api_call(cl, OrganizationTeamSyncing, 'POST', UNSYNCED_TEAM_PARAMS, config) + + # Ensure the team is now synced. + sync_info = model.team.get_team_sync_information(UNSYNCED_TEAM_PARAMS['orgname'], + UNSYNCED_TEAM_PARAMS['teamname']) + assert sync_info is not None + assert json.loads(sync_info.config) == config + + # Remove the syncing. + conduct_api_call(cl, OrganizationTeamSyncing, 'DELETE', UNSYNCED_TEAM_PARAMS, None) + + # Ensure the team is no longer synced. + sync_info = model.team.get_team_sync_information(UNSYNCED_TEAM_PARAMS['orgname'], + UNSYNCED_TEAM_PARAMS['teamname']) + assert sync_info is None + + +def test_team_member_sync_info(client): + with mock_ldap() as ldap: + with patch('endpoints.api.team.authentication', ldap): + # Check for an unsynced team, with superuser. + with client_with_identity('devtable', client) as cl: + resp = conduct_api_call(cl, TeamMemberList, 'GET', UNSYNCED_TEAM_PARAMS) + assert 'can_sync' in resp.json + assert resp.json['can_sync']['service'] == 'ldap' + + assert 'synced' not in resp.json + + # Check for an unsynced team, with non-superuser. + with client_with_identity('randomuser', client) as cl: + resp = conduct_api_call(cl, TeamMemberList, 'GET', UNSYNCED_TEAM_PARAMS) + assert 'can_sync' not in resp.json + assert 'synced' not in resp.json + + # Check for a synced team, with superuser. + with client_with_identity('devtable', client) as cl: + resp = conduct_api_call(cl, TeamMemberList, 'GET', SYNCED_TEAM_PARAMS) + assert 'can_sync' in resp.json + assert resp.json['can_sync']['service'] == 'ldap' + + assert 'synced' in resp.json + assert 'last_updated' in resp.json['synced'] + assert 'group_dn' in resp.json['synced']['config'] + + # Check for a synced team, with non-superuser. + with client_with_identity('randomuser', client) as cl: + resp = conduct_api_call(cl, TeamMemberList, 'GET', SYNCED_TEAM_PARAMS) + assert 'can_sync' not in resp.json + + assert 'synced' in resp.json + assert 'last_updated' not in resp.json['synced'] + assert 'config' not in resp.json['synced'] + + +def test_organization_teams_sync_bool(client): + with mock_ldap() as ldap: + with patch('endpoints.api.organization.authentication', ldap): + # Ensure synced teams are marked as such in the organization teams list. + with client_with_identity('devtable', client) as cl: + resp = conduct_api_call(cl, Organization, 'GET', {'orgname': 'sellnsmall'}) + + assert not resp.json['teams']['owners']['is_synced'] + + assert resp.json['teams']['synced']['is_synced'] diff --git a/endpoints/oauth/test/test_login.py b/endpoints/oauth/test/test_login.py index 04b510aef..f081f84b1 100644 --- a/endpoints/oauth/test/test_login.py +++ b/endpoints/oauth/test/test_login.py @@ -1,11 +1,9 @@ import pytest -from endpoints.oauth.login import _conduct_oauth_login - -from oauth.services.github import GithubOAuthService - from data import model, database from data.users import get_users_handler, DatabaseUsers +from endpoints.oauth.login import _conduct_oauth_login +from oauth.services.github import GithubOAuthService from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file from test.test_ldap import mock_ldap diff --git a/initdb.py b/initdb.py index 03bb9131c..a7e278823 100644 --- a/initdb.py +++ b/initdb.py @@ -657,6 +657,9 @@ def populate_database(minimal=False, with_storage=False): liborg = model.organization.create_organization('library', 'quay+library@devtable.com', new_user_1) liborg.save() + thirdorg = model.organization.create_organization('sellnsmall', 'quay+sell@devtable.com', new_user_1) + thirdorg.save() + model.user.create_robot('coolrobot', org) oauth_app_1 = model.oauth.create_application(org, 'Some Test App', 'http://localhost:8000', @@ -700,6 +703,19 @@ def populate_database(minimal=False, with_storage=False): model.team.add_user_to_team(creatorbot, creators) model.team.add_user_to_team(creatoruser, creators) + sell_owners = model.team.get_organization_team('sellnsmall', 'owners') + sell_owners.description = 'Owners have unfettered access across the entire org.' + sell_owners.save() + + model.team.add_user_to_team(new_user_4, sell_owners) + + sync_config = {'group_dn': 'cn=Test-Group,ou=Users', 'group_id': 'somegroupid'} + synced_team = model.team.create_team('synced', org, 'member', 'Some synced team.') + model.team.set_team_syncing(synced_team, 'ldap', sync_config) + + another_synced_team = model.team.create_team('synced', thirdorg, 'member', 'Some synced team.') + model.team.set_team_syncing(another_synced_team, 'ldap', {'group_dn': 'cn=Test-Group,ou=Users'}) + __generate_repository(with_storage, new_user_1, 'superwide', None, False, [], [(10, [], 'latest2'), (2, [], 'latest3'), diff --git a/static/css/core-ui.css b/static/css/core-ui.css index 217d31de1..36b2ce342 100644 --- a/static/css/core-ui.css +++ b/static/css/core-ui.css @@ -1521,7 +1521,7 @@ a:focus { top: 11px; left: 12px; font-size: 22px; - color: #E4C212; + color: #FCA657; } .co-alert.co-alert-danger { @@ -1566,6 +1566,14 @@ a:focus { left: 19px; } +.co-alert-inline:before { + position: relative !important; + top: auto !important; + left: auto !important; + vertical-align: middle; + margin-right: 10px; +} + .co-alert-popin-warning { margin-left: 10px; } @@ -1579,6 +1587,14 @@ a:focus { } } +.co-alert-inline { + border: 0px; + display: inline-block; + background-color: transparent !important; + margin: 0px; + padding: 4px; +} + .co-list-table tr td:first-child { font-weight: bold; padding-right: 10px; diff --git a/static/css/directives/ui/teams-manager.css b/static/css/directives/ui/teams-manager.css index a93a844ac..4c7c6e6b5 100644 --- a/static/css/directives/ui/teams-manager.css +++ b/static/css/directives/ui/teams-manager.css @@ -111,3 +111,7 @@ color: #aaa; font-size: 14px; } + +.teams-manager .fa-refresh { + color: #aaa; +} diff --git a/static/css/pages/team-view.css b/static/css/pages/team-view.css index 86cb7b6de..dad2fccab 100644 --- a/static/css/pages/team-view.css +++ b/static/css/pages/team-view.css @@ -3,6 +3,18 @@ position: relative; } +.team-view .team-sync-table { + margin-bottom: 20px; +} + +.team-view .team-sync-table td { + padding: 6px; +} + +.team-view .team-sync-table td:first-child { + font-weight: bold; +} + .team-view .team-title { vertical-align: middle; margin-right: 10px; @@ -14,10 +26,10 @@ margin-left: 6px; } -.team-view .team-view-header { +.team-view .team-view-header, .team-view .team-sync-header { border-bottom: 1px solid #eee; - margin-bottom: 10px; - padding-bottom: 10px; + margin-bottom: 20px; + padding-bottom: 20px; } .team-view .team-view-header button i.fa { diff --git a/static/directives/config/config-setup-tool.html b/static/directives/config/config-setup-tool.html index ead4609bf..226a7f8da 100644 --- a/static/directives/config/config-setup-tool.html +++ b/static/directives/config/config-setup-tool.html @@ -194,7 +194,7 @@ > + validator="validateHostname(value)"> @@ -553,7 +553,7 @@ prevent passwords from being saved as plaintext by the Docker client. - +
+ + + + + + + + +
Authentication: @@ -565,6 +565,28 @@
Team synchronization: +
+ Enable Team Synchronization Support +
+
+ If enabled, organization administrators who are also superusers can set teams to have their membership synchronized with a backing group in {{ config.AUTHENTICATION_TYPE }}. +
+
Resynchronization duration: + +
+ The duration before a team must be re-synchronized. Must be expressed in a duration string form: 30m, 1h, 1d. +
+
@@ -758,7 +780,7 @@ Administrator DN Password: -
+
Note: This will be stored in plaintext inside the config.yaml, so setting up a dedicated account or using a password hash is highly recommended. diff --git a/static/directives/team-view-add.html b/static/directives/team-view-add.html index 88c747d2f..91aa154bb 100644 --- a/static/directives/team-view-add.html +++ b/static/directives/team-view-add.html @@ -1,20 +1,26 @@
Inviting team member
- Search by registry username, robot account name or enter an email address to invite - Search by registry username or robot account name + + Search by registry username, robot account name or enter an email address to invite + Search by registry username or robot account name + + + Search by robot account name. Users must be added in {{ syncInfo.service }}. +
diff --git a/static/directives/teams-manager.html b/static/directives/teams-manager.html index 714b5711a..338e9fb72 100644 --- a/static/directives/teams-manager.html +++ b/static/directives/teams-manager.html @@ -41,6 +41,7 @@ + @@ -65,6 +66,9 @@ + - +
Team Name
+ + @@ -97,7 +101,8 @@ + role-changed="setRole(role, team.name)" roles="teamRoles" + read-only="!organization.is_admin"> diff --git a/static/js/directives/ui/teams-manager.js b/static/js/directives/ui/teams-manager.js index 7235c91f2..88e926318 100644 --- a/static/js/directives/ui/teams-manager.js +++ b/static/js/directives/ui/teams-manager.js @@ -12,8 +12,10 @@ angular.module('quay').directive('teamsManager', function () { 'organization': '=organization', 'isEnabled': '=isEnabled' }, - controller: function($scope, $element, ApiService, $timeout, UserService, TableService, UIService) { + controller: function($scope, $element, ApiService, $timeout, UserService, TableService, UIService, Config, Features) { $scope.TableService = TableService; + $scope.Config = Config; + $scope.Features = Features; $scope.options = { 'predicate': 'ordered_team_index', diff --git a/static/js/pages/team-view.js b/static/js/pages/team-view.js index 0215ab080..91135163c 100644 --- a/static/js/pages/team-view.js +++ b/static/js/pages/team-view.js @@ -14,12 +14,14 @@ var teamname = $routeParams.teamname; var orgname = $routeParams.orgname; + $scope.context = {}; $scope.orgname = orgname; $scope.teamname = teamname; $scope.addingMember = false; $scope.memberMap = null; $scope.allowEmail = Features.MAILING; $scope.feedback = null; + $scope.allowedEntities = ['user', 'robot']; $rootScope.title = 'Loading...'; @@ -146,6 +148,39 @@ }, ApiService.errorDisplay('Cannot remove team member')); }; + $scope.getServiceName = function(service) { + switch (service) { + case 'ldap': + return 'LDAP'; + + case 'keystone': + return 'Keystone Auth'; + + case 'jwtauthn': + return 'External JWT Auth'; + + default: + return synced.service; + } + }; + + $scope.getAddPlaceholder = function(email, synced) { + var kinds = []; + + if (!synced) { + kinds.push('registered user'); + } + + kinds.push('robot'); + + if (email && !synced) { + kinds.push('email address'); + } + + kind_string = kinds.join(', ') + return 'Add a ' + kind_string + ' to the team'; + }; + $scope.updateForDescription = function(content) { $scope.organization.teams[teamname].description = content; @@ -166,6 +201,48 @@ }); }; + $scope.showEnableSyncing = function() { + $scope.enableSyncingInfo = { + 'service_info': $scope.canSync, + 'config': {} + }; + }; + + $scope.showDisableSyncing = function() { + msg = 'Are you sure you want to disable group syncing on this team? ' + + 'The team will once again become editable.'; + bootbox.confirm(msg, function(result) { + if (result) { + $scope.disableSyncing(); + } + }); + }; + + $scope.disableSyncing = function() { + var params = { + 'orgname': orgname, + 'teamname': teamname + }; + + var errorHandler = ApiService.errorDisplay('Could not disable team syncing'); + ApiService.disableOrganizationTeamSync(null, params).then(function(resp) { + loadMembers(); + }, errorHandler); + }; + + $scope.enableSyncing = function(config, callback) { + var params = { + 'orgname': orgname, + 'teamname': teamname + }; + + var errorHandler = ApiService.errorDisplay('Cannot enable team syncing', callback); + ApiService.enableOrganizationTeamSync(config, params).then(function(resp) { + loadMembers(); + callback(true); + }, errorHandler); + }; + var loadOrganization = function() { $scope.orgResource = ApiService.getOrganizationAsResource({'orgname': orgname}).get(function(org) { $scope.organization = org; @@ -187,6 +264,9 @@ $scope.membersResource = ApiService.getOrganizationTeamMembersAsResource(params).get(function(resp) { $scope.members = resp.members; $scope.canEditMembers = resp.can_edit; + $scope.canSync = resp.can_sync; + $scope.syncInfo = resp.synced; + $scope.allowedEntities = resp.synced ? ['robot'] : ['user', 'robot']; $('.info-icon').popover({ 'trigger': 'hover', diff --git a/static/partials/team-view.html b/static/partials/team-view.html index 5be2d58c5..b046f8d2b 100644 --- a/static/partials/team-view.html +++ b/static/partials/team-view.html @@ -18,6 +18,44 @@
+
+
Directory Synchronization
+

Directory synchronization allows this team's user membership to be backed by a group in {{ getServiceName(canSync.service) }}.

+ +
+ + +
+
+ This team is synchronized with a group in {{ getServiceName(syncInfo.service) }} and its user membership is therefore read-only. +
+ +
+
Directory Synchronization
+ + + + + + + + + + +
Bound to group: +
+ {{ syncInfo.config.group_dn }} +
+
+ {{ syncInfo.config.group_id }} +
+
Last Updated: at {{ syncInfo.last_updated | amDateFormat:'dddd, MMMM Do YYYY, h:mm:ss a' }}Never
+ + +
You must be an admin of this organization to disable team synchronization
+
+
+
Team Description
@@ -33,12 +71,15 @@
-
Team Members
+
Team Members
This team has no members.
-
- Click the "Add Team Member" button above to add or invite team members. +
+ Enter a user or robot above to add or invite to the team. +
+
+ This team is synchronized with an external group defined in {{ getServiceName(syncInfo.service) }}. To add a user to this team, add them in the backing group. To add a robot account to this team, enter them above.
@@ -46,7 +87,7 @@
Team Members Team Members (defined in {{ getServiceName(syncInfo.service) }})
- + Remove {{ member.name }} @@ -122,6 +163,29 @@ + +
+
Please note that once team syncing is enabled, the team's user membership from within will be read-only.
+
+
+
+ Enter the distinguished name of the group, relative to {{ enableSyncingInfo.service_info.base_dn }}: + +
+
+ Enter the Keystone group ID: + +
+
+
+
+ +