Merge pull request #2387 from coreos-inc/team-sync

Team synchronization support in Quay Enterprise

config.py

@@ -432,3 +432,8 @@ class DefaultConfig(object):
 
   # Maximum size allowed for layers in the registry.
   MAXIMUM_LAYER_SIZE = '20G'
+
+  # Feature Flag: Whether team syncing from the backing auth is enabled.
+  FEATURE_TEAM_SYNCING = False
+  TEAM_RESYNC_STALE_TIME = '30m'
+  TEAM_SYNC_WORKER_FREQUENCY = 60 # seconds

data/database.py

@@ -451,7 +451,7 @@ class User(BaseModel):
                                  TagManifest, AccessToken, OAuthAccessToken, BlobUpload,
                                  RepositoryNotification, OAuthAuthorizationCode,
                                  RepositoryActionCount, TagManifestLabel, Tag,
-                                 ManifestLabel, BlobUploading} | beta_classes
+                                 ManifestLabel, BlobUploading, TeamSync} | beta_classes
 
       delete_instance_filtered(self, User, delete_nullable, skip_transitive_deletes)
 
@@ -526,6 +526,15 @@ class LoginService(BaseModel):
   name = CharField(unique=True, index=True)
 
 
+class TeamSync(BaseModel):
+  team = ForeignKeyField(Team, unique=True)
+
+  transaction_id = CharField()
+  last_updated = DateTimeField(null=True, index=True)
+  service = ForeignKeyField(LoginService)
+  config = JSONField()
+
+
 class FederatedLogin(BaseModel):
   user = QuayUserField(allows_robots=True, index=True)
   service = ForeignKeyField(LoginService)

data/migrations/versions/be8d1c402ce0_add_teamsync_table.py

@@ -0,0 +1,39 @@
+"""Add TeamSync table
+
+Revision ID: be8d1c402ce0
+Revises: a6c463dfb9fe
+Create Date: 2017-02-23 13:34:52.356812
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'be8d1c402ce0'
+down_revision = 'a6c463dfb9fe'
+
+from alembic import op
+import sqlalchemy as sa
+from util.migrate import UTF8LongText
+
+def upgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('teamsync',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('team_id', sa.Integer(), nullable=False),
+    sa.Column('transaction_id', sa.String(length=255), nullable=False),
+    sa.Column('last_updated', sa.DateTime(), nullable=True),
+    sa.Column('service_id', sa.Integer(), nullable=False),
+    sa.Column('config', UTF8LongText(), nullable=False),
+    sa.ForeignKeyConstraint(['service_id'], ['loginservice.id'], name=op.f('fk_teamsync_service_id_loginservice')),
+    sa.ForeignKeyConstraint(['team_id'], ['team.id'], name=op.f('fk_teamsync_team_id_team')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_teamsync'))
+    )
+    op.create_index('teamsync_last_updated', 'teamsync', ['last_updated'], unique=False)
+    op.create_index('teamsync_service_id', 'teamsync', ['service_id'], unique=False)
+    op.create_index('teamsync_team_id', 'teamsync', ['team_id'], unique=True)
+    ### end Alembic commands ###
+
+
+def downgrade(tables):
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('teamsync')
+    ### end Alembic commands ###

data/model/team.py

@@ -1,9 +1,15 @@
-from data.database import Team, TeamMember, TeamRole, User, TeamMemberInvite, RepositoryPermission
+import json
+import uuid
+
+from datetime import datetime
+from peewee import fn
+
+from data.database import (Team, TeamMember, TeamRole, User, TeamMemberInvite, RepositoryPermission,
+                           TeamSync, LoginService, FederatedLogin, db_random_func, db_transaction)
 from data.model import (DataModelException, InvalidTeamException, UserAlreadyInTeam,
-                        InvalidTeamMemberException, user, _basequery)
+                        InvalidTeamMemberException, _basequery)
 from data.text import prefix_search
 from util.validation import validate_username
-from peewee import fn, JOIN_LEFT_OUTER
 from util.morecollections import AttrDict
 
 
@@ -188,7 +194,7 @@ def get_matching_teams(team_prefix, organization):
   return query.limit(10)
 
 
-def get_teams_within_org(organization):
+def get_teams_within_org(organization, has_external_auth=False):
   """ Returns a AttrDict of team info (id, name, description), its role under the org,
       the number of repositories on which it has permission, and the number of members.
   """
@@ -205,6 +211,8 @@ def get_teams_within_org(organization):
 
       'repo_count': 0,
       'member_count': 0,
+
+      'is_synced': False,
     }
 
   teams = {team.id: _team_view(team) for team in query}
@@ -232,6 +240,12 @@ def get_teams_within_org(organization):
   for member_tuple in members_tuples:
     teams[member_tuple[0]]['member_count'] = member_tuple[1]
 
+  # Add syncing information.
+  if has_external_auth:
+    sync_query = TeamSync.select(TeamSync.team).where(TeamSync.team << teams.keys())
+    for team_sync in sync_query:
+      teams[team_sync.team_id]['is_synced'] = True
+
   return [AttrDict(team_info) for team_info in teams.values()]
 
 
@@ -369,3 +383,121 @@ def confirm_team_invite(code, user_obj):
   team = found.team
   inviter = found.inviter
   return (team, inviter)
+
+
+def get_federated_team_member_mapping(team, login_service_name):
+  """ Returns a dict of all federated IDs for all team members in the team whose users are
+      bound to the login service within the given name. The dictionary is from federated service
+      identifier (username) to their Quay User table ID.
+  """
+  login_service = LoginService.get(name=login_service_name)
+
+  query = (FederatedLogin
+           .select(FederatedLogin.service_ident, User.id)
+           .join(User)
+           .join(TeamMember)
+           .join(Team)
+           .where(Team.id == team, User.robot == False, FederatedLogin.service == login_service))
+  return dict(query.tuples())
+
+
+def list_team_users(team):
+  """ Returns an iterator of all the *users* found in a team. Does not include robots. """
+  return (User
+          .select()
+          .join(TeamMember)
+          .join(Team)
+          .where(Team.id == team, User.robot == False))
+
+
+def list_team_robots(team):
+  """ Returns an iterator of all the *robots* found in a team. Does not include users. """
+  return (User
+          .select()
+          .join(TeamMember)
+          .join(Team)
+          .where(Team.id == team, User.robot == True))
+
+
+def set_team_syncing(team, login_service_name, config):
+  """ Sets the given team to sync to the given service using the given config. """
+  login_service = LoginService.get(name=login_service_name)
+  return TeamSync.create(team=team, transaction_id='', service=login_service,
+                         config=json.dumps(config))
+
+
+def remove_team_syncing(orgname, teamname):
+  """ Removes syncing on the team matching the given organization name and team name. """
+  existing = get_team_sync_information(orgname, teamname)
+  if existing:
+    existing.delete_instance()
+
+
+def get_stale_team(stale_timespan):
+  """ Returns a team that is setup to sync to an external group, and who has not been synced in
+      now - stale_timespan. Returns None if none found.
+  """
+  stale_at = datetime.now() - stale_timespan
+
+  try:
+    candidates = (TeamSync
+                  .select(TeamSync.id)
+                  .where((TeamSync.last_updated <= stale_at) | (TeamSync.last_updated >> None))
+                  .limit(500)
+                  .alias('candidates'))
+
+    found = (TeamSync
+             .select(candidates.c.id)
+             .from_(candidates)
+             .order_by(db_random_func())
+             .get())
+
+    if found is None:
+      return
+
+    return TeamSync.select(TeamSync, Team).join(Team).where(TeamSync.id == found.id).get()
+  except TeamSync.DoesNotExist:
+    return None
+
+
+def get_team_sync_information(orgname, teamname):
+  """ Returns the team syncing information for the team with the given name under the organization
+      with the given name or None if none.
+  """
+  query = (TeamSync
+           .select(TeamSync, LoginService)
+           .join(Team)
+           .join(User)
+           .switch(TeamSync)
+           .join(LoginService)
+           .where(Team.name == teamname, User.organization == True, User.username == orgname))
+
+  try:
+    return query.get()
+  except TeamSync.DoesNotExist:
+    return None
+
+
+def update_sync_status(team_sync_info):
+  """ Attempts to update the transaction ID and last updated time on a TeamSync object. If the
+      transaction ID on the entry in the DB does not match that found on the object, this method
+      returns False, which indicates another caller updated it first.
+  """
+  new_transaction_id = str(uuid.uuid4())
+  query = (TeamSync
+           .update(transaction_id=new_transaction_id, last_updated=datetime.now())
+           .where(TeamSync.id == team_sync_info.id,
+                  TeamSync.transaction_id == team_sync_info.transaction_id))
+  return query.execute() == 1
+
+
+def delete_members_not_present(team, member_id_set):
+  """ Deletes all members of the given team that are not found in the member ID set. """
+  with db_transaction():
+    user_ids = set([u.id for u in list_team_users(team)])
+    to_delete = list(user_ids - member_id_set)
+    if to_delete:
+      query = TeamMember.delete().where(TeamMember.team == team, TeamMember.user << to_delete)
+      return query.execute()
+
+  return 0

data/users/__init__.py

@@ -175,6 +175,12 @@ class UserAuthentication(object):
     """
     return self.state.link_user(username_or_email)
 
+  def get_and_link_federated_user_info(self, user_info):
+    """ Returns a tuple containing the database user record linked to the given UserInformation
+        pair and any error that occurred when trying to link the user.
+    """
+    return self.state.get_and_link_federated_user_info(user_info)
+
   def confirm_existing_user(self, username, password):
     """ Verifies that the given password matches to the given DB username. Unlike
         verify_credentials, this call first translates the DB user via the FederatedLogin table
@@ -186,6 +192,28 @@ class UserAuthentication(object):
     """ Verifies that the given username and password credentials are valid. """
     return self.state.verify_credentials(username_or_email, password)
 
+  def check_group_lookup_args(self, group_lookup_args):
+    """ Verifies that the given group lookup args point to a valid group. Returns a tuple consisting
+        of a boolean status and an error message (if any).
+    """
+    return self.state.check_group_lookup_args(group_lookup_args)
+
+  def service_metadata(self):
+    """ Returns a dictionary of extra metadata to present to *superusers* about this auth engine.
+        For example, LDAP returns the base DN so we can display to the user during sync setup.
+    """
+    return self.state.service_metadata()
+
+  def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False):
+    """ Returns a tuple of an iterator over all the members of the group matching the given lookup
+        args dictionary, or the error that occurred if the initial call failed or is unsupported.
+        The format of the lookup args dictionary is specific to the implementation.
+        Each result in the iterator is a tuple of (UserInformation, error_message), and only
+        one will be not-None.
+    """
+    return self.state.iterate_group_members(group_lookup_args, page_size=page_size,
+                                            disable_pagination=disable_pagination)
+
   def verify_and_link_user(self, username_or_email, password, basic_auth=False):
     """ Verifies that the given username and password credentials are valid and, if so,
         creates or links the database user to the federated identity. """

data/users/database.py

@@ -24,7 +24,22 @@ class DatabaseUsers(object):
     """ Never used since all users being added are already, by definition, in the database. """
     return (None, 'Unsupported for this authentication system')
 
+  def get_and_link_federated_user_info(self, user_info):
+    """ Never used since all users being added are already, by definition, in the database. """
+    return (None, 'Unsupported for this authentication system')
+
   def query_users(self, query, limit):
     """ No need to implement, as we already query for users directly in the database. """
     return (None, '', '')
 
+  def check_group_lookup_args(self, group_lookup_args):
+    """ Never used since all groups, by definition, are in the database. """
+    return (False, 'Not supported')
+
+  def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False):
+    """ Never used since all groups, by definition, are in the database. """
+    return (None, 'Not supported')
+
+  def service_metadata(self):
+    """ Never used since database has no metadata """
+    return {}

data/users/externalldap.py

@@ -2,6 +2,8 @@ import ldap
 import logging
 import os
 
+from ldap.controls import SimplePagedResultsControl
+
 from collections import namedtuple
 from data.users.federated import FederatedUsers, UserInformation
 from util.itertoolrecipes import take
@@ -10,6 +12,7 @@ logger = logging.getLogger(__name__)
 
 _DEFAULT_NETWORK_TIMEOUT = 10.0 # seconds
 _DEFAULT_TIMEOUT = 10.0 # seconds
+_DEFAULT_PAGE_SIZE = 1000
 
 
 class LDAPConnectionBuilder(object):
@@ -63,7 +66,7 @@ class LDAPUsers(FederatedUsers):
 
   def __init__(self, ldap_uri, base_dn, admin_dn, admin_passwd, user_rdn, uid_attr, email_attr,
                allow_tls_fallback=False, secondary_user_rdns=None, requires_email=True,
-               timeout=None, network_timeout=None):
+               timeout=None, network_timeout=None, force_no_pagination=False):
     super(LDAPUsers, self).__init__('ldap', requires_email)
 
     self._ldap = LDAPConnectionBuilder(ldap_uri, admin_dn, admin_passwd, allow_tls_fallback,
@@ -73,6 +76,7 @@ class LDAPUsers(FederatedUsers):
     self._email_attr = email_attr
     self._allow_tls_fallback = allow_tls_fallback
     self._requires_email = requires_email
+    self._force_no_pagination = force_no_pagination
 
     # Note: user_rdn is a list of RDN pieces (for historical reasons), and secondary_user_rds
     # is a list of RDN strings.
@@ -84,6 +88,7 @@ class LDAPUsers(FederatedUsers):
 
     # Create the set of full DN paths.
     self._user_dns = [get_full_rdn(relative_dn) for relative_dn in relative_user_dns]
+    self._base_dn = ','.join(base_dn)
 
   def _get_ldap_referral_dn(self, referral_exception):
     logger.debug('Got referral: %s', referral_exception.args[0])
@@ -174,7 +179,7 @@ class LDAPUsers(FederatedUsers):
     with_mail = [result for result in with_dns if result.attrs.get(self._email_attr)]
     return (with_mail[0] if with_mail else with_dns[0], None)
 
-  def _credential_for_user(self, response):
+  def _build_user_information(self, response):
     if not response.get(self._uid_attr):
       return (None, 'Missing uid field "%s" in user record' % self._uid_attr)
 
@@ -194,7 +199,7 @@ class LDAPUsers(FederatedUsers):
 
     logger.debug('Found user for LDAP username or email %s', username_or_email)
     _, found_response = found_user
-    return self._credential_for_user(found_response)
+    return self._build_user_information(found_response)
 
   def query_users(self, query, limit=20):
     """ Queries LDAP for matching users. """
@@ -208,7 +213,7 @@ class LDAPUsers(FederatedUsers):
 
     final_results = []
     for result in results[0:limit]:
-      credentials, err_msg = self._credential_for_user(result.attrs)
+      credentials, err_msg = self._build_user_information(result.attrs)
       if err_msg is not None:
         continue
 
@@ -253,4 +258,87 @@ class LDAPUsers(FederatedUsers):
       logger.debug('Invalid LDAP credentials')
       return (None, 'Invalid password')
 
-    return self._credential_for_user(found_response)
+    return self._build_user_information(found_response)
+
+  def service_metadata(self):
+    return {
+      'base_dn': self._base_dn,
+    }
+
+  def check_group_lookup_args(self, group_lookup_args, disable_pagination=False):
+    if not group_lookup_args.get('group_dn'):
+      return (False, 'Missing group_dn')
+
+    (it, err) = self.iterate_group_members(group_lookup_args, page_size=1,
+                                           disable_pagination=disable_pagination)
+    if err is not None:
+      return (False, err)
+
+    if not list(it):
+      return (False, 'Group does not exist or is empty')
+
+    return (True, None)
+
+  def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False):
+    try:
+      with self._ldap.get_connection():
+        pass
+    except ldap.INVALID_CREDENTIALS:
+      return (None, 'LDAP Admin dn or password is invalid')
+
+    group_dn = group_lookup_args['group_dn']
+    page_size = page_size or _DEFAULT_PAGE_SIZE
+    return (self._iterate_members(group_dn, page_size, disable_pagination), None)
+
+  def _iterate_members(self, group_dn, page_size, disable_pagination):
+    has_pagination = not(self._force_no_pagination or disable_pagination)
+    with self._ldap.get_connection() as conn:
+      lc = ldap.controls.libldap.SimplePagedResultsControl(criticality=True, size=page_size,
+                                                           cookie='')
+
+      search_flt = '(memberOf=%s,%s)' % (group_dn, self._base_dn)
+      attributes = [self._uid_attr, self._email_attr]
+
+      for user_search_dn in self._user_dns:
+        # Conduct the initial search for users that are a member of the group.
+        if has_pagination:
+          msgid = conn.search_ext(user_search_dn, ldap.SCOPE_SUBTREE, search_flt, serverctrls=[lc],
+                                  attrlist=attributes)
+        else:
+          msgid = conn.search(user_search_dn, ldap.SCOPE_SUBTREE, search_flt, attrlist=attributes)
+
+        while True:
+          if has_pagination:
+            _, rdata, _, serverctrls = conn.result3(msgid)
+          else:
+            _, rdata = conn.result(msgid)
+
+          # Yield any users found.
+          for userdata in rdata:
+            yield self._build_user_information(userdata[1])
+
+          # If pagination is disabled, nothing more to do.
+          if not has_pagination:
+            break
+
+          # Filter down the controls with which the server responded, looking for the paging
+          # control type. If not found, then the server does not support pagination and we already
+          # got all of the results.
+          pctrls = [control for control in serverctrls
+                    if control.controlType == ldap.controls.SimplePagedResultsControl.controlType]
+
+          if pctrls:
+            # Server supports pagination. Update the cookie so the next search finds the next page,
+            # then conduct the next search.
+            cookie = lc.cookie = pctrls[0].cookie
+            if cookie:
+                msgid = conn.search_ext(user_search_dn, ldap.SCOPE_SUBTREE, search_flt,
+                                        serverctrls=[lc], attrlist=attributes)
+                continue
+            else:
+              # No additional results.
+              break
+          else:
+            # Pagination is not supported.
+            logger.debug('Pagination is not supported for this LDAP server')
+            break

data/users/federated.py

@@ -37,7 +37,63 @@ class FederatedUsers(object):
     """ If implemented, get_user must be implemented as well. """
     return (None, 'Not supported')
 
-  def _get_federated_user(self, username, email):
+  def link_user(self, username_or_email):
+    (user_info, err_msg) = self.get_user(username_or_email)
+    if user_info is None:
+      return (None, err_msg)
+
+    return self.get_and_link_federated_user_info(user_info)
+
+  def get_and_link_federated_user_info(self, user_info):
+    return self._get_and_link_federated_user_info(user_info.username, user_info.email)
+
+  def verify_and_link_user(self, username_or_email, password):
+    """ Verifies the given credentials and, if valid, creates/links a database user to the
+        associated federated service.
+    """
+    (credentials, err_msg) = self.verify_credentials(username_or_email, password)
+    if credentials is None:
+      return (None, err_msg)
+
+    return self._get_and_link_federated_user_info(credentials.username, credentials.email)
+
+  def confirm_existing_user(self, username, password):
+    """ Confirms that the given *database* username and service password are valid for the linked
+        service. This method is used when the federated service's username is not known.
+    """
+    db_user = model.user.get_user(username)
+    if not db_user:
+      return (None, 'Invalid user')
+
+    federated_login = model.user.lookup_federated_login(db_user, self._federated_service)
+    if not federated_login:
+      return (None, 'Invalid user')
+
+    (credentials, err_msg) = self.verify_credentials(federated_login.service_ident, password)
+    if credentials is None:
+      return (None, err_msg)
+
+    return (db_user, None)
+
+  def service_metadata(self):
+    """ Returns a dictionary of extra metadata to present to *superusers* about this auth engine.
+        For example, LDAP returns the base DN so we can display to the user during sync setup.
+    """
+    return {}
+
+  def check_group_lookup_args(self, group_lookup_args):
+    """ Verifies that the given group lookup args point to a valid group. Returns a tuple consisting
+        of a boolean status and an error message (if any).
+    """
+    return (False, 'Not supported')
+
+  def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False):
+    """ Returns an iterator over all the members of the group matching the given lookup args
+        dictionary. The format of the lookup args dictionary is specific to the implementation.
+    """
+    return (None, 'Not supported')
+
+  def _get_and_link_federated_user_info(self, username, email):
     db_user = model.user.verify_federated_login(self._federated_service, username)
     if not db_user:
       # We must create the user in our db
@@ -58,43 +114,8 @@ class FederatedUsers(object):
                                                  prompts=prompts)
     else:
       # Update the db attributes from the federated service.
-      if email:
+      if email and db_user.email != email:
         db_user.email = email
         db_user.save()
 
     return (db_user, None)
-
-  def link_user(self, username_or_email):
-    (credentials, err_msg) = self.get_user(username_or_email)
-    if credentials is None:
-      return (None, err_msg)
-
-    return self._get_federated_user(credentials.username, credentials.email)
-
-  def verify_and_link_user(self, username_or_email, password):
-    """ Verifies the given credentials and, if valid, creates/links a database user to the
-        associated federated service.
-    """
-    (credentials, err_msg) = self.verify_credentials(username_or_email, password)
-    if credentials is None:
-      return (None, err_msg)
-
-    return self._get_federated_user(credentials.username, credentials.email)
-
-  def confirm_existing_user(self, username, password):
-    """ Confirms that the given *database* username and service password are valid for the linked
-        service. This method is used when the federated service's username is not known.
-    """
-    db_user = model.user.get_user(username)
-    if not db_user:
-      return (None, 'Invalid user')
-
-    federated_login = model.user.lookup_federated_login(db_user, self._federated_service)
-    if not federated_login:
-      return (None, 'Invalid user')
-
-    (credentials, err_msg) = self.verify_credentials(federated_login.service_ident, password)
-    if credentials is None:
-      return (None, err_msg)
-
-    return (db_user, None)

data/users/keystone.py

@@ -5,6 +5,7 @@ from keystoneclient.v2_0 import client as kclient
 from keystoneclient.v3 import client as kv3client
 from keystoneclient.exceptions import AuthorizationFailure as KeystoneAuthorizationFailure
 from keystoneclient.exceptions import Unauthorized as KeystoneUnauthorized
+from keystoneclient.exceptions import NotFound as KeystoneNotFound
 from data.users.federated import FederatedUsers, UserInformation
 from util.itertoolrecipes import take
 
@@ -83,6 +84,11 @@ class KeystoneV3Users(FederatedUsers):
     self.debug = os.environ.get('USERS_DEBUG') == '1'
     self.requires_email = requires_email
 
+  def _get_admin_client(self):
+    return kv3client.Client(username=self.admin_username, password=self.admin_password,
+                            tenant_name=self.admin_tenant, auth_url=self.auth_url,
+                            timeout=self.timeout, debug=self.debug)
+
   def verify_credentials(self, username_or_email, password):
     try:
       keystone_client = kv3client.Client(username=username_or_email, password=password,
@@ -116,6 +122,46 @@ class KeystoneV3Users(FederatedUsers):
 
     return (user, None)
 
+  def check_group_lookup_args(self, group_lookup_args):
+    if not group_lookup_args.get('group_id'):
+      return (False, 'Missing group_id')
+
+    group_id = group_lookup_args['group_id']
+    return self._check_group(group_id)
+
+  def _check_group(self, group_id):
+    try:
+      return (bool(self._get_admin_client().groups.get(group_id)), None)
+    except KeystoneNotFound:
+      return (False, 'Group not found')
+    except KeystoneAuthorizationFailure as kaf:
+      logger.exception('Keystone auth failure for admin user for group lookup %s', group_id)
+      return (False, kaf.message or 'Invalid admin username or password')
+    except KeystoneUnauthorized as kut:
+      logger.exception('Keystone unauthorized for admin user for group lookup %s', group_id)
+      return (False, kut.message or 'Invalid admin username or password')
+
+  def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False):
+    group_id = group_lookup_args['group_id']
+
+    (status, err) = self._check_group(group_id)
+    if not status:
+      return (None, err)
+
+    try:
+      user_info_iterator = self._get_admin_client().users.list(group=group_id)
+      def iterator():
+        for user in user_info_iterator:
+          yield (self._user_info(user), None)
+
+      return (iterator(), None)
+    except KeystoneAuthorizationFailure as kaf:
+      logger.exception('Keystone auth failure for admin user for group lookup %s', group_id)
+      return (False, kaf.message or 'Invalid admin username or password')
+    except KeystoneUnauthorized as kut:
+      logger.exception('Keystone unauthorized for admin user for group lookup %s', group_id)
+      return (False, kut.message or 'Invalid admin username or password')
+
   @staticmethod
   def _user_info(user):
     email = user.email if hasattr(user, 'email') else None
@@ -126,10 +172,7 @@ class KeystoneV3Users(FederatedUsers):
       return ([], self.federated_service, None)
 
     try:
-      keystone_client = kv3client.Client(username=self.admin_username, password=self.admin_password,
-                                         tenant_name=self.admin_tenant, auth_url=self.auth_url,
-                                         timeout=self.timeout, debug=self.debug)
-      found_users = list(take(limit, keystone_client.users.list(name=query)))
+      found_users = list(take(limit, self._get_admin_client().users.list(name=query)))
       logger.debug('For Keystone query %s found users: %s', query, found_users)
       if not found_users:
         return ([], self.federated_service, None)

data/users/teamsync.py

@@ -0,0 +1,135 @@
+import logging
+import json
+
+from data import model
+
+logger = logging.getLogger(__name__)
+
+
+MAX_TEAMS_PER_ITERATION = 500
+
+
+def sync_teams_to_groups(authentication, stale_cutoff):
+  """ Performs team syncing by looking up any stale team(s) found, and performing the sync
+      operation on them.
+  """
+  logger.debug('Looking up teams to sync to groups')
+
+  sync_team_tried = set()
+  while len(sync_team_tried) < MAX_TEAMS_PER_ITERATION:
+    # Find a stale team.
+    stale_team_sync = model.team.get_stale_team(stale_cutoff)
+    if not stale_team_sync:
+      logger.debug('No additional stale team found; sleeping')
+      return
+
+    # Make sure we don't try to reprocess a team on this iteration.
+    if stale_team_sync.id in sync_team_tried:
+      break
+
+    sync_team_tried.add(stale_team_sync.id)
+
+    # Sync the team.
+    sync_successful = sync_team(authentication, stale_team_sync)
+    if not sync_successful:
+      return
+
+
+def sync_team(authentication, stale_team_sync):
+  """ Performs synchronization of a team (as referenced by the TeamSync stale_team_sync).
+      Returns True on success and False otherwise.
+  """
+  sync_config = json.loads(stale_team_sync.config)
+  logger.info('Syncing team `%s` under organization %s via %s (#%s)', stale_team_sync.team.name,
+              stale_team_sync.team.organization.username, sync_config, stale_team_sync.team_id,
+              extra={'team': stale_team_sync.team_id, 'sync_config': sync_config})
+
+  # Load all the existing members of the team in Quay that are bound to the auth service.
+  existing_users = model.team.get_federated_team_member_mapping(stale_team_sync.team,
+                                                                authentication.federated_service)
+
+  logger.debug('Existing membership of %s for team `%s` under organization %s via %s (#%s)',
+               len(existing_users), stale_team_sync.team.name,
+               stale_team_sync.team.organization.username, sync_config, stale_team_sync.team_id,
+               extra={'team': stale_team_sync.team_id, 'sync_config': sync_config,
+                      'existing_member_count': len(existing_users)})
+
+  # Load all the members of the team from the authenication system.
+  (member_iterator, err) = authentication.iterate_group_members(sync_config)
+  if err is not None:
+    logger.error('Got error when trying to iterate group members with config %s: %s',
+                 sync_config, err)
+    return False
+
+  # Collect all the members currently found in the group, adding them to the team as we go
+  # along.
+  group_membership = set()
+  for (member_info, err) in member_iterator:
+    if err is not None:
+      logger.error('Got error when trying to construct a member: %s', err)
+      continue
+
+    # If the member is already in the team, nothing more to do.
+    if member_info.username in existing_users:
+      logger.debug('Member %s already in team `%s` under organization %s via %s (#%s)',
+                   member_info.username, stale_team_sync.team.name,
+                   stale_team_sync.team.organization.username, sync_config,
+                   stale_team_sync.team_id,
+                   extra={'team': stale_team_sync.team_id, 'sync_config': sync_config,
+                          'member': member_info.username})
+
+      group_membership.add(existing_users[member_info.username])
+      continue
+
+    # Retrieve the Quay user associated with the member info.
+    (quay_user, err) = authentication.get_and_link_federated_user_info(member_info)
+    if err is not None:
+      logger.error('Could not link external user %s to an internal user: %s',
+                   member_info.username, err,
+                   extra={'team': stale_team_sync.team_id, 'sync_config': sync_config,
+                          'member': member_info.username, 'error': err})
+      continue
+
+    # Add the user to the membership set.
+    group_membership.add(quay_user.id)
+
+    # Add the user to the team.
+    try:
+      logger.info('Adding member %s to team `%s` under organization %s via %s (#%s)',
+                  quay_user.username, stale_team_sync.team.name,
+                  stale_team_sync.team.organization.username, sync_config,
+                  stale_team_sync.team_id,
+                  extra={'team': stale_team_sync.team_id, 'sync_config': sync_config,
+                         'member': quay_user.username})
+
+      model.team.add_user_to_team(quay_user, stale_team_sync.team)
+    except model.UserAlreadyInTeam:
+      # If the user is already present, nothing more to do for them.
+      pass
+
+  # Update the transaction and last_updated time of the team sync. Only if it matches
+  # the current value will we then perform the deletion step.
+  got_transaction_handle = model.team.update_sync_status(stale_team_sync)
+  if not got_transaction_handle:
+    # Another worker updated this team. Nothing more to do.
+    logger.debug('Another worker synced team `%s` under organization %s via %s (#%s)',
+                 stale_team_sync.team.name,
+                 stale_team_sync.team.organization.username, sync_config,
+                 stale_team_sync.team_id,
+                 extra={'team': stale_team_sync.team_id, 'sync_config': sync_config})
+    return True
+
+  # Delete any team members not found in the backing auth system.
+  logger.debug('Deleting stale members for team `%s` under organization %s via %s (#%s)',
+               stale_team_sync.team.name, stale_team_sync.team.organization.username,
+               sync_config, stale_team_sync.team_id,
+               extra={'team': stale_team_sync.team_id, 'sync_config': sync_config})
+
+  deleted = model.team.delete_members_not_present(stale_team_sync.team, group_membership)
+
+  # Done!
+  logger.info('Finishing sync for team `%s` under organization %s via %s (#%s): %s deleted',
+              stale_team_sync.team.name, stale_team_sync.team.organization.username,
+              sync_config, stale_team_sync.team_id, deleted,
+              extra={'team': stale_team_sync.team_id, 'sync_config': sync_config})
+  return True

data/users/test/test_teamsync.py

@@ -0,0 +1,260 @@
+from datetime import datetime, timedelta
+
+import pytest
+
+from data import model, database
+from data.users.federated import FederatedUsers, UserInformation
+from data.users.teamsync import sync_team, sync_teams_to_groups
+from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file
+from test.test_ldap import mock_ldap
+from test.test_keystone_auth import fake_keystone
+from util.names import parse_robot_username
+
+_FAKE_AUTH = 'fake'
+
+class FakeUsers(FederatedUsers):
+  def __init__(self, group_members):
+    super(FakeUsers, self).__init__(_FAKE_AUTH, False)
+    self.group_tuples = [(m, None) for m in group_members]
+
+  def iterate_group_members(self, group_lookup_args, page_size=None, disable_pagination=False):
+    return (self.group_tuples, None)
+
+
+@pytest.mark.parametrize('starting_membership,group_membership,expected_membership', [
+  # Empty team + single member in group => Single member in team.
+  ([],
+   [
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+   ],
+   ['someuser']),
+
+  # Team with a Quay user + empty group => empty team.
+  ([('someuser', None)],
+   [],
+   []),
+
+  # Team with an existing external user + user is in the group => no changes.
+  ([
+    ('someuser', 'someuser'),
+   ],
+   [
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+   ],
+   ['someuser']),
+
+  # Team with an existing external user (with a different Quay username) + user is in the group.
+  # => no changes
+  ([
+    ('anotherquayname', 'someuser'),
+   ],
+   [
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+   ],
+   ['someuser']),
+
+  # Team missing a few members that are in the group => members added.
+  ([('someuser', 'someuser')],
+   [
+     UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'),
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+     UserInformation('thirduser', 'thirduser', 'thirduser@devtable.com'),
+   ],
+   ['anotheruser', 'someuser', 'thirduser']),
+
+  # Team has a few extra members no longer in the group => members removed.
+  ([
+    ('anotheruser', 'anotheruser'),
+    ('someuser', 'someuser'),
+    ('thirduser', 'thirduser'),
+    ('nontestuser', None),
+   ],
+   [
+     UserInformation('thirduser', 'thirduser', 'thirduser@devtable.com'),
+   ],
+   ['thirduser']),
+
+  # Team has different membership than the group => members added and removed.
+  ([
+    ('anotheruser', 'anotheruser'),
+    ('someuser', 'someuser'),
+    ('nontestuser', None),
+   ],
+   [
+     UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'),
+     UserInformation('missinguser', 'missinguser', 'missinguser@devtable.com'),
+   ],
+   ['anotheruser', 'missinguser']),
+
+  # Team has same membership but some robots => robots remain and no other changes.
+  ([
+    ('someuser', 'someuser'),
+    ('buynlarge+anotherbot', None),
+    ('buynlarge+somerobot', None),
+   ],
+   [
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+   ],
+   ['someuser', 'buynlarge+somerobot', 'buynlarge+anotherbot']),
+
+  # Team has an extra member and some robots => member removed and robots remain.
+  ([
+    ('someuser', 'someuser'),
+    ('buynlarge+anotherbot', None),
+    ('buynlarge+somerobot', None),
+   ],
+   [
+     # No members.
+   ],
+   ['buynlarge+somerobot', 'buynlarge+anotherbot']),
+
+  # Team has a different member and some robots => member changed and robots remain.
+  ([
+    ('someuser', 'someuser'),
+    ('buynlarge+anotherbot', None),
+    ('buynlarge+somerobot', None),
+   ],
+   [
+     UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'),
+   ],
+   ['anotheruser', 'buynlarge+somerobot', 'buynlarge+anotherbot']),
+
+  # Team with an existing external user (with a different Quay username) + user is in the group.
+  # => no changes and robots remain.
+  ([
+    ('anotherquayname', 'someuser'),
+    ('buynlarge+anotherbot', None),
+   ],
+   [
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+   ],
+   ['someuser', 'buynlarge+anotherbot']),
+
+  # Team which returns the same member twice, as pagination in some engines (like LDAP) is not
+  # stable.
+  ([],
+   [
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+     UserInformation('anotheruser', 'anotheruser', 'anotheruser@devtable.com'),
+     UserInformation('someuser', 'someuser', 'someuser@devtable.com'),
+   ],
+   ['anotheruser', 'someuser']),
+])
+def test_syncing(starting_membership, group_membership, expected_membership, app):
+  org = model.organization.get_organization('buynlarge')
+
+  # Necessary for the fake auth entries to be created in FederatedLogin.
+  database.LoginService.create(name=_FAKE_AUTH)
+
+  # Assert the team is empty, so we have a clean slate.
+  sync_team_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  assert len(list(model.team.list_team_users(sync_team_info.team))) == 0
+
+  # Add the existing starting members to the team.
+  for starting_member in starting_membership:
+    (quay_username, fakeauth_username) = starting_member
+    if '+' in quay_username:
+      # Add a robot.
+      (_, shortname) = parse_robot_username(quay_username)
+      robot, _ = model.user.create_robot(shortname, org)
+      model.team.add_user_to_team(robot, sync_team_info.team)
+    else:
+      email = quay_username + '@devtable.com'
+
+      if fakeauth_username is None:
+        quay_user = model.user.create_user_noverify(quay_username, email)
+      else:
+        quay_user = model.user.create_federated_user(quay_username, email, _FAKE_AUTH,
+                                                     fakeauth_username, False)
+
+      model.team.add_user_to_team(quay_user, sync_team_info.team)
+
+  # Call syncing on the team.
+  fake_auth = FakeUsers(group_membership)
+  assert sync_team(fake_auth, sync_team_info)
+
+  # Ensure the last updated time and transaction_id's have changed.
+  updated_sync_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  assert updated_sync_info.last_updated is not None
+  assert updated_sync_info.transaction_id != sync_team_info.transaction_id
+
+  users_expected = set([name for name in expected_membership if '+' not in name])
+  robots_expected = set([name for name in expected_membership if '+' in name])
+  assert len(users_expected) + len(robots_expected) == len(expected_membership)
+
+  # Check that the team's users match those expected.
+  service_user_map = model.team.get_federated_team_member_mapping(sync_team_info.team, _FAKE_AUTH)
+  assert set(service_user_map.keys()) == users_expected
+
+  quay_users = model.team.list_team_users(sync_team_info.team)
+  assert len(quay_users) == len(users_expected)
+
+  for quay_user in quay_users:
+    fakeauth_record = model.user.lookup_federated_login(quay_user, _FAKE_AUTH)
+    assert fakeauth_record is not None
+    assert fakeauth_record.service_ident in users_expected
+    assert service_user_map[fakeauth_record.service_ident] == quay_user.id
+
+  # Check that the team's robots match those expected.
+  robots_found = set([r.username for r in model.team.list_team_robots(sync_team_info.team)])
+  assert robots_expected == robots_found
+
+
+def test_sync_teams_to_groups(app):
+  # Necessary for the fake auth entries to be created in FederatedLogin.
+  database.LoginService.create(name=_FAKE_AUTH)
+
+  # Assert the team has not yet been updated.
+  sync_team_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  assert sync_team_info.last_updated is None
+
+  # Call to sync all teams.
+  fake_auth = FakeUsers([])
+  sync_teams_to_groups(fake_auth, timedelta(seconds=1))
+
+  # Ensure the team was synced.
+  updated_sync_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  assert updated_sync_info.last_updated is not None
+  assert updated_sync_info.transaction_id != sync_team_info.transaction_id
+
+  # Set the stale threshold to a high amount and ensure the team is not resynced.
+  current_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  current_info.last_updated = datetime.now() - timedelta(seconds=2)
+  current_info.save()
+
+  sync_teams_to_groups(fake_auth, timedelta(seconds=120))
+
+  third_sync_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  assert third_sync_info.last_updated == current_info.last_updated
+  assert third_sync_info.transaction_id == updated_sync_info.transaction_id
+
+  # Set the stale threshold to 10 seconds, and ensure the team is resynced, after making it
+  # "updated" 20s ago.
+  current_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  current_info.last_updated = datetime.now() - timedelta(seconds=20)
+  current_info.save()
+
+  sync_teams_to_groups(fake_auth, timedelta(seconds=10))
+
+  fourth_sync_info = model.team.get_team_sync_information('buynlarge', 'synced')
+  assert fourth_sync_info.transaction_id != updated_sync_info.transaction_id
+
+
+@pytest.mark.parametrize('auth_system_builder,config', [
+  (mock_ldap, {'group_dn': 'cn=AwesomeFolk'}),
+  (fake_keystone, {'group_id': 'somegroupid'}),
+])
+def test_teamsync_end_to_end(auth_system_builder, config, app):
+  with auth_system_builder() as auth:
+    # Create an new team to sync.
+    org = model.organization.get_organization('buynlarge')
+    new_synced_team = model.team.create_team('synced2', org, 'member', 'Some synced team.')
+    sync_team_info = model.team.set_team_syncing(new_synced_team, auth.federated_service, config)
+
+    # Sync the team.
+    assert sync_team(auth, sync_team_info)
+
+    # Ensure we now have members.
+    msg = 'Auth system: %s' % auth.federated_service
+    sync_team_info = model.team.get_team_sync_information('buynlarge', 'synced2')
+    assert len(list(model.team.list_team_users(sync_team_info.team))) > 0, msg

endpoints/api/organization.py

@@ -6,7 +6,7 @@ from flask import request
 
 import features
 
-from app import billing as stripe, avatar, all_queues
+from app import billing as stripe, avatar, all_queues, authentication
 from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error,
                            related_user_resource, internal_only, require_user_admin, log_action,
                            show_if, path_param, require_scope, require_fresh_login)
@@ -33,6 +33,8 @@ def team_view(orgname, team):
 
     'repo_count': team.repo_count,
     'member_count': team.member_count,
+
+    'is_synced': team.is_synced,
   }
 
 
@@ -157,7 +159,8 @@ class Organization(ApiResource):
 
     teams = None
     if OrganizationMemberPermission(orgname).can():
-      teams = model.team.get_teams_within_org(org)
+      has_syncing = features.TEAM_SYNCING and bool(authentication.federated_service)
+      teams = model.team.get_teams_within_org(org, has_syncing)
 
     return org_view(org, teams)
 

endpoints/api/team.py

@@ -1,19 +1,27 @@
 """ Create, list and manage an organization's teams. """
 
+import json
+
+from functools import wraps
+
 from flask import request
 
 import features
 
-from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error,
-                           log_action, internal_only, require_scope, path_param, query_param,
-                           truthy_bool, parse_args, require_user_admin, show_if)
-from endpoints.exception import Unauthorized, NotFound
-from auth.permissions import AdministerOrganizationPermission, ViewTeamPermission
+from app import avatar, authentication
+from auth.permissions import (AdministerOrganizationPermission, ViewTeamPermission,
+                              SuperUserPermission)
+
 from auth.auth_context import get_authenticated_user
 from auth import scopes
 from data import model
+from endpoints.api import (resource, nickname, ApiResource, validate_json_request, request_error,
+                           log_action, internal_only, require_scope, path_param, query_param,
+                           truthy_bool, parse_args, require_user_admin, show_if, format_date,
+                           verify_not_prod, require_fresh_login)
+from endpoints.exception import Unauthorized, NotFound, InvalidRequest
 from util.useremails import send_org_invite_email
-from app import avatar
+from util.names import parse_robot_username
 
 def permission_view(permission):
   return {
@@ -24,7 +32,6 @@ def permission_view(permission):
     'role': permission.role.name
   }
 
-
 def try_accept_invite(code, user):
   (team, inviter) = model.team.confirm_team_invite(code, user)
 
@@ -40,7 +47,6 @@ def try_accept_invite(code, user):
 
   return team
 
-
 def handle_addinvite_team(inviter, team, user=None, email=None):
   requires_invite = features.MAILING and features.REQUIRE_TEAM_INVITE
   invite = model.team.add_or_invite_to_team(inviter, team, user, email,
@@ -82,7 +88,6 @@ def member_view(member, invited=False):
     'invited': invited,
   }
 
-
 def invite_view(invite):
   if invite.user:
     return member_view(invite.user, invited=True)
@@ -94,6 +99,30 @@ def invite_view(invite):
       'invited': True
     }
 
+def disallow_for_synced_team(except_robots=False):
+  """ Disallows the decorated operation for a team that is marked as being synced from an internal
+      auth provider such as LDAP. If except_robots is True, then the operation is allowed if the
+      member specified on the operation is a robot account.
+  """
+  def inner(func):
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+      # Team syncing can only be enabled if we have a federated service.
+      if features.TEAM_SYNCING and authentication.federated_service:
+        orgname = kwargs['orgname']
+        teamname = kwargs['teamname']
+        if model.team.get_team_sync_information(orgname, teamname):
+          if not except_robots or not parse_robot_username(kwargs.get('membername', '')):
+            raise InvalidRequest('Cannot call this method on an auth-synced team')
+
+      return func(self, *args, **kwargs)
+    return wrapper
+  return inner
+
+
+disallow_nonrobots_for_synced_team = disallow_for_synced_team(except_robots=True)
+disallow_all_for_synced_team = disallow_for_synced_team(except_robots=False)
+
 
 @resource('/v1/organization/<orgname>/team/<teamname>')
 @path_param('orgname', 'The name of the organization')
@@ -180,6 +209,58 @@ class OrganizationTeam(ApiResource):
     raise Unauthorized()
 
 
+@resource('/v1/organization/<orgname>/team/<teamname>/syncing')
+@path_param('orgname', 'The name of the organization')
+@path_param('teamname', 'The name of the team')
+@show_if(features.TEAM_SYNCING)
+class OrganizationTeamSyncing(ApiResource):
+  """ Resource for managing syncing of a team by a backing group. """
+  @require_scope(scopes.ORG_ADMIN)
+  @require_scope(scopes.SUPERUSER)
+  @nickname('enableOrganizationTeamSync')
+  @verify_not_prod
+  @require_fresh_login
+  def post(self, orgname, teamname):
+    # User must be both the org admin AND a superuser.
+    if SuperUserPermission().can() and AdministerOrganizationPermission(orgname).can():
+      try:
+        team = model.team.get_organization_team(orgname, teamname)
+      except model.InvalidTeamException:
+        raise NotFound()
+
+      config = request.get_json()
+
+      # Ensure that the specified config points to a valid group.
+      status, err = authentication.check_group_lookup_args(config)
+      if not status:
+        raise InvalidRequest('Could not sync to group: %s' % err)
+
+      # Set the team's syncing config.
+      model.team.set_team_syncing(team, authentication.federated_service, config)
+
+      return team_view(orgname, team)
+
+    raise Unauthorized()
+
+  @require_scope(scopes.ORG_ADMIN)
+  @require_scope(scopes.SUPERUSER)
+  @nickname('disableOrganizationTeamSync')
+  @verify_not_prod
+  @require_fresh_login
+  def delete(self, orgname, teamname):
+    # User must be both the org admin AND a superuser.
+    if SuperUserPermission().can() and AdministerOrganizationPermission(orgname).can():
+      try:
+        team = model.team.get_organization_team(orgname, teamname)
+      except model.InvalidTeamException:
+        raise NotFound()
+
+      model.team.remove_team_syncing(orgname, teamname)
+      return team_view(orgname, team)
+
+    raise Unauthorized()
+
+
 @resource('/v1/organization/<orgname>/team/<teamname>/members')
 @path_param('orgname', 'The name of the organization')
 @path_param('teamname', 'The name of the team')
@@ -211,9 +292,29 @@ class TeamMemberList(ApiResource):
       data = {
         'name': teamname,
         'members': [member_view(m) for m in members] + [invite_view(i) for i in invites],
-        'can_edit': edit_permission.can()
+        'can_edit': edit_permission.can(),
       }
 
+      if features.TEAM_SYNCING and authentication.federated_service:
+        if SuperUserPermission().can() and AdministerOrganizationPermission(orgname).can():
+          data['can_sync'] = {
+            'service': authentication.federated_service,
+          }
+
+          data['can_sync'].update(authentication.service_metadata())
+
+        sync_info = model.team.get_team_sync_information(orgname, teamname)
+        if sync_info is not None:
+          data['synced'] = {
+            'service': sync_info.service.name,
+          }
+
+          if SuperUserPermission().can():
+            data['synced'].update({
+              'last_updated': format_date(sync_info.last_updated),
+              'config': json.loads(sync_info.config),
+            })
+
       return data
 
     raise Unauthorized()
@@ -228,6 +329,7 @@ class TeamMember(ApiResource):
 
   @require_scope(scopes.ORG_ADMIN)
   @nickname('updateOrganizationTeamMember')
+  @disallow_nonrobots_for_synced_team
   def put(self, orgname, teamname, membername):
     """ Adds or invites a member to an existing team. """
     permission = AdministerOrganizationPermission(orgname)
@@ -265,6 +367,7 @@ class TeamMember(ApiResource):
 
   @require_scope(scopes.ORG_ADMIN)
   @nickname('deleteOrganizationTeamMember')
+  @disallow_nonrobots_for_synced_team
   def delete(self, orgname, teamname, membername):
     """ Delete a member of a team. If the user is merely invited to join
         the team, then the invite is removed instead.
@@ -308,6 +411,7 @@ class InviteTeamMember(ApiResource):
   """ Resource for inviting a team member via email address. """
   @require_scope(scopes.ORG_ADMIN)
   @nickname('inviteTeamMemberEmail')
+  @disallow_all_for_synced_team
   def put(self, orgname, teamname, email):
     """ Invites an email address to an existing team. """
     permission = AdministerOrganizationPermission(orgname)
@@ -407,7 +511,7 @@ class TeamMemberInvite(ApiResource):
   @nickname('declineOrganizationTeamInvite')
   @require_user_admin
   def delete(self, code):
-    """ Delete an existing member of a team. """
+    """ Delete an existing invitation to join a team. """
     (team, inviter) = model.team.delete_team_invite(code, user_obj=get_authenticated_user())
 
     model.notification.delete_matching_notifications(get_authenticated_user(), 'org_team_invite',

endpoints/api/test/shared.py

@@ -2,7 +2,6 @@ import datetime
 import json
 
 from contextlib import contextmanager
-
 from data import model
 from endpoints.api import api
 

endpoints/api/test/test_security.py

@@ -1,5 +1,7 @@
 import pytest
 
+from endpoints.api import api
+from endpoints.api.team import OrganizationTeamSyncing
 from endpoints.api.test.shared import client_with_identity, conduct_api_call
 from endpoints.api.superuser import SuperUserRepositoryBuildLogs, SuperUserRepositoryBuildResource
 from endpoints.api.superuser import SuperUserRepositoryBuildStatus
@@ -9,6 +11,16 @@ TEAM_PARAMS = {'orgname': 'buynlarge', 'teamname': 'owners'}
 BUILD_PARAMS = {'build_uuid': 'test-1234'}
 
 @pytest.mark.parametrize('resource,method,params,body,identity,expected', [
+  (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, None, 403),
+  (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, 'freshuser', 403),
+  (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, 'reader', 403),
+  (OrganizationTeamSyncing, 'POST', TEAM_PARAMS, {}, 'devtable', 400),
+
+  (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, None, 403),
+  (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, 'freshuser', 403),
+  (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, 'reader', 403),
+  (OrganizationTeamSyncing, 'DELETE', TEAM_PARAMS, {}, 'devtable', 200),
+
   (SuperUserRepositoryBuildLogs, 'GET', BUILD_PARAMS, None, None, 401),
   (SuperUserRepositoryBuildLogs, 'GET', BUILD_PARAMS, None, 'freshuser', 403),
   (SuperUserRepositoryBuildLogs, 'GET', BUILD_PARAMS, None, 'reader', 403),

endpoints/api/test/test_team.py

@@ -0,0 +1,87 @@
+import json
+
+from mock import patch
+
+from data import model
+from endpoints.api import api
+from endpoints.api.test.shared import client_with_identity, conduct_api_call
+from endpoints.api.team import OrganizationTeamSyncing, TeamMemberList
+from endpoints.api.organization import Organization
+from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file
+from test.test_ldap import mock_ldap
+
+SYNCED_TEAM_PARAMS = {'orgname': 'sellnsmall', 'teamname': 'synced'}
+UNSYNCED_TEAM_PARAMS = {'orgname': 'sellnsmall', 'teamname': 'owners'}
+
+def test_team_syncing(client):
+  with mock_ldap() as ldap:
+    with patch('endpoints.api.team.authentication', ldap):
+      with client_with_identity('devtable', client) as cl:
+        config = {
+          'group_dn': 'cn=AwesomeFolk',
+        }
+
+        conduct_api_call(cl, OrganizationTeamSyncing, 'POST', UNSYNCED_TEAM_PARAMS, config)
+
+        # Ensure the team is now synced.
+        sync_info = model.team.get_team_sync_information(UNSYNCED_TEAM_PARAMS['orgname'],
+                                                         UNSYNCED_TEAM_PARAMS['teamname'])
+        assert sync_info is not None
+        assert json.loads(sync_info.config) == config
+
+        # Remove the syncing.
+        conduct_api_call(cl, OrganizationTeamSyncing, 'DELETE', UNSYNCED_TEAM_PARAMS, None)
+
+        # Ensure the team is no longer synced.
+        sync_info = model.team.get_team_sync_information(UNSYNCED_TEAM_PARAMS['orgname'],
+                                                         UNSYNCED_TEAM_PARAMS['teamname'])
+        assert sync_info is None
+
+
+def test_team_member_sync_info(client):
+  with mock_ldap() as ldap:
+    with patch('endpoints.api.team.authentication', ldap):
+      # Check for an unsynced team, with superuser.
+      with client_with_identity('devtable', client) as cl:
+        resp = conduct_api_call(cl, TeamMemberList, 'GET', UNSYNCED_TEAM_PARAMS)
+        assert 'can_sync' in resp.json
+        assert resp.json['can_sync']['service'] == 'ldap'
+
+        assert 'synced' not in resp.json
+
+      # Check for an unsynced team, with non-superuser.
+      with client_with_identity('randomuser', client) as cl:
+        resp = conduct_api_call(cl, TeamMemberList, 'GET', UNSYNCED_TEAM_PARAMS)
+        assert 'can_sync' not in resp.json
+        assert 'synced' not in resp.json
+
+      # Check for a synced team, with superuser.
+      with client_with_identity('devtable', client) as cl:
+        resp = conduct_api_call(cl, TeamMemberList, 'GET', SYNCED_TEAM_PARAMS)
+        assert 'can_sync' in resp.json
+        assert resp.json['can_sync']['service'] == 'ldap'
+
+        assert 'synced' in resp.json
+        assert 'last_updated' in resp.json['synced']
+        assert 'group_dn' in resp.json['synced']['config']
+
+      # Check for a synced team, with non-superuser.
+      with client_with_identity('randomuser', client) as cl:
+        resp = conduct_api_call(cl, TeamMemberList, 'GET', SYNCED_TEAM_PARAMS)
+        assert 'can_sync' not in resp.json
+
+        assert 'synced' in resp.json
+        assert 'last_updated' not in resp.json['synced']
+        assert 'config' not in resp.json['synced']
+
+
+def test_organization_teams_sync_bool(client):
+  with mock_ldap() as ldap:
+    with patch('endpoints.api.organization.authentication', ldap):
+      # Ensure synced teams are marked as such in the organization teams list.
+      with client_with_identity('devtable', client) as cl:
+        resp = conduct_api_call(cl, Organization, 'GET', {'orgname': 'sellnsmall'})
+
+        assert not resp.json['teams']['owners']['is_synced']
+
+        assert resp.json['teams']['synced']['is_synced']

endpoints/oauth/test/test_login.py

@@ -1,11 +1,9 @@
 import pytest
 
-from endpoints.oauth.login import _conduct_oauth_login
-
-from oauth.services.github import GithubOAuthService
-
 from data import model, database
 from data.users import get_users_handler, DatabaseUsers
+from endpoints.oauth.login import _conduct_oauth_login
+from oauth.services.github import GithubOAuthService
 from test.fixtures import app, appconfig, database_uri, init_db_path, sqlitedb_file
 from test.test_ldap import mock_ldap
 

initdb.py

@@ -657,6 +657,9 @@ def populate_database(minimal=False, with_storage=False):
   liborg = model.organization.create_organization('library', 'quay+library@devtable.com', new_user_1)
   liborg.save()
 
+  thirdorg = model.organization.create_organization('sellnsmall', 'quay+sell@devtable.com', new_user_1)
+  thirdorg.save()
+
   model.user.create_robot('coolrobot', org)
 
   oauth_app_1 = model.oauth.create_application(org, 'Some Test App', 'http://localhost:8000',
@@ -700,6 +703,19 @@ def populate_database(minimal=False, with_storage=False):
   model.team.add_user_to_team(creatorbot, creators)
   model.team.add_user_to_team(creatoruser, creators)
 
+  sell_owners = model.team.get_organization_team('sellnsmall', 'owners')
+  sell_owners.description = 'Owners have unfettered access across the entire org.'
+  sell_owners.save()
+
+  model.team.add_user_to_team(new_user_4, sell_owners)
+
+  sync_config = {'group_dn': 'cn=Test-Group,ou=Users', 'group_id': 'somegroupid'}
+  synced_team = model.team.create_team('synced', org, 'member', 'Some synced team.')
+  model.team.set_team_syncing(synced_team, 'ldap', sync_config)
+
+  another_synced_team = model.team.create_team('synced', thirdorg, 'member', 'Some synced team.')
+  model.team.set_team_syncing(another_synced_team, 'ldap', {'group_dn': 'cn=Test-Group,ou=Users'})
+
   __generate_repository(with_storage, new_user_1, 'superwide', None, False, [],
                         [(10, [], 'latest2'),
                          (2, [], 'latest3'),

static/css/core-ui.css

@@ -1521,7 +1521,7 @@ a:focus {
   top: 11px;
   left: 12px;
   font-size: 22px;
-  color: #E4C212;
+  color: #FCA657;
 }
 
 .co-alert.co-alert-danger {
@@ -1566,6 +1566,14 @@ a:focus {
   left: 19px;
 }
 
+.co-alert-inline:before {
+  position: relative !important;
+  top: auto !important;
+  left: auto !important;
+  vertical-align: middle;
+  margin-right: 10px;
+}
+
 .co-alert-popin-warning {
   margin-left: 10px;
 }
@@ -1579,6 +1587,14 @@ a:focus {
   }
 }
 
+.co-alert-inline {
+  border: 0px;
+  display: inline-block;
+  background-color: transparent !important;
+  margin: 0px;
+  padding: 4px;
+}
+
 .co-list-table tr td:first-child {
   font-weight: bold;
   padding-right: 10px;

static/css/directives/ui/teams-manager.css

@@ -111,3 +111,7 @@
   color: #aaa;
   font-size: 14px;
 }
+
+.teams-manager .fa-refresh {
+  color: #aaa;
+}

static/css/pages/team-view.css

@@ -3,6 +3,18 @@
   position: relative;
 }
 
+.team-view .team-sync-table {
+  margin-bottom: 20px;
+}
+
+.team-view .team-sync-table td {
+  padding: 6px;
+}
+
+.team-view .team-sync-table td:first-child {
+  font-weight: bold;
+}
+
 .team-view .team-title {
   vertical-align: middle;
   margin-right: 10px;
@@ -14,10 +26,10 @@
   margin-left: 6px;
 }
 
-.team-view .team-view-header {
+.team-view .team-view-header, .team-view .team-sync-header {
   border-bottom: 1px solid #eee;
-  margin-bottom: 10px;
-  padding-bottom: 10px;
+  margin-bottom: 20px;
+  padding-bottom: 20px;
 }
 
 .team-view .team-view-header button i.fa {

static/directives/config/config-setup-tool.html

@@ -194,7 +194,7 @@
               <span class="config-string-field" binding="mapped.redis.host"
                     placeholder="The redis server hostname"
                     pattern="{{ HOSTNAME_REGEX }}"
-                    validator="validateHostname(value)">></span>
+                    validator="validateHostname(value)"></span>
             </td>
           </tr>
           <tr>
@@ -553,7 +553,7 @@
           prevent passwords from being saved as plaintext by the Docker client.
         </div>
 
-        <table class="config-table">
+        <table class="config-table" style="margin-bottom: 20px;">
           <tr>
             <td class="non-input">Authentication:</td>
             <td>
@@ -565,6 +565,28 @@
               </select>
             </td>
           </tr>
+
+          <tr ng-if="config.AUTHENTICATION_TYPE == 'LDAP' || config.AUTHENTICATION_TYPE == 'Keystone'">
+            <td>Team synchronization:</td>
+            <td>
+              <div class="config-bool-field" binding="config.FEATURE_TEAM_SYNCING">
+                Enable Team Synchronization Support
+              </div>
+              <div class="help-text">
+                If enabled, organization administrators who are also superusers can set teams to have their membership synchronized with a backing group in {{ config.AUTHENTICATION_TYPE }}.
+              </div>
+            </td>
+          </tr>
+          <tr ng-if="(config.AUTHENTICATION_TYPE == 'LDAP' || config.AUTHENTICATION_TYPE == 'Keystone') && config.FEATURE_TEAM_SYNCING">
+          <td>Resynchronization duration:</td>
+          <td>
+            <span class="config-string-field" binding="config.TEAM_RESYNC_STALE_TIME"
+                  pattern="[0-9]+(m|h|d|s)"></span>
+            <div class="help-text">
+            The duration before a team must be re-synchronized. Must be expressed in a duration string form: <code>30m</code>, <code>1h</code>, <code>1d</code>.
+            </div>
+          </td>
+          </tr>
         </table>
 
         <!--  Keystone Authentication -->
@@ -758,7 +780,7 @@
           <tr>
             <td>Administrator DN Password:</td>
             <td>
-              <div class="co-alert co-alert-warning">
+              <div class="co-alert co-alert-warning" style="margin-bottom: 10px;">
                 Note: This will be stored in
                 <strong>plaintext</strong> inside the config.yaml, so setting up a dedicated account or using
                 <a href="http://tools.ietf.org/id/draft-stroeder-hashed-userpassword-values-01.html" ng-safenewtab>a password hash</a> is <strong>highly</strong> recommended.

static/directives/team-view-add.html

@@ -1,20 +1,26 @@
 <div class="team-view-add-element" focusable-popover-content>
   <div class="entity-search"
-       namespace="orgname" placeholder="allowEmail ? 'Add a registered user, robot or email address to the team' : 'Add a registered user or robot to the team'"
+       namespace="orgname"
+       placeholder="getAddPlaceholder(allowEmail, syncInfo)"
        entity-selected="addNewMember(entity)"
        email-selected="inviteEmail(email)"
        current-entity="selectedMember"
        auto-clear="true"
-       allowed-entities="['user', 'robot']"
+       allowed-entities="allowedEntities"
        pull-right="true"
-       allow-emails="allowEmail"
+       allow-emails="allowEmail && syncInfo"
        email-message="Press enter to invite the entered e-mail address to this team"
        ng-show="!addingMember"></div>
   <div ng-show="addingMember">
     <div class="cor-loader-inline"></div> Inviting team member
   </div>
   <div class="help-text" ng-show="!addingMember">
-  <span ng-if="allowEmail">Search by registry username, robot account name or enter an email address to invite</span>
-  <span ng-if="!allowEmail">Search by registry username or robot account name</span>
+  <span ng-if="!syncInfo">
+    <span ng-if="allowEmail">Search by registry username, robot account name or enter an email address to invite</span>
+    <span ng-if="!allowEmail">Search by registry username or robot account name</span>
+  </span>
+  <span ng-if="syncInfo">
+    Search by robot account name. Users must be added in {{ syncInfo.service }}.
+  </span>
   </div>
 </div>

static/directives/teams-manager.html

@@ -41,6 +41,7 @@
 
     <table class="co-table" style="margin-top: 10px;">
       <thead>
+        <td class="options-col" ng-if="::Config.AUTHENTICATION_TYPE != 'Database' && Features.TEAM_SYNCING"></td>
         <td ng-class="TableService.tablePredicateClass('name', options.predicate, options.reverse)">
           <a ng-click="TableService.orderBy('name', options)">Team Name</a>
         </td>
@@ -65,6 +66,9 @@
        <tr class="co-checkable-row"
            ng-repeat="team in orderedTeams.visibleEntries"
            bindonce>
+        <td class="options-col" ng-if="::Config.AUTHENTICATION_TYPE != 'Database' && Features.TEAM_SYNCING">
+          <i class="fa fa-refresh" ng-if="team.is_synced" data-title="Team is synchronized with a backing group" bs-tooltip></i>
+        </td>
         <td style="white-space: nowrap;">
           <span class="avatar" data="team.avatar" size="24"></span>
           <span bo-show="team.can_view">
@@ -97,7 +101,8 @@
         </td>
         <td>
           <span class="role-group" current-role="team.role" pull-left="true"
-                role-changed="setRole(role, team.name)" roles="teamRoles"></span>
+                role-changed="setRole(role, team.name)" roles="teamRoles"
+                read-only="!organization.is_admin"></span>
         </td>
         <td>
           <span class="cor-options-menu" ng-show="organization.is_admin">

static/js/directives/ui/teams-manager.js

@@ -12,8 +12,10 @@ angular.module('quay').directive('teamsManager', function () {
       'organization': '=organization',
       'isEnabled': '=isEnabled'
     },
-    controller: function($scope, $element, ApiService, $timeout, UserService, TableService, UIService) {
+    controller: function($scope, $element, ApiService, $timeout, UserService, TableService, UIService, Config, Features) {
       $scope.TableService = TableService;
+      $scope.Config = Config;
+      $scope.Features = Features;
 
       $scope.options = {
         'predicate': 'ordered_team_index',

static/js/pages/team-view.js

@@ -14,12 +14,14 @@
     var teamname = $routeParams.teamname;
     var orgname = $routeParams.orgname;
 
+    $scope.context = {};
     $scope.orgname = orgname;
     $scope.teamname = teamname;
     $scope.addingMember = false;
     $scope.memberMap = null;
     $scope.allowEmail = Features.MAILING;
     $scope.feedback = null;
+    $scope.allowedEntities = ['user', 'robot'];
 
     $rootScope.title = 'Loading...';
 
@@ -146,6 +148,39 @@
       }, ApiService.errorDisplay('Cannot remove team member'));
     };
 
+    $scope.getServiceName = function(service) {
+      switch (service) {
+        case 'ldap':
+          return 'LDAP';
+
+        case 'keystone':
+          return 'Keystone Auth';
+
+        case 'jwtauthn':
+          return 'External JWT Auth';
+
+        default:
+          return synced.service;
+      }
+    };
+
+    $scope.getAddPlaceholder = function(email, synced) {
+      var kinds = [];
+
+      if (!synced) {
+        kinds.push('registered user');
+      }
+
+      kinds.push('robot');
+
+      if (email && !synced) {
+        kinds.push('email address');
+      }
+
+      kind_string = kinds.join(', ')
+      return 'Add a ' + kind_string + ' to the team';
+    };
+
     $scope.updateForDescription = function(content) {
       $scope.organization.teams[teamname].description = content;
 
@@ -166,6 +201,48 @@
       });
     };
 
+    $scope.showEnableSyncing = function() {
+      $scope.enableSyncingInfo = {
+        'service_info': $scope.canSync,
+        'config': {}
+      };
+    };
+
+    $scope.showDisableSyncing = function() {
+      msg = 'Are you sure you want to disable group syncing on this team? ' +
+            'The team will once again become editable.';
+      bootbox.confirm(msg, function(result) {
+        if (result) {
+          $scope.disableSyncing();
+        }
+      });
+    };
+
+    $scope.disableSyncing = function() {
+      var params = {
+        'orgname': orgname,
+        'teamname': teamname
+      };
+
+      var errorHandler = ApiService.errorDisplay('Could not disable team syncing');
+      ApiService.disableOrganizationTeamSync(null, params).then(function(resp) {
+        loadMembers();
+      }, errorHandler);
+    };
+
+    $scope.enableSyncing = function(config, callback) {
+      var params = {
+        'orgname': orgname,
+        'teamname': teamname
+      };
+
+      var errorHandler = ApiService.errorDisplay('Cannot enable team syncing', callback);
+      ApiService.enableOrganizationTeamSync(config, params).then(function(resp) {
+        loadMembers();
+        callback(true);
+      }, errorHandler);
+    };
+
     var loadOrganization = function() {
       $scope.orgResource = ApiService.getOrganizationAsResource({'orgname': orgname}).get(function(org) {
         $scope.organization = org;
@@ -187,6 +264,9 @@
       $scope.membersResource = ApiService.getOrganizationTeamMembersAsResource(params).get(function(resp) {
         $scope.members = resp.members;
         $scope.canEditMembers = resp.can_edit;
+        $scope.canSync = resp.can_sync;
+        $scope.syncInfo = resp.synced;
+        $scope.allowedEntities = resp.synced ? ['robot'] : ['user', 'robot'];
 
         $('.info-icon').popover({
           'trigger': 'hover',

static/partials/team-view.html

@@ -18,6 +18,44 @@
     <div class="co-main-content-panel">
       <div class="feedback-bar" feedback="feedback"></div>
 
+      <div class="team-sync-header" ng-if="canSync && !syncInfo">
+        <div class="section-header">Directory Synchronization</div>
+        <p>Directory synchronization allows this team's user membership to be backed by a group in {{ getServiceName(canSync.service) }}.</p>
+        <button class="btn btn-primary" ng-click="showEnableSyncing()">Enable Directory Synchronization</button>
+      </div>
+
+      <!-- Sync Header -->
+      <div ng-if="syncInfo">
+        <div class="co-alert co-alert-info">
+        This team is synchronized with a group in <strong>{{ getServiceName(syncInfo.service) }}</strong> and its user membership is therefore <strong>read-only</strong>.
+        </div>
+
+        <div class="team-sync-header" ng-if="syncInfo.config">
+          <div class="section-header">Directory Synchronization</div>
+          <table class="team-sync-table">
+          <tr>
+            <td>Bound to group:</td>
+            <td>
+              <div ng-if="syncInfo.service == 'ldap'">
+                <code>{{ syncInfo.config.group_dn }}</code>
+              </div>
+              <div ng-if="syncInfo.service == 'keystone'">
+                <code>{{ syncInfo.config.group_id }}</code>
+              </div>
+            </td>
+          </tr>
+          <tr>
+            <td>Last Updated:</td>
+            <td ng-if="syncInfo.last_updated"><span am-time-ago="syncInfo.last_updated"></span> at {{ syncInfo.last_updated | amDateFormat:'dddd, MMMM Do YYYY, h:mm:ss a' }}</td>
+            <td ng-if="!syncInfo.last_updated" style="color: #aaa;">Never</td>
+          </tr>
+          </table>
+
+          <button class="btn btn-default" ng-click="showDisableSyncing()" ng-if="canSync">Remove Synchronization</button>
+          <div ng-if="!canSync" class="co-alert co-alert-warning co-alert-inline">You must be an admin of this organization to disable team synchronization</div>
+        </div>
+      </div>
+
       <!-- Description -->
       <div class="section-header">Team Description</div>
       <div class="team-view-header">
@@ -33,12 +71,15 @@
           <div ng-include="'/static/directives/team-view-add.html'" style="max-width: 500px;"></div>
         </div>
       </div>
-      <div class="section-header">Team Members</div>
+      <div class="section-header" style="margin-bottom: 55px;">Team Members</div>
 
       <div class="empty" ng-if="!members.length">
         <div class="empty-primary-msg">This team has no members.</div>
-        <div class="empty-secondary-msg">
-          Click the "Add Team Member" button above to add or invite team members.
+        <div class="empty-secondary-msg" ng-if="!syncInfo">
+          Enter a user or robot above to add or invite to the team.
+        </div>
+        <div class="empty-secondary-msg" ng-if="syncInfo">
+          This team is synchronized with an external group defined in {{ getServiceName(syncInfo.service) }}. To add a user to this team, add them in the backing group. To add a robot account to this team, enter them above.
         </div>
       </div>
 
@@ -46,7 +87,7 @@
         <!-- Team Members -->
         <tr class="co-table-header-row"
             ng-if="(members | filter: filterFunction(false, false)).length">
-          <td colspan="3"><i class="fa fa-user"></i> Team Members</td>
+          <td colspan="3"><i class="fa fa-user"></i> Team Members <span ng-if="syncInfo">(defined in {{ getServiceName(syncInfo.service) }})</span></td>
         </tr>
 
         <tr class="indented-row"
@@ -56,7 +97,7 @@
                   show-avatar="true" avatar-size="24"></span>
           </td>
           <td class="options-col">
-            <span class="cor-options-menu" ng-if="canEditMembers">
+            <span class="cor-options-menu" ng-if="canEditMembers && !syncInfo">
               <span class="cor-option" option-click="removeMember(member.name)">
                 <i class="fa fa-times"></i> Remove {{ member.name }}
               </span>
@@ -122,6 +163,29 @@
   </div>
 </div>
 
+<!-- Directory binding dialog -->
+<div class="cor-confirm-dialog"
+   dialog-context="enableSyncingInfo"
+   dialog-action="enableSyncing(info.config, callback)"
+   dialog-title="Enable Directory Syncing"
+   dialog-action-title="Enable Group Sync"
+   dialog-form="context.syncform">
+   <div class="co-alert co-alert-warning">Please note that once team syncing is enabled, the team's user membership from within <span class="registry-name"></span> will be read-only.</div>
+   <form name="context.syncform" class="co-single-field-dialog">
+     <div ng-switch on="enableSyncingInfo.service_info.service">
+      <div ng-switch-when="ldap">
+        Enter the distinguished name of the group, relative to <code>{{ enableSyncingInfo.service_info.base_dn }}</code>:
+        <input type="text" class="form-control" placeholder="Group DN" ng-model="enableSyncingInfo.config.group_dn" required>
+      </div>
+      <div ng-switch-when="keystone">
+        Enter the Keystone group ID:
+        <input type="text" class="form-control" placeholder="Group ID" ng-model="enableSyncingInfo.config.group_id" required>
+      </div>
+     </div>
+   </form>
+</div>
+
+
 <!-- Modal message dialog -->
 <div class="modal fade" id="cannotChangeTeamModal">
   <div class="modal-dialog">

test/test_api_usage.py

@@ -7,6 +7,7 @@ import time
 import re
 import json as py_json
 
+from mock import patch
 from StringIO import StringIO
 from calendar import timegm
 from contextlib import contextmanager
@@ -77,6 +78,7 @@ from endpoints.api.suconfig import (SuperUserRegistryStatus, SuperUserConfig, Su
                                     SuperUserCreateInitialSuperUser)
 from endpoints.api.manifest import RepositoryManifestLabels, ManageRepositoryManifestLabel
 from test.test_ssl_util import generate_test_cert
+from util.morecollections import AttrDict
 
 
 try:
@@ -1005,7 +1007,7 @@ class TestConductSearch(ApiTestCase):
     json = self.getJsonResponse(ConductSearch,
                                 params=dict(query='owners'))
 
-    self.assertEquals(2, len(json['results']))
+    self.assertEquals(3, len(json['results']))
     self.assertEquals(json['results'][0]['kind'], 'team')
     self.assertEquals(json['results'][0]['name'], 'owners')
 
@@ -1592,6 +1594,54 @@ class TestUpdateOrganizationTeamMember(ApiTestCase):
       self.assertNotEqual(membername, member['name'])
 
 
+  def test_updatemembers_syncedteam(self):
+    self.login(ADMIN_ACCESS_USER)
+
+    with patch('endpoints.api.team.authentication', AttrDict({'federated_service': 'foobar'})):
+      # Add the user to a non-synced team, which should succeed.
+      self.putJsonResponse(TeamMember,
+                           params=dict(orgname=ORGANIZATION, teamname='owners',
+                                       membername=READ_ACCESS_USER))
+
+      # Remove the user from the non-synced team, which should succeed.
+      self.deleteEmptyResponse(TeamMember,
+                               params=dict(orgname=ORGANIZATION, teamname='owners',
+                                           membername=READ_ACCESS_USER))
+
+      # Attempt to add the user to a synced team, which should fail.
+      self.putResponse(TeamMember,
+                       params=dict(orgname=ORGANIZATION, teamname='synced',
+                                   membername=READ_ACCESS_USER),
+                       expected_code=400)
+
+      # Attempt to remove the user from the synced team, which should fail.
+      self.deleteResponse(TeamMember,
+                          params=dict(orgname=ORGANIZATION, teamname='synced',
+                                      membername=READ_ACCESS_USER),
+                          expected_code=400)
+
+      # Add a robot to the synced team, which should succeed.
+      self.putJsonResponse(TeamMember,
+                           params=dict(orgname=ORGANIZATION, teamname='synced',
+                                       membername=ORGANIZATION + '+coolrobot'))
+
+      # Remove the robot from the non-synced team, which should succeed.
+      self.deleteEmptyResponse(TeamMember,
+                               params=dict(orgname=ORGANIZATION, teamname='synced',
+                                           membername=ORGANIZATION + '+coolrobot'))
+
+      # Invite a team member to a non-synced team, which should succeed.
+      self.putJsonResponse(InviteTeamMember,
+                           params=dict(orgname=ORGANIZATION, teamname='owners',
+                                       email='someguy+new@devtable.com'))
+
+      # Attempt to invite a team member to a synced team, which should fail.
+      self.putResponse(InviteTeamMember,
+                       params=dict(orgname=ORGANIZATION, teamname='synced',
+                                   email='someguy+new@devtable.com'),
+                       expected_code=400)
+
+
 class TestAcceptTeamMemberInvite(ApiTestCase):
   def test_accept(self):
     self.login(ADMIN_ACCESS_USER)

test/test_keystone_auth.py

@@ -7,14 +7,14 @@ import requests
 from flask import Flask, request, abort, make_response
 from contextlib import contextmanager
 
-from helpers import liveserver_app
+from test.helpers import liveserver_app
 from data.users.keystone import get_keystone_users
 from initdb import setup_database_for_testing, finished_database_for_testing
 
 _PORT_NUMBER = 5001
 
 @contextmanager
-def fake_keystone(version, requires_email=True):
+def fake_keystone(version=3, requires_email=True):
   """ Context manager which instantiates and runs a webserver with a fake Keystone implementation,
       until the result is yielded.
 
@@ -47,6 +47,23 @@ def _create_app(requires_email=True):
     {'username': 'some.neat.user', 'name': 'Neat User', 'password': 'foobar'},
   ]
 
+  groups = [
+    {'id': 'somegroupid', 'name': 'somegroup', 'description': 'Hi there!',
+     'members': ['adminuser', 'cool.user']},
+  ]
+
+  def _get_user(username):
+    for user in users:
+      if user['username'] == username:
+        user_data = {}
+        user_data['id'] = username
+        user_data['name'] = username
+        if requires_email:
+          user_data['email'] = username + '@example.com'
+        return user_data
+
+    return None
+
   ks_app = Flask('testks')
   ks_app.config['SERVER_HOSTNAME'] = 'localhost:%s' % _PORT_NUMBER
   if os.environ.get('DEBUG') == 'true':
@@ -66,6 +83,35 @@ def _create_app(requires_email=True):
 
     abort(404)
 
+  @ks_app.route('/v3/identity/groups/<groupid>/users', methods=['GET'])
+  def getv3groupmembers(groupid):
+    for group in groups:
+      if group['id'] == groupid:
+        group_data = {
+          "links": {},
+          "users": [_get_user(username) for username in group['members']],
+        }
+
+        return json.dumps(group_data)
+
+    abort(404)
+
+  @ks_app.route('/v3/identity/groups/<groupid>', methods=['GET'])
+  def getv3group(groupid):
+    for group in groups:
+      if group['id'] == groupid:
+        group_data = {
+          "description": group['description'],
+          "domain_id": "default",
+          "id": groupid,
+          "links": {},
+          "name": group['name'],
+        }
+
+        return json.dumps({'group': group_data})
+
+    abort(404)
+
   @ks_app.route('/v3/identity/users/<userid>', methods=['GET'])
   def getv3user(userid):
     for user in users:
@@ -321,6 +367,32 @@ class KeystoneV3AuthTests(KeystoneAuthTestsMixin, unittest.TestCase):
       self.assertIsNotNone(result)
       self.assertEquals('cool_user', result.username)
 
+  def test_check_group_lookup_args(self):
+    with self.fake_keystone() as keystone:
+      (status, err) = keystone.check_group_lookup_args({})
+      self.assertFalse(status)
+      self.assertEquals('Missing group_id', err)
+
+      (status, err) = keystone.check_group_lookup_args({'group_id': 'unknownid'})
+      self.assertFalse(status)
+      self.assertEquals('Group not found', err)
+
+      (status, err) = keystone.check_group_lookup_args({'group_id': 'somegroupid'})
+      self.assertTrue(status)
+      self.assertIsNone(err)
+
+  def test_iterate_group_members(self):
+    with self.fake_keystone() as keystone:
+      (itt, err) = keystone.iterate_group_members({'group_id': 'somegroupid'})
+      self.assertIsNone(err)
+
+      results = list(itt)
+      results.sort()
+
+      self.assertEquals(2, len(results))
+      self.assertEquals('adminuser', results[0][0].id)
+      self.assertEquals('cool.user', results[1][0].id)
+
 
 if __name__ == '__main__':
   unittest.main()

test/test_ldap.py

@@ -1,5 +1,7 @@
 import unittest
 
+import ldap
+
 from app import app
 from initdb import setup_database_for_testing, finished_database_for_testing
 from data.users import LDAPUsers
@@ -34,18 +36,25 @@ def mock_ldap(requires_email=True):
       'dc': ['quay', 'io'],
       'ou': 'otheremployees'
     },
+    'cn=AwesomeFolk,dc=quay,dc=io': {
+      'dc': ['quay', 'io'],
+      'cn': 'AwesomeFolk'
+    },
     'uid=testy,ou=employees,dc=quay,dc=io': {
       'dc': ['quay', 'io'],
       'ou': 'employees',
-      'uid': 'testy',
-      'userPassword': ['password']
+      'uid': ['testy'],
+      'userPassword': ['password'],
+      'mail': ['bar@baz.com'],
+      'memberOf': ['cn=AwesomeFolk,dc=quay,dc=io'],
     },
     'uid=someuser,ou=employees,dc=quay,dc=io': {
       'dc': ['quay', 'io'],
       'ou': 'employees',
       'uid': ['someuser'],
       'userPassword': ['somepass'],
-      'mail': ['foo@bar.com']
+      'mail': ['foo@bar.com'],
+      'memberOf': ['cn=AwesomeFolk,dc=quay,dc=io'],
     },
     'uid=nomail,ou=employees,dc=quay,dc=io': {
       'dc': ['quay', 'io'],
@@ -116,6 +125,45 @@ def mock_ldap(requires_email=True):
     obj.search_s.seed('ou=employees,dc=quay,dc=io', 2, '(|(uid=unknown*)(mail=unknown*))')([])
     obj.search_s.seed('ou=otheremployees,dc=quay,dc=io', 2,
                       '(|(uid=unknown*)(mail=unknown*))')([])
+
+    obj._results = {}
+
+    def result3(messageid):
+      if messageid is None:
+        return None, [], None, None
+
+      return obj._results[messageid]
+
+    def search_ext(user_search_dn, scope, search_flt, serverctrls=None, attrlist=None):
+      if scope != ldap.SCOPE_SUBTREE:
+        return None
+
+      if not serverctrls:
+        return None
+
+      page_control = serverctrls[0]
+      if page_control.controlType != ldap.controls.SimplePagedResultsControl.controlType:
+        return None
+
+      msgid = obj.search(user_search_dn, scope, search_flt, attrlist=attrlist)
+      _, rdata = obj.result(msgid)
+
+      msgid = 'messageid'
+      cookie = int(page_control.cookie) if page_control.cookie else 0
+
+      results = rdata[cookie:cookie+page_control.size]
+      cookie = cookie + page_control.size
+      if cookie > len(results):
+        page_control.cookie = None
+      else:
+        page_control.cookie = cookie
+
+      obj._results['messageid'] = (None, results, None, [page_control])
+      return msgid
+
+    obj.search_ext = search_ext
+    obj.result3 = result3
+
     return obj
 
   mockldap.start()
@@ -301,6 +349,71 @@ class TestLDAP(unittest.TestCase):
                        requires_email=False, timeout=5)
       ldap.query_users('cool')
 
+  def test_iterate_group_members(self):
+    with mock_ldap() as ldap:
+      (it, err) = ldap.iterate_group_members({'group_dn': 'cn=AwesomeFolk'},
+                                             disable_pagination=True)
+      self.assertIsNone(err)
+
+      results = list(it)
+      self.assertEquals(2, len(results))
+
+      first = results[0][0]
+      second = results[1][0]
+
+      if first.id == 'testy':
+        testy, someuser = first, second
+      else:
+        testy, someuser = second, first
+
+      self.assertEquals('testy', testy.id)
+      self.assertEquals('testy', testy.username)
+      self.assertEquals('bar@baz.com', testy.email)
+
+      self.assertEquals('someuser', someuser.id)
+      self.assertEquals('someuser', someuser.username)
+      self.assertEquals('foo@bar.com', someuser.email)
+
+  def test_iterate_group_members_with_pagination(self):
+    with mock_ldap() as ldap:
+      (it, err) = ldap.iterate_group_members({'group_dn': 'cn=AwesomeFolk'}, page_size=1)
+      self.assertIsNone(err)
+
+      results = list(it)
+      self.assertEquals(2, len(results))
+
+      first = results[0][0]
+      second = results[1][0]
+
+      if first.id == 'testy':
+        testy, someuser = first, second
+      else:
+        testy, someuser = second, first
+
+      self.assertEquals('testy', testy.id)
+      self.assertEquals('testy', testy.username)
+      self.assertEquals('bar@baz.com', testy.email)
+
+      self.assertEquals('someuser', someuser.id)
+      self.assertEquals('someuser', someuser.username)
+      self.assertEquals('foo@bar.com', someuser.email)
+
+  def test_check_group_lookup_args(self):
+    with mock_ldap() as ldap:
+      (result, err) = ldap.check_group_lookup_args({'group_dn': 'cn=invalid'},
+                                                   disable_pagination=True)
+      self.assertFalse(result)
+      self.assertIsNotNone(err)
+
+      (result, err) = ldap.check_group_lookup_args({'group_dn': 'cn=AwesomeFolk'},
+                                                   disable_pagination=True)
+      self.assertTrue(result)
+      self.assertIsNone(err)
+
+  def test_metadata(self):
+    with mock_ldap() as ldap:
+      assert 'base_dn' in ldap.service_metadata()
+
 if __name__ == '__main__':
   unittest.main()
 

test/testconfig.py

@@ -94,3 +94,4 @@ class TestConfig(DefaultConfig):
   RECAPTCHA_SECRET_KEY = 'somesecretkey'
 
   FEATURE_APP_REGISTRY = True
+  FEATURE_TEAM_SYNCING = True

util/config/configutil.py

@@ -76,3 +76,4 @@ def add_enterprise_config_defaults(config_obj, current_secret_key, hostname):
   config_obj['PREFERRED_URL_SCHEME'] = config_obj.get('PREFERRED_URL_SCHEME', 'http')
   config_obj['ENTERPRISE_LOGO_URL'] = config_obj.get(
     'ENTERPRISE_LOGO_URL', '/static/img/quay-logo.png')
+  config_obj['TEAM_RESYNC_STALE_TIME'] = '60m'

workers/teamsyncworker.py

@@ -0,0 +1,40 @@
+import logging
+import time
+
+import features
+
+from app import app, authentication
+from data.users.teamsync import sync_teams_to_groups
+from workers.worker import Worker
+from util.timedeltastring import convert_to_timedelta
+
+logger = logging.getLogger(__name__)
+
+WORKER_FREQUENCY = app.config.get('TEAM_SYNC_WORKER_FREQUENCY', 60)
+STALE_CUTOFF = convert_to_timedelta(app.config.get('TEAM_RESYNC_STALE_TIME', '30m'))
+
+class TeamSynchronizationWorker(Worker):
+  """ Worker which synchronizes teams with their backing groups in LDAP/Keystone/etc.
+  """
+  def __init__(self):
+    super(TeamSynchronizationWorker, self).__init__()
+    self.add_operation(self._sync_teams_to_groups, WORKER_FREQUENCY)
+
+  def _sync_teams_to_groups(self):
+    sync_teams_to_groups(authentication, STALE_CUTOFF)
+
+
+def main():
+  logging.config.fileConfig('conf/logging_debug.conf', disable_existing_loggers=False)
+
+  if not features.TEAM_SYNCING or not authentication.federated_service:
+    logger.debug('Team syncing is disabled; sleeping')
+    while True:
+      time.sleep(100000)
+
+  worker = TeamSynchronizationWorker()
+  worker.start()
+
+
+if __name__ == "__main__":
+  main()