Merge pull request #3024 from coreos-inc/manageable-robots

Manageable robots epic

data/database.py

@@ -440,6 +440,8 @@ class User(BaseModel):
   location = CharField(null=True)
 
   maximum_queued_builds_count = IntegerField(null=True)
+  creation_date = DateTimeField(default=datetime.utcnow, null=True)
+  last_accessed = DateTimeField(null=True, index=True)
 
   def delete_instance(self, recursive=False, delete_nullable=False):
     # If we are deleting a robot account, only execute the subset of queries necessary.
@@ -477,6 +479,12 @@ class User(BaseModel):
 Namespace = User.alias()
 
 
+class RobotAccountMetadata(BaseModel):
+  robot_account = QuayUserField(index=True, allows_robots=True, unique=True)
+  description = CharField()
+  unstructured_json = JSONField()
+
+
 class DeletedNamespace(BaseModel):
   namespace = QuayUserField(index=True, allows_robots=False, unique=True)
   marked = DateTimeField(default=datetime.now)

data/migrations/versions/0cf50323c78b_add_creation_date_to_user_table.py

@@ -0,0 +1,26 @@
+"""Add creation date to User table
+
+Revision ID: 0cf50323c78b
+Revises: 87fbbc224f10
+Create Date: 2018-03-09 13:19:41.903196
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0cf50323c78b'
+down_revision = '87fbbc224f10'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('user', sa.Column('creation_date', sa.DateTime(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('user', 'creation_date')
+    # ### end Alembic commands ###

data/migrations/versions/224ce4c72c2f_add_last_accessed_field_to_user_table.py

@@ -0,0 +1,28 @@
+"""Add last_accessed field to User table
+
+Revision ID: 224ce4c72c2f
+Revises: b547bc139ad8
+Create Date: 2018-03-12 22:44:07.070490
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '224ce4c72c2f'
+down_revision = 'b547bc139ad8'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('user', sa.Column('last_accessed', sa.DateTime(), nullable=True))
+    op.create_index('user_last_accessed', 'user', ['last_accessed'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index('user_last_accessed', table_name='user')
+    op.drop_column('user', 'last_accessed')
+    # ### end Alembic commands ###

data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py

@@ -0,0 +1,35 @@
+"""Add RobotAccountMetadata table
+
+Revision ID: b547bc139ad8
+Revises: 0cf50323c78b
+Create Date: 2018-03-09 15:50:48.298880
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'b547bc139ad8'
+down_revision = '0cf50323c78b'
+
+from alembic import op
+import sqlalchemy as sa
+from util.migrate import UTF8CharField
+
+
+def upgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('robotaccountmetadata',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('robot_account_id', sa.Integer(), nullable=False),
+    sa.Column('description', UTF8CharField(length=255), nullable=False),
+    sa.Column('unstructured_json', sa.Text(), nullable=False),
+    sa.ForeignKeyConstraint(['robot_account_id'], ['user.id'], name=op.f('fk_robotaccountmetadata_robot_account_id_user')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_robotaccountmetadata'))
+    )
+    op.create_index('robotaccountmetadata_robot_account_id', 'robotaccountmetadata', ['robot_account_id'], unique=True)
+    # ### end Alembic commands ###
+
+
+def downgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('robotaccountmetadata')
+    # ### end Alembic commands ###

data/model/_basequery.py

@@ -1,11 +1,17 @@
-from peewee import fn
+import logging
+
+from peewee import fn, PeeweeException
 from cachetools import lru_cache
 
-from data.model import DataModelException
+from datetime import datetime, timedelta
+
+from data.model import DataModelException, config
 from data.database import (Repository, User, Team, TeamMember, RepositoryPermission, TeamRole,
                            Namespace, Visibility, ImageStorage, Image, RepositoryKind,
                            db_for_update)
 
+logger = logging.getLogger(__name__)
+
 def reduce_as_tree(queries_to_reduce):
   """ This method will split a list of queries into halves recursively until we reach individual
       queries, at which point it will start unioning the queries, or the already unioned subqueries.
@@ -164,3 +170,36 @@ def calculate_image_aggregate_size(ancestors_str, image_size, parent_image):
     return None
 
   return ancestor_size + image_size
+
+
+def update_last_accessed(token_or_user):
+  """ Updates the `last_accessed` field on the given token or user. If the existing field's value
+      is within the configured threshold, the update is skipped. """
+  threshold = timedelta(seconds=config.app_config.get('LAST_ACCESSED_UPDATE_THRESHOLD_S', 120))
+  if (token_or_user.last_accessed is not None and
+      datetime.utcnow() - token_or_user.last_accessed < threshold):
+    # Skip updating, as we don't want to put undue pressure on the database.
+    return
+
+  model_class = token_or_user.__class__
+  last_accessed = datetime.utcnow()
+
+  try:
+    (model_class
+     .update(last_accessed=last_accessed)
+     .where(model_class.id == token_or_user.id)
+     .execute())
+    token_or_user.last_accessed = last_accessed
+  except PeeweeException as ex:
+    # If there is any form of DB exception, only fail if strict logging is enabled.
+    strict_logging_disabled = config.app_config.get('ALLOW_PULLS_WITHOUT_STRICT_LOGGING')
+    if strict_logging_disabled:
+      data = {
+        'exception': ex,
+        'token_or_user': token_or_user.id,
+        'class': str(model_class),
+      }
+
+      logger.exception('update last_accessed for token/user failed', extra=data)
+    else:
+      raise

data/model/appspecifictoken.py

@@ -3,10 +3,10 @@ import logging
 from datetime import datetime
 
 from cachetools import lru_cache
-from peewee import PeeweeException
 
 from data.database import AppSpecificAuthToken, User, db_transaction
 from data.model import config
+from data.model._basequery import update_last_accessed
 from util.timedeltastring import convert_to_timedelta
 
 logger = logging.getLogger(__name__)
@@ -104,23 +104,7 @@ def access_valid_token(token_code):
                       ((AppSpecificAuthToken.expiration > datetime.now()) |
                        (AppSpecificAuthToken.expiration >> None)))
                .get())
+      update_last_accessed(token)
+      return token
     except AppSpecificAuthToken.DoesNotExist:
       return None
-
-    token.last_accessed = datetime.now()
-
-    try:
-      token.save()
-    except PeeweeException as ex:
-      strict_logging_disabled = config.app_config.get('ALLOW_PULLS_WITHOUT_STRICT_LOGGING')
-      if strict_logging_disabled:
-        data = {
-          'exception': ex,
-          'token': token.id,
-        }
-
-        logger.exception('update last_accessed for token failed', extra=data)
-      else:
-        raise
-
-  return token

data/model/user.py

@@ -14,7 +14,8 @@ from data.database import (User, LoginService, FederatedLogin, RepositoryPermiss
                            EmailConfirmation, Role, db_for_update, random_string_generator,
                            UserRegion, ImageStorageLocation,
                            ServiceKeyApproval, OAuthApplication, RepositoryBuildTrigger,
-                           UserPromptKind, UserPrompt, UserPromptTypes, DeletedNamespace)
+                           UserPromptKind, UserPrompt, UserPromptTypes, DeletedNamespace,
+                           RobotAccountMetadata)
 from data.model import (DataModelException, InvalidPasswordException, InvalidRobotException,
                         InvalidUsernameException, InvalidEmailAddressException,
                         TooManyLoginAttemptsException, db_transaction,
@@ -172,7 +173,7 @@ def change_username(user_id, new_username):
     user = db_for_update(User.select().where(User.id == user_id)).get()
 
     # Rename the robots
-    for robot in db_for_update(_list_entity_robots(user.username)):
+    for robot in db_for_update(_list_entity_robots(user.username, include_metadata=False)):
       _, robot_shortname = parse_robot_username(robot.username)
       new_robot_name = format_robot_username(new_username, robot_shortname)
       robot.username = new_robot_name
@@ -231,7 +232,7 @@ def update_enabled(user, set_enabled):
   user.save()
 
 
-def create_robot(robot_shortname, parent):
+def create_robot(robot_shortname, parent, description='', unstructured_metadata=None):
   (username_valid, username_issue) = validate_username(robot_shortname)
   if not username_valid:
     raise InvalidRobotException('The name for the robot \'%s\' is invalid: %s' %
@@ -245,33 +246,48 @@ def create_robot(robot_shortname, parent):
     msg = 'Existing robot with name: %s' % username
     logger.info(msg)
     raise InvalidRobotException(msg)
-
   except User.DoesNotExist:
     pass
 
+  service = LoginService.get(name='quayrobot')
   try:
-    created = User.create(username=username, robot=True)
+    with db_transaction():
+      created = User.create(username=username, robot=True)
+      password = created.email
 
-    service = LoginService.get(name='quayrobot')
-    password = created.email
-    FederatedLogin.create(user=created, service=service,
-                          service_ident=password)
-
-    return created, password
+      FederatedLogin.create(user=created, service=service, service_ident=password)
+      RobotAccountMetadata.create(robot_account=created, description=description[0:255],
+                                  unstructured_json=unstructured_metadata or {})
+      return created, password
   except Exception as ex:
     raise DataModelException(ex.message)
 
 
-def get_robot(robot_shortname, parent):
+def get_or_create_robot_metadata(robot):
+  defaults = dict(description='', unstructured_json='{}')
+  metadata, _ = RobotAccountMetadata.get_or_create(robot_account=robot, defaults=defaults)
+  return metadata
+
+
+def update_robot_metadata(robot, description='', unstructured_json=None):
+  """ Updates the description and user-specified unstructured metadata associated
+      with a robot account to that specified. """
+  metadata = get_or_create_robot_metadata(robot)
+  metadata.description = description
+  metadata.unstructured_json = unstructured_json or metadata.unstructured_json or {}
+  metadata.save()
+  
+
+def get_robot_and_metadata(robot_shortname, parent):
   robot_username = format_robot_username(parent.username, robot_shortname)
-  robot = lookup_robot(robot_username)
-  return robot, robot.email
+  robot, metadata = lookup_robot_and_metadata(robot_username)
+  return robot, robot.email, metadata
 
 
 def lookup_robot(robot_username):
   try:
     return (User
-            .select()
+            .select(User, FederatedLogin)
             .join(FederatedLogin)
             .join(LoginService)
             .where(LoginService.name == 'quayrobot', User.username == robot_username,
@@ -281,6 +297,11 @@ def lookup_robot(robot_username):
     raise InvalidRobotException('Could not find robot with username: %s' % robot_username)
 
 
+def lookup_robot_and_metadata(robot_username):
+  robot = lookup_robot(robot_username)
+  return robot, get_or_create_robot_metadata(robot)
+
+
 def get_matching_robots(name_prefix, username, limit=10):
   admined_orgs = (_basequery.get_user_organizations(username)
                   .switch(Team)
@@ -328,12 +349,15 @@ def verify_robot(robot_username, password):
   if not owner.enabled:
     raise InvalidRobotException('This user has been disabled. Please contact your administrator.')
 
+  # Mark that the robot was accessed.
+  _basequery.update_last_accessed(robot)
+
   return robot
 
 def regenerate_robot_token(robot_shortname, parent):
   robot_username = format_robot_username(parent.username, robot_shortname)
 
-  robot = lookup_robot(robot_username)
+  robot, metadata = lookup_robot_and_metadata(robot_username)
   password = random_string_generator(length=64)()
   robot.email = password
   robot.uuid = str(uuid4())
@@ -345,7 +369,7 @@ def regenerate_robot_token(robot_shortname, parent):
   login.save()
   robot.save()
 
-  return robot, password
+  return robot, password, metadata
 
 def delete_robot(robot_username):
   try:
@@ -362,20 +386,28 @@ def list_namespace_robots(namespace):
   return _list_entity_robots(namespace)
 
 
-def _list_entity_robots(entity_name):
+def _list_entity_robots(entity_name, include_metadata=True):
   """ Return the list of robots for the specified entity. This MUST return a query, not a
       materialized list so that callers can use db_for_update.
-      """
-  return (User
-          .select()
-          .join(FederatedLogin)
-          .where(User.robot == True, User.username ** (entity_name + '+%')))
+  """
+  query = (User
+           .select(User, FederatedLogin)
+           .join(FederatedLogin)
+           .where(User.robot == True, User.username ** (entity_name + '+%')))
+
+  if include_metadata:
+    query = (query.switch(User)
+                  .join(RobotAccountMetadata, JOIN_LEFT_OUTER)
+                  .select(User, FederatedLogin, RobotAccountMetadata))
+
+  return query
 
 
 def list_entity_robot_permission_teams(entity_name, include_permissions=False):
   query = (_list_entity_robots(entity_name))
 
-  fields = [User.username, FederatedLogin.service_ident]
+  fields = [User.username, User.creation_date, User.last_accessed, FederatedLogin.service_ident,
+            RobotAccountMetadata.description, RobotAccountMetadata.unstructured_json]
   if include_permissions:
     query = (query
              .join(RepositoryPermission, JOIN_LEFT_OUTER,
@@ -449,6 +481,10 @@ def verify_federated_login(service_id, service_ident):
              .switch(FederatedLogin).join(User)
              .where(FederatedLogin.service_ident == service_ident, LoginService.name == service_id)
              .get())
+
+    # Mark that the user was accessed.
+    _basequery.update_last_accessed(found.user)
+
     return found.user
   except FederatedLogin.DoesNotExist:
     return None
@@ -714,6 +750,9 @@ def verify_user(username_or_email, password):
        .where(User.id == fetched.id)
        .execute())
 
+    # Mark that the user was accessed.
+    _basequery.update_last_accessed(fetched)
+
     # Return the valid user.
     return fetched
 
@@ -955,10 +994,10 @@ def get_pull_credentials(robotname):
     return None
 
   return {
-      'username': robot.username,
-      'password': login_info.service_ident,
-      'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'],
-                                   config.app_config['SERVER_HOSTNAME']),
+    'username': robot.username,
+    'password': login_info.service_ident,
+    'registry': '%s://%s/v1/' % (config.app_config['PREFERRED_URL_SCHEME'],
+                                 config.app_config['SERVER_HOSTNAME']),
   }
 
 def get_region_locations(user):