Add support for metadata on robot accounts

Fixes https://jira.coreos.com/browse/QUAY-847
Fixes https://jira.coreos.com/browse/QUAY-816

data/database.py

@@ -479,6 +479,12 @@ class User(BaseModel):
 Namespace = User.alias()
 
 
+class RobotAccountMetadata(BaseModel):
+  robot_account = QuayUserField(index=True, allows_robots=True, unique=True)
+  description = CharField()
+  unstructured_json = JSONField()
+
+
 class DeletedNamespace(BaseModel):
   namespace = QuayUserField(index=True, allows_robots=False, unique=True)
   marked = DateTimeField(default=datetime.now)

data/migrations/versions/b547bc139ad8_add_robotaccountmetadata_table.py

@@ -0,0 +1,35 @@
+"""Add RobotAccountMetadata table
+
+Revision ID: b547bc139ad8
+Revises: 0cf50323c78b
+Create Date: 2018-03-09 15:50:48.298880
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'b547bc139ad8'
+down_revision = '0cf50323c78b'
+
+from alembic import op
+import sqlalchemy as sa
+from util.migrate import UTF8CharField
+
+
+def upgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('robotaccountmetadata',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('robot_account_id', sa.Integer(), nullable=False),
+    sa.Column('description', UTF8CharField(length=255), nullable=False),
+    sa.Column('unstructured_json', sa.Text(), nullable=False),
+    sa.ForeignKeyConstraint(['robot_account_id'], ['user.id'], name=op.f('fk_robotaccountmetadata_robot_account_id_user')),
+    sa.PrimaryKeyConstraint('id', name=op.f('pk_robotaccountmetadata'))
+    )
+    op.create_index('robotaccountmetadata_robot_account_id', 'robotaccountmetadata', ['robot_account_id'], unique=True)
+    # ### end Alembic commands ###
+
+
+def downgrade(tables):
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('robotaccountmetadata')
+    # ### end Alembic commands ###

data/model/user.py

@@ -14,7 +14,8 @@ from data.database import (User, LoginService, FederatedLogin, RepositoryPermiss
                            EmailConfirmation, Role, db_for_update, random_string_generator,
                            UserRegion, ImageStorageLocation,
                            ServiceKeyApproval, OAuthApplication, RepositoryBuildTrigger,
-                           UserPromptKind, UserPrompt, UserPromptTypes, DeletedNamespace)
+                           UserPromptKind, UserPrompt, UserPromptTypes, DeletedNamespace,
+                           RobotAccountMetadata)
 from data.model import (DataModelException, InvalidPasswordException, InvalidRobotException,
                         InvalidUsernameException, InvalidEmailAddressException,
                         TooManyLoginAttemptsException, db_transaction,
@@ -231,7 +232,7 @@ def update_enabled(user, set_enabled):
   user.save()
 
 
-def create_robot(robot_shortname, parent):
+def create_robot(robot_shortname, parent, description='', unstructured_metadata=None):
   (username_valid, username_issue) = validate_username(robot_shortname)
   if not username_valid:
     raise InvalidRobotException('The name for the robot \'%s\' is invalid: %s' %
@@ -245,33 +246,59 @@ def create_robot(robot_shortname, parent):
     msg = 'Existing robot with name: %s' % username
     logger.info(msg)
     raise InvalidRobotException(msg)
-
   except User.DoesNotExist:
     pass
 
+  service = LoginService.get(name='quayrobot')
   try:
-    created = User.create(username=username, robot=True)
+    with db_transaction():
+      created = User.create(username=username, robot=True)
+      password = created.email
 
-    service = LoginService.get(name='quayrobot')
+      FederatedLogin.create(user=created, service=service, service_ident=password)
-    password = created.email
+      RobotAccountMetadata.create(robot_account=created, description=description[0:255],
-    FederatedLogin.create(user=created, service=service,
+                                  unstructured_json=unstructured_metadata or {})
-                          service_ident=password)
+      return created, password
-
-    return created, password
   except Exception as ex:
     raise DataModelException(ex.message)
 
 
+def get_or_create_robot_metadata(robot):
+  try:
+    return RobotAccountMetadata.get(robot_account=robot)
+  except RobotAccountMetadata.DoesNotExist:
+    try:
+      return RobotAccountMetadata.create(robot_account=robot, description='',
+                                         unstructured_json='{}')
+    except IntegrityError:
+      return RobotAccountMetadata.get(robot_account=robot)
+  
+
+def update_robot_metadata(robot, description='', unstructured_json=None):
+  """ Updates the description and user-specified unstructured metadata associated
+      with a robot account to that specified. """
+  metadata = get_or_create_robot_metadata(robot)
+  metadata.description = description
+  metadata.unstructured_json = unstructured_json or metadata.unstructured_json or {}
+  metadata.save()
+
+
 def get_robot(robot_shortname, parent):
   robot_username = format_robot_username(parent.username, robot_shortname)
   robot = lookup_robot(robot_username)
   return robot, robot.email
 
 
+def get_robot_and_metadata(robot_shortname, parent):
+  robot_username = format_robot_username(parent.username, robot_shortname)
+  robot, metadata = lookup_robot_and_metadata(robot_username)
+  return robot, robot.email, metadata
+
+
 def lookup_robot(robot_username):
   try:
     return (User
-            .select()
+            .select(User, FederatedLogin)
             .join(FederatedLogin)
             .join(LoginService)
             .where(LoginService.name == 'quayrobot', User.username == robot_username,
@@ -281,6 +308,11 @@ def lookup_robot(robot_username):
     raise InvalidRobotException('Could not find robot with username: %s' % robot_username)
 
 
+def lookup_robot_and_metadata(robot_username):
+  robot = lookup_robot(robot_username)
+  return robot, get_or_create_robot_metadata(robot)
+
+
 def get_matching_robots(name_prefix, username, limit=10):
   admined_orgs = (_basequery.get_user_organizations(username)
                   .switch(Team)
@@ -333,7 +365,7 @@ def verify_robot(robot_username, password):
 def regenerate_robot_token(robot_shortname, parent):
   robot_username = format_robot_username(parent.username, robot_shortname)
 
-  robot = lookup_robot(robot_username)
+  robot, metadata = lookup_robot_and_metadata(robot_username)
   password = random_string_generator(length=64)()
   robot.email = password
   robot.uuid = str(uuid4())
@@ -345,7 +377,7 @@ def regenerate_robot_token(robot_shortname, parent):
   login.save()
   robot.save()
 
-  return robot, password
+  return robot, password, metadata
 
 def delete_robot(robot_username):
   try:
@@ -367,15 +399,18 @@ def _list_entity_robots(entity_name):
       materialized list so that callers can use db_for_update.
       """
   return (User
-          .select()
+          .select(User, FederatedLogin, RobotAccountMetadata)
           .join(FederatedLogin)
+          .switch(User)
+          .join(RobotAccountMetadata, JOIN_LEFT_OUTER)
           .where(User.robot == True, User.username ** (entity_name + '+%')))
 
 
 def list_entity_robot_permission_teams(entity_name, include_permissions=False):
   query = (_list_entity_robots(entity_name))
 
-  fields = [User.username, User.creation_date, FederatedLogin.service_ident]
+  fields = [User.username, User.creation_date, FederatedLogin.service_ident,
+            RobotAccountMetadata.description, RobotAccountMetadata.unstructured_json]
   if include_permissions:
     query = (query
              .join(RepositoryPermission, JOIN_LEFT_OUTER,

endpoints/api/__init__.py

@@ -321,7 +321,7 @@ def require_scope(scope_object):
   return wrapper
 
 
-def validate_json_request(schema_name):
+def validate_json_request(schema_name, optional=False):
   def wrapper(func):
     @add_method_metadata('request_schema', schema_name)
     @wraps(func)
@@ -330,9 +330,10 @@ def validate_json_request(schema_name):
       try:
         json_data = request.get_json()
         if json_data is None:
-          raise InvalidRequest('Missing JSON body')
+          if not optional:
-
+            raise InvalidRequest('Missing JSON body')
-        validate(json_data, schema)
+        else:
+          validate(json_data, schema)
         return func(self, *args, **kwargs)
       except ValidationError as ex:
         raise InvalidRequest(ex.message)

endpoints/api/robot.py

@@ -2,14 +2,31 @@
 
 from endpoints.api import (resource, nickname, ApiResource, log_action, related_user_resource,
                            require_user_admin, require_scope, path_param, parse_args,
-                           truthy_bool, query_param)
+                           truthy_bool, query_param, validate_json_request)
 from endpoints.api.robot_models_pre_oci import pre_oci_model as model
 from endpoints.exception import Unauthorized
 from auth.permissions import AdministerOrganizationPermission, OrganizationMemberPermission
 from auth.auth_context import get_authenticated_user
 from auth import scopes
 from util.names import format_robot_username
-from flask import abort
+from flask import abort, request
+
+
+CREATE_ROBOT_SCHEMA = {
+  'type': 'object',
+  'description': 'Optional data for creating a robot',
+  'properties': {
+    'description': {
+      'type': 'string',
+      'description': 'Optional text description for the robot',
+      'maxLength': 255,
+    },
+    'unstructured_metadata': {
+      'type': 'object',
+      'description': 'Optional unstructured metadata for the robot',
+    },
+  },
+}
 
 
 def robots_list(prefix, include_permissions=False):
@@ -38,6 +55,9 @@ class UserRobotList(ApiResource):
             'The short name for the robot, without any user or organization prefix')
 class UserRobot(ApiResource):
   """ Resource for managing a user's robots. """
+  schemas = {
+    'CreateRobot': CREATE_ROBOT_SCHEMA,
+  }
 
   @require_user_admin
   @nickname('getUserRobot')
@@ -45,16 +65,23 @@ class UserRobot(ApiResource):
     """ Returns the user's robot with the specified name. """
     parent = get_authenticated_user()
     robot = model.get_user_robot(robot_shortname, parent)
-    return robot.to_dict()
+    return robot.to_dict(include_metadata=True)
 
   @require_user_admin
   @nickname('createUserRobot')
+  @validate_json_request('CreateRobot', optional=True)
   def put(self, robot_shortname):
     """ Create a new user robot with the specified name. """
     parent = get_authenticated_user()
-    robot = model.create_user_robot(robot_shortname, parent)
+    create_data = request.get_json() or {}
-    log_action('create_robot', parent.username, {'robot': robot_shortname})
+    robot = model.create_user_robot(robot_shortname, parent, create_data.get('description'),
-    return robot.to_dict(), 201
+                                    create_data.get('unstructured_metadata'))
+    log_action('create_robot', parent.username, {
+      'robot': robot_shortname,
+      'description': create_data.get('description'),
+      'unstructured_metadata': create_data.get('unstructured_metadata'),
+    })
+    return robot.to_dict(include_metadata=True), 201
 
   @require_user_admin
   @nickname('deleteUserRobot')
@@ -82,6 +109,7 @@ class OrgRobotList(ApiResource):
     """ List the organization's robots. """
     permission = OrganizationMemberPermission(orgname)
     if permission.can():
+      include_metadata = AdministerOrganizationPermission(orgname).can()
       return robots_list(orgname, include_permissions=parsed_args.get('permissions', False))
 
     raise Unauthorized()
@@ -94,6 +122,9 @@ class OrgRobotList(ApiResource):
 @related_user_resource(UserRobot)
 class OrgRobot(ApiResource):
   """ Resource for managing an organization's robots. """
+  schemas = {
+    'CreateRobot': CREATE_ROBOT_SCHEMA,
+  }
 
   @require_scope(scopes.ORG_ADMIN)
   @nickname('getOrgRobot')
@@ -102,19 +133,26 @@ class OrgRobot(ApiResource):
     permission = AdministerOrganizationPermission(orgname)
     if permission.can():
       robot = model.get_org_robot(robot_shortname, orgname)
-      return robot.to_dict()
+      return robot.to_dict(include_metadata=True)
 
     raise Unauthorized()
 
   @require_scope(scopes.ORG_ADMIN)
   @nickname('createOrgRobot')
+  @validate_json_request('CreateRobot', optional=True)
   def put(self, orgname, robot_shortname):
     """ Create a new robot in the organization. """
     permission = AdministerOrganizationPermission(orgname)
     if permission.can():
-      robot = model.create_org_robot(robot_shortname, orgname)
+      create_data = request.get_json() or {}
-      log_action('create_robot', orgname, {'robot': robot_shortname})
+      robot = model.create_org_robot(robot_shortname, orgname, create_data.get('description'),
-      return robot.to_dict(), 201
+                                     create_data.get('unstructured_metadata'))
+      log_action('create_robot', orgname, {
+        'robot': robot_shortname,
+        'description': create_data.get('description'),
+        'unstructured_metadata': create_data.get('unstructured_metadata'),
+      })
+      return robot.to_dict(include_metadata=True), 201
 
     raise Unauthorized()
 

endpoints/api/robot_models_interface.py

@@ -41,6 +41,7 @@ class RobotWithPermissions(
     'created',
     'teams',
     'repository_names',
+    'description',
   ])):
   """
   RobotWithPermissions is a list of robot entries.
@@ -49,7 +50,7 @@ class RobotWithPermissions(
   :type created: datetime|None
   :type teams: [Team]
   :type repository_names: [string]
-
+  :type description: string
   """
 
   def to_dict(self):
@@ -58,7 +59,8 @@ class RobotWithPermissions(
       'token': self.password,
       'created': format_date(self.created) if self.created is not None else None,
       'teams': [team.to_dict() for team in self.teams],
-      'repositories': self.repository_names
+      'repositories': self.repository_names,
+      'description': self.description,
     }
 
 
@@ -67,22 +69,31 @@ class Robot(
     'name',
     'password',
     'created',
+    'description',
+    'unstructured_metadata',
   ])):
   """
   Robot represents a robot entity.
   :type name: string
   :type password: string
   :type created: datetime|None
-
+  :type description: string
+  :type unstructured_metadata: dict
   """
 
-  def to_dict(self):
+  def to_dict(self, include_metadata=False):
-    return {
+    data = {
       'name': self.name,
       'token': self.password,
       'created': format_date(self.created) if self.created is not None else None,
+      'description': self.description,
     }
 
+    if include_metadata:
+      data['unstructured_metadata'] = self.unstructured_metadata
+    
+    return data
+
 
 @add_metaclass(ABCMeta)
 class RobotInterface(object):

endpoints/api/robot_models_pre_oci.py

@@ -1,6 +1,6 @@
 from app import avatar
 from data import model
-from data.database import User, FederatedLogin, Team as TeamTable, Repository
+from data.database import User, FederatedLogin, Team as TeamTable, Repository, RobotAccountMetadata
 from endpoints.api.robot_models_interface import (RobotInterface, Robot, RobotWithPermissions, Team,
                                                   Permission)
 
@@ -24,14 +24,18 @@ class RobotPreOCIModel(RobotInterface):
           'name': robot_name,
           'token': robot_tuple.get(FederatedLogin.service_ident),
           'created': robot_tuple.get(User.creation_date),
+          'description': robot_tuple.get(RobotAccountMetadata.description),
+          'unstructured_metadata': robot_tuple.get(RobotAccountMetadata.unstructured_json),
         }
 
         if include_permissions:
           robot_dict.update({
             'teams': [],
-            'repositories': []
+            'repositories': [],
           })
-      robots[robot_name] = Robot(robot_dict['name'], robot_dict['token'], robot_dict['created'])
+
+      robots[robot_name] = Robot(robot_dict['name'], robot_dict['token'], robot_dict['created'],
+                                 robot_dict['description'], robot_dict['unstructured_metadata'])
       if include_permissions:
         team_name = robot_tuple.get(TeamTable.name)
         repository_name = robot_tuple.get(Repository.name)
@@ -51,39 +55,48 @@ class RobotPreOCIModel(RobotInterface):
             robot_dict['repositories'].append(repository_name)
         robots[robot_name] = RobotWithPermissions(robot_dict['name'], robot_dict['token'],
                                                   robot_dict['created'], robot_dict['teams'],
-                                                  robot_dict['repositories'])
+                                                  robot_dict['repositories'],
+                                                  robot_dict['description'])
 
     return robots.values()
 
   def regenerate_user_robot_token(self, robot_shortname, owning_user):
-    robot, password = model.user.regenerate_robot_token(robot_shortname, owning_user)
+    robot, password, metadata = model.user.regenerate_robot_token(robot_shortname, owning_user)
-    return Robot(robot.username, password, robot.creation_date)
+    return Robot(robot.username, password, robot.creation_date, metadata.description,
+                 metadata.unstructured_json)
 
   def regenerate_org_robot_token(self, robot_shortname, orgname):
     parent = model.organization.get_organization(orgname)
-    robot, password = model.user.regenerate_robot_token(robot_shortname, parent)
+    robot, password, metadata = model.user.regenerate_robot_token(robot_shortname, parent)
-    return Robot(robot.username, password, robot.creation_date)
+    return Robot(robot.username, password, robot.creation_date, metadata.description,
+                 metadata.unstructured_json)
 
   def delete_robot(self, robot_username):
     model.user.delete_robot(robot_username)
 
-  def create_user_robot(self, robot_shortname, owning_user):
+  def create_user_robot(self, robot_shortname, owning_user, description, unstructured_metadata):
-    robot, password = model.user.create_robot(robot_shortname, owning_user)
+    robot, password = model.user.create_robot(robot_shortname, owning_user, description or '',
-    return Robot(robot.username, password, robot.creation_date)
+                                              unstructured_metadata)
+    return Robot(robot.username, password, robot.creation_date, description or '',
+                 unstructured_metadata)
 
-  def create_org_robot(self, robot_shortname, orgname):
+  def create_org_robot(self, robot_shortname, orgname, description, unstructured_metadata):
     parent = model.organization.get_organization(orgname)
-    robot, password = model.user.create_robot(robot_shortname, parent)
+    robot, password = model.user.create_robot(robot_shortname, parent, description or '',
-    return Robot(robot.username, password, robot.creation_date)
+                                              unstructured_metadata)
+    return Robot(robot.username, password, robot.creation_date, description or '',
+                 unstructured_metadata)
 
   def get_org_robot(self, robot_shortname, orgname):
     parent = model.organization.get_organization(orgname)
-    robot, password = model.user.get_robot(robot_shortname, parent)
+    robot, password, metadata = model.user.get_robot_and_metadata(robot_shortname, parent)
-    return Robot(robot.username, password, robot.creation_date)
+    return Robot(robot.username, password, robot.creation_date, metadata.description,
+                 metadata.unstructured_json)
 
   def get_user_robot(self, robot_shortname, owning_user):
-    robot, password = model.user.get_robot(robot_shortname, owning_user)
+    robot, password, metadata = model.user.get_robot_and_metadata(robot_shortname, owning_user)
-    return Robot(robot.username, password, robot.creation_date)
+    return Robot(robot.username, password, robot.creation_date, metadata.description,
+                 metadata.unstructured_json)
 
 
 pre_oci_model = RobotPreOCIModel()

endpoints/api/test/test_robot.py

@@ -0,0 +1,38 @@
+import pytest
+import json
+
+from data import model
+from endpoints.api import api
+from endpoints.api.test.shared import conduct_api_call
+from endpoints.api.robot import UserRobot, OrgRobot
+from endpoints.test.shared import client_with_identity
+
+from test.test_ldap import mock_ldap
+
+from test.fixtures import *
+
+@pytest.mark.parametrize('endpoint', [
+  UserRobot,
+  OrgRobot,
+])
+@pytest.mark.parametrize('body', [
+  {},
+  {'description': 'this is a description'},
+  {'unstructured_metadata': {'foo': 'bar'}},
+  {'description': 'this is a description', 'unstructured_metadata': {'foo': 'bar'}},
+])
+def test_create_robot_with_metadata(endpoint, body, client):
+  with client_with_identity('devtable', client) as cl:
+    # Create the robot with the specified body.
+    conduct_api_call(cl, endpoint, 'PUT', {'orgname': 'buynlarge', 'robot_shortname': 'somebot'},
+                     body, expected_code=201)
+
+    # Ensure the create succeeded.
+    resp = conduct_api_call(cl, endpoint, 'GET', {
+      'orgname': 'buynlarge',
+      'robot_shortname': 'somebot',
+    })
+
+    body = body or {}
+    assert resp.json['description'] == (body.get('description') or '')
+    assert resp.json['unstructured_metadata'] == (body.get('unstructured_metadata') or {})