8a212728a3
This will allow customers to request their usage logs for a repository or an entire namespace, and we can export the logs in a manner that doesn't absolutely destroy the database, with every step along the way timed.
340 lines
13 KiB
Python
340 lines
13 KiB
Python
""" Access usage logs for organizations or repositories. """
|
|
import json
|
|
import uuid
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
from flask import request
|
|
|
|
from app import export_action_logs_queue
|
|
from endpoints.api import (resource, nickname, ApiResource, query_param, parse_args,
|
|
RepositoryParamResource, require_repo_admin, related_user_resource,
|
|
format_date, require_user_admin, path_param, require_scope, page_support,
|
|
validate_json_request, InvalidRequest)
|
|
from data import model as data_model
|
|
from endpoints.api.logs_models_pre_oci import pre_oci_model as model
|
|
from endpoints.exception import Unauthorized, NotFound
|
|
from auth.permissions import AdministerOrganizationPermission
|
|
from auth.auth_context import get_authenticated_user
|
|
from auth import scopes
|
|
|
|
LOGS_PER_PAGE = 20
|
|
SERVICE_LEVEL_LOG_KINDS = set(['service_key_create', 'service_key_approve', 'service_key_delete',
|
|
'service_key_modify', 'service_key_extend', 'service_key_rotate'])
|
|
|
|
|
|
def _validate_logs_arguments(start_time, end_time):
|
|
if start_time:
|
|
try:
|
|
start_time = datetime.strptime(start_time + ' UTC', '%m/%d/%Y %Z')
|
|
except ValueError:
|
|
start_time = None
|
|
|
|
if not start_time:
|
|
start_time = datetime.today() - timedelta(7) # One week
|
|
|
|
if end_time:
|
|
try:
|
|
end_time = datetime.strptime(end_time + ' UTC', '%m/%d/%Y %Z')
|
|
end_time = end_time + timedelta(days=1)
|
|
except ValueError:
|
|
end_time = None
|
|
|
|
if not end_time:
|
|
end_time = datetime.today()
|
|
|
|
return start_time, end_time
|
|
|
|
|
|
def get_logs(start_time, end_time, performer_name=None, repository_name=None, namespace_name=None,
|
|
page_token=None, ignore=None):
|
|
(start_time, end_time) = _validate_logs_arguments(start_time, end_time)
|
|
|
|
kinds = model.get_log_entry_kinds()
|
|
log_entry_page = model.get_logs_query(start_time, end_time, performer_name, repository_name,
|
|
namespace_name, ignore, page_token)
|
|
|
|
include_namespace = namespace_name is None and repository_name is None
|
|
|
|
return {
|
|
'start_time': format_date(start_time),
|
|
'end_time': format_date(end_time),
|
|
'logs': [log.to_dict(kinds, include_namespace) for log in log_entry_page.logs],
|
|
}, log_entry_page.next_page_token
|
|
|
|
|
|
def get_aggregate_logs(start_time, end_time, performer_name=None, repository=None, namespace=None,
|
|
ignore=None):
|
|
(start_time, end_time) = _validate_logs_arguments(start_time, end_time)
|
|
|
|
kinds = model.get_log_entry_kinds()
|
|
aggregated_logs = model.get_aggregated_logs(start_time, end_time, performer_name=performer_name,
|
|
repository_name=repository, namespace_name=namespace,
|
|
ignore=ignore)
|
|
|
|
return {
|
|
'aggregated': [log.to_dict(kinds, start_time) for log in aggregated_logs]
|
|
}
|
|
|
|
|
|
@resource('/v1/repository/<apirepopath:repository>/logs')
|
|
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
|
class RepositoryLogs(RepositoryParamResource):
|
|
""" Resource for fetching logs for the specific repository. """
|
|
|
|
@require_repo_admin
|
|
@nickname('listRepoLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. The time should be formatted "%m/%d/%Y" in UTC.', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. The time should be formatted "%m/%d/%Y" in UTC.', type=str)
|
|
@page_support()
|
|
def get(self, namespace, repository, page_token, parsed_args):
|
|
""" List the logs for the specified repository. """
|
|
if model.repo_exists(namespace, repository) is False:
|
|
raise NotFound()
|
|
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
return get_logs(start_time, end_time, repository_name=repository, page_token=page_token,
|
|
namespace_name=namespace)
|
|
|
|
|
|
@resource('/v1/user/logs')
|
|
class UserLogs(ApiResource):
|
|
""" Resource for fetching logs for the current user. """
|
|
|
|
@require_user_admin
|
|
@nickname('listUserLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('performer', 'Username for which to filter logs.', type=str)
|
|
@page_support()
|
|
def get(self, parsed_args, page_token):
|
|
""" List the logs for the current user. """
|
|
performer_name = parsed_args['performer']
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
|
|
user = get_authenticated_user()
|
|
return get_logs(start_time, end_time, performer_name=performer_name,
|
|
namespace_name=user.username, page_token=page_token,
|
|
ignore=SERVICE_LEVEL_LOG_KINDS)
|
|
|
|
|
|
@resource('/v1/organization/<orgname>/logs')
|
|
@path_param('orgname', 'The name of the organization')
|
|
@related_user_resource(UserLogs)
|
|
class OrgLogs(ApiResource):
|
|
""" Resource for fetching logs for the entire organization. """
|
|
|
|
@nickname('listOrgLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('performer', 'Username for which to filter logs.', type=str)
|
|
@page_support()
|
|
@require_scope(scopes.ORG_ADMIN)
|
|
def get(self, orgname, page_token, parsed_args):
|
|
""" List the logs for the specified organization. """
|
|
permission = AdministerOrganizationPermission(orgname)
|
|
if permission.can():
|
|
performer_name = parsed_args['performer']
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
|
|
return get_logs(start_time, end_time, namespace_name=orgname, performer_name=performer_name,
|
|
page_token=page_token, ignore=SERVICE_LEVEL_LOG_KINDS)
|
|
|
|
raise Unauthorized()
|
|
|
|
|
|
@resource('/v1/repository/<apirepopath:repository>/aggregatelogs')
|
|
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
|
class RepositoryAggregateLogs(RepositoryParamResource):
|
|
""" Resource for fetching aggregated logs for the specific repository. """
|
|
|
|
@require_repo_admin
|
|
@nickname('getAggregateRepoLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
|
|
def get(self, namespace, repository, parsed_args):
|
|
""" Returns the aggregated logs for the specified repository. """
|
|
if model.repo_exists(namespace, repository) is False:
|
|
raise NotFound()
|
|
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
return get_aggregate_logs(start_time, end_time, repository=repository, namespace=namespace)
|
|
|
|
|
|
@resource('/v1/user/aggregatelogs')
|
|
class UserAggregateLogs(ApiResource):
|
|
""" Resource for fetching aggregated logs for the current user. """
|
|
|
|
@require_user_admin
|
|
@nickname('getAggregateUserLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('performer', 'Username for which to filter logs.', type=str)
|
|
def get(self, parsed_args):
|
|
""" Returns the aggregated logs for the current user. """
|
|
performer_name = parsed_args['performer']
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
|
|
user = get_authenticated_user()
|
|
return get_aggregate_logs(start_time, end_time, performer_name=performer_name,
|
|
namespace=user.username, ignore=SERVICE_LEVEL_LOG_KINDS)
|
|
|
|
|
|
@resource('/v1/organization/<orgname>/aggregatelogs')
|
|
@path_param('orgname', 'The name of the organization')
|
|
@related_user_resource(UserLogs)
|
|
class OrgAggregateLogs(ApiResource):
|
|
""" Resource for fetching aggregate logs for the entire organization. """
|
|
|
|
@nickname('getAggregateOrgLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('performer', 'Username for which to filter logs.', type=str)
|
|
@require_scope(scopes.ORG_ADMIN)
|
|
def get(self, orgname, parsed_args):
|
|
""" Gets the aggregated logs for the specified organization. """
|
|
permission = AdministerOrganizationPermission(orgname)
|
|
if permission.can():
|
|
performer_name = parsed_args['performer']
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
|
|
return get_aggregate_logs(start_time, end_time, namespace=orgname,
|
|
performer_name=performer_name, ignore=SERVICE_LEVEL_LOG_KINDS)
|
|
|
|
raise Unauthorized()
|
|
|
|
|
|
def queue_logs_export(start_time, end_time, options, namespace_name, repository_name=None):
|
|
export_id = str(uuid.uuid4())
|
|
namespace = data_model.user.get_namespace_user(namespace_name)
|
|
if namespace is None:
|
|
raise InvalidRequest('Unknown namespace')
|
|
|
|
repository = None
|
|
if repository_name is not None:
|
|
repository = data_model.repository.get_repository(namespace_name, repository_name)
|
|
if repository is None:
|
|
raise InvalidRequest('Unknown repository')
|
|
|
|
callback_url = options.get('callback_url')
|
|
if callback_url:
|
|
if not callback_url.startswith('https://') and not callback_url.startswith('http://'):
|
|
raise InvalidRequest('Invalid callback URL')
|
|
|
|
export_action_logs_queue.put([namespace_name], json.dumps({
|
|
'export_id': export_id,
|
|
'repository_id': repository.id if repository else None,
|
|
'namespace_id': namespace.id,
|
|
'namespace_name': namespace.username,
|
|
'repository_name': repository.name if repository else None,
|
|
'start_time': start_time,
|
|
'end_time': end_time,
|
|
'callback_url': callback_url,
|
|
'callback_email': options.get('callback_email'),
|
|
}), retries_remaining=3)
|
|
|
|
return {
|
|
'export_id': export_id,
|
|
}
|
|
|
|
|
|
EXPORT_LOGS_SCHEMA = {
|
|
'type': 'object',
|
|
'description': 'Configuration for an export logs operation',
|
|
'properties': {
|
|
'callback_url': {
|
|
'type': 'string',
|
|
'description': 'The callback URL to invoke with a link to the exported logs',
|
|
},
|
|
'callback_email': {
|
|
'type': 'string',
|
|
'description': 'The e-mail address at which to e-mail a link to the exported logs',
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
@resource('/v1/repository/<apirepopath:repository>/exportlogs')
|
|
@path_param('repository', 'The full path of the repository. e.g. namespace/name')
|
|
class ExportRepositoryLogs(RepositoryParamResource):
|
|
""" Resource for exporting the logs for the specific repository. """
|
|
schemas = {
|
|
'ExportLogs': EXPORT_LOGS_SCHEMA
|
|
}
|
|
|
|
@require_repo_admin
|
|
@nickname('exportRepoLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs (%m/%d/%Y %Z)', type=str)
|
|
@validate_json_request('ExportLogs')
|
|
def post(self, namespace, repository, parsed_args):
|
|
""" Queues an export of the logs for the specified repository. """
|
|
if model.repo_exists(namespace, repository) is False:
|
|
raise NotFound()
|
|
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
return queue_logs_export(start_time, end_time, request.get_json(), namespace,
|
|
repository_name=repository)
|
|
|
|
|
|
@resource('/v1/user/exportlogs')
|
|
class ExportUserLogs(ApiResource):
|
|
""" Resource for exporting the logs for the current user repository. """
|
|
schemas = {
|
|
'ExportLogs': EXPORT_LOGS_SCHEMA
|
|
}
|
|
|
|
@require_user_admin
|
|
@nickname('exportUserLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@validate_json_request('ExportLogs')
|
|
def post(self, parsed_args):
|
|
""" Returns the aggregated logs for the current user. """
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
|
|
user = get_authenticated_user()
|
|
return queue_logs_export(start_time, end_time, request.get_json(), user.username)
|
|
|
|
|
|
@resource('/v1/organization/<orgname>/exportlogs')
|
|
@path_param('orgname', 'The name of the organization')
|
|
@related_user_resource(ExportUserLogs)
|
|
class ExportOrgLogs(ApiResource):
|
|
""" Resource for exporting the logs for an entire organization. """
|
|
schemas = {
|
|
'ExportLogs': EXPORT_LOGS_SCHEMA
|
|
}
|
|
|
|
@nickname('exportOrgLogs')
|
|
@parse_args()
|
|
@query_param('starttime', 'Earliest time from which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@query_param('endtime', 'Latest time to which to get logs. (%m/%d/%Y %Z)', type=str)
|
|
@require_scope(scopes.ORG_ADMIN)
|
|
@validate_json_request('ExportLogs')
|
|
def post(self, orgname, parsed_args):
|
|
""" Gets the aggregated logs for the specified organization. """
|
|
permission = AdministerOrganizationPermission(orgname)
|
|
if permission.can():
|
|
start_time = parsed_args['starttime']
|
|
end_time = parsed_args['endtime']
|
|
|
|
return queue_logs_export(start_time, end_time, request.get_json(), orgname)
|
|
|
|
raise Unauthorized()
|