import logging from requests_aws4auth import AWS4Auth from elasticsearch import RequestsHttpConnection from elasticsearch_dsl import Index, Document, Integer, Date, Text, Ip from elasticsearch_dsl.connections import connections logger = logging.getLogger(__name__) ELASTICSEARCH_CONNECTION_ALIAS = 'LogEntry' INDEX_NAME_PREFIX = 'logentry_' class LogEntry(Document): random_id = Integer() # random id is the tie-breaker for sorting in pagination. # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-search-after.html kind_id = Integer() account_id = Integer() performer_id = Integer() repository_id = Integer() ip = Ip() metadata_json = Text() datetime = Date() @classmethod def init_with_template(cls, index_prefix=INDEX_NAME_PREFIX, index_settings=None): """ Create the index template, and populate LogEntry's mapping and index settings. """ wildcard_index = Index(name=index_prefix + '*', using=ELASTICSEARCH_CONNECTION_ALIAS) wildcard_index.settings(**(index_settings or {})) wildcard_index.document(cls) cls._index = wildcard_index cls._index_prefix = index_prefix index_template = wildcard_index.as_template(index_prefix) index_template.save(using=ELASTICSEARCH_CONNECTION_ALIAS) def save(self, **kwargs): # We group the logs based on year and month as different indexes, so that # dropping those indexes based on retention range ~1 month is easy. kwargs['index'] = self.datetime.strftime(self._index_prefix + '%Y-%m') return super(LogEntry, self).save(**kwargs) class AWSElasticsearchLogs(object): """ Model for logs operations stored in an AWS Elasticsearch cluster. """ def __init__(self, host, port, aws_access_key, aws_secret_key, aws_region, index_prefix=INDEX_NAME_PREFIX, index_settings=None): # for options in index_settings, please refer to: # https://www.elastic.co/guide/en/elasticsearch/guide/master/_index_settings.html # some index settings are set at index creation time, and therefore, you should NOT # change those settings once the index is set. assert len(index_prefix) >= 1 http_auth = None if aws_access_key and aws_secret_key: http_auth = AWS4Auth(aws_access_key, aws_secret_key, aws_region, 'es') else: logger.warn("Connecting to AWS Elasticsearch without HTTP AUTH") connections.create_connection( alias=ELASTICSEARCH_CONNECTION_ALIAS, hosts=[{ 'host': host, 'port': port }], http_auth=http_auth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, ) LogEntry.init_with_template(index_prefix, index_settings)