035541c6f2
- Implement logs model using Elasticsearch with tests - Implement transition model using both elasticsearch and database model - Add LOGS_MODEL configuration to choose which to use. Co-authored-by: Sida Chen <sidchen@redhat.com> Co-authored-by: Kenny Lee Sin Cheong <kenny.lee@redhat.com>
75 lines
2.7 KiB
Python
75 lines
2.7 KiB
Python
import logging
|
|
from requests_aws4auth import AWS4Auth
|
|
from elasticsearch import RequestsHttpConnection
|
|
from elasticsearch_dsl import Index, Document, Integer, Date, Text, Ip
|
|
from elasticsearch_dsl.connections import connections
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
ELASTICSEARCH_CONNECTION_ALIAS = 'LogEntry'
|
|
INDEX_NAME_PREFIX = 'logentry_'
|
|
|
|
|
|
class LogEntry(Document):
|
|
random_id = Integer() # random id is the tie-breaker for sorting in pagination.
|
|
# https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-search-after.html
|
|
kind_id = Integer()
|
|
account_id = Integer()
|
|
performer_id = Integer()
|
|
repository_id = Integer()
|
|
ip = Ip()
|
|
metadata_json = Text()
|
|
datetime = Date()
|
|
|
|
@classmethod
|
|
def init_with_template(cls, index_prefix=INDEX_NAME_PREFIX, index_settings=None):
|
|
"""
|
|
Create the index template, and populate LogEntry's mapping and index settings.
|
|
"""
|
|
wildcard_index = Index(name=index_prefix + '*', using=ELASTICSEARCH_CONNECTION_ALIAS)
|
|
wildcard_index.settings(**(index_settings or {}))
|
|
wildcard_index.document(cls)
|
|
|
|
cls._index = wildcard_index
|
|
cls._index_prefix = index_prefix
|
|
index_template = wildcard_index.as_template(index_prefix)
|
|
index_template.save(using=ELASTICSEARCH_CONNECTION_ALIAS)
|
|
|
|
def save(self, **kwargs):
|
|
# We group the logs based on year and month as different indexes, so that
|
|
# dropping those indexes based on retention range ~1 month is easy.
|
|
kwargs['index'] = self.datetime.strftime(self._index_prefix + '%Y-%m')
|
|
return super(LogEntry, self).save(**kwargs)
|
|
|
|
|
|
class AWSElasticsearchLogs(object):
|
|
"""
|
|
Model for logs operations stored in an AWS Elasticsearch cluster.
|
|
"""
|
|
|
|
def __init__(self, host, port, aws_access_key, aws_secret_key, aws_region,
|
|
index_prefix=INDEX_NAME_PREFIX, index_settings=None):
|
|
# for options in index_settings, please refer to:
|
|
# https://www.elastic.co/guide/en/elasticsearch/guide/master/_index_settings.html
|
|
# some index settings are set at index creation time, and therefore, you should NOT
|
|
# change those settings once the index is set.
|
|
assert len(index_prefix) >= 1
|
|
http_auth = None
|
|
if aws_access_key and aws_secret_key:
|
|
http_auth = AWS4Auth(aws_access_key, aws_secret_key, aws_region, 'es')
|
|
else:
|
|
logger.warn("Connecting to AWS Elasticsearch without HTTP AUTH")
|
|
|
|
connections.create_connection(
|
|
alias=ELASTICSEARCH_CONNECTION_ALIAS,
|
|
hosts=[{
|
|
'host': host,
|
|
'port': port
|
|
}],
|
|
http_auth=http_auth,
|
|
use_ssl=True,
|
|
verify_certs=True,
|
|
connection_class=RequestsHttpConnection,
|
|
)
|
|
|
|
LogEntry.init_with_template(index_prefix, index_settings)
|