Implement logs model using Elasticsearch

- Implement logs model using Elasticsearch with tests
- Implement transition model using both elasticsearch and database model
- Add LOGS_MODEL configuration to choose which to use.

Co-authored-by: Sida Chen <sidchen@redhat.com>
Co-authored-by: Kenny Lee Sin Cheong <kenny.lee@redhat.com>

data/logs_model/elastic_logs.py

@@ -0,0 +1,75 @@
+import logging
+from requests_aws4auth import AWS4Auth
+from elasticsearch import RequestsHttpConnection
+from elasticsearch_dsl import Index, Document, Integer, Date, Text, Ip
+from elasticsearch_dsl.connections import connections
+
+logger = logging.getLogger(__name__)
+
+ELASTICSEARCH_CONNECTION_ALIAS = 'LogEntry'
+INDEX_NAME_PREFIX = 'logentry_'
+
+
+class LogEntry(Document):
+  random_id = Integer()  # random id is the tie-breaker for sorting in pagination.
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-search-after.html
+  kind_id = Integer()
+  account_id = Integer()
+  performer_id = Integer()
+  repository_id = Integer()
+  ip = Ip()
+  metadata_json = Text()
+  datetime = Date()
+
+  @classmethod
+  def init_with_template(cls, index_prefix=INDEX_NAME_PREFIX, index_settings=None):
+    """
+    Create the index template, and populate LogEntry's mapping and index settings.
+    """
+    wildcard_index = Index(name=index_prefix + '*', using=ELASTICSEARCH_CONNECTION_ALIAS)
+    wildcard_index.settings(**(index_settings or {}))
+    wildcard_index.document(cls)
+
+    cls._index = wildcard_index
+    cls._index_prefix = index_prefix
+    index_template = wildcard_index.as_template(index_prefix)
+    index_template.save(using=ELASTICSEARCH_CONNECTION_ALIAS)
+
+  def save(self, **kwargs):
+    # We group the logs based on year and month as different indexes, so that
+    # dropping those indexes based on retention range ~1 month is easy.
+    kwargs['index'] = self.datetime.strftime(self._index_prefix + '%Y-%m')
+    return super(LogEntry, self).save(**kwargs)
+
+
+class AWSElasticsearchLogs(object):
+  """
+  Model for logs operations stored in an AWS Elasticsearch cluster.
+  """
+
+  def __init__(self, host, port, aws_access_key, aws_secret_key, aws_region,
+               index_prefix=INDEX_NAME_PREFIX, index_settings=None):
+    # for options in index_settings, please refer to:
+    # https://www.elastic.co/guide/en/elasticsearch/guide/master/_index_settings.html
+    # some index settings are set at index creation time, and therefore, you should NOT
+    # change those settings once the index is set.
+    assert len(index_prefix) >= 1
+    http_auth = None
+    if aws_access_key and aws_secret_key:
+      http_auth = AWS4Auth(aws_access_key, aws_secret_key, aws_region, 'es')
+    else:
+      logger.warn("Connecting to AWS Elasticsearch without HTTP AUTH")
+
+    connections.create_connection(
+      alias=ELASTICSEARCH_CONNECTION_ALIAS,
+      hosts=[{
+        'host': host,
+        'port': port
+      }],
+      http_auth=http_auth,
+      use_ssl=True,
+      verify_certs=True,
+      connection_class=RequestsHttpConnection,
+    )
+
+    LogEntry.init_with_template(index_prefix, index_settings)