400 lines
8.6 KiB
Python
400 lines
8.6 KiB
Python
# -*- coding: utf-8 -*-
|
|
import json
|
|
|
|
from datetime import datetime
|
|
from dateutil.parser import parse
|
|
|
|
from data.logs_model.datatypes import LogEntriesPage, Log, AggregatedLogCount
|
|
|
|
|
|
def _status(d, code=200):
|
|
return {"status_code": code, "content": json.dumps(d)}
|
|
|
|
|
|
def _shards(d, total=5, failed=0, successful=5):
|
|
d.update({"_shards": {"total": total, "failed": failed, "successful": successful}})
|
|
return d
|
|
|
|
|
|
def _hits(hits):
|
|
return {"hits": {"total": len(hits), "max_score": None, "hits": hits}}
|
|
|
|
|
|
INDEX_LIST_RESPONSE_HIT1_HIT2 = _status({
|
|
"logentry_2018-03-08": {},
|
|
"logentry_2018-04-02": {}
|
|
})
|
|
|
|
|
|
INDEX_LIST_RESPONSE_HIT2 = _status({
|
|
"logentry_2018-04-02": {}
|
|
})
|
|
|
|
|
|
INDEX_LIST_RESPONSE = _status({
|
|
"logentry_2019-01-01": {},
|
|
"logentry_2017-03-08": {},
|
|
"logentry_2018-03-08": {},
|
|
"logentry_2018-04-02": {}
|
|
})
|
|
|
|
|
|
DEFAULT_TEMPLATE_RESPONSE = _status({"acknowledged": True})
|
|
INDEX_RESPONSE_2019_01_01 = _status(
|
|
_shards({
|
|
"_index": "logentry_2019-01-01",
|
|
"_type": "_doc",
|
|
"_id": "1",
|
|
"_version": 1,
|
|
"_seq_no": 0,
|
|
"_primary_term": 1,
|
|
"result": "created"
|
|
}))
|
|
|
|
INDEX_RESPONSE_2017_03_08 = _status(
|
|
_shards({
|
|
"_index": "logentry_2017-03-08",
|
|
"_type": "_doc",
|
|
"_id": "1",
|
|
"_version": 1,
|
|
"_seq_no": 0,
|
|
"_primary_term": 1,
|
|
"result": "created"
|
|
}))
|
|
|
|
FAILURE_400 = _status({}, 400)
|
|
|
|
INDEX_REQUEST_2019_01_01 = [
|
|
"logentry_2019-01-01", {
|
|
"account_id":
|
|
1,
|
|
"repository_id":
|
|
1,
|
|
"ip":
|
|
"192.168.1.1",
|
|
"random_id":
|
|
233,
|
|
"datetime":
|
|
"2019-01-01T03:30:00",
|
|
"metadata_json": json.loads("{\"\\ud83d\\ude02\": \"\\ud83d\\ude02\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\", \"key\": \"value\", \"time\": 1520479800}"),
|
|
"performer_id":
|
|
1,
|
|
"kind_id":
|
|
1
|
|
}
|
|
]
|
|
|
|
INDEX_REQUEST_2017_03_08 = [
|
|
"logentry_2017-03-08", {
|
|
"repository_id":
|
|
1,
|
|
"account_id":
|
|
1,
|
|
"ip":
|
|
"192.168.1.1",
|
|
"random_id":
|
|
233,
|
|
"datetime":
|
|
"2017-03-08T03:30:00",
|
|
"metadata_json": json.loads("{\"\\ud83d\\ude02\": \"\\ud83d\\ude02\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\", \"key\": \"value\", \"time\": 1520479800}"),
|
|
"performer_id":
|
|
1,
|
|
"kind_id":
|
|
2
|
|
}
|
|
]
|
|
|
|
_hit1 = {
|
|
"_index": "logentry_2018-03-08",
|
|
"_type": "doc",
|
|
"_id": "1",
|
|
"_score": None,
|
|
"_source": {
|
|
"random_id":
|
|
233,
|
|
"kind_id":
|
|
1,
|
|
"account_id":
|
|
1,
|
|
"performer_id":
|
|
1,
|
|
"repository_id":
|
|
1,
|
|
"ip":
|
|
"192.168.1.1",
|
|
"metadata_json":
|
|
"{\"\\ud83d\\ude02\": \"\\ud83d\\ude02\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\", \"key\": \"value\", \"time\": 1520479800}",
|
|
"datetime":
|
|
"2018-03-08T03:30",
|
|
},
|
|
"sort": [1520479800000, 233]
|
|
}
|
|
|
|
_hit2 = {
|
|
"_index": "logentry_2018-04-02",
|
|
"_type": "doc",
|
|
"_id": "2",
|
|
"_score": None,
|
|
"_source": {
|
|
"random_id":
|
|
233,
|
|
"kind_id":
|
|
2,
|
|
"account_id":
|
|
1,
|
|
"performer_id":
|
|
1,
|
|
"repository_id":
|
|
1,
|
|
"ip":
|
|
"192.168.1.2",
|
|
"metadata_json":
|
|
"{\"\\ud83d\\ude02\": \"\\ud83d\\ude02\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\", \"key\": \"value\", \"time\": 1522639800}",
|
|
"datetime":
|
|
"2018-04-02T03:30",
|
|
},
|
|
"sort": [1522639800000, 233]
|
|
}
|
|
|
|
_log1 = Log(
|
|
"{\"\\ud83d\\ude02\": \"\\ud83d\\ude02\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\", \"key\": \"value\", \"time\": 1520479800}",
|
|
"192.168.1.1", parse("2018-03-08T03:30"), "user1.email", "user1.username", "user1.robot",
|
|
"user1.organization", "user1.username", "user1.email", "user1.robot", 1)
|
|
_log2 = Log(
|
|
"{\"\\ud83d\\ude02\": \"\\ud83d\\ude02\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\\ud83d\\udc4c\", \"key\": \"value\", \"time\": 1522639800}",
|
|
"192.168.1.2", parse("2018-04-02T03:30"), "user1.email", "user1.username", "user1.robot",
|
|
"user1.organization", "user1.username", "user1.email", "user1.robot", 2)
|
|
|
|
SEARCH_RESPONSE_START = _status(_shards(_hits([_hit1, _hit2])))
|
|
SEARCH_RESPONSE_END = _status(_shards(_hits([_hit2])))
|
|
SEARCH_REQUEST_START = {
|
|
"sort": [{
|
|
"datetime": "desc"
|
|
}, {
|
|
"random_id.keyword": "desc"
|
|
}],
|
|
"query": {
|
|
"bool": {
|
|
"filter": [{
|
|
"term": {
|
|
"performer_id": 1
|
|
}
|
|
}, {
|
|
"term": {
|
|
"repository_id": 1
|
|
}
|
|
}]
|
|
}
|
|
},
|
|
"size": 2
|
|
}
|
|
SEARCH_REQUEST_END = {
|
|
"sort": [{
|
|
"datetime": "desc"
|
|
}, {
|
|
"random_id.keyword": "desc"
|
|
}],
|
|
"query": {
|
|
"bool": {
|
|
"filter": [{
|
|
"term": {
|
|
"performer_id": 1
|
|
}
|
|
}, {
|
|
"term": {
|
|
"repository_id": 1
|
|
}
|
|
}]
|
|
}
|
|
},
|
|
"search_after": [1520479800000, 233],
|
|
"size": 2
|
|
}
|
|
SEARCH_REQUEST_FILTER = {
|
|
"sort": [{
|
|
"datetime": "desc"
|
|
}, {
|
|
"random_id.keyword": "desc"
|
|
}],
|
|
"query": {
|
|
"bool": {
|
|
"filter": [{
|
|
"term": {
|
|
"performer_id": 1
|
|
}
|
|
}, {
|
|
"term": {
|
|
"repository_id": 1
|
|
}
|
|
}, {
|
|
"bool": {
|
|
"must_not": [{
|
|
"terms": {
|
|
"kind_id": [1]
|
|
}
|
|
}]
|
|
}
|
|
}]
|
|
}
|
|
},
|
|
"size": 2
|
|
}
|
|
SEARCH_PAGE_TOKEN = {
|
|
"datetime": datetime(2018, 3, 8, 3, 30).isoformat(),
|
|
"random_id": 233,
|
|
"page_number": 1
|
|
}
|
|
SEARCH_PAGE_START = LogEntriesPage(logs=[_log1], next_page_token=SEARCH_PAGE_TOKEN)
|
|
SEARCH_PAGE_END = LogEntriesPage(logs=[_log2], next_page_token=None)
|
|
SEARCH_PAGE_EMPTY = LogEntriesPage([], None)
|
|
|
|
AGGS_RESPONSE = _status(
|
|
_shards({
|
|
"hits": {
|
|
"total": 4,
|
|
"max_score": None,
|
|
"hits": []
|
|
},
|
|
"aggregations": {
|
|
"by_id": {
|
|
"doc_count_error_upper_bound":
|
|
0,
|
|
"sum_other_doc_count":
|
|
0,
|
|
"buckets": [{
|
|
"key": 2,
|
|
"doc_count": 3,
|
|
"by_date": {
|
|
"buckets": [{
|
|
"key_as_string": "2009-11-12T00:00:00.000Z",
|
|
"key": 1257984000000,
|
|
"doc_count": 1
|
|
}, {
|
|
"key_as_string": "2009-11-13T00:00:00.000Z",
|
|
"key": 1258070400000,
|
|
"doc_count": 0
|
|
}, {
|
|
"key_as_string": "2009-11-14T00:00:00.000Z",
|
|
"key": 1258156800000,
|
|
"doc_count": 2
|
|
}]
|
|
}
|
|
}, {
|
|
"key": 1,
|
|
"doc_count": 1,
|
|
"by_date": {
|
|
"buckets": [{
|
|
"key_as_string": "2009-11-15T00:00:00.000Z",
|
|
"key": 1258243200000,
|
|
"doc_count": 1
|
|
}]
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
}))
|
|
|
|
AGGS_REQUEST = {
|
|
"query": {
|
|
"bool": {
|
|
"filter": [{
|
|
"term": {
|
|
"performer_id": 1
|
|
}
|
|
}, {
|
|
"term": {
|
|
"repository_id": 1
|
|
}
|
|
}, {
|
|
"bool": {
|
|
"must_not": [{
|
|
"terms": {
|
|
"kind_id": [2]
|
|
}
|
|
}]
|
|
}
|
|
}],
|
|
"must": [{
|
|
"range": {
|
|
"datetime": {
|
|
"lt": "2018-04-08T03:30:00",
|
|
"gte": "2018-03-08T03:30:00"
|
|
}
|
|
}
|
|
}]
|
|
}
|
|
},
|
|
"aggs": {
|
|
"by_id": {
|
|
"terms": {
|
|
"field": "kind_id"
|
|
},
|
|
"aggs": {
|
|
"by_date": {
|
|
"date_histogram": {
|
|
"field": "datetime",
|
|
"interval": "day"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"size": 0
|
|
}
|
|
|
|
AGGS_COUNT = [
|
|
AggregatedLogCount(1, 1, parse("2009-11-15T00:00:00.000")),
|
|
AggregatedLogCount(2, 1, parse("2009-11-12T00:00:00.000")),
|
|
AggregatedLogCount(2, 2, parse("2009-11-14T00:00:00.000"))
|
|
]
|
|
|
|
COUNT_REQUEST = {
|
|
"query": {
|
|
"bool": {
|
|
"filter": [{
|
|
"term": {
|
|
"repository_id": 1
|
|
}
|
|
}]
|
|
}
|
|
}
|
|
}
|
|
COUNT_RESPONSE = _status(_shards({
|
|
"count": 1,
|
|
}))
|
|
|
|
# assume there are 2 pages
|
|
_scroll_id = "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAACEmFkk1aGlTRzdSUWllejZmYTlEYTN3SVEAAAAAAAAhJRZJNWhpU0c3UlFpZXo2ZmE5RGEzd0lRAAAAAAAAHtAWLWZpaFZXVzVSTy1OTXA5V3MwcHZrZwAAAAAAAB7RFi1maWhWV1c1Uk8tTk1wOVdzMHB2a2cAAAAAAAAhJxZJNWhpU0c3UlFpZXo2ZmE5RGEzd0lR"
|
|
|
|
|
|
def _scroll(d):
|
|
d["_scroll_id"] = _scroll_id
|
|
return d
|
|
|
|
|
|
SCROLL_CREATE = _status(_shards(_scroll(_hits([_hit1]))))
|
|
SCROLL_GET = _status(_shards(_scroll(_hits([_hit2]))))
|
|
SCROLL_GET_2 = _status(_shards(_scroll(_hits([]))))
|
|
SCROLL_DELETE = _status({"succeeded": True, "num_freed": 5})
|
|
SCROLL_LOGS = [[_log1], [_log2]]
|
|
|
|
SCROLL_REQUESTS = [
|
|
[
|
|
"5m", 1, {
|
|
"sort": "_doc",
|
|
"query": {
|
|
"range": {
|
|
"datetime": {
|
|
"lt": "2018-04-02T00:00:00",
|
|
"gte": "2018-03-08T00:00:00"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
[{"scroll": "5m", "scroll_id": _scroll_id}],
|
|
[{"scroll":"5m", "scroll_id": _scroll_id}],
|
|
[{"scroll_id": [_scroll_id]}],
|
|
]
|
|
|
|
SCROLL_RESPONSES = [SCROLL_CREATE, SCROLL_GET, SCROLL_GET_2, SCROLL_DELETE]
|