Temporarily change to storing logs in a new LogEntry2 table
This will prevent us from running out of auto-incrementing ID values until such time as we can upgrade to peewee 3 and change the field type to a BigInt Fixes https://jira.coreos.com/browse/QUAY-943
This commit is contained in:
Joseph Schorr
parent
66b4e45929
commit
a007332d4c
13 changed files with 201 additions and 113 deletions
|
|
@ -7,7 +7,7 @@ from datetime import datetime, timedelta
|
|||
from cachetools import lru_cache
|
||||
|
||||
import data
|
||||
from data.database import LogEntry, LogEntryKind, User, RepositoryActionCount, db
|
||||
from data.database import LogEntry, LogEntry2, LogEntryKind, User, RepositoryActionCount, db
|
||||
from data.model import config, user, DataModelException
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -16,27 +16,29 @@ ACTIONS_ALLOWED_WITHOUT_AUDIT_LOGGING = ['pull_repo']
|
|||
|
||||
|
||||
def _logs_query(selections, start_time, end_time, performer=None, repository=None, namespace=None,
|
||||
ignore=None):
|
||||
joined = (LogEntry.select(*selections).switch(LogEntry)
|
||||
.where(LogEntry.datetime >= start_time, LogEntry.datetime < end_time))
|
||||
ignore=None, model=LogEntry):
|
||||
""" Returns a query for selecting logs from the table, with various options and filters. """
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
joined = (model.select(*selections).switch(model)
|
||||
.where(model.datetime >= start_time, model.datetime < end_time))
|
||||
|
||||
if repository:
|
||||
joined = joined.where(LogEntry.repository == repository)
|
||||
joined = joined.where(model.repository == repository)
|
||||
|
||||
if performer:
|
||||
joined = joined.where(LogEntry.performer == performer)
|
||||
joined = joined.where(model.performer == performer)
|
||||
|
||||
if namespace and not repository:
|
||||
namespace_user = user.get_user_or_org(namespace)
|
||||
if namespace_user is None:
|
||||
raise DataModelException('Invalid namespace requested')
|
||||
|
||||
joined = joined.where(LogEntry.account == namespace_user.id)
|
||||
joined = joined.where(model.account == namespace_user.id)
|
||||
|
||||
if ignore:
|
||||
kind_map = get_log_entry_kinds()
|
||||
ignore_ids = [kind_map[kind_name] for kind_name in ignore]
|
||||
joined = joined.where(~(LogEntry.kind << ignore_ids))
|
||||
joined = joined.where(~(model.kind << ignore_ids))
|
||||
|
||||
return joined
|
||||
|
||||
|
|
@ -57,29 +59,35 @@ def _get_log_entry_kind(name):
|
|||
|
||||
|
||||
def get_aggregated_logs(start_time, end_time, performer=None, repository=None, namespace=None,
|
||||
ignore=None):
|
||||
date = db.extract_date('day', LogEntry.datetime)
|
||||
selections = [LogEntry.kind, date.alias('day'), fn.Count(LogEntry.id).alias('count')]
|
||||
query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore)
|
||||
return query.group_by(date, LogEntry.kind)
|
||||
ignore=None, model=LogEntry):
|
||||
""" Returns the count of logs, by kind and day, for the logs matching the given filters. """
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
date = db.extract_date('day', model.datetime)
|
||||
selections = [model.kind, date.alias('day'), fn.Count(model.id).alias('count')]
|
||||
query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore,
|
||||
model=model)
|
||||
return query.group_by(date, model.kind)
|
||||
|
||||
|
||||
def get_logs_query(start_time, end_time, performer=None, repository=None, namespace=None,
|
||||
ignore=None):
|
||||
ignore=None, model=LogEntry):
|
||||
""" Returns the logs matching the given filters. """
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
Performer = User.alias()
|
||||
Account = User.alias()
|
||||
selections = [LogEntry, Performer]
|
||||
selections = [model, Performer]
|
||||
|
||||
if namespace is None and repository is None:
|
||||
selections.append(Account)
|
||||
|
||||
query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore)
|
||||
query = (query.switch(LogEntry).join(Performer, JOIN_LEFT_OUTER,
|
||||
on=(LogEntry.performer == Performer.id).alias('performer')))
|
||||
query = _logs_query(selections, start_time, end_time, performer, repository, namespace, ignore,
|
||||
model=model)
|
||||
query = (query.switch(model).join(Performer, JOIN_LEFT_OUTER,
|
||||
on=(model.performer == Performer.id).alias('performer')))
|
||||
|
||||
if namespace is None and repository is None:
|
||||
query = (query.switch(LogEntry).join(Account, JOIN_LEFT_OUTER,
|
||||
on=(LogEntry.account == Account.id).alias('account')))
|
||||
query = (query.switch(model).join(Account, JOIN_LEFT_OUTER,
|
||||
on=(model.account == Account.id).alias('account')))
|
||||
|
||||
return query
|
||||
|
||||
|
|
@ -93,6 +101,7 @@ def _json_serialize(obj):
|
|||
|
||||
def log_action(kind_name, user_or_organization_name, performer=None, repository=None, ip=None,
|
||||
metadata={}, timestamp=None):
|
||||
""" Logs an entry in the LogEntry2 table. """
|
||||
if not timestamp:
|
||||
timestamp = datetime.today()
|
||||
|
||||
|
|
@ -123,7 +132,7 @@ def log_action(kind_name, user_or_organization_name, performer=None, repository=
|
|||
}
|
||||
|
||||
try:
|
||||
LogEntry.create(**log_data)
|
||||
LogEntry2.create(**log_data)
|
||||
except PeeweeException as ex:
|
||||
strict_logging_disabled = config.app_config.get('ALLOW_PULLS_WITHOUT_STRICT_LOGGING')
|
||||
if strict_logging_disabled and kind_name in ACTIONS_ALLOWED_WITHOUT_AUDIT_LOGGING:
|
||||
|
|
@ -132,39 +141,47 @@ def log_action(kind_name, user_or_organization_name, performer=None, repository=
|
|||
raise
|
||||
|
||||
|
||||
def get_stale_logs_start_id():
|
||||
def get_stale_logs_start_id(model):
|
||||
""" Gets the oldest log entry. """
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
try:
|
||||
return (LogEntry.select(LogEntry.id).order_by(LogEntry.id).limit(1).tuples())[0][0]
|
||||
return (model.select(model.id).order_by(model.id).limit(1).tuples())[0][0]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
|
||||
def get_stale_logs_cutoff_id(cutoff_date):
|
||||
def get_stale_logs_cutoff_id(cutoff_date, model):
|
||||
""" Gets the most recent ID created before the cutoff_date. """
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
try:
|
||||
return (LogEntry.select(fn.Max(LogEntry.id)).where(LogEntry.datetime <= cutoff_date)
|
||||
return (model.select(fn.Max(model.id)).where(model.datetime <= cutoff_date)
|
||||
.tuples())[0][0]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
|
||||
def get_stale_logs(start_id, end_id):
|
||||
def get_stale_logs(start_id, end_id, model):
|
||||
""" Returns all the logs with IDs between start_id and end_id inclusively. """
|
||||
return LogEntry.select().where((LogEntry.id >= start_id), (LogEntry.id <= end_id))
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
return model.select().where((model.id >= start_id), (model.id <= end_id))
|
||||
|
||||
|
||||
def delete_stale_logs(start_id, end_id):
|
||||
def delete_stale_logs(start_id, end_id, model):
|
||||
""" Deletes all the logs with IDs between start_id and end_id. """
|
||||
LogEntry.delete().where((LogEntry.id >= start_id), (LogEntry.id <= end_id)).execute()
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
model.delete().where((model.id >= start_id), (model.id <= end_id)).execute()
|
||||
|
||||
|
||||
def get_repository_action_counts(repo, start_date):
|
||||
""" Returns the daily aggregated action counts for the given repository, starting at the given
|
||||
start date.
|
||||
"""
|
||||
return RepositoryActionCount.select().where(RepositoryActionCount.repository == repo,
|
||||
RepositoryActionCount.date >= start_date)
|
||||
|
||||
|
||||
def get_repositories_action_sums(repository_ids):
|
||||
""" Returns a map from repository ID to total actions within that repository in the last week. """
|
||||
if not repository_ids:
|
||||
return {}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ from collections import namedtuple
|
|||
from peewee import IntegrityError
|
||||
|
||||
from datetime import date, timedelta, datetime
|
||||
from data.database import (Repository, LogEntry, RepositoryActionCount, RepositorySearchScore,
|
||||
db_random_func, fn)
|
||||
from data.database import (Repository, LogEntry, LogEntry2, RepositoryActionCount,
|
||||
RepositorySearchScore, db_random_func, fn)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -52,13 +52,16 @@ def count_repository_actions(to_count):
|
|||
today = date.today()
|
||||
yesterday = today - timedelta(days=1)
|
||||
|
||||
actions = (LogEntry
|
||||
.select()
|
||||
.where(LogEntry.repository == to_count,
|
||||
LogEntry.datetime >= yesterday,
|
||||
LogEntry.datetime < today)
|
||||
.count())
|
||||
# TODO(LogMigrate): Remove the branch once we're back on LogEntry only.
|
||||
def lookup_action_count(model):
|
||||
return (model
|
||||
.select()
|
||||
.where(model.repository == to_count,
|
||||
model.datetime >= yesterday,
|
||||
model.datetime < today)
|
||||
.count())
|
||||
|
||||
actions = lookup_action_count(LogEntry) + lookup_action_count(LogEntry2)
|
||||
try:
|
||||
RepositoryActionCount.create(repository=to_count, date=yesterday, count=actions)
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import pytest
|
||||
|
||||
from data.database import LogEntry, User
|
||||
from data.database import LogEntry, LogEntry2, User
|
||||
from data.model import config as _config
|
||||
from data.model.log import log_action
|
||||
|
||||
|
|
@ -21,8 +21,8 @@ def logentry_kind():
|
|||
|
||||
@pytest.fixture()
|
||||
def logentry(logentry_kind):
|
||||
with patch('data.database.LogEntry.create', spec=True):
|
||||
yield LogEntry
|
||||
with patch('data.database.LogEntry2.create', spec=True):
|
||||
yield LogEntry2
|
||||
|
||||
@pytest.fixture()
|
||||
def user():
|
||||
|
|
|
|||
Reference in a new issue