Cache startup queries
This commit is contained in:
parent
b4a824aa38
commit
113900d3eb
9 changed files with 81 additions and 34 deletions
|
@ -57,6 +57,7 @@ type Config struct {
|
|||
FirebaseKeyFile string
|
||||
CacheFile string
|
||||
CacheDuration time.Duration
|
||||
CacheStartupQueries string
|
||||
AuthFile string
|
||||
AuthDefaultRead bool
|
||||
AuthDefaultWrite bool
|
||||
|
|
|
@ -88,18 +88,6 @@ const (
|
|||
selectAttachmentsExpiredQuery = `SELECT mid FROM messages WHERE attachment_expires > 0 AND attachment_expires < ?`
|
||||
)
|
||||
|
||||
// Performance & setup queries (see https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
|
||||
// - Write-ahead log (speeds up reads)
|
||||
// - Only sync on WAL checkpoint
|
||||
// - Temporary indices in memory
|
||||
const (
|
||||
setupQueries = `
|
||||
pragma journal_mode = WAL;
|
||||
pragma synchronous = normal;
|
||||
pragma temp_store = memory;
|
||||
`
|
||||
)
|
||||
|
||||
// Schema management queries
|
||||
const (
|
||||
currentSchemaVersion = 7
|
||||
|
@ -198,12 +186,12 @@ type messageCache struct {
|
|||
}
|
||||
|
||||
// newSqliteCache creates a SQLite file-backed cache
|
||||
func newSqliteCache(filename string, nop bool) (*messageCache, error) {
|
||||
func newSqliteCache(filename, startupQueries string, nop bool) (*messageCache, error) {
|
||||
db, err := sql.Open("sqlite3", filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := setupCacheDB(db); err != nil {
|
||||
if err := setupCacheDB(db, startupQueries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &messageCache{
|
||||
|
@ -214,13 +202,13 @@ func newSqliteCache(filename string, nop bool) (*messageCache, error) {
|
|||
|
||||
// newMemCache creates an in-memory cache
|
||||
func newMemCache() (*messageCache, error) {
|
||||
return newSqliteCache(createMemoryFilename(), false)
|
||||
return newSqliteCache(createMemoryFilename(), "", false)
|
||||
}
|
||||
|
||||
// newNopCache creates an in-memory cache that discards all messages;
|
||||
// it is always empty and can be used if caching is entirely disabled
|
||||
func newNopCache() (*messageCache, error) {
|
||||
return newSqliteCache(createMemoryFilename(), true)
|
||||
return newSqliteCache(createMemoryFilename(), "", true)
|
||||
}
|
||||
|
||||
// createMemoryFilename creates a unique memory filename to use for the SQLite backend.
|
||||
|
@ -511,10 +499,12 @@ func readMessages(rows *sql.Rows) ([]*message, error) {
|
|||
return messages, nil
|
||||
}
|
||||
|
||||
func setupCacheDB(db *sql.DB) error {
|
||||
// Performance: WAL mode, only sync on WAL checkpoints
|
||||
if _, err := db.Exec(setupQueries); err != nil {
|
||||
return err
|
||||
func setupCacheDB(db *sql.DB, startupQueries string) error {
|
||||
// Run startup queries
|
||||
if startupQueries != "" {
|
||||
if _, err := db.Exec(startupQueries); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// If 'messages' table does not exist, this must be a new database
|
||||
|
|
|
@ -378,7 +378,7 @@ func TestSqliteCache_Migration_From0(t *testing.T) {
|
|||
require.Nil(t, db.Close())
|
||||
|
||||
// Create cache to trigger migration
|
||||
c := newSqliteTestCacheFromFile(t, filename)
|
||||
c := newSqliteTestCacheFromFile(t, filename, "")
|
||||
checkSchemaVersion(t, c.db)
|
||||
|
||||
messages, err := c.Messages("mytopic", sinceAllMessages, false)
|
||||
|
@ -424,7 +424,7 @@ func TestSqliteCache_Migration_From1(t *testing.T) {
|
|||
require.Nil(t, db.Close())
|
||||
|
||||
// Create cache to trigger migration
|
||||
c := newSqliteTestCacheFromFile(t, filename)
|
||||
c := newSqliteTestCacheFromFile(t, filename, "")
|
||||
checkSchemaVersion(t, c.db)
|
||||
|
||||
// Add delayed message
|
||||
|
@ -443,6 +443,37 @@ func TestSqliteCache_Migration_From1(t *testing.T) {
|
|||
require.Equal(t, 11, len(messages))
|
||||
}
|
||||
|
||||
func TestSqliteCache_StartupQueries_WAL(t *testing.T) {
|
||||
filename := newSqliteTestCacheFile(t)
|
||||
startupQueries := `pragma journal_mode = WAL;
|
||||
pragma synchronous = normal;
|
||||
pragma temp_store = memory;`
|
||||
db, err := newSqliteCache(filename, startupQueries, false)
|
||||
require.Nil(t, err)
|
||||
require.Nil(t, db.AddMessage(newDefaultMessage("mytopic", "some message")))
|
||||
require.FileExists(t, filename)
|
||||
require.FileExists(t, filename+"-wal")
|
||||
require.FileExists(t, filename+"-shm")
|
||||
}
|
||||
|
||||
func TestSqliteCache_StartupQueries_None(t *testing.T) {
|
||||
filename := newSqliteTestCacheFile(t)
|
||||
startupQueries := ""
|
||||
db, err := newSqliteCache(filename, startupQueries, false)
|
||||
require.Nil(t, err)
|
||||
require.Nil(t, db.AddMessage(newDefaultMessage("mytopic", "some message")))
|
||||
require.FileExists(t, filename)
|
||||
require.NoFileExists(t, filename+"-wal")
|
||||
require.NoFileExists(t, filename+"-shm")
|
||||
}
|
||||
|
||||
func TestSqliteCache_StartupQueries_Fail(t *testing.T) {
|
||||
filename := newSqliteTestCacheFile(t)
|
||||
startupQueries := `xx error`
|
||||
_, err := newSqliteCache(filename, startupQueries, false)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func checkSchemaVersion(t *testing.T, db *sql.DB) {
|
||||
rows, err := db.Query(`SELECT version FROM schemaVersion`)
|
||||
require.Nil(t, err)
|
||||
|
@ -468,7 +499,7 @@ func TestMemCache_NopCache(t *testing.T) {
|
|||
}
|
||||
|
||||
func newSqliteTestCache(t *testing.T) *messageCache {
|
||||
c, err := newSqliteCache(newSqliteTestCacheFile(t), false)
|
||||
c, err := newSqliteCache(newSqliteTestCacheFile(t), "", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
@ -479,8 +510,8 @@ func newSqliteTestCacheFile(t *testing.T) string {
|
|||
return filepath.Join(t.TempDir(), "cache.db")
|
||||
}
|
||||
|
||||
func newSqliteTestCacheFromFile(t *testing.T, filename string) *messageCache {
|
||||
c, err := newSqliteCache(filename, false)
|
||||
func newSqliteTestCacheFromFile(t *testing.T, filename, startupQueries string) *messageCache {
|
||||
c, err := newSqliteCache(filename, startupQueries, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
@ -158,7 +158,7 @@ func createMessageCache(conf *Config) (*messageCache, error) {
|
|||
if conf.CacheDuration == 0 {
|
||||
return newNopCache()
|
||||
} else if conf.CacheFile != "" {
|
||||
return newSqliteCache(conf.CacheFile, false)
|
||||
return newSqliteCache(conf.CacheFile, conf.CacheStartupQueries, false)
|
||||
}
|
||||
return newMemCache()
|
||||
}
|
||||
|
|
|
@ -37,14 +37,22 @@
|
|||
#
|
||||
# firebase-key-file: <filename>
|
||||
|
||||
# If set, messages are cached in a local SQLite database instead of only in-memory. This
|
||||
# allows for service restarts without losing messages in support of the since= parameter.
|
||||
# If "cache-file" is set, messages are cached in a local SQLite database instead of only in-memory.
|
||||
# This allows for service restarts without losing messages in support of the since= parameter.
|
||||
#
|
||||
# The "cache-duration" parameter defines the duration for which messages will be buffered
|
||||
# before they are deleted. This is required to support the "since=..." and "poll=1" parameter.
|
||||
# To disable the cache entirely (on-disk/in-memory), set "cache-duration" to 0.
|
||||
# The cache file is created automatically, provided that the correct permissions are set.
|
||||
#
|
||||
# The "cache-startup-queries" parameter allows you to run commands when the database is initialized,
|
||||
# e.g. to enable WAL mode (see https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)).
|
||||
# Example:
|
||||
# cache-startup-queries: |
|
||||
# pragma journal_mode = WAL;
|
||||
# pragma synchronous = normal;
|
||||
# pragma temp_store = memory;
|
||||
#
|
||||
# Debian/RPM package users:
|
||||
# Use /var/cache/ntfy/cache.db as cache file to avoid permission issues. The package
|
||||
# creates this folder for you.
|
||||
|
@ -55,6 +63,7 @@
|
|||
#
|
||||
# cache-file: <filename>
|
||||
# cache-duration: "12h"
|
||||
# cache-startup-queries:
|
||||
|
||||
# If set, access to the ntfy server and API can be controlled on a granular level using
|
||||
# the 'ntfy user' and 'ntfy access' commands. See the --help pages for details, or check the docs.
|
||||
|
|
|
@ -1415,6 +1415,7 @@ func TestServer_PublishWhileUpdatingStatsWithLotsOfMessages(t *testing.T) {
|
|||
count := 50000
|
||||
c := newTestConfig(t)
|
||||
c.TotalTopicLimit = 50001
|
||||
c.CacheStartupQueries = "pragma journal_mode = WAL; pragma synchronous = normal; pragma temp_store = memory;"
|
||||
s := newTestServer(t, c)
|
||||
|
||||
// Add lots of messages
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue