Cache startup queries

server/config.go

@@ -57,6 +57,7 @@ type Config struct {
 	FirebaseKeyFile                      string
 	CacheFile                            string
 	CacheDuration                        time.Duration
+	CacheStartupQueries                  string
 	AuthFile                             string
 	AuthDefaultRead                      bool
 	AuthDefaultWrite                     bool

server/message_cache.go

@@ -88,18 +88,6 @@ const (
 	selectAttachmentsExpiredQuery   = `SELECT mid FROM messages WHERE attachment_expires > 0 AND attachment_expires < ?`
 )
 
-// Performance & setup queries (see https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
-// - Write-ahead log (speeds up reads)
-// - Only sync on WAL checkpoint
-// - Temporary indices in memory
-const (
-	setupQueries = `
-		pragma journal_mode = WAL;
-		pragma synchronous = normal;
-		pragma temp_store = memory;
-	`
-)
-
 // Schema management queries
 const (
 	currentSchemaVersion          = 7
@@ -198,12 +186,12 @@ type messageCache struct {
 }
 
 // newSqliteCache creates a SQLite file-backed cache
-func newSqliteCache(filename string, nop bool) (*messageCache, error) {
+func newSqliteCache(filename, startupQueries string, nop bool) (*messageCache, error) {
 	db, err := sql.Open("sqlite3", filename)
 	if err != nil {
 		return nil, err
 	}
-	if err := setupCacheDB(db); err != nil {
+	if err := setupCacheDB(db, startupQueries); err != nil {
 		return nil, err
 	}
 	return &messageCache{
@@ -214,13 +202,13 @@ func newSqliteCache(filename string, nop bool) (*messageCache, error) {
 
 // newMemCache creates an in-memory cache
 func newMemCache() (*messageCache, error) {
-	return newSqliteCache(createMemoryFilename(), false)
+	return newSqliteCache(createMemoryFilename(), "", false)
 }
 
 // newNopCache creates an in-memory cache that discards all messages;
 // it is always empty and can be used if caching is entirely disabled
 func newNopCache() (*messageCache, error) {
-	return newSqliteCache(createMemoryFilename(), true)
+	return newSqliteCache(createMemoryFilename(), "", true)
 }
 
 // createMemoryFilename creates a unique memory filename to use for the SQLite backend.
@@ -511,10 +499,12 @@ func readMessages(rows *sql.Rows) ([]*message, error) {
 	return messages, nil
 }
 
-func setupCacheDB(db *sql.DB) error {
-	// Performance: WAL mode, only sync on WAL checkpoints
-	if _, err := db.Exec(setupQueries); err != nil {
-		return err
+func setupCacheDB(db *sql.DB, startupQueries string) error {
+	// Run startup queries
+	if startupQueries != "" {
+		if _, err := db.Exec(startupQueries); err != nil {
+			return err
+		}
 	}
 
 	// If 'messages' table does not exist, this must be a new database

server/message_cache_test.go

@@ -378,7 +378,7 @@ func TestSqliteCache_Migration_From0(t *testing.T) {
 	require.Nil(t, db.Close())
 
 	// Create cache to trigger migration
-	c := newSqliteTestCacheFromFile(t, filename)
+	c := newSqliteTestCacheFromFile(t, filename, "")
 	checkSchemaVersion(t, c.db)
 
 	messages, err := c.Messages("mytopic", sinceAllMessages, false)
@@ -424,7 +424,7 @@ func TestSqliteCache_Migration_From1(t *testing.T) {
 	require.Nil(t, db.Close())
 
 	// Create cache to trigger migration
-	c := newSqliteTestCacheFromFile(t, filename)
+	c := newSqliteTestCacheFromFile(t, filename, "")
 	checkSchemaVersion(t, c.db)
 
 	// Add delayed message
@@ -443,6 +443,37 @@ func TestSqliteCache_Migration_From1(t *testing.T) {
 	require.Equal(t, 11, len(messages))
 }
 
+func TestSqliteCache_StartupQueries_WAL(t *testing.T) {
+	filename := newSqliteTestCacheFile(t)
+	startupQueries := `pragma journal_mode = WAL; 
+pragma synchronous = normal; 
+pragma temp_store = memory;`
+	db, err := newSqliteCache(filename, startupQueries, false)
+	require.Nil(t, err)
+	require.Nil(t, db.AddMessage(newDefaultMessage("mytopic", "some message")))
+	require.FileExists(t, filename)
+	require.FileExists(t, filename+"-wal")
+	require.FileExists(t, filename+"-shm")
+}
+
+func TestSqliteCache_StartupQueries_None(t *testing.T) {
+	filename := newSqliteTestCacheFile(t)
+	startupQueries := ""
+	db, err := newSqliteCache(filename, startupQueries, false)
+	require.Nil(t, err)
+	require.Nil(t, db.AddMessage(newDefaultMessage("mytopic", "some message")))
+	require.FileExists(t, filename)
+	require.NoFileExists(t, filename+"-wal")
+	require.NoFileExists(t, filename+"-shm")
+}
+
+func TestSqliteCache_StartupQueries_Fail(t *testing.T) {
+	filename := newSqliteTestCacheFile(t)
+	startupQueries := `xx error`
+	_, err := newSqliteCache(filename, startupQueries, false)
+	require.Error(t, err)
+}
+
 func checkSchemaVersion(t *testing.T, db *sql.DB) {
 	rows, err := db.Query(`SELECT version FROM schemaVersion`)
 	require.Nil(t, err)
@@ -468,7 +499,7 @@ func TestMemCache_NopCache(t *testing.T) {
 }
 
 func newSqliteTestCache(t *testing.T) *messageCache {
-	c, err := newSqliteCache(newSqliteTestCacheFile(t), false)
+	c, err := newSqliteCache(newSqliteTestCacheFile(t), "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -479,8 +510,8 @@ func newSqliteTestCacheFile(t *testing.T) string {
 	return filepath.Join(t.TempDir(), "cache.db")
 }
 
-func newSqliteTestCacheFromFile(t *testing.T, filename string) *messageCache {
-	c, err := newSqliteCache(filename, false)
+func newSqliteTestCacheFromFile(t *testing.T, filename, startupQueries string) *messageCache {
+	c, err := newSqliteCache(filename, startupQueries, false)
 	if err != nil {
 		t.Fatal(err)
 	}

server/server.go

@@ -158,7 +158,7 @@ func createMessageCache(conf *Config) (*messageCache, error) {
 	if conf.CacheDuration == 0 {
 		return newNopCache()
 	} else if conf.CacheFile != "" {
-		return newSqliteCache(conf.CacheFile, false)
+		return newSqliteCache(conf.CacheFile, conf.CacheStartupQueries, false)
 	}
 	return newMemCache()
 }

server/server.yml

@@ -37,14 +37,22 @@
 #
 # firebase-key-file: <filename>
 
-# If set, messages are cached in a local SQLite database instead of only in-memory. This
-# allows for service restarts without losing messages in support of the since= parameter.
+# If "cache-file" is set, messages are cached in a local SQLite database instead of only in-memory.
+# This allows for service restarts without losing messages in support of the since= parameter.
 #
 # The "cache-duration" parameter defines the duration for which messages will be buffered
 # before they are deleted. This is required to support the "since=..." and "poll=1" parameter.
 # To disable the cache entirely (on-disk/in-memory), set "cache-duration" to 0.
 # The cache file is created automatically, provided that the correct permissions are set.
 #
+# The "cache-startup-queries" parameter allows you to run commands when the database is initialized,
+# e.g. to enable WAL mode (see https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)).
+# Example:
+#    cache-startup-queries: |
+#       pragma journal_mode = WAL;
+#       pragma synchronous = normal;
+#       pragma temp_store = memory;
+#
 # Debian/RPM package users:
 #   Use /var/cache/ntfy/cache.db as cache file to avoid permission issues. The package
 #   creates this folder for you.
@@ -55,6 +63,7 @@
 #
 # cache-file: <filename>
 # cache-duration: "12h"
+# cache-startup-queries:
 
 # If set, access to the ntfy server and API can be controlled on a granular level using
 # the 'ntfy user' and 'ntfy access' commands. See the --help pages for details, or check the docs.

server/server_test.go

@@ -1415,6 +1415,7 @@ func TestServer_PublishWhileUpdatingStatsWithLotsOfMessages(t *testing.T) {
 	count := 50000
 	c := newTestConfig(t)
 	c.TotalTopicLimit = 50001
+	c.CacheStartupQueries = "pragma journal_mode = WAL; pragma synchronous = normal; pragma temp_store = memory;"
 	s := newTestServer(t, c)
 
 	// Add lots of messages