Attachments limits; working visitor limit

This commit is contained in:
Philipp Heckel 2022-01-07 14:49:28 +01:00
parent 70aefc2e48
commit c45a28e6af
9 changed files with 287 additions and 186 deletions

View file

@ -21,7 +21,8 @@ var flagsServe = []cli.Flag{
altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-file", Aliases: []string{"C"}, EnvVars: []string{"NTFY_CACHE_FILE"}, Usage: "cache file used for message caching"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "cache-file", Aliases: []string{"C"}, EnvVars: []string{"NTFY_CACHE_FILE"}, Usage: "cache file used for message caching"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "cache-duration", Aliases: []string{"b"}, EnvVars: []string{"NTFY_CACHE_DURATION"}, Value: server.DefaultCacheDuration, Usage: "buffer messages for this time to allow `since` requests"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "cache-duration", Aliases: []string{"b"}, EnvVars: []string{"NTFY_CACHE_DURATION"}, Value: server.DefaultCacheDuration, Usage: "buffer messages for this time to allow `since` requests"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-cache-dir", EnvVars: []string{"NTFY_ATTACHMENT_CACHE_DIR"}, Usage: "cache directory for attached files"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-cache-dir", EnvVars: []string{"NTFY_ATTACHMENT_CACHE_DIR"}, Usage: "cache directory for attached files"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-size-limit", Aliases: []string{"A"}, EnvVars: []string{"NTFY_ATTACHMENT_SIZE_LIMIT"}, DefaultText: "15M", Usage: "attachment size limit (e.g. 10k, 2M)"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-total-size-limit", Aliases: []string{"A"}, EnvVars: []string{"NTFY_ATTACHMENT_TOTAL_SIZE_LIMIT"}, DefaultText: "1G", Usage: "limit of the on-disk attachment cache"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "attachment-file-size-limit", Aliases: []string{"Y"}, EnvVars: []string{"NTFY_ATTACHMENT_FILE_SIZE_LIMIT"}, DefaultText: "15M", Usage: "per-file attachment size limit (e.g. 300k, 2M, 100M)"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "keepalive-interval", Aliases: []string{"k"}, EnvVars: []string{"NTFY_KEEPALIVE_INTERVAL"}, Value: server.DefaultKeepaliveInterval, Usage: "interval of keepalive messages"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "keepalive-interval", Aliases: []string{"k"}, EnvVars: []string{"NTFY_KEEPALIVE_INTERVAL"}, Value: server.DefaultKeepaliveInterval, Usage: "interval of keepalive messages"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "manager-interval", Aliases: []string{"m"}, EnvVars: []string{"NTFY_MANAGER_INTERVAL"}, Value: server.DefaultManagerInterval, Usage: "interval of for message pruning and stats printing"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "manager-interval", Aliases: []string{"m"}, EnvVars: []string{"NTFY_MANAGER_INTERVAL"}, Value: server.DefaultManagerInterval, Usage: "interval of for message pruning and stats printing"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-sender-addr", EnvVars: []string{"NTFY_SMTP_SENDER_ADDR"}, Usage: "SMTP server address (host:port) for outgoing emails"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-sender-addr", EnvVars: []string{"NTFY_SMTP_SENDER_ADDR"}, Usage: "SMTP server address (host:port) for outgoing emails"}),
@ -33,6 +34,7 @@ var flagsServe = []cli.Flag{
altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-server-addr-prefix", EnvVars: []string{"NTFY_SMTP_SERVER_ADDR_PREFIX"}, Usage: "SMTP email address prefix for topics to prevent spam (e.g. 'ntfy-')"}), altsrc.NewStringFlag(&cli.StringFlag{Name: "smtp-server-addr-prefix", EnvVars: []string{"NTFY_SMTP_SERVER_ADDR_PREFIX"}, Usage: "SMTP email address prefix for topics to prevent spam (e.g. 'ntfy-')"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "global-topic-limit", Aliases: []string{"T"}, EnvVars: []string{"NTFY_GLOBAL_TOPIC_LIMIT"}, Value: server.DefaultTotalTopicLimit, Usage: "total number of topics allowed"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "global-topic-limit", Aliases: []string{"T"}, EnvVars: []string{"NTFY_GLOBAL_TOPIC_LIMIT"}, Value: server.DefaultTotalTopicLimit, Usage: "total number of topics allowed"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-subscription-limit", EnvVars: []string{"NTFY_VISITOR_SUBSCRIPTION_LIMIT"}, Value: server.DefaultVisitorSubscriptionLimit, Usage: "number of subscriptions per visitor"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-subscription-limit", EnvVars: []string{"NTFY_VISITOR_SUBSCRIPTION_LIMIT"}, Value: server.DefaultVisitorSubscriptionLimit, Usage: "number of subscriptions per visitor"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "visitor-attachment-total-size-limit", EnvVars: []string{"NTFY_VISITOR_ATTACHMENT_TOTAL_SIZE_LIMIT"}, Value: "50M", Usage: "total storage limit used for attachments per visitor"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-request-limit-burst", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_BURST"}, Value: server.DefaultVisitorRequestLimitBurst, Usage: "initial limit of requests per visitor"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-request-limit-burst", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_BURST"}, Value: server.DefaultVisitorRequestLimitBurst, Usage: "initial limit of requests per visitor"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "visitor-request-limit-replenish", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_REPLENISH"}, Value: server.DefaultVisitorRequestLimitReplenish, Usage: "interval at which burst limit is replenished (one per x)"}), altsrc.NewDurationFlag(&cli.DurationFlag{Name: "visitor-request-limit-replenish", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_REPLENISH"}, Value: server.DefaultVisitorRequestLimitReplenish, Usage: "interval at which burst limit is replenished (one per x)"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-email-limit-burst", EnvVars: []string{"NTFY_VISITOR_EMAIL_LIMIT_BURST"}, Value: server.DefaultVisitorEmailLimitBurst, Usage: "initial limit of e-mails per visitor"}), altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-email-limit-burst", EnvVars: []string{"NTFY_VISITOR_EMAIL_LIMIT_BURST"}, Value: server.DefaultVisitorEmailLimitBurst, Usage: "initial limit of e-mails per visitor"}),
@ -72,7 +74,8 @@ func execServe(c *cli.Context) error {
cacheFile := c.String("cache-file") cacheFile := c.String("cache-file")
cacheDuration := c.Duration("cache-duration") cacheDuration := c.Duration("cache-duration")
attachmentCacheDir := c.String("attachment-cache-dir") attachmentCacheDir := c.String("attachment-cache-dir")
attachmentSizeLimitStr := c.String("attachment-size-limit") attachmentTotalSizeLimitStr := c.String("attachment-total-size-limit")
attachmentFileSizeLimitStr := c.String("attachment-file-size-limit")
keepaliveInterval := c.Duration("keepalive-interval") keepaliveInterval := c.Duration("keepalive-interval")
managerInterval := c.Duration("manager-interval") managerInterval := c.Duration("manager-interval")
smtpSenderAddr := c.String("smtp-sender-addr") smtpSenderAddr := c.String("smtp-sender-addr")
@ -82,8 +85,9 @@ func execServe(c *cli.Context) error {
smtpServerListen := c.String("smtp-server-listen") smtpServerListen := c.String("smtp-server-listen")
smtpServerDomain := c.String("smtp-server-domain") smtpServerDomain := c.String("smtp-server-domain")
smtpServerAddrPrefix := c.String("smtp-server-addr-prefix") smtpServerAddrPrefix := c.String("smtp-server-addr-prefix")
globalTopicLimit := c.Int("global-topic-limit") totalTopicLimit := c.Int("global-topic-limit")
visitorSubscriptionLimit := c.Int("visitor-subscription-limit") visitorSubscriptionLimit := c.Int("visitor-subscription-limit")
visitorAttachmentTotalSizeLimitStr := c.String("visitor-attachment-total-size-limit")
visitorRequestLimitBurst := c.Int("visitor-request-limit-burst") visitorRequestLimitBurst := c.Int("visitor-request-limit-burst")
visitorRequestLimitReplenish := c.Duration("visitor-request-limit-replenish") visitorRequestLimitReplenish := c.Duration("visitor-request-limit-replenish")
visitorEmailLimitBurst := c.Int("visitor-email-limit-burst") visitorEmailLimitBurst := c.Int("visitor-email-limit-burst")
@ -111,14 +115,18 @@ func execServe(c *cli.Context) error {
return errors.New("if smtp-server-listen is set, smtp-server-domain must also be set") return errors.New("if smtp-server-listen is set, smtp-server-domain must also be set")
} }
// Convert // Convert sizes to bytes
attachmentSizeLimit := server.DefaultAttachmentSizeLimit attachmentTotalSizeLimit, err := parseSize(attachmentTotalSizeLimitStr, server.DefaultAttachmentTotalSizeLimit)
if attachmentSizeLimitStr != "" { if err != nil {
var err error return err
attachmentSizeLimit, err = util.ParseSize(attachmentSizeLimitStr) }
if err != nil { attachmentFileSizeLimit, err := parseSize(attachmentFileSizeLimitStr, server.DefaultAttachmentFileSizeLimit)
return err if err != nil {
} return err
}
visitorAttachmentTotalSizeLimit, err := parseSize(visitorAttachmentTotalSizeLimitStr, server.DefaultVisitorAttachmentTotalSizeLimit)
if err != nil {
return err
} }
// Run server // Run server
@ -132,7 +140,8 @@ func execServe(c *cli.Context) error {
conf.CacheFile = cacheFile conf.CacheFile = cacheFile
conf.CacheDuration = cacheDuration conf.CacheDuration = cacheDuration
conf.AttachmentCacheDir = attachmentCacheDir conf.AttachmentCacheDir = attachmentCacheDir
conf.AttachmentSizeLimit = attachmentSizeLimit conf.AttachmentTotalSizeLimit = attachmentTotalSizeLimit
conf.AttachmentFileSizeLimit = attachmentFileSizeLimit
conf.KeepaliveInterval = keepaliveInterval conf.KeepaliveInterval = keepaliveInterval
conf.ManagerInterval = managerInterval conf.ManagerInterval = managerInterval
conf.SMTPSenderAddr = smtpSenderAddr conf.SMTPSenderAddr = smtpSenderAddr
@ -142,8 +151,9 @@ func execServe(c *cli.Context) error {
conf.SMTPServerListen = smtpServerListen conf.SMTPServerListen = smtpServerListen
conf.SMTPServerDomain = smtpServerDomain conf.SMTPServerDomain = smtpServerDomain
conf.SMTPServerAddrPrefix = smtpServerAddrPrefix conf.SMTPServerAddrPrefix = smtpServerAddrPrefix
conf.TotalTopicLimit = globalTopicLimit conf.TotalTopicLimit = totalTopicLimit
conf.VisitorSubscriptionLimit = visitorSubscriptionLimit conf.VisitorSubscriptionLimit = visitorSubscriptionLimit
conf.VisitorAttachmentTotalSizeLimit = visitorAttachmentTotalSizeLimit
conf.VisitorRequestLimitBurst = visitorRequestLimitBurst conf.VisitorRequestLimitBurst = visitorRequestLimitBurst
conf.VisitorRequestLimitReplenish = visitorRequestLimitReplenish conf.VisitorRequestLimitReplenish = visitorRequestLimitReplenish
conf.VisitorEmailLimitBurst = visitorEmailLimitBurst conf.VisitorEmailLimitBurst = visitorEmailLimitBurst
@ -159,3 +169,14 @@ func execServe(c *cli.Context) error {
log.Printf("Exiting.") log.Printf("Exiting.")
return nil return nil
} }
func parseSize(s string, defaultValue int64) (v int64, err error) {
if s == "" {
return defaultValue, nil
}
v, err = util.ParseSize(s)
if err != nil {
return 0, err
}
return v, nil
}

View file

@ -661,22 +661,31 @@ Here's an example that will open Reddit when the notification is clicked:
## Send files + URLs ## Send files + URLs
``` ```
- Uploaded attachment
- External attachment
- Preview without attachment
# Send attachment
curl -T image.jpg ntfy.sh/howdy curl -T image.jpg ntfy.sh/howdy
# Send attachment with custom message and filename
curl \ curl \
-T flower.jpg \ -T flower.jpg \
-H "Message: Here's a flower for you" \ -H "Message: Here's a flower for you" \
-H "Filename: flower.jpg" \ -H "Filename: flower.jpg" \
ntfy.sh/howdy ntfy.sh/howdy
# Send attachment from another URL, with custom preview and message
curl \ curl \
-T files.zip \ -H "Attachment: https://example.com/files.zip" \
-H "Preview: https://example.com/filespreview.jpg" \
"ntfy.sh/howdy?m=Important+documents+attached" "ntfy.sh/howdy?m=Important+documents+attached"
# Send normal message with external image
curl \ curl \
-d "A link for you" \ -H "Image: https://example.com/someimage.jpg" \
-H "Link: https://unifiedpush.org" \ "ntfy.sh/howdy?m=Important+documents+attached"
"ntfy.sh/howdy"
``` ```
## E-mail notifications ## E-mail notifications

View file

@ -20,4 +20,5 @@ type cache interface {
Topics() (map[string]*topic, error) Topics() (map[string]*topic, error)
Prune(olderThan time.Time) error Prune(olderThan time.Time) error
MarkPublished(m *message) error MarkPublished(m *message) error
AttachmentsSize(owner string) (int64, error)
} }

View file

@ -125,6 +125,20 @@ func (c *memCache) Prune(olderThan time.Time) error {
return nil return nil
} }
func (c *memCache) AttachmentsSize(owner string) (int64, error) {
c.mu.Lock()
defer c.mu.Unlock()
var size int64
for topic := range c.messages {
for _, m := range c.messages[topic] {
if m.Attachment != nil && m.Attachment.Owner == owner {
size += m.Attachment.Size
}
}
}
return size, nil
}
func (c *memCache) pruneTopic(topic string, olderThan time.Time) { func (c *memCache) pruneTopic(topic string, olderThan time.Time) {
messages := make([]*message, 0) messages := make([]*message, 0)
for _, m := range c.messages[topic] { for _, m := range c.messages[topic] {

View file

@ -27,32 +27,32 @@ const (
attachment_type TEXT NOT NULL, attachment_type TEXT NOT NULL,
attachment_size INT NOT NULL, attachment_size INT NOT NULL,
attachment_expires INT NOT NULL, attachment_expires INT NOT NULL,
attachment_preview_url TEXT NOT NULL,
attachment_url TEXT NOT NULL, attachment_url TEXT NOT NULL,
attachment_owner TEXT NOT NULL,
published INT NOT NULL published INT NOT NULL
); );
CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic); CREATE INDEX IF NOT EXISTS idx_topic ON messages (topic);
COMMIT; COMMIT;
` `
insertMessageQuery = ` insertMessageQuery = `
INSERT INTO messages (id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url, published) INSERT INTO messages (id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner, published)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
` `
pruneMessagesQuery = `DELETE FROM messages WHERE time < ? AND published = 1` pruneMessagesQuery = `DELETE FROM messages WHERE time < ? AND published = 1`
selectMessagesSinceTimeQuery = ` selectMessagesSinceTimeQuery = `
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner
FROM messages FROM messages
WHERE topic = ? AND time >= ? AND published = 1 WHERE topic = ? AND time >= ? AND published = 1
ORDER BY time ASC ORDER BY time ASC
` `
selectMessagesSinceTimeIncludeScheduledQuery = ` selectMessagesSinceTimeIncludeScheduledQuery = `
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner
FROM messages FROM messages
WHERE topic = ? AND time >= ? WHERE topic = ? AND time >= ?
ORDER BY time ASC ORDER BY time ASC
` `
selectMessagesDueQuery = ` selectMessagesDueQuery = `
SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_preview_url, attachment_url SELECT id, time, topic, message, title, priority, tags, click, attachment_name, attachment_type, attachment_size, attachment_expires, attachment_url, attachment_owner
FROM messages FROM messages
WHERE time <= ? AND published = 0 WHERE time <= ? AND published = 0
` `
@ -60,6 +60,7 @@ const (
selectMessagesCountQuery = `SELECT COUNT(*) FROM messages` selectMessagesCountQuery = `SELECT COUNT(*) FROM messages`
selectMessageCountForTopicQuery = `SELECT COUNT(*) FROM messages WHERE topic = ?` selectMessageCountForTopicQuery = `SELECT COUNT(*) FROM messages WHERE topic = ?`
selectTopicsQuery = `SELECT topic FROM messages GROUP BY topic` selectTopicsQuery = `SELECT topic FROM messages GROUP BY topic`
selectAttachmentsSizeQuery = `SELECT IFNULL(SUM(attachment_size), 0) FROM messages WHERE attachment_owner = ?`
) )
// Schema management queries // Schema management queries
@ -97,7 +98,7 @@ const (
ALTER TABLE messages ADD COLUMN attachment_type TEXT NOT NULL DEFAULT(''); ALTER TABLE messages ADD COLUMN attachment_type TEXT NOT NULL DEFAULT('');
ALTER TABLE messages ADD COLUMN attachment_size INT NOT NULL DEFAULT('0'); ALTER TABLE messages ADD COLUMN attachment_size INT NOT NULL DEFAULT('0');
ALTER TABLE messages ADD COLUMN attachment_expires INT NOT NULL DEFAULT('0'); ALTER TABLE messages ADD COLUMN attachment_expires INT NOT NULL DEFAULT('0');
ALTER TABLE messages ADD COLUMN attachment_preview_url TEXT NOT NULL DEFAULT(''); ALTER TABLE messages ADD COLUMN attachment_owner TEXT NOT NULL DEFAULT('');
ALTER TABLE messages ADD COLUMN attachment_url TEXT NOT NULL DEFAULT(''); ALTER TABLE messages ADD COLUMN attachment_url TEXT NOT NULL DEFAULT('');
COMMIT; COMMIT;
` `
@ -128,15 +129,15 @@ func (c *sqliteCache) AddMessage(m *message) error {
} }
published := m.Time <= time.Now().Unix() published := m.Time <= time.Now().Unix()
tags := strings.Join(m.Tags, ",") tags := strings.Join(m.Tags, ",")
var attachmentName, attachmentType, attachmentPreviewURL, attachmentURL string var attachmentName, attachmentType, attachmentURL, attachmentOwner string
var attachmentSize, attachmentExpires int64 var attachmentSize, attachmentExpires int64
if m.Attachment != nil { if m.Attachment != nil {
attachmentName = m.Attachment.Name attachmentName = m.Attachment.Name
attachmentType = m.Attachment.Type attachmentType = m.Attachment.Type
attachmentSize = m.Attachment.Size attachmentSize = m.Attachment.Size
attachmentExpires = m.Attachment.Expires attachmentExpires = m.Attachment.Expires
attachmentPreviewURL = m.Attachment.PreviewURL
attachmentURL = m.Attachment.URL attachmentURL = m.Attachment.URL
attachmentOwner = m.Attachment.Owner
} }
_, err := c.db.Exec( _, err := c.db.Exec(
insertMessageQuery, insertMessageQuery,
@ -152,8 +153,8 @@ func (c *sqliteCache) AddMessage(m *message) error {
attachmentType, attachmentType,
attachmentSize, attachmentSize,
attachmentExpires, attachmentExpires,
attachmentPreviewURL,
attachmentURL, attachmentURL,
attachmentOwner,
published, published,
) )
return err return err
@ -232,14 +233,32 @@ func (c *sqliteCache) Prune(olderThan time.Time) error {
return err return err
} }
func (c *sqliteCache) AttachmentsSize(owner string) (int64, error) {
rows, err := c.db.Query(selectAttachmentsSizeQuery, owner)
if err != nil {
return 0, err
}
defer rows.Close()
var size int64
if !rows.Next() {
return 0, errors.New("no rows found")
}
if err := rows.Scan(&size); err != nil {
return 0, err
} else if err := rows.Err(); err != nil {
return 0, err
}
return size, nil
}
func readMessages(rows *sql.Rows) ([]*message, error) { func readMessages(rows *sql.Rows) ([]*message, error) {
defer rows.Close() defer rows.Close()
messages := make([]*message, 0) messages := make([]*message, 0)
for rows.Next() { for rows.Next() {
var timestamp, attachmentSize, attachmentExpires int64 var timestamp, attachmentSize, attachmentExpires int64
var priority int var priority int
var id, topic, msg, title, tagsStr, click, attachmentName, attachmentType, attachmentPreviewURL, attachmentURL string var id, topic, msg, title, tagsStr, click, attachmentName, attachmentType, attachmentURL, attachmentOwner string
if err := rows.Scan(&id, &timestamp, &topic, &msg, &title, &priority, &tagsStr, &click, &attachmentName, &attachmentType, &attachmentSize, &attachmentExpires, &attachmentPreviewURL, &attachmentURL); err != nil { if err := rows.Scan(&id, &timestamp, &topic, &msg, &title, &priority, &tagsStr, &click, &attachmentName, &attachmentType, &attachmentSize, &attachmentExpires, &attachmentOwner, &attachmentURL); err != nil {
return nil, err return nil, err
} }
var tags []string var tags []string
@ -249,12 +268,12 @@ func readMessages(rows *sql.Rows) ([]*message, error) {
var att *attachment var att *attachment
if attachmentName != "" && attachmentURL != "" { if attachmentName != "" && attachmentURL != "" {
att = &attachment{ att = &attachment{
Name: attachmentName, Name: attachmentName,
Type: attachmentType, Type: attachmentType,
Size: attachmentSize, Size: attachmentSize,
Expires: attachmentExpires, Expires: attachmentExpires,
PreviewURL: attachmentPreviewURL, URL: attachmentURL,
URL: attachmentURL, Owner: attachmentOwner,
} }
} }
messages = append(messages, &message{ messages = append(messages, &message{

View file

@ -13,9 +13,9 @@ const (
DefaultAtSenderInterval = 10 * time.Second DefaultAtSenderInterval = 10 * time.Second
DefaultMinDelay = 10 * time.Second DefaultMinDelay = 10 * time.Second
DefaultMaxDelay = 3 * 24 * time.Hour DefaultMaxDelay = 3 * 24 * time.Hour
DefaultMessageLimit = 4096 // Bytes DefaultMessageLimit = 4096 // Bytes
DefaultAttachmentSizeLimit = int64(15 * 1024 * 1024) DefaultAttachmentTotalSizeLimit = int64(1024 * 1024 * 1024) // 1 GB
DefaultAttachmentSizePreviewMax = 20 * 1024 * 1024 // Bytes DefaultAttachmentFileSizeLimit = int64(15 * 1024 * 1024) // 15 MB
DefaultAttachmentExpiryDuration = 3 * time.Hour DefaultAttachmentExpiryDuration = 3 * time.Hour
DefaultFirebaseKeepaliveInterval = 3 * time.Hour // Not too frequently to save battery DefaultFirebaseKeepaliveInterval = 3 * time.Hour // Not too frequently to save battery
) )
@ -33,80 +33,78 @@ const (
DefaultVisitorRequestLimitReplenish = 10 * time.Second DefaultVisitorRequestLimitReplenish = 10 * time.Second
DefaultVisitorEmailLimitBurst = 16 DefaultVisitorEmailLimitBurst = 16
DefaultVisitorEmailLimitReplenish = time.Hour DefaultVisitorEmailLimitReplenish = time.Hour
DefaultVisitorAttachmentBytesLimitBurst = 50 * 1024 * 1024 DefaultVisitorAttachmentTotalSizeLimit = 50 * 1024 * 1024
DefaultVisitorAttachmentBytesLimitReplenish = time.Hour DefaultVisitorAttachmentBytesLimitReplenish = time.Hour
) )
// Config is the main config struct for the application. Use New to instantiate a default config struct. // Config is the main config struct for the application. Use New to instantiate a default config struct.
type Config struct { type Config struct {
BaseURL string BaseURL string
ListenHTTP string ListenHTTP string
ListenHTTPS string ListenHTTPS string
KeyFile string KeyFile string
CertFile string CertFile string
FirebaseKeyFile string FirebaseKeyFile string
CacheFile string CacheFile string
CacheDuration time.Duration CacheDuration time.Duration
AttachmentCacheDir string AttachmentCacheDir string
AttachmentSizeLimit int64 AttachmentTotalSizeLimit int64
AttachmentSizePreviewMax int64 AttachmentFileSizeLimit int64
AttachmentExpiryDuration time.Duration AttachmentExpiryDuration time.Duration
KeepaliveInterval time.Duration KeepaliveInterval time.Duration
ManagerInterval time.Duration ManagerInterval time.Duration
AtSenderInterval time.Duration AtSenderInterval time.Duration
FirebaseKeepaliveInterval time.Duration FirebaseKeepaliveInterval time.Duration
SMTPSenderAddr string SMTPSenderAddr string
SMTPSenderUser string SMTPSenderUser string
SMTPSenderPass string SMTPSenderPass string
SMTPSenderFrom string SMTPSenderFrom string
SMTPServerListen string SMTPServerListen string
SMTPServerDomain string SMTPServerDomain string
SMTPServerAddrPrefix string SMTPServerAddrPrefix string
MessageLimit int MessageLimit int
MinDelay time.Duration MinDelay time.Duration
MaxDelay time.Duration MaxDelay time.Duration
TotalTopicLimit int TotalTopicLimit int
TotalAttachmentSizeLimit int64 TotalAttachmentSizeLimit int64
VisitorSubscriptionLimit int VisitorSubscriptionLimit int
VisitorRequestLimitBurst int VisitorAttachmentTotalSizeLimit int64
VisitorRequestLimitReplenish time.Duration VisitorRequestLimitBurst int
VisitorEmailLimitBurst int VisitorRequestLimitReplenish time.Duration
VisitorEmailLimitReplenish time.Duration VisitorEmailLimitBurst int
VisitorAttachmentBytesLimitBurst int64 VisitorEmailLimitReplenish time.Duration
VisitorAttachmentBytesLimitReplenish time.Duration BehindProxy bool
BehindProxy bool
} }
// NewConfig instantiates a default new server config // NewConfig instantiates a default new server config
func NewConfig() *Config { func NewConfig() *Config {
return &Config{ return &Config{
BaseURL: "", BaseURL: "",
ListenHTTP: DefaultListenHTTP, ListenHTTP: DefaultListenHTTP,
ListenHTTPS: "", ListenHTTPS: "",
KeyFile: "", KeyFile: "",
CertFile: "", CertFile: "",
FirebaseKeyFile: "", FirebaseKeyFile: "",
CacheFile: "", CacheFile: "",
CacheDuration: DefaultCacheDuration, CacheDuration: DefaultCacheDuration,
AttachmentCacheDir: "", AttachmentCacheDir: "",
AttachmentSizeLimit: DefaultAttachmentSizeLimit, AttachmentTotalSizeLimit: DefaultAttachmentTotalSizeLimit,
AttachmentSizePreviewMax: DefaultAttachmentSizePreviewMax, AttachmentFileSizeLimit: DefaultAttachmentFileSizeLimit,
AttachmentExpiryDuration: DefaultAttachmentExpiryDuration, AttachmentExpiryDuration: DefaultAttachmentExpiryDuration,
KeepaliveInterval: DefaultKeepaliveInterval, KeepaliveInterval: DefaultKeepaliveInterval,
ManagerInterval: DefaultManagerInterval, ManagerInterval: DefaultManagerInterval,
MessageLimit: DefaultMessageLimit, MessageLimit: DefaultMessageLimit,
MinDelay: DefaultMinDelay, MinDelay: DefaultMinDelay,
MaxDelay: DefaultMaxDelay, MaxDelay: DefaultMaxDelay,
AtSenderInterval: DefaultAtSenderInterval, AtSenderInterval: DefaultAtSenderInterval,
FirebaseKeepaliveInterval: DefaultFirebaseKeepaliveInterval, FirebaseKeepaliveInterval: DefaultFirebaseKeepaliveInterval,
TotalTopicLimit: DefaultTotalTopicLimit, TotalTopicLimit: DefaultTotalTopicLimit,
VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit, VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit,
VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst, VisitorAttachmentTotalSizeLimit: DefaultVisitorAttachmentTotalSizeLimit,
VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish, VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst,
VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst, VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish,
VisitorEmailLimitReplenish: DefaultVisitorEmailLimitReplenish, VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst,
VisitorAttachmentBytesLimitBurst: DefaultVisitorAttachmentBytesLimitBurst, VisitorEmailLimitReplenish: DefaultVisitorEmailLimitReplenish,
VisitorAttachmentBytesLimitReplenish: DefaultVisitorAttachmentBytesLimitReplenish, BehindProxy: false,
BehindProxy: false,
} }
} }

88
server/file_cache.go Normal file
View file

@ -0,0 +1,88 @@
package server
import (
"errors"
"heckel.io/ntfy/util"
"io"
"log"
"os"
"path/filepath"
"regexp"
"sync"
)
var (
fileIDRegex = regexp.MustCompile(`^[-_A-Za-z0-9]+$`)
errInvalidFileID = errors.New("invalid file ID")
)
type fileCache struct {
dir string
totalSizeCurrent int64
totalSizeLimit int64
fileSizeLimit int64
mu sync.Mutex
}
func newFileCache(dir string, totalSizeLimit int64, fileSizeLimit int64) (*fileCache, error) {
if err := os.MkdirAll(dir, 0700); err != nil {
return nil, err
}
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
var size int64
for _, e := range entries {
info, err := e.Info()
if err != nil {
return nil, err
}
size += info.Size()
}
return &fileCache{
dir: dir,
totalSizeCurrent: size,
totalSizeLimit: totalSizeLimit,
fileSizeLimit: fileSizeLimit,
}, nil
}
func (c *fileCache) Write(id string, in io.Reader, limiters ...*util.Limiter) (int64, error) {
if !fileIDRegex.MatchString(id) {
return 0, errInvalidFileID
}
file := filepath.Join(c.dir, id)
f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return 0, err
}
defer f.Close()
log.Printf("remaining total: %d", c.remainingTotalSize())
limiters = append(limiters, util.NewLimiter(c.remainingTotalSize()), util.NewLimiter(c.fileSizeLimit))
limitWriter := util.NewLimitWriter(f, limiters...)
size, err := io.Copy(limitWriter, in)
if err != nil {
os.Remove(file)
return 0, err
}
if err := f.Close(); err != nil {
os.Remove(file)
return 0, err
}
c.mu.Lock()
c.totalSizeCurrent += size
c.mu.Unlock()
return size, nil
}
func (c *fileCache) remainingTotalSize() int64 {
c.mu.Lock()
defer c.mu.Unlock()
remaining := c.totalSizeLimit - c.totalSizeCurrent
if remaining < 0 {
return 0
}
return remaining
}

View file

@ -31,12 +31,12 @@ type message struct {
} }
type attachment struct { type attachment struct {
Name string `json:"name"` Name string `json:"name"`
Type string `json:"type,omitempty"` Type string `json:"type,omitempty"`
Size int64 `json:"size,omitempty"` Size int64 `json:"size,omitempty"`
Expires int64 `json:"expires,omitempty"` Expires int64 `json:"expires,omitempty"`
PreviewURL string `json:"preview_url,omitempty"` URL string `json:"url"`
URL string `json:"url"` Owner string `json:"-"` // IP address of uploader, used for rate limiting
} }
// messageEncoder is a function that knows how to encode a message // messageEncoder is a function that knows how to encode a message

View file

@ -9,7 +9,6 @@ import (
firebase "firebase.google.com/go" firebase "firebase.google.com/go"
"firebase.google.com/go/messaging" "firebase.google.com/go/messaging"
"fmt" "fmt"
"github.com/disintegration/imaging"
"github.com/emersion/go-smtp" "github.com/emersion/go-smtp"
"google.golang.org/api/option" "google.golang.org/api/option"
"heckel.io/ntfy/util" "heckel.io/ntfy/util"
@ -45,6 +44,7 @@ type Server struct {
mailer mailer mailer mailer
messages int64 messages int64
cache cache cache cache
fileCache *fileCache
closeChan chan bool closeChan chan bool
mu sync.Mutex mu sync.Mutex
} }
@ -101,8 +101,7 @@ var (
staticRegex = regexp.MustCompile(`^/static/.+`) staticRegex = regexp.MustCompile(`^/static/.+`)
docsRegex = regexp.MustCompile(`^/docs(|/.*)$`) docsRegex = regexp.MustCompile(`^/docs(|/.*)$`)
fileRegex = regexp.MustCompile(`^/file/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`) fileRegex = regexp.MustCompile(`^/file/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`)
previewRegex = regexp.MustCompile(`^/preview/([-_A-Za-z0-9]{1,64})(?:\.[A-Za-z0-9]{1,16})?$`) disallowedTopics = []string{"docs", "static", "file"}
disallowedTopics = []string{"docs", "static", "file", "preview"}
templateFnMap = template.FuncMap{ templateFnMap = template.FuncMap{
"durationToHuman": util.DurationToHuman, "durationToHuman": util.DurationToHuman,
@ -124,7 +123,6 @@ var (
docsStaticCached = &util.CachingEmbedFS{ModTime: time.Now(), FS: docsStaticFs} docsStaticCached = &util.CachingEmbedFS{ModTime: time.Now(), FS: docsStaticFs}
errHTTPNotFound = &errHTTP{40401, http.StatusNotFound, "page not found", ""} errHTTPNotFound = &errHTTP{40401, http.StatusNotFound, "page not found", ""}
errHTTPNotFoundTooLarge = &errHTTP{40402, http.StatusNotFound, "page not found: preview not available, file too large", ""}
errHTTPTooManyRequestsLimitRequests = &errHTTP{42901, http.StatusTooManyRequests, "limit reached: too many requests, please be nice", "https://ntfy.sh/docs/publish/#limitations"} errHTTPTooManyRequestsLimitRequests = &errHTTP{42901, http.StatusTooManyRequests, "limit reached: too many requests, please be nice", "https://ntfy.sh/docs/publish/#limitations"}
errHTTPTooManyRequestsLimitEmails = &errHTTP{42902, http.StatusTooManyRequests, "limit reached: too many emails, please be nice", "https://ntfy.sh/docs/publish/#limitations"} errHTTPTooManyRequestsLimitEmails = &errHTTP{42902, http.StatusTooManyRequests, "limit reached: too many emails, please be nice", "https://ntfy.sh/docs/publish/#limitations"}
errHTTPTooManyRequestsLimitSubscriptions = &errHTTP{42903, http.StatusTooManyRequests, "limit reached: too many active subscriptions, please be nice", "https://ntfy.sh/docs/publish/#limitations"} errHTTPTooManyRequestsLimitSubscriptions = &errHTTP{42903, http.StatusTooManyRequests, "limit reached: too many active subscriptions, please be nice", "https://ntfy.sh/docs/publish/#limitations"}
@ -174,18 +172,21 @@ func New(conf *Config) (*Server, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
var fileCache *fileCache
if conf.AttachmentCacheDir != "" { if conf.AttachmentCacheDir != "" {
if err := os.MkdirAll(conf.AttachmentCacheDir, 0700); err != nil { fileCache, err = newFileCache(conf.AttachmentCacheDir, conf.AttachmentTotalSizeLimit, conf.AttachmentFileSizeLimit)
if err != nil {
return nil, err return nil, err
} }
} }
return &Server{ return &Server{
config: conf, config: conf,
cache: cache, cache: cache,
firebase: firebaseSubscriber, fileCache: fileCache,
mailer: mailer, firebase: firebaseSubscriber,
topics: topics, mailer: mailer,
visitors: make(map[string]*visitor), topics: topics,
visitors: make(map[string]*visitor),
}, nil }, nil
} }
@ -234,7 +235,6 @@ func createFirebaseSubscriber(conf *Config) (subscriber, error) {
data["attachment_type"] = m.Attachment.Type data["attachment_type"] = m.Attachment.Type
data["attachment_size"] = fmt.Sprintf("%d", m.Attachment.Size) data["attachment_size"] = fmt.Sprintf("%d", m.Attachment.Size)
data["attachment_expires"] = fmt.Sprintf("%d", m.Attachment.Expires) data["attachment_expires"] = fmt.Sprintf("%d", m.Attachment.Expires)
data["attachment_preview_url"] = m.Attachment.PreviewURL
data["attachment_url"] = m.Attachment.URL data["attachment_url"] = m.Attachment.URL
} }
} }
@ -355,8 +355,6 @@ func (s *Server) handleInternal(w http.ResponseWriter, r *http.Request) error {
return s.handleDocs(w, r) return s.handleDocs(w, r)
} else if r.Method == http.MethodGet && fileRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" { } else if r.Method == http.MethodGet && fileRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" {
return s.withRateLimit(w, r, s.handleFile) return s.withRateLimit(w, r, s.handleFile)
} else if r.Method == http.MethodGet && previewRegex.MatchString(r.URL.Path) && s.config.AttachmentCacheDir != "" {
return s.withRateLimit(w, r, s.handlePreview)
} else if r.Method == http.MethodOptions { } else if r.Method == http.MethodOptions {
return s.handleOptions(w, r) return s.handleOptions(w, r)
} else if r.Method == http.MethodGet && topicPathRegex.MatchString(r.URL.Path) { } else if r.Method == http.MethodGet && topicPathRegex.MatchString(r.URL.Path) {
@ -436,39 +434,6 @@ func (s *Server) handleFile(w http.ResponseWriter, r *http.Request, _ *visitor)
return err return err
} }
func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request, _ *visitor) error {
if s.config.AttachmentCacheDir == "" {
return errHTTPInternalError
}
matches := previewRegex.FindStringSubmatch(r.URL.Path)
if len(matches) != 2 {
return errHTTPInternalErrorInvalidFilePath
}
messageID := matches[1]
file := filepath.Join(s.config.AttachmentCacheDir, messageID)
stat, err := os.Stat(file)
if err != nil {
return errHTTPNotFound
}
if stat.Size() > s.config.AttachmentSizePreviewMax {
return errHTTPNotFoundTooLarge
}
img, err := imaging.Open(file)
if err != nil {
return err
}
var width, height int
if width >= height {
width = 200
height = int(float32(img.Bounds().Dy()) / float32(img.Bounds().Dx()) * float32(width))
} else {
height = 200
width = int(float32(img.Bounds().Dx()) / float32(img.Bounds().Dy()) * float32(height))
}
preview := imaging.Resize(img, width, height, imaging.Lanczos)
return imaging.Encode(w, preview, imaging.JPEG, imaging.JPEGQuality(80))
}
func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error { func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visitor) error {
t, err := s.topicFromPath(r.URL.Path) t, err := s.topicFromPath(r.URL.Path)
if err != nil { if err != nil {
@ -482,7 +447,7 @@ func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visito
filename := readParam(r, "x-filename", "filename", "file", "f") filename := readParam(r, "x-filename", "filename", "file", "f")
if filename == "" && !body.LimitReached && utf8.Valid(body.PeakedBytes) { if filename == "" && !body.LimitReached && utf8.Valid(body.PeakedBytes) {
m.Message = strings.TrimSpace(string(body.PeakedBytes)) m.Message = strings.TrimSpace(string(body.PeakedBytes))
} else if s.config.AttachmentCacheDir != "" { } else if s.fileCache != nil {
if err := s.writeAttachment(r, v, m, body); err != nil { if err := s.writeAttachment(r, v, m, body); err != nil {
return err return err
} }
@ -601,48 +566,34 @@ func readParam(r *http.Request, names ...string) string {
} }
func (s *Server) writeAttachment(r *http.Request, v *visitor, m *message, body *util.PeakedReadCloser) error { func (s *Server) writeAttachment(r *http.Request, v *visitor, m *message, body *util.PeakedReadCloser) error {
if s.config.AttachmentCacheDir == "" {
return errHTTPBadRequestInvalidMessage
}
contentType := http.DetectContentType(body.PeakedBytes) contentType := http.DetectContentType(body.PeakedBytes)
ext := util.ExtensionByType(contentType) ext := util.ExtensionByType(contentType)
fileURL := fmt.Sprintf("%s/file/%s%s", s.config.BaseURL, m.ID, ext) fileURL := fmt.Sprintf("%s/file/%s%s", s.config.BaseURL, m.ID, ext)
previewURL := ""
if strings.HasPrefix(contentType, "image/") {
previewURL = fmt.Sprintf("%s/preview/%s%s", s.config.BaseURL, m.ID, ext)
}
filename := readParam(r, "x-filename", "filename", "file", "f") filename := readParam(r, "x-filename", "filename", "file", "f")
if filename == "" { if filename == "" {
filename = fmt.Sprintf("attachment%s", ext) filename = fmt.Sprintf("attachment%s", ext)
} }
file := filepath.Join(s.config.AttachmentCacheDir, m.ID) // TODO do not allowed delayed delivery for attachments
f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) visitorAttachmentsSize, err := s.cache.AttachmentsSize(v.ip)
if err != nil { if err != nil {
return err return err
} }
defer f.Close() remainingVisitorAttachmentSize := s.config.VisitorAttachmentTotalSizeLimit - visitorAttachmentsSize
maxSizeLimiter := util.NewLimiter(s.config.AttachmentSizeLimit) //FIXME visitor limit log.Printf("remaining visitor: %d", remainingVisitorAttachmentSize)
limitWriter := util.NewLimitWriter(f, maxSizeLimiter) size, err := s.fileCache.Write(m.ID, body, util.NewLimiter(remainingVisitorAttachmentSize))
size, err := io.Copy(limitWriter, body) if err == util.ErrLimitReached {
if err != nil { return errHTTPBadRequestMessageTooLarge
os.Remove(file) } else if err != nil {
if err == util.ErrLimitReached {
return errHTTPBadRequestMessageTooLarge
}
return err
}
if err := f.Close(); err != nil {
os.Remove(file)
return err return err
} }
m.Message = fmt.Sprintf("You received a file: %s", filename) // May be overwritten later m.Message = fmt.Sprintf("You received a file: %s", filename) // May be overwritten later
m.Attachment = &attachment{ m.Attachment = &attachment{
Name: filename, Name: filename,
Type: contentType, Type: contentType,
Size: size, Size: size,
Expires: time.Now().Add(s.config.AttachmentExpiryDuration).Unix(), Expires: time.Now().Add(s.config.AttachmentExpiryDuration).Unix(),
PreviewURL: previewURL, URL: fileURL,
URL: fileURL, Owner: v.ip, // Important for attachment rate limiting
} }
return nil return nil
} }