vbatts/ntfy
1
0
Fork 0
Code Pull requests Activity

Daily traffic limit

Browse source
This commit is contained in:
Philipp Heckel 2022-01-12 18:52:07 -05:00
parent c76e55a1c8
commit aa94410308
7 changed files with 170 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
cmd/serve.go
View file

@ -2,11 +2,13 @@ package cmd
import (
"errors"
"fmt"
"github.com/urfave/cli/v2"
"github.com/urfave/cli/v2/altsrc"
"heckel.io/ntfy/server"
"heckel.io/ntfy/util"
"log"
"math"
"time"
)
@ -36,6 +38,7 @@ var flagsServe = []cli.Flag{
altsrc.NewIntFlag(&cli.IntFlag{Name: "global-topic-limit", Aliases: []string{"T"}, EnvVars: []string{"NTFY_GLOBAL_TOPIC_LIMIT"}, Value: server.DefaultTotalTopicLimit, Usage: "total number of topics allowed"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-subscription-limit", EnvVars: []string{"NTFY_VISITOR_SUBSCRIPTION_LIMIT"}, Value: server.DefaultVisitorSubscriptionLimit, Usage: "number of subscriptions per visitor"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "visitor-attachment-total-size-limit", EnvVars: []string{"NTFY_VISITOR_ATTACHMENT_TOTAL_SIZE_LIMIT"}, Value: "50M", Usage: "total storage limit used for attachments per visitor"}),
altsrc.NewStringFlag(&cli.StringFlag{Name: "visitor-attachment-daily-traffic-limit", EnvVars: []string{"NTFY_VISITOR_ATTACHMENT_DAILY_TRAFFIC_LIMIT"}, Value: "500M", Usage: "total daily attachment download/upload traffic limit per visitor"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-request-limit-burst", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_BURST"}, Value: server.DefaultVisitorRequestLimitBurst, Usage: "initial limit of requests per visitor"}),
altsrc.NewDurationFlag(&cli.DurationFlag{Name: "visitor-request-limit-replenish", EnvVars: []string{"NTFY_VISITOR_REQUEST_LIMIT_REPLENISH"}, Value: server.DefaultVisitorRequestLimitReplenish, Usage: "interval at which burst limit is replenished (one per x)"}),
altsrc.NewIntFlag(&cli.IntFlag{Name: "visitor-email-limit-burst", EnvVars: []string{"NTFY_VISITOR_EMAIL_LIMIT_BURST"}, Value: server.DefaultVisitorEmailLimitBurst, Usage: "initial limit of e-mails per visitor"}),
@ -90,6 +93,7 @@ func execServe(c *cli.Context) error {
totalTopicLimit := c.Int("global-topic-limit")
visitorSubscriptionLimit := c.Int("visitor-subscription-limit")
visitorAttachmentTotalSizeLimitStr := c.String("visitor-attachment-total-size-limit")
visitorAttachmentDailyTrafficLimitStr := c.String("visitor-attachment-daily-traffic-limit")
visitorRequestLimitBurst := c.Int("visitor-request-limit-burst")
visitorRequestLimitReplenish := c.Duration("visitor-request-limit-replenish")
visitorEmailLimitBurst := c.Int("visitor-email-limit-burst")
@ -130,6 +134,12 @@ func execServe(c *cli.Context) error {
if err != nil {
return err
}
visitorAttachmentDailyTrafficLimit, err := parseSize(visitorAttachmentDailyTrafficLimitStr, server.DefaultVisitorAttachmentDailyTrafficLimit)
if err != nil {
return err
} else if visitorAttachmentDailyTrafficLimit > math.MaxInt {
return fmt.Errorf("config option visitor-attachment-daily-traffic-limit must be lower than %d", math.MaxInt)
}
// Run server
conf := server.NewConfig()
@ -157,6 +167,7 @@ func execServe(c *cli.Context) error {
conf.TotalTopicLimit = totalTopicLimit
conf.VisitorSubscriptionLimit = visitorSubscriptionLimit
conf.VisitorAttachmentTotalSizeLimit = visitorAttachmentTotalSizeLimit
conf.VisitorAttachmentDailyTrafficLimit = int(visitorAttachmentDailyTrafficLimit)
conf.VisitorRequestLimitBurst = visitorRequestLimitBurst
conf.VisitorRequestLimitReplenish = visitorRequestLimitReplenish
conf.VisitorEmailLimitBurst = visitorEmailLimitBurst

12
docs/config.md
View file

@ -274,7 +274,7 @@ firebase-key-file: "/etc/ntfy/ntfy-sh-firebase-adminsdk-ahnce-9f4d6f14b5.json"
By default, ntfy runs without authentication, so it is vitally important that we protect the server from abuse or overload.
There are various limits and rate limits in place that you can use to configure the server. Let's do the easy ones first:
* `global-topic-limit` defines the total number of topics before the server rejects new topics. It defaults to 5000.
* `global-topic-limit` defines the total number of topics before the server rejects new topics. It defaults to 15,000.
* `visitor-subscription-limit` is the number of subscriptions (open connections) per visitor. This value defaults to 30.
A **visitor** is identified by its IP address (or the `X-Forwarded-For` header if `behind-proxy` is set). All config
@ -309,7 +309,7 @@ Depending on *how you run it*, here are a few limits that are relevant:
### For systemd services
If you're running ntfy in a systemd service (e.g. for .deb/.rpm packages), the main limiting factor is the
`LimitNOFILE` setting in the systemd unit. The default open files limit for `ntfy.service` is 10000. You can override it
`LimitNOFILE` setting in the systemd unit. The default open files limit for `ntfy.service` is 10,000. You can override it
by creating a `/etc/systemd/system/ntfy.service.d/override.conf` file. As far as I can tell, `/etc/security/limits.conf`
is not relevant.
@ -322,7 +322,7 @@ is not relevant.
### Outside of systemd
If you're running outside systemd, you may want to adjust your `/etc/security/limits.conf` file to
increase the `nofile` setting. Here's an example that increases the limit to 5000. You can find out the current setting
increase the `nofile` setting. Here's an example that increases the limit to 5,000. You can find out the current setting
by running `ulimit -n`, or manually override it temporarily by running `ulimit -n 50000`.
=== "/etc/security/limits.conf"
@ -423,12 +423,16 @@ variable before running the `ntfy` command (e.g. `export NTFY_LISTEN_HTTP=:80`).
| `smtp-server-addr-prefix` | `NTFY_SMTP_SERVER_ADDR_PREFIX` | `[ip]:port` | - | Optional prefix for the e-mail addresses to prevent spam, e.g. `ntfy-` |
| `keepalive-interval` | `NTFY_KEEPALIVE_INTERVAL` | *duration* | 55s | Interval in which keepalive messages are sent to the client. This is to prevent intermediaries closing the connection for inactivity. Note that the Android app has a hardcoded timeout at 77s, so it should be less than that. |
| `manager-interval` | `$NTFY_MANAGER_INTERVAL` | *duration* | 1m | Interval in which the manager prunes old messages, deletes topics and prints the stats. |
| `global-topic-limit` | `NTFY_GLOBAL_TOPIC_LIMIT` | *number* | 5000 | Rate limiting: Total number of topics before the server rejects new topics. |
| `global-topic-limit` | `NTFY_GLOBAL_TOPIC_LIMIT` | *number* | 15,000 | Rate limiting: Total number of topics before the server rejects new topics. |
| `visitor-subscription-limit` | `NTFY_VISITOR_SUBSCRIPTION_LIMIT` | *number* | 30 | Rate limiting: Number of subscriptions per visitor (IP address) |
| `visitor-request-limit-burst` | `NTFY_VISITOR_REQUEST_LIMIT_BURST` | *number* | 60 | Allowed GET/PUT/POST requests per second, per visitor. This setting is the initial bucket of requests each visitor has |
| `visitor-request-limit-replenish` | `NTFY_VISITOR_REQUEST_LIMIT_REPLENISH` | *duration* | 10s | Strongly related to `visitor-request-limit-burst`: The rate at which the bucket is refilled |
| `visitor-email-limit-burst` | `NTFY_VISITOR_EMAIL_LIMIT_BURST` | *number* | 16 |Initial limit of e-mails per visitor |
| `visitor-email-limit-replenish` | `NTFY_VISITOR_EMAIL_LIMIT_REPLENISH` | *duration* | 1h | Strongly related to `visitor-email-limit-burst`: The rate at which the bucket is refilled |
xx daily traffic limit
xx DefaultVisitorAttachmentTotalSizeLimit
xx attachment cache dir
xx attachment
The format for a *duration* is: `<number>(smh)`, e.g. 30s, 20m or 1h.

30
server/config.go
View file

@ -4,7 +4,7 @@ import (
"time"
)
// Defines default config settings
// Defines default config settings (excluding limits, see below)
const (
DefaultListenHTTP = ":80"
DefaultCacheDuration = 12 * time.Hour
@ -13,27 +13,35 @@ const (
DefaultAtSenderInterval = 10 * time.Second
DefaultMinDelay = 10 * time.Second
DefaultMaxDelay = 3 * 24 * time.Hour
DefaultMessageLimit = 4096 // Bytes
DefaultAttachmentTotalSizeLimit = int64(1024 * 1024 * 1024) // 1 GB
DefaultAttachmentFileSizeLimit = int64(15 * 1024 * 1024) // 15 MB
DefaultAttachmentExpiryDuration = 3 * time.Hour
DefaultFirebaseKeepaliveInterval = 3 * time.Hour // Not too frequently to save battery
)
// Defines all the limits
// Defines all global and per-visitor limits
// - message size limit: the max number of bytes for a message
// - total topic limit: max number of topics overall
// - various attachment limits
const (
DefaultMessageLengthLimit = 4096 // Bytes
DefaultTotalTopicLimit = 15000
DefaultAttachmentTotalSizeLimit = int64(10 * 1024 * 1024 * 1024) // 10 GB
DefaultAttachmentFileSizeLimit = int64(15 * 1024 * 1024) // 15 MB
DefaultAttachmentExpiryDuration = 3 * time.Hour
)
// Defines all per-visitor limits
// - per visitor subscription limit: max number of subscriptions (active HTTP connections) per per-visitor/IP
// - per visitor request limit: max number of PUT/GET/.. requests (here: 60 requests bucket, replenished at a rate of one per 10 seconds)
// - per visitor email limit: max number of emails (here: 16 email bucket, replenished at a rate of one per hour)
// - per visitor attachment size limit:
// - per visitor attachment size limit: total per-visitor attachment size in bytes to be stored on the server
// - per visitor attachment daily traffic limit: number of bytes that can be transferred to/from the server
const (
DefaultTotalTopicLimit = 5000
DefaultVisitorSubscriptionLimit = 30
DefaultVisitorRequestLimitBurst = 60
DefaultVisitorRequestLimitReplenish = 10 * time.Second
DefaultVisitorEmailLimitBurst = 16
DefaultVisitorEmailLimitReplenish = time.Hour
DefaultVisitorAttachmentTotalSizeLimit = 50 * 1024 * 1024
DefaultVisitorAttachmentTotalSizeLimit = 100 * 1024 * 1024 // 100 MB
DefaultVisitorAttachmentDailyTrafficLimit = 500 * 1024 * 1024 // 500 MB
)
// Config is the main config struct for the application. Use New to instantiate a default config struct.
@ -68,6 +76,7 @@ type Config struct {
TotalAttachmentSizeLimit int64
VisitorSubscriptionLimit int
VisitorAttachmentTotalSizeLimit int64
VisitorAttachmentDailyTrafficLimit int
VisitorRequestLimitBurst int
VisitorRequestLimitReplenish time.Duration
VisitorEmailLimitBurst int
@ -92,7 +101,7 @@ func NewConfig() *Config {
AttachmentExpiryDuration: DefaultAttachmentExpiryDuration,
KeepaliveInterval: DefaultKeepaliveInterval,
ManagerInterval: DefaultManagerInterval,
MessageLimit: DefaultMessageLimit,
MessageLimit: DefaultMessageLengthLimit,
MinDelay: DefaultMinDelay,
MaxDelay: DefaultMaxDelay,
AtSenderInterval: DefaultAtSenderInterval,
@ -100,6 +109,7 @@ func NewConfig() *Config {
TotalTopicLimit: DefaultTotalTopicLimit,
VisitorSubscriptionLimit: DefaultVisitorSubscriptionLimit,
VisitorAttachmentTotalSizeLimit: DefaultVisitorAttachmentTotalSizeLimit,
VisitorAttachmentDailyTrafficLimit: DefaultVisitorAttachmentDailyTrafficLimit,
VisitorRequestLimitBurst: DefaultVisitorRequestLimitBurst,
VisitorRequestLimitReplenish: DefaultVisitorRequestLimitReplenish,
VisitorEmailLimitBurst: DefaultVisitorEmailLimitBurst,

12
server/server.go
View file

@ -139,12 +139,13 @@ var (
errHTTPBadRequestTopicInvalid = &errHTTP{40009, http.StatusBadRequest, "invalid topic: path invalid", ""}
errHTTPBadRequestTopicDisallowed = &errHTTP{40010, http.StatusBadRequest, "invalid topic: topic name is disallowed", ""}
errHTTPBadRequestMessageNotUTF8 = &errHTTP{40011, http.StatusBadRequest, "invalid message: message must be UTF-8 encoded", ""}
errHTTPBadRequestAttachmentTooLarge = &errHTTP{40012, http.StatusBadRequest, "invalid request: attachment too large", ""}
errHTTPBadRequestAttachmentTooLarge = &errHTTP{40012, http.StatusBadRequest, "invalid request: attachment too large, or traffic limit reached", ""}
errHTTPBadRequestAttachmentURLInvalid = &errHTTP{40013, http.StatusBadRequest, "invalid request: attachment URL is invalid", ""}
errHTTPBadRequestAttachmentURLPeakGeneral = &errHTTP{40014, http.StatusBadRequest, "invalid request: attachment URL peak failed", ""}
errHTTPBadRequestAttachmentURLPeakNon2xx = &errHTTP{40015, http.StatusBadRequest, "invalid request: attachment URL peak failed with non-2xx status code", ""}
errHTTPBadRequestAttachmentsDisallowed = &errHTTP{40016, http.StatusBadRequest, "invalid request: attachments not allowed", ""}
errHTTPBadRequestAttachmentsExpiryBeforeDelivery = &errHTTP{40017, http.StatusBadRequest, "invalid request: attachment expiry before delayed delivery date", ""}
errHTTPTooManyRequestsAttachmentTrafficLimit = &errHTTP{42901, http.StatusTooManyRequests, "too many requests: daily traffic limit reached", "https://ntfy.sh/docs/publish/#limitations"}
errHTTPInternalError = &errHTTP{50001, http.StatusInternalServerError, "internal server error", ""}
errHTTPInternalErrorInvalidFilePath = &errHTTP{50002, http.StatusInternalServerError, "internal server error: invalid file path", ""}
)
@ -341,7 +342,7 @@ func (s *Server) handle(w http.ResponseWriter, r *http.Request) {
if e, ok = err.(*errHTTP); !ok {
e = errHTTPInternalError
}
log.Printf("[%s] %s - %d - %s", r.RemoteAddr, r.Method, e.HTTPCode, err.Error())
log.Printf("[%s] %s - %d - %d - %s", r.RemoteAddr, r.Method, e.HTTPCode, e.Code, err.Error())
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Access-Control-Allow-Origin", "*") // CORS, allow cross-origin requests
w.WriteHeader(e.HTTPCode)
@ -417,7 +418,7 @@ func (s *Server) handleDocs(w http.ResponseWriter, r *http.Request) error {
return nil
}
func (s *Server) handleFile(w http.ResponseWriter, r *http.Request, _ *visitor) error {
func (s *Server) handleFile(w http.ResponseWriter, r *http.Request, v *visitor) error {
if s.config.AttachmentCacheDir == "" {
return errHTTPInternalError
}
@ -431,6 +432,9 @@ func (s *Server) handleFile(w http.ResponseWriter, r *http.Request, _ *visitor)
if err != nil {
return errHTTPNotFound
}
if err := v.TrafficLimiter().Allow(stat.Size()); err != nil {
return errHTTPTooManyRequestsAttachmentTrafficLimit
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
f, err := os.Open(file)
if err != nil {
@ -648,7 +652,7 @@ func (s *Server) handleBodyAsAttachment(r *http.Request, v *visitor, m *message,
if m.Message == "" {
m.Message = fmt.Sprintf(defaultAttachmentMessage, m.Attachment.Name)
}
m.Attachment.Size, err = s.fileCache.Write(m.ID, body, util.NewFixedLimiter(remainingVisitorAttachmentSize))
m.Attachment.Size, err = s.fileCache.Write(m.ID, body, v.TrafficLimiter(), util.NewFixedLimiter(remainingVisitorAttachmentSize))
if err == util.ErrLimitReached {
return errHTTPBadRequestAttachmentTooLarge
} else if err != nil {

2
server/server.yml
View file

@ -87,7 +87,7 @@
# Rate limiting: Total number of topics before the server rejects new topics.
#
# global-topic-limit: 5000
# global-topic-limit: 15000
# Rate limiting: Number of subscriptions per visitor (IP address)
#

49
server/server_test.go
View file

@ -826,7 +826,6 @@ func TestServer_PublishAttachmentAndPrune(t *testing.T) {
// Publish and make sure we can retrieve it
response := request(t, s, "PUT", "/mytopic", content, nil)
println(response.Body.String())
msg := toMessage(t, response.Body.String())
require.Contains(t, msg.Attachment.URL, "http://127.0.0.1:12345/file/")
file := filepath.Join(s.config.AttachmentCacheDir, msg.ID)
@ -845,6 +844,54 @@ func TestServer_PublishAttachmentAndPrune(t *testing.T) {
require.Equal(t, 404, response.Code)
}
func TestServer_PublishAttachmentTrafficLimit(t *testing.T) {
content := util.RandomString(5000) // > 4096
c := newTestConfig(t)
c.VisitorAttachmentDailyTrafficLimit = 5*5000 + 123 // A little more than 1 upload and 3 downloads
s := newTestServer(t, c)
// Publish attachment
response := request(t, s, "PUT", "/mytopic", content, nil)
msg := toMessage(t, response.Body.String())
require.Contains(t, msg.Attachment.URL, "http://127.0.0.1:12345/file/")
// Get it 4 times successfully
path := strings.TrimPrefix(msg.Attachment.URL, "http://127.0.0.1:12345")
for i := 1; i <= 4; i++ { // 4 successful downloads
response = request(t, s, "GET", path, "", nil)
require.Equal(t, 200, response.Code)
require.Equal(t, content, response.Body.String())
}
// And then fail with a 429
response = request(t, s, "GET", path, "", nil)
err := toHTTPError(t, response.Body.String())
require.Equal(t, 429, response.Code)
require.Equal(t, 42901, err.Code)
}
func TestServer_PublishAttachmentTrafficLimitUploadOnly(t *testing.T) {
content := util.RandomString(5000) // > 4096
c := newTestConfig(t)
c.VisitorAttachmentDailyTrafficLimit = 5*5000 + 500 // 5 successful uploads
s := newTestServer(t, c)
// 5 successful uploads
for i := 1; i <= 5; i++ {
response := request(t, s, "PUT", "/mytopic", content, nil)
msg := toMessage(t, response.Body.String())
require.Contains(t, msg.Attachment.URL, "http://127.0.0.1:12345/file/")
}
// And a failed one
response := request(t, s, "PUT", "/mytopic", content, nil)
err := toHTTPError(t, response.Body.String())
require.Equal(t, 400, response.Code)
require.Equal(t, 40012, err.Code)
}
func newTestConfig(t *testing.T) *Config {
conf := NewConfig()
conf.BaseURL = "http://127.0.0.1:12345"

10
server/visitor.go
View file

@ -24,8 +24,9 @@ type visitor struct {
config *Config
ip string
requests *rate.Limiter
subscriptions util.Limiter
emails *rate.Limiter
subscriptions util.Limiter
traffic util.Limiter
seen time.Time
mu sync.Mutex
}
@ -35,8 +36,9 @@ func newVisitor(conf *Config, ip string) *visitor {
config: conf,
ip: ip,
requests: rate.NewLimiter(rate.Every(conf.VisitorRequestLimitReplenish), conf.VisitorRequestLimitBurst),
subscriptions: util.NewFixedLimiter(int64(conf.VisitorSubscriptionLimit)),
emails: rate.NewLimiter(rate.Every(conf.VisitorEmailLimitReplenish), conf.VisitorEmailLimitBurst),
subscriptions: util.NewFixedLimiter(int64(conf.VisitorSubscriptionLimit)),
traffic: util.NewBytesLimiter(conf.VisitorAttachmentDailyTrafficLimit, 24*time.Hour),
seen: time.Now(),
}
}
@ -80,6 +82,10 @@ func (v *visitor) Keepalive() {
v.seen = time.Now()
}
func (v *visitor) TrafficLimiter() util.Limiter {
return v.traffic
}
func (v *visitor) Stale() bool {
v.mu.Lock()
defer v.mu.Unlock()