vbatts/ntfy
1
0
Fork 0
Code Pull requests Activity

Polish the poll_request stuff

Browse source
This commit is contained in:
Philipp Heckel 2022-05-27 20:30:20 -04:00
parent 6a43c1a126
commit 96bb357435
9 changed files with 328 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

2
server/config.go
View file

@ -69,7 +69,7 @@ type Config struct {
AtSenderInterval time.Duration
FirebaseKeepaliveInterval time.Duration
FirebasePollInterval time.Duration
ForwardPollURL string
UpstreamBaseURL string
SMTPSenderAddr string
SMTPSenderUser string
SMTPSenderPass string

95
server/server.go
View file

@ -440,40 +440,13 @@ func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visito
}
}
if s.firebase != nil && firebase && !delayed {
go func() {
if err := s.firebase(m); err != nil {
log.Printf("[%s] FB - Unable to publish to Firebase: %v", v.ip, err.Error())
}
}()
go s.sendToFirebase(v, m)
}
if s.mailer != nil && email != "" && !delayed {
go func() {
if err := s.mailer.Send(v.ip, email, m); err != nil {
log.Printf("[%s] MAIL - Unable to send email: %v", v.ip, err.Error())
}
}()
go s.sendEmail(v, m, email)
}
if s.config.ForwardPollURL != "" {
go func() {
topicURL := fmt.Sprintf("%s/%s", s.config.BaseURL, m.Topic)
topicHash := fmt.Sprintf("%x", sha256.Sum256([]byte(topicURL)))
forwardURL := fmt.Sprintf("%s/%s", s.config.ForwardPollURL, topicHash)
log.Printf("forwarding: topicURL %s, to upstream url %s", topicURL, forwardURL)
req, err := http.NewRequest("POST", forwardURL, strings.NewReader(""))
if err != nil {
log.Printf("[%s] FWD - Unable to forward poll request: %v", v.ip, err.Error())
return
}
req.Header.Set("X-Poll-ID", m.ID)
response, err := http.DefaultClient.Do(req)
if err != nil {
log.Printf("[%s] FWD - Unable to forward poll request: %v", v.ip, err.Error())
return
} else if response.StatusCode != http.StatusOK {
log.Printf("[%s] FWD - Unable to forward poll request, unexpected status: %d", v.ip, response.StatusCode)
return
}
}()
if s.config.UpstreamBaseURL != "" {
go s.forwardPollRequest(v, m)
}
if cache {
if err := s.messageCache.AddMessage(m); err != nil {
@ -491,6 +464,38 @@ func (s *Server) handlePublish(w http.ResponseWriter, r *http.Request, v *visito
return nil
}
func (s *Server) sendToFirebase(v *visitor, m *message) {
if err := s.firebase(m); err != nil {
log.Printf("[%s] FB - Unable to publish to Firebase: %v", v.ip, err.Error())
}
}
func (s *Server) sendEmail(v *visitor, m *message, email string) {
if err := s.mailer.Send(v.ip, email, m); err != nil {
log.Printf("[%s] MAIL - Unable to send email: %v", v.ip, err.Error())
}
}
func (s *Server) forwardPollRequest(v *visitor, m *message) {
topicURL := fmt.Sprintf("%s/%s", s.config.BaseURL, m.Topic)
topicHash := fmt.Sprintf("%x", sha256.Sum256([]byte(topicURL)))
forwardURL := fmt.Sprintf("%s/%s", s.config.UpstreamBaseURL, topicHash)
req, err := http.NewRequest("POST", forwardURL, strings.NewReader(""))
if err != nil {
log.Printf("[%s] FWD - Unable to forward poll request: %v", v.ip, err.Error())
return
}
req.Header.Set("X-Poll-ID", m.ID)
response, err := http.DefaultClient.Do(req)
if err != nil {
log.Printf("[%s] FWD - Unable to forward poll request: %v", v.ip, err.Error())
return
} else if response.StatusCode != http.StatusOK {
log.Printf("[%s] FWD - Unable to forward poll request, unexpected status: %d", v.ip, response.StatusCode)
return
}
}
func (s *Server) parsePublishParams(r *http.Request, v *visitor, m *message) (cache bool, firebase bool, email string, unifiedpush bool, err error) {
cache = readBoolParam(r, true, "x-cache", "cache")
firebase = readBoolParam(r, true, "x-firebase", "firebase")
@ -587,29 +592,31 @@ func (s *Server) parsePublishParams(r *http.Request, v *visitor, m *message) (ca
// handlePublishBody consumes the PUT/POST body and decides whether the body is an attachment or the message.
//
// 1. curl -T somebinarydata.bin "ntfy.sh/mytopic?up=1"
// 1. curl -X POST -H "Poll: 1234" ntfy.sh/...
// If a message is flagged as poll request, the body does not matter and is discarded
// 2. curl -T somebinarydata.bin "ntfy.sh/mytopic?up=1"
// If body is binary, encode as base64, if not do not encode
// 2. curl -H "Attach: http://example.com/file.jpg" ntfy.sh/mytopic
// 3. curl -H "Attach: http://example.com/file.jpg" ntfy.sh/mytopic
// Body must be a message, because we attached an external URL
// 3. curl -T short.txt -H "Filename: short.txt" ntfy.sh/mytopic
// 4. curl -T short.txt -H "Filename: short.txt" ntfy.sh/mytopic
// Body must be attachment, because we passed a filename
// 4. curl -T file.txt ntfy.sh/mytopic
// If file.txt is <= 4096 (message limit) and valid UTF-8, treat it as a message
// 5. curl -T file.txt ntfy.sh/mytopic
// If file.txt is <= 4096 (message limit) and valid UTF-8, treat it as a message
// 6. curl -T file.txt ntfy.sh/mytopic
// If file.txt is > message limit, treat it as an attachment
func (s *Server) handlePublishBody(r *http.Request, v *visitor, m *message, body *util.PeekedReadCloser, unifiedpush bool) error {
if m.Event == pollRequestEvent {
return nil // Ignore body
if m.Event == pollRequestEvent { // Case 1
return nil
} else if unifiedpush {
return s.handleBodyAsMessageAutoDetect(m, body) // Case 1
return s.handleBodyAsMessageAutoDetect(m, body) // Case 2
} else if m.Attachment != nil && m.Attachment.URL != "" {
return s.handleBodyAsTextMessage(m, body) // Case 2
return s.handleBodyAsTextMessage(m, body) // Case 3
} else if m.Attachment != nil && m.Attachment.Name != "" {
return s.handleBodyAsAttachment(r, v, m, body) // Case 3
return s.handleBodyAsAttachment(r, v, m, body) // Case 4
} else if !body.LimitReached && utf8.Valid(body.PeekedBytes) {
return s.handleBodyAsTextMessage(m, body) // Case 4
return s.handleBodyAsTextMessage(m, body) // Case 5
}
return s.handleBodyAsAttachment(r, v, m, body) // Case 5
return s.handleBodyAsAttachment(r, v, m, body) // Case 6
}
func (s *Server) handleBodyAsMessageAutoDetect(m *message, body *util.PeekedReadCloser) error {
@ -745,7 +752,6 @@ func (s *Server) handleSubscribeHTTP(w http.ResponseWriter, r *http.Request, v *
w.Header().Set("Access-Control-Allow-Origin", "*") // CORS, allow cross-origin requests
w.Header().Set("Content-Type", contentType+"; charset=utf-8") // Android/Volley client needs charset!
if poll {
log.Printf("polling %#v", r.URL)
return s.sendOldMessages(topics, since, scheduled, sub)
}
subscriberIDs := make([]int, 0)
@ -1114,7 +1120,6 @@ func (s *Server) runFirebaseKeepaliver() {
log.Printf("error sending Firebase keepalive message to %s: %s", firebaseControlTopic, err.Error())
}
case <-time.After(s.config.FirebasePollInterval):
log.Printf("Sending to timer topic %s", firebasePollTopic)
if err := s.firebase(newKeepaliveMessage(firebasePollTopic)); err != nil {
log.Printf("error sending Firebase keepalive message to %s: %s", firebasePollTopic, err.Error())
}

12
server/server.yml
View file

@ -135,6 +135,18 @@
#
# web-root: app
# Server URL of a Firebase/APNS-connected ntfy server (likely "https://ntfy.sh").
#
# iOS users:
# If you use the iOS ntfy app, you MUST configure this to receive timely notifications. You'll like want this:
# upstream-base-url: "https://ntfy.sh"
#
# If set, all incoming messages will publish a "poll_request" message to the configured upstream server, containing
# the message ID of the original message, instructing the iOS app to poll this server for the actual message contents.
# This is to prevent the upstream server and Firebase/APNS from being able to read the message.
#
# upstream-base-url:
# Rate limiting: Total number of topics before the server rejects new topics.
#
# global-topic-limit: 15000

188
server/server_firebase.go
View file

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
firebase "firebase.google.com/go"
@ -18,39 +17,6 @@ const (
fcmApnsBodyMessageLimit = 100
)
// maybeTruncateFCMMessage performs best-effort truncation of FCM messages.
// The docs say the limit is 4000 characters, but during testing it wasn't quite clear
// what fields matter; so we're just capping the serialized JSON to 4000 bytes.
func maybeTruncateFCMMessage(m *messaging.Message) *messaging.Message {
s, err := json.Marshal(m)
if err != nil {
return m
}
if len(s) > fcmMessageLimit {
over := len(s) - fcmMessageLimit + 16 // = len("truncated":"1",), sigh ...
message, ok := m.Data["message"]
if ok && len(message) > over {
m.Data["truncated"] = "1"
m.Data["message"] = message[:len(message)-over]
}
}
return m
}
// maybeTruncateAPNSBodyMessage truncates the body for APNS.
//
// The "body" of the push notification can contain the entire message, which would count doubly for the overall length
// of the APNS payload. I set a limit of 100 characters before truncating the notification "body" with ellipsis.
// The message would not be changed (unless truncated for being too long). Note: if the payload is too large (>4KB),
// APNS will simply reject / discard the notification, meaning it will never arrive on the iOS device.
func maybeTruncateAPNSBodyMessage(s string) string {
if len(s) >= fcmApnsBodyMessageLimit {
over := len(s) - fcmApnsBodyMessageLimit + 3 // len("...")
return s[:len(s)-over] + "..."
}
return s
}
func createFirebaseSubscriber(credentialsFile string, auther auth.Auther) (subscriber, error) {
fb, err := firebase.NewApp(context.Background(), nil, option.WithCredentialsFile(credentialsFile))
if err != nil {
@ -65,12 +31,31 @@ func createFirebaseSubscriber(credentialsFile string, auther auth.Auther) (subsc
if err != nil {
return err
}
log.Printf("Sending %#v %#v", m, fbm)
_, err = msg.Send(context.Background(), fbm)
return err
}, nil
}
// toFirebaseMessage converts a message to a Firebase message.
//
// Normal messages ("message"):
// - For Android, we can receive data messages from Firebase and process them as code, so we just send all fields
// in the "data" attribute. In the Android app, we then turn those into a notification and display it.
// - On iOS, we are not allowed to receive data-only messages, so we build messages with an "alert" (with title and
// message), and still send the rest of the data along in the "aps" attribute. We can then locally modify the
// message in the Notification Service Extension.
//
// Keepalive messages ("keepalive"):
// - On Android, we subscribe to the "~control" topic, which is used to restart the foreground service (if it died,
// e.g. after an app update). We send these keepalive messages regularly (see Config.FirebaseKeepaliveInterval).
// - On iOS, we subscribe to the "~poll" topic, which is used to poll all topics regularly. This is because iOS
// does not allow any background or scheduled activity at all.
//
// Poll request messages ("poll_request"):
// - Normal messages are turned into poll request messages if anonymous users are not allowed to read the message.
// On Android, this will trigger the app to poll the topic and thereby displaying new messages.
// - If UpstreamBaseURL is set, messages are forwarded as poll requests to an upstream server and then forwarded
// to Firebase here. This is mainly for iOS to support self-hosted servers.
func toFirebaseMessage(m *message, auther auth.Auther) (*messaging.Message, error) {
var data map[string]string // Mostly matches https://ntfy.sh/docs/subscribe/api/#json-message-format
var apnsConfig *messaging.APNSConfig
@ -82,24 +67,7 @@ func toFirebaseMessage(m *message, auther auth.Auther) (*messaging.Message, erro
"event": m.Event,
"topic": m.Topic,
}
// Silent notification; only 2-3 per hour are allowed; delivery not guaranteed
// See https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/pushing_background_updates_to_your_app
apnsData := make(map[string]interface{})
for k, v := range data {
apnsData[k] = v
}
apnsConfig = &messaging.APNSConfig{
Headers: map[string]string{
"apns-push-type": "background",
"apns-priority": "5",
},
Payload: &messaging.APNSPayload{
Aps: &messaging.Aps{
ContentAvailable: true,
},
CustomData: apnsData,
},
}
apnsConfig = createAPNSBackgroundConfig(data)
case pollRequestEvent:
data = map[string]string{
"id": m.ID,
@ -109,22 +77,7 @@ func toFirebaseMessage(m *message, auther auth.Auther) (*messaging.Message, erro
"message": m.Message,
"poll_id": m.PollID,
}
apnsData := make(map[string]interface{})
for k, v := range data {
apnsData[k] = v
}
apnsConfig = &messaging.APNSConfig{
Payload: &messaging.APNSPayload{
CustomData: apnsData,
Aps: &messaging.Aps{
MutableContent: true,
Alert: &messaging.ApsAlert{
Title: m.Title,
Body: maybeTruncateAPNSBodyMessage(m.Message),
},
},
},
}
apnsConfig = createAPNSAlertConfig(m, data)
case messageEvent:
allowForward := true
if auther != nil {
@ -157,22 +110,7 @@ func toFirebaseMessage(m *message, auther auth.Auther) (*messaging.Message, erro
data["attachment_expires"] = fmt.Sprintf("%d", m.Attachment.Expires)
data["attachment_url"] = m.Attachment.URL
}
apnsData := make(map[string]interface{})
for k, v := range data {
apnsData[k] = v
}
apnsConfig = &messaging.APNSConfig{
Payload: &messaging.APNSPayload{
CustomData: apnsData,
Aps: &messaging.Aps{
MutableContent: true,
Alert: &messaging.ApsAlert{
Title: m.Title,
Body: maybeTruncateAPNSBodyMessage(m.Message),
},
},
},
}
apnsConfig = createAPNSAlertConfig(m, data)
} else {
// If anonymous read for a topic is not allowed, we cannot send the message along
// via Firebase. Instead, we send a "poll_request" message, asking the client to poll.
@ -182,6 +120,7 @@ func toFirebaseMessage(m *message, auther auth.Auther) (*messaging.Message, erro
"event": pollRequestEvent,
"topic": m.Topic,
}
// TODO Handle APNS?
}
}
var androidConfig *messaging.AndroidConfig
@ -197,3 +136,82 @@ func toFirebaseMessage(m *message, auther auth.Auther) (*messaging.Message, erro
APNS: apnsConfig,
}), nil
}
// maybeTruncateFCMMessage performs best-effort truncation of FCM messages.
// The docs say the limit is 4000 characters, but during testing it wasn't quite clear
// what fields matter; so we're just capping the serialized JSON to 4000 bytes.
func maybeTruncateFCMMessage(m *messaging.Message) *messaging.Message {
s, err := json.Marshal(m)
if err != nil {
return m
}
if len(s) > fcmMessageLimit {
over := len(s) - fcmMessageLimit + 16 // = len("truncated":"1",), sigh ...
message, ok := m.Data["message"]
if ok && len(message) > over {
m.Data["truncated"] = "1"
m.Data["message"] = message[:len(message)-over]
}
}
return m
}
// createAPNSAlertConfig creates an APNS config for iOS notifications that show up as an alert (only relevant for iOS).
// We must set the Alert struct ("alert"), and we need to set MutableContent ("mutable-content"), so the Notification Service
// Extension in iOS can modify the message.
func createAPNSAlertConfig(m *message, data map[string]string) *messaging.APNSConfig {
apnsData := make(map[string]interface{})
for k, v := range data {
apnsData[k] = v
}
return &messaging.APNSConfig{
Payload: &messaging.APNSPayload{
CustomData: apnsData,
Aps: &messaging.Aps{
MutableContent: true,
Alert: &messaging.ApsAlert{
Title: m.Title,
Body: maybeTruncateAPNSBodyMessage(m.Message),
},
},
},
}
}
// createAPNSBackgroundConfig creates an APNS config for a silent background message (only relevant for iOS). Apple only
// allows us to send 2-3 of these notifications per hour, and delivery not guaranteed. We use this only for the ~poll
// topic, which triggers the iOS app to poll all topics for changes.
//
// See https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/pushing_background_updates_to_your_app
func createAPNSBackgroundConfig(data map[string]string) *messaging.APNSConfig {
apnsData := make(map[string]interface{})
for k, v := range data {
apnsData[k] = v
}
return &messaging.APNSConfig{
Headers: map[string]string{
"apns-push-type": "background",
"apns-priority": "5",
},
Payload: &messaging.APNSPayload{
Aps: &messaging.Aps{
ContentAvailable: true,
},
CustomData: apnsData,
},
}
}
// maybeTruncateAPNSBodyMessage truncates the body for APNS.
//
// The "body" of the push notification can contain the entire message, which would count doubly for the overall length
// of the APNS payload. I set a limit of 100 characters before truncating the notification "body" with ellipsis.
// The message would not be changed (unless truncated for being too long). Note: if the payload is too large (>4KB),
// APNS will simply reject / discard the notification, meaning it will never arrive on the iOS device.
func maybeTruncateAPNSBodyMessage(s string) string {
if len(s) >= fcmApnsBodyMessageLimit {
over := len(s) - fcmApnsBodyMessageLimit + 3 // len("...")
return s[:len(s)-over] + "..."
}
return s
}

118
server/server_firebase_test.go
View file

@ -32,6 +32,23 @@ func TestToFirebaseMessage_Keepalive(t *testing.T) {
require.Nil(t, err)
require.Equal(t, "mytopic", fbm.Topic)
require.Nil(t, fbm.Android)
require.Equal(t, &messaging.APNSConfig{
Headers: map[string]string{
"apns-push-type": "background",
"apns-priority": "5",
},
Payload: &messaging.APNSPayload{
Aps: &messaging.Aps{
ContentAvailable: true,
},
CustomData: map[string]interface{}{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
"event": m.Event,
"topic": m.Topic,
},
},
}, fbm.APNS)
require.Equal(t, map[string]string{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
@ -46,6 +63,23 @@ func TestToFirebaseMessage_Open(t *testing.T) {
require.Nil(t, err)
require.Equal(t, "mytopic", fbm.Topic)
require.Nil(t, fbm.Android)
require.Equal(t, &messaging.APNSConfig{
Headers: map[string]string{
"apns-push-type": "background",
"apns-priority": "5",
},
Payload: &messaging.APNSPayload{
Aps: &messaging.Aps{
ContentAvailable: true,
},
CustomData: map[string]interface{}{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
"event": m.Event,
"topic": m.Topic,
},
},
}, fbm.APNS)
require.Equal(t, map[string]string{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
@ -60,6 +94,25 @@ func TestToFirebaseMessage_Message_Normal_Allowed(t *testing.T) {
m.Tags = []string{"tag 1", "tag2"}
m.Click = "https://google.com"
m.Title = "some title"
m.Actions = []*action{
{
ID: "123",
Action: "view",
Label: "Open page",
Clear: true,
URL: "https://ntfy.sh",
},
{
ID: "456",
Action: "http",
Label: "Close door",
URL: "https://door.com/close",
Method: "PUT",
Headers: map[string]string{
"really": "yes",
},
},
}
m.Attachment = &attachment{
Name: "some file.jpg",
Type: "image/jpeg",
@ -74,6 +127,35 @@ func TestToFirebaseMessage_Message_Normal_Allowed(t *testing.T) {
require.Equal(t, &messaging.AndroidConfig{
Priority: "high",
}, fbm.Android)
require.Equal(t, &messaging.APNSConfig{
Payload: &messaging.APNSPayload{
Aps: &messaging.Aps{
MutableContent: true,
Alert: &messaging.ApsAlert{
Title: "some title",
Body: "this is a message",
},
},
CustomData: map[string]interface{}{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
"event": "message",
"topic": "mytopic",
"priority": "4",
"tags": strings.Join(m.Tags, ","),
"click": "https://google.com",
"title": "some title",
"message": "this is a message",
"actions": `[{"id":"123","action":"view","label":"Open page","clear":true,"url":"https://ntfy.sh"},{"id":"456","action":"http","label":"Close door","clear":false,"url":"https://door.com/close","method":"PUT","headers":{"really":"yes"}}]`,
"encoding": "",
"attachment_name": "some file.jpg",
"attachment_type": "image/jpeg",
"attachment_size": "12345",
"attachment_expires": "98765543",
"attachment_url": "https://example.com/file.jpg",
},
},
}, fbm.APNS)
require.Equal(t, map[string]string{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
@ -84,6 +166,7 @@ func TestToFirebaseMessage_Message_Normal_Allowed(t *testing.T) {
"click": "https://google.com",
"title": "some title",
"message": "this is a message",
"actions": `[{"id":"123","action":"view","label":"Open page","clear":true,"url":"https://ntfy.sh"},{"id":"456","action":"http","label":"Close door","clear":false,"url":"https://door.com/close","method":"PUT","headers":{"really":"yes"}}]`,
"encoding": "",
"attachment_name": "some file.jpg",
"attachment_type": "image/jpeg",
@ -112,6 +195,41 @@ func TestToFirebaseMessage_Message_Normal_Not_Allowed(t *testing.T) {
}, fbm.Data)
}
func TestToFirebaseMessage_PollRequest(t *testing.T) {
m := newPollRequestMessage("mytopic", "fOv6k1QbCzo6")
fbm, err := toFirebaseMessage(m, nil)
require.Nil(t, err)
require.Equal(t, "mytopic", fbm.Topic)
require.Nil(t, fbm.Android)
require.Equal(t, &messaging.APNSConfig{
Payload: &messaging.APNSPayload{
Aps: &messaging.Aps{
MutableContent: true,
Alert: &messaging.ApsAlert{
Title: "",
Body: "New message",
},
},
CustomData: map[string]interface{}{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
"event": "poll_request",
"topic": "mytopic",
"message": "New message",
"poll_id": "fOv6k1QbCzo6",
},
},
}, fbm.APNS)
require.Equal(t, map[string]string{
"id": m.ID,
"time": fmt.Sprintf("%d", m.Time),
"event": "poll_request",
"topic": "mytopic",
"message": "New message",
"poll_id": "fOv6k1QbCzo6",
}, fbm.Data)
}
func TestMaybeTruncateFCMMessage(t *testing.T) {
origMessage := strings.Repeat("this is a long string", 300)
origFCMMessage := &messaging.Message{