Merge 091af996d8 into 550cd3e1f8

'software', when used as a noun, is not a 'countable' type, and it does not get an article like 'a'.  It's like 'traffic'.

README.md

@@ -48,7 +48,7 @@ __________________
 
 
 ## Developers
-listmonk is a free and open source software licensed under AGPLv3. If you are interested in contributing, refer to the [developer setup](https://listmonk.app/docs/developer-setup). The backend is written in Go and the frontend is Vue with Buefy for UI. 
+listmonk is free and open source software licensed under AGPLv3. If you are interested in contributing, refer to the [developer setup](https://listmonk.app/docs/developer-setup). The backend is written in Go and the frontend is Vue with Buefy for UI. 
 
 
 ## License

cmd/bounce.go

@@ -191,6 +191,23 @@ func handleBounceWebhook(c echo.Context) error {
 		}
 		bounces = append(bounces, bs...)
 
+	// ForwardEmail.
+	case service == "forwardemail" && app.constants.BounceForwardemailEnabled:
+		var (
+			sig = c.Request().Header.Get("X-Webhook-Signature")
+		)
+
+		bs, err := app.bounce.Forwardemail.ProcessBounce([]byte(sig), rawReq)
+		if err != nil {
+			app.log.Printf("error processing forwardemail notification: %v", err)
+			if _, ok := err.(*echo.HTTPError); ok {
+				return err
+			}
+
+			return echo.NewHTTPError(http.StatusBadRequest, app.i18n.T("globals.messages.invalidData"))
+		}
+		bounces = append(bounces, bs...)
+
 	// Postmark.
 	case service == "postmark" && app.constants.BouncePostmarkEnabled:
 		bs, err := app.bounce.Postmark.ProcessBounce(rawReq, c)

cmd/init.go

@@ -106,10 +106,11 @@ type constants struct {
 		Extensions []string
 	}
 
-	BounceWebhooksEnabled bool
-	BounceSESEnabled      bool
-	BounceSendgridEnabled bool
-	BouncePostmarkEnabled bool
+	BounceWebhooksEnabled     bool
+	BounceSESEnabled          bool
+	BounceSendgridEnabled     bool
+	BounceForwardemailEnabled bool
+	BouncePostmarkEnabled     bool
 }
 
 type notifTpls struct {
@@ -418,6 +419,7 @@ func initConstants() *constants {
 	c.BounceWebhooksEnabled = ko.Bool("bounce.webhooks_enabled")
 	c.BounceSESEnabled = ko.Bool("bounce.ses_enabled")
 	c.BounceSendgridEnabled = ko.Bool("bounce.sendgrid_enabled")
+	c.BounceForwardemailEnabled = ko.Bool("bounce.forwardemail_enabled")
 	c.BouncePostmarkEnabled = ko.Bool("bounce.postmark.enabled")
 
 	b := md5.Sum([]byte(time.Now().String()))
@@ -666,10 +668,12 @@ func initNotifTemplates(path string, fs stuffbin.FileSystem, i *i18n.I18n, cs *c
 // for incoming bounce events.
 func initBounceManager(app *App) *bounce.Manager {
 	opt := bounce.Opt{
-		WebhooksEnabled: ko.Bool("bounce.webhooks_enabled"),
-		SESEnabled:      ko.Bool("bounce.ses_enabled"),
-		SendgridEnabled: ko.Bool("bounce.sendgrid_enabled"),
-		SendgridKey:     ko.String("bounce.sendgrid_key"),
+		WebhooksEnabled:     ko.Bool("bounce.webhooks_enabled"),
+		SESEnabled:          ko.Bool("bounce.ses_enabled"),
+		SendgridEnabled:     ko.Bool("bounce.sendgrid_enabled"),
+		SendgridKey:         ko.String("bounce.sendgrid_key"),
+		ForwardemailEnabled: ko.Bool("bounce.forwardemail_enabled"),
+		ForwardemailKey:     ko.String("bounce.forwardemail_key"),
 		Postmark: struct {
 			Enabled  bool
 			Username string

cmd/settings.go

@@ -67,6 +67,8 @@ func handleGetSettings(c echo.Context) error {
 	for i := 0; i < len(s.Messengers); i++ {
 		s.Messengers[i].Password = strings.Repeat(pwdMask, utf8.RuneCountInString(s.Messengers[i].Password))
 	}
+
+	s.ForwardemailKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.ForwardemailKey))
 	s.UploadS3AwsSecretAccessKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.UploadS3AwsSecretAccessKey))
 	s.SendgridKey = strings.Repeat(pwdMask, utf8.RuneCountInString(s.SendgridKey))
 	s.SecurityCaptchaSecret = strings.Repeat(pwdMask, utf8.RuneCountInString(s.SecurityCaptchaSecret))
@@ -195,6 +197,9 @@ func handleUpdateSettings(c echo.Context) error {
 	if set.SendgridKey == "" {
 		set.SendgridKey = cur.SendgridKey
 	}
+	if set.ForwardemailKey == "" {
+		set.ForwardemailKey = cur.ForwardemailKey
+	}
 	if set.BouncePostmark.Password == "" {
 		set.BouncePostmark.Password = cur.BouncePostmark.Password
 	}

docs/docs/content/bounces.md

@@ -42,11 +42,12 @@ curl -u 'username:password' -X POST 'http://localhost:9000/webhooks/bounce' \
 ## External webhooks
 listmonk supports receiving bounce webhook events from the following SMTP providers.
 
-| Endpoint                                                  | Description                            | More info                                                                                                             |
-|:----------------------------------------------------------|:---------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
-| `https://listmonk.yoursite.com/webhooks/service/ses`      | Amazon (AWS) SES                       | See below                                                                                                             |
-| `https://listmonk.yoursite.com/webhooks/service/sendgrid` | Sendgrid / Twilio Signed event webhook | [More info](https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features) |
-| `https://listmonk.yoursite.com/webhooks/service/postmark` | Postmark webhook                       | [More info](https://postmarkapp.com/developer/webhooks/webhooks-overview)                                             |
+| Endpoint                                                      | Description                            | More info                                                                                                             |
+|:--------------------------------------------------------------|:---------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
+| `https://listmonk.yoursite.com/webhooks/service/ses`          | Amazon (AWS) SES                       | See below                                                                                                             |
+| `https://listmonk.yoursite.com/webhooks/service/sendgrid`     | Sendgrid / Twilio Signed event webhook | [More info](https://docs.sendgrid.com/for-developers/tracking-events/getting-started-event-webhook-security-features) |
+| `https://listmonk.yoursite.com/webhooks/service/postmark`     | Postmark webhook                       | [More info](https://postmarkapp.com/developer/webhooks/webhooks-overview)                                             |
+| `https://listmonk.yoursite.com/webhooks/service/forwardemail` | Forward Email webhook                   | [More info](https://forwardemail.net/en/faq#do-you-support-bounce-webhooks)                                                  |
 
 ## Amazon Simple Email Service (SES)
 

docs/swagger/collections.yaml

@@ -2703,6 +2703,8 @@ components:
               type: string
             settings.bounces.enableSendgrid:
               type: string
+            settings.bounces.enableForwardemail:
+              type: string
             settings.bounces.enablePostmark:
               type: string
             settings.bounces.enableWebhooks:
@@ -2723,6 +2725,8 @@ components:
               type: string
             settings.bounces.sendgridKey:
               type: string
+            settings.bounces.forwardemailKey:
+              type: string
             settings.bounces.postmarkUsername:
               type: string
             settings.bounces.postmarkUsernameHelp:
@@ -3406,6 +3410,10 @@ components:
           type: boolean
         bounce.sendgrid_key:
           type: string
+        bounce.forwardemail_enabled:
+          type: boolean
+        bounce.forwardemail_key:
+          type: string
         bounce.postmark_enabled:
           type: boolean
         bounce.postmark_username:

frontend/src/views/Settings.vue

@@ -166,6 +166,12 @@ export default Vue.extend({
         hasDummy = 'postmark';
       }
 
+      if (this.isDummy(form['bounce.forwardemail_key'])) {
+        form['bounce.forwardemail_key'] = '';
+      } else if (this.hasDummy(form['bounce.forwardemail_key'])) {
+        hasDummy = 'forwardemail';
+      }
+
       for (let i = 0; i < form.messengers.length; i += 1) {
         // If it's the dummy UI password placeholder, ignore it.
         if (this.isDummy(form.messengers[i].password)) {

frontend/src/views/settings/bounces.vue

@@ -59,6 +59,20 @@
             </b-field>
           </div>
         </div>
+        <div class="columns">
+          <div class="column is-3">
+            <b-field :label="$t('settings.bounces.enableForwardemail')">
+              <b-switch v-model="data['bounce.forwardemail_enabled']" name="forwardemail_enabled" :native-value="true"
+                data-cy="btn-enable-bounce-forwardemail" />
+            </b-field>
+          </div>
+          <div class="column">
+            <b-field :label="$t('settings.bounces.forwardemailKey')" :message="$t('globals.messages.passwordChange')">
+              <b-input v-model="data['bounce.forwardemail_key']" type="password" :disabled="!data['bounce.forwardemail_enabled']"
+                name="forwardemail_enabled" :native-value="true" data-cy="btn-enable-bounce-forwardemail" />
+            </b-field>
+          </div>
+        </div>
         <div class="columns">
           <div class="column is-3">
             <b-field :label="$t('settings.bounces.enableSendgrid')">

frontend/src/views/settings/smtp.vue

@@ -67,6 +67,7 @@
               </div>
             </div><!-- auth -->
             <div class="smtp-shortcuts is-size-7">
+              <a href="#" @click.prevent="() => fillSettings(n, 'forwardemail')">Forward Email</a>
               <a href="#" @click.prevent="() => fillSettings(n, 'gmail')">Gmail</a>
               <a href="#" @click.prevent="() => fillSettings(n, 'ses')">Amazon SES</a>
               <a href="#" @click.prevent="() => fillSettings(n, 'mailgun')">Mailgun</a>
@@ -220,6 +221,9 @@ const smtpTemplates = {
   sendgrid: {
     host: 'smtp.sendgrid.net', port: 465, auth_protocol: 'login', tls_type: 'TLS',
   },
+  forwardemail: {
+    host: 'smtp.forwardemail.net', port: 465, auth_protocol: 'login', tls_type: 'TLS',
+  },
   postmark: {
     host: 'smtp.postmarkapp.com', port: 587, auth_protocol: 'cram', tls_type: 'STARTTLS',
   },

i18n/en.json

@@ -369,12 +369,14 @@
     "settings.bounces.enable": "Enable bounce processing",
     "settings.bounces.enableMailbox": "Enable bounce mailbox",
     "settings.bounces.enablePostmark": "Enable Postmark",
+    "settings.bounces.enableForwardemail": "Enable Forward Email",
     "settings.bounces.enableSES": "Enable SES",
     "settings.bounces.enableSendgrid": "Enable SendGrid",
     "settings.bounces.enableWebhooks": "Enable bounce webhooks",
     "settings.bounces.enabled": "Enabled",
     "settings.bounces.folder": "Folder",
     "settings.bounces.folderHelp": "Name of the IMAP folder to scan. Eg: Inbox.",
+    "settings.bounces.forwardemailKey": "Forward Email Key",
     "settings.bounces.invalidScanInterval": "Bounce scan interval should be minimum 1 minute.",
     "settings.bounces.name": "Bounces",
     "settings.bounces.none": "None",

internal/bounce/bounce.go

@@ -26,14 +26,16 @@ type Mailbox interface {
 
 // Opt represents bounce processing options.
 type Opt struct {
-	MailboxEnabled  bool        `json:"mailbox_enabled"`
-	MailboxType     string      `json:"mailbox_type"`
-	Mailbox         mailbox.Opt `json:"mailbox"`
-	WebhooksEnabled bool        `json:"webhooks_enabled"`
-	SESEnabled      bool        `json:"ses_enabled"`
-	SendgridEnabled bool        `json:"sendgrid_enabled"`
-	SendgridKey     string      `json:"sendgrid_key"`
-	Postmark        struct {
+	MailboxEnabled      bool        `json:"mailbox_enabled"`
+	MailboxType         string      `json:"mailbox_type"`
+	Mailbox             mailbox.Opt `json:"mailbox"`
+	WebhooksEnabled     bool        `json:"webhooks_enabled"`
+	SESEnabled          bool        `json:"ses_enabled"`
+	SendgridEnabled     bool        `json:"sendgrid_enabled"`
+	SendgridKey         string      `json:"sendgrid_key"`
+	ForwardemailEnabled bool        `json:"forwardemail_enabled"`
+	ForwardemailKey     string      `json:"forwardemail_key"`
+	Postmark            struct {
 		Enabled  bool
 		Username string
 		Password string
@@ -44,14 +46,15 @@ type Opt struct {
 
 // Manager handles e-mail bounces.
 type Manager struct {
-	queue    chan models.Bounce
-	mailbox  Mailbox
-	SES      *webhooks.SES
-	Sendgrid *webhooks.Sendgrid
-	Postmark *webhooks.Postmark
-	queries  *Queries
-	opt      Opt
-	log      *log.Logger
+	queue        chan models.Bounce
+	mailbox      Mailbox
+	SES          *webhooks.SES
+	Sendgrid     *webhooks.Sendgrid
+	Postmark     *webhooks.Postmark
+	Forwardemail *webhooks.Forwardemail
+	queries      *Queries
+	opt          Opt
+	log          *log.Logger
 }
 
 // Queries contains the queries.
@@ -93,6 +96,11 @@ func New(opt Opt, q *Queries, lo *log.Logger) (*Manager, error) {
 			}
 		}
 
+		if opt.ForwardemailEnabled {
+			fe := webhooks.NewForwardemail([]byte(opt.ForwardemailKey))
+			m.Forwardemail = fe
+		}
+
 		if opt.Postmark.Enabled {
 			m.Postmark = webhooks.NewPostmark(opt.Postmark.Username, opt.Postmark.Password)
 		}

internal/bounce/webhooks/forwardemail.go

@@ -0,0 +1,87 @@
+package webhooks
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/knadh/listmonk/models"
+)
+
+type BounceDetails struct {
+	Action   string `json:"action"`
+	Message  string `json:"message"`
+	Category string `json:"category"`
+	Code     int    `json:"code"`
+	Status   string `json:"status"`
+	Line     int    `json:"line"`
+}
+
+type forwardemailNotif struct {
+	EmailID         string            `json:"email_id"`
+	ListID          string            `json:"list_id"`
+	ListUnsubscribe string            `json:"list_unsubscribe"`
+	FeedbackID      string            `json:"feedback_id"`
+	Recipient       string            `json:"recipient"`
+	Message         string            `json:"message"`
+	Response        string            `json:"response"`
+	ResponseCode    int               `json:"response_code"`
+	TruthSource     string            `json:"truth_source"`
+	Headers         map[string]string `json:"headers"`
+	Bounce          BounceDetails     `json:"bounce"`
+	BouncedAt       time.Time         `json:"bounced_at"`
+}
+
+// Forwardemail handles webhook notifications (mainly bounce notifications).
+type Forwardemail struct {
+	hmacKey []byte
+}
+
+func NewForwardemail(key []byte) *Forwardemail {
+	return &Forwardemail{hmacKey: key}
+}
+
+// ProcessBounce processes Forward Email bounce notifications and returns one object.
+func (p *Forwardemail) ProcessBounce(sig, b []byte) ([]models.Bounce, error) {
+	key := []byte(p.hmacKey)
+
+	mac := hmac.New(sha256.New, key)
+
+	mac.Write(b)
+
+	signature := mac.Sum(nil)
+
+	if subtle.ConstantTimeCompare(signature, []byte(sig)) != 1 {
+		return nil, fmt.Errorf("invalid signature")
+	}
+
+	var n forwardemailNotif
+	if err := json.Unmarshal(b, &n); err != nil {
+		return nil, fmt.Errorf("error unmarshalling Forwardemail notification: %v", err)
+	}
+
+	typ := models.BounceTypeSoft
+	// TODO: support `typ = models.BounceTypeComplaint` in future
+	switch n.Bounce.Category {
+	case "block", "recipient", "virus", "spam":
+		typ = models.BounceTypeHard
+	}
+
+	campUUID := ""
+	if v, ok := n.Headers["X-Listmonk-Campaign"]; ok {
+		campUUID = v
+	}
+
+	return []models.Bounce{{
+		Email:        strings.ToLower(n.Recipient),
+		CampaignUUID: campUUID,
+		Type:         typ,
+		Source:       "forwardemail",
+		Meta:         json.RawMessage(b),
+		CreatedAt:    n.BouncedAt,
+	}}, nil
+}

models/settings.go

@@ -99,7 +99,9 @@ type Settings struct {
 		Username string `json:"username"`
 		Password string `json:"password"`
 	} `json:"bounce.postmark"`
-	BounceBoxes []struct {
+	ForwardemailEnabled bool   `json:"bounce.forwardemail_enabled"`
+	ForwardemailKey     string `json:"bounce.forwardemail_key"`
+	BounceBoxes         []struct {
 		UUID          string `json:"uuid"`
 		Enabled       bool   `json:"enabled"`
 		Type          string `json:"type"`

schema.sql

@@ -268,6 +268,8 @@ INSERT INTO settings (key, value) VALUES
     ('bounce.enabled', 'false'),
     ('bounce.webhooks_enabled', 'false'),
     ('bounce.actions', '{"soft": {"count": 2, "action": "none"}, "hard": {"count": 1, "action": "blocklist"}, "complaint" : {"count": 1, "action": "blocklist"}}'),
+    ('bounce.forwardemail_enabled', 'false'),
+    ('bounce.forwardemail_key', '""'),
     ('bounce.ses_enabled', 'false'),
     ('bounce.sendgrid_enabled', 'false'),
     ('bounce.sendgrid_key', '""'),