Create signed email addresses for VERP emails

app/config.py

@@ -74,7 +74,6 @@ MONITORING_EMAIL = os.environ.get("MONITORING_EMAIL")
 # VERP: mail_from set to BOUNCE_PREFIX + email_log.id + BOUNCE_SUFFIX
 BOUNCE_PREFIX = os.environ.get("BOUNCE_PREFIX") or "bounce+"
 BOUNCE_SUFFIX = os.environ.get("BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
-BOUNCE_EMAIL = BOUNCE_PREFIX + "{}" + BOUNCE_SUFFIX
 
 # Used for VERP during reply phase. It's similar to BOUNCE_PREFIX.
 # It's needed when sending emails from custom domain to respect DMARC.
@@ -93,9 +92,6 @@ TRANSACTIONAL_BOUNCE_PREFIX = (
 TRANSACTIONAL_BOUNCE_SUFFIX = (
     os.environ.get("TRANSACTIONAL_BOUNCE_SUFFIX") or f"+@{EMAIL_DOMAIN}"
 )
-TRANSACTIONAL_BOUNCE_EMAIL = (
-    TRANSACTIONAL_BOUNCE_PREFIX + "{}" + TRANSACTIONAL_BOUNCE_SUFFIX
-)
 
 try:
     MAX_NB_EMAIL_FREE_PLAN = int(os.environ["MAX_NB_EMAIL_FREE_PLAN"])
@@ -169,6 +165,8 @@ DB_URI = os.environ["DB_URI"]
 
 # Flask secret
 FLASK_SECRET = os.environ["FLASK_SECRET"]
+if not FLASK_SECRET:
+    raise RuntimeError("FLASK_SECRET is empty. Please define it.")
 SESSION_COOKIE_NAME = "slapp"
 MAILBOX_SECRET = FLASK_SECRET + "mailbox"
 CUSTOM_ALIAS_SECRET = FLASK_SECRET + "custom_alias"
@@ -426,6 +424,18 @@ ZENDESK_ENABLED = "ZENDESK_ENABLED" in os.environ
 
 DMARC_CHECK_ENABLED = "DMARC_CHECK_ENABLED" in os.environ
 
+# Bounces can happen after 5 days
+VERP_MESSAGE_LIFETIME = 5 * 865400
+VERP_PREFIX = os.environ.get("VERP_PREFIX") or "sl"
+# Generate with python3 -c 'import secrets; print(secrets.token_hex(28))'
+VERP_EMAIL_SECRET = os.environ.get("VERP_EMAIL_SECRET") or (
+    FLASK_SECRET + "pleasegenerateagoodrandomtoken"
+)
+if len(VERP_EMAIL_SECRET) < 32:
+    raise RuntimeError(
+        "Please, set VERP_EMAIL_SECRET to a random string at least 32 chars long"
+    )
+
 
 def get_allowed_redirect_domains() -> List[str]:
     allowed_domains = sl_getenv("ALLOWED_REDIRECT_DOMAINS", list)

app/email_utils.py

@@ -1,5 +1,7 @@
 import base64
 import enum
+import hmac
+import json
 import os
 import quopri
 import random
@@ -46,13 +48,15 @@ from app.config import (
     LANDING_PAGE_URL,
     EMAIL_DOMAIN,
     ALERT_DIRECTORY_DISABLED_ALIAS_CREATION,
-    TRANSACTIONAL_BOUNCE_EMAIL,
     ALERT_SPF,
     ALERT_INVALID_TOTP_LOGIN,
     TEMP_DIR,
     ALIAS_AUTOMATIC_DISABLE,
     RSPAMD_SIGN_DKIM,
     NOREPLY,
+    VERP_PREFIX,
+    VERP_MESSAGE_LIFETIME,
+    VERP_EMAIL_SECRET,
 )
 from app.db import Session
 from app.dns_utils import get_mx_domains
@@ -71,6 +75,7 @@ from app.models import (
     IgnoreBounceSender,
     InvalidMailboxDomain,
     DmarcCheckResult,
+    VerpType,
     SpamdResult,
     SPFCheckResult,
 )
@@ -324,7 +329,7 @@ def send_email(
 
     # use a different envelope sender for each transactional email (aka VERP)
     sl_sendmail(
-        TRANSACTIONAL_BOUNCE_EMAIL.format(transaction.id),
+        generate_verp_email(VerpType.transactional, transaction.id),
         to_email,
         msg,
         retries=retries,
@@ -1466,3 +1471,49 @@ def get_spamd_result(msg: Message) -> Optional[SpamdResult]:
 
     newrelic.agent.record_custom_event("SpamdCheck", spamd_result.event_data())
     return spamd_result
+
+
+def generate_verp_email(
+    verp_type: VerpType, object_id: int, sender_domain: Optional[str]
+) -> str:
+    # Encoded as a list to minimize size of email address
+    data = [verp_type.bounce_forward.value, object_id, int(time.time())]
+    json_payload = json.dumps(data).encode("utf-8")
+    # Signing without itsdangereous because it uses base64 that includes +/= symbols and lower and upper case letters.
+    # We need to encode in base32
+    payload_hmac = hmac.new(
+        VERP_EMAIL_SECRET.encode("utf-8"), json_payload, "shake128"
+    ).digest()
+    encoded_payload = base64.b32encode(json_payload).rstrip(b"=").decode("utf-8")
+    encoded_signature = base64.b32encode(payload_hmac).rstrip(b"=").decode("utf-8")
+    return "{}.{}.{}@{}".format(
+        VERP_PREFIX, encoded_payload, encoded_signature, sender_domain or EMAIL_DOMAIN
+    ).lower()
+
+
+# This method processes the email address, checks if it's a signed verp email generated by us to receive bounces
+# and extracts the type of verp email and associated email log id/transactional email id stored as object_id
+def get_verp_info_from_email(email: str) -> Optional[Tuple[VerpType, int]]:
+    idx = email.find("@")
+    if idx == -1:
+        return None
+    username = email[:idx]
+    fields = username.split(".")
+    if len(fields) != 3 or fields[0] != VERP_PREFIX:
+        return None
+    padding = 8 - (len(fields[1]) % 8)
+    payload = base64.b32decode(fields[1].encode("utf-8").upper() + (b"=" * padding))
+    padding = 8 - (len(fields[2]) % 8)
+    signature = base64.b32decode(fields[2].encode("utf-8").upper() + (b"=" * padding))
+    expected_signature = hmac.new(
+        VERP_EMAIL_SECRET.encode("utf-8"), payload, "shake128"
+    ).digest()
+    if expected_signature != signature:
+        return None
+    data = json.loads(payload)
+    # verp type, object_id, time
+    if len(data) != 3:
+        return None
+    if data[2] > time.time() + VERP_MESSAGE_LIFETIME:
+        return None
+    return VerpType(data[0]), data[1]

app/models.py

@@ -294,6 +294,12 @@ class SpamdResult:
         return {"header": "present", "dmarc": self.dmarc, "spf": self.spf}
 
 
+class VerpType(EnumE):
+    bounce_forward = 0
+    bounce_reply = 1
+    transactional = 2
+
+
 class Hibp(Base, ModelMixin):
     __tablename__ = "hibp"
     name = sa.Column(sa.String(), nullable=False, unique=True, index=True)

email_handler.py

@@ -72,7 +72,6 @@ from app.config import (
     PGP_SENDER_PRIVATE_KEY,
     ALERT_BOUNCE_EMAIL_REPLY_PHASE,
     NOREPLY,
-    BOUNCE_EMAIL,
     BOUNCE_PREFIX,
     BOUNCE_SUFFIX,
     TRANSACTIONAL_BOUNCE_PREFIX,
@@ -131,6 +130,8 @@ from app.email_utils import (
     get_mailbox_bounce_info,
     save_email_for_debugging,
     get_spamd_result,
+    generate_verp_email,
+    get_verp_info_from_email,
 )
 from app.errors import (
     NonReverseAliasInReplyPhase,
@@ -963,7 +964,7 @@ def forward_email_to_mailbox(
     try:
         sl_sendmail(
             # use a different envelope sender for each forward (aka VERP)
-            BOUNCE_EMAIL.format(email_log.id),
+            generate_verp_email(VerpType.bounce_forward, email_log.id),
             mailbox.email,
             msg,
             envelope.mail_options,
@@ -1246,12 +1247,9 @@ def handle_reply(envelope, msg: Message, rcpt_to: str) -> (bool, str):
     if should_add_dkim_signature(alias_domain):
         add_dkim_signature(msg, alias_domain)
 
-    # generate a mail_from for VERP
-    verp_mail_from = f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+{email_log.id}+@{alias_domain}"
-
     try:
         sl_sendmail(
-            verp_mail_from,
+            generate_verp_email(VerpType.bounce_reply, email_log.id, alias_domain),
             contact.website_email,
             msg,
             envelope.mail_options,
@@ -2092,11 +2090,13 @@ def handle_unsubscribe_user(user_id: int, mail_from: str) -> str:
     return status.E202
 
 
-def handle_transactional_bounce(envelope: Envelope, msg, rcpt_to):
+def handle_transactional_bounce(
+    envelope: Envelope, msg, rcpt_to, transactional_id=None
+):
     LOG.d("handle transactional bounce sent to %s", rcpt_to)
 
     # parse the TransactionalEmail
-    transactional_id = parse_id_from_bounce(rcpt_to)
+    transactional_id = transactional_id or parse_id_from_bounce(rcpt_to)
     transactional = TransactionalEmail.get(transactional_id)
 
     # a transaction might have been deleted in delete_logs()
@@ -2285,15 +2285,18 @@ def handle(envelope: Envelope, msg: Message) -> str:
         return handle_unsubscribe(envelope, msg)
 
     # region mail sent to VERP
+    verp_info = get_verp_info_from_email(rcpt_tos[0])
 
     # sent to transactional VERP. Either bounce emails or out-of-office
     if (
         len(rcpt_tos) == 1
         and rcpt_tos[0].startswith(TRANSACTIONAL_BOUNCE_PREFIX)
         and rcpt_tos[0].endswith(TRANSACTIONAL_BOUNCE_SUFFIX)
-    ):
+    ) or (verp_info and verp_info[0] == VerpType.transactional):
         if is_bounce(envelope, msg):
-            handle_transactional_bounce(envelope, msg, rcpt_tos[0])
+            handle_transactional_bounce(
+                envelope, msg, rcpt_tos[0], verp_info and verp_info[1]
+            )
             return status.E205
         elif is_automatic_out_of_office(msg):
             LOG.d(
@@ -2308,8 +2311,8 @@ def handle(envelope: Envelope, msg: Message) -> str:
         len(rcpt_tos) == 1
         and rcpt_tos[0].startswith(BOUNCE_PREFIX)
         and rcpt_tos[0].endswith(BOUNCE_SUFFIX)
-    ):
-        email_log_id = parse_id_from_bounce(rcpt_tos[0])
+    ) or (verp_info and verp_info[0] == VerpType.bounce_forward):
+        email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0])
         email_log = EmailLog.get(email_log_id)
 
         if not email_log:
@@ -2324,10 +2327,12 @@ def handle(envelope: Envelope, msg: Message) -> str:
             raise VERPForward
 
     # sent to reply VERP, can be either bounce or out-of-office
-    if len(rcpt_tos) == 1 and rcpt_tos[0].startswith(
-        f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+"
+    if (
+        len(rcpt_tos) == 1
+        and rcpt_tos[0].startswith(f"{BOUNCE_PREFIX_FOR_REPLY_PHASE}+")
+        or (verp_info and verp_info[0] == VerpType.bounce_reply)
     ):
-        email_log_id = parse_id_from_bounce(rcpt_tos[0])
+        email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(rcpt_tos[0])
         email_log = EmailLog.get(email_log_id)
 
         if not email_log:
@@ -2346,12 +2351,13 @@ def handle(envelope: Envelope, msg: Message) -> str:
             )
 
     # iCloud returns the bounce with mail_from=bounce+{email_log_id}+@simplelogin.co, rcpt_to=alias
+    verp_info = get_verp_info_from_email(mail_from[0])
     if (
         len(rcpt_tos) == 1
         and mail_from.startswith(BOUNCE_PREFIX)
         and mail_from.endswith(BOUNCE_SUFFIX)
-    ):
-        email_log_id = parse_id_from_bounce(mail_from)
+    ) or (verp_info and verp_info[0] == VerpType.bounce_forward):
+        email_log_id = (verp_info and verp_info[1]) or parse_id_from_bounce(mail_from)
         email_log = EmailLog.get(email_log_id)
         alias = Alias.get_by(email=rcpt_tos[0])
         LOG.w(

tests/test_email_utils.py

@@ -5,7 +5,7 @@ from email.message import EmailMessage
 import arrow
 import pytest
 
-from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, BOUNCE_EMAIL, ROOT_DIR
+from app.config import MAX_ALERT_24H, EMAIL_DOMAIN, ROOT_DIR
 from app.db import Session
 from app.email_utils import (
     get_email_domain_part,
@@ -37,6 +37,8 @@ from app.email_utils import (
     get_mailbox_bounce_info,
     is_invalid_mailbox_domain,
     get_spamd_result,
+    generate_verp_email,
+    get_verp_info_from_email,
 )
 from app.models import (
     User,
@@ -47,6 +49,7 @@ from app.models import (
     IgnoreBounceSender,
     InvalidMailboxDomain,
     DmarcCheckResult,
+    VerpType,
 )
 
 # flake8: noqa: E101, W191
@@ -739,7 +742,6 @@ def test_should_disable_bounce_consecutive_days(flask_client):
 def test_parse_id_from_bounce():
     assert parse_id_from_bounce("bounces+1234+@local") == 1234
     assert parse_id_from_bounce("anything+1234+@local") == 1234
-    assert parse_id_from_bounce(BOUNCE_EMAIL.format(1234)) == 1234
 
 
 def test_get_queue_id():
@@ -823,3 +825,11 @@ def test_dmarc_result_na():
 def test_dmarc_result_bad_policy():
     msg = load_eml_file("dmarc_bad_policy.eml")
     assert DmarcCheckResult.bad_policy == get_spamd_result(msg).dmarc
+
+
+def test_generate_verp_email():
+    generated_email = generate_verp_email(VerpType.bounce_forward, 1, "somewhere.net")
+    print(generated_email)
+    info = get_verp_info_from_email(generated_email.lower())
+    assert info[0] == VerpType.bounce_forward
+    assert info[1] == 1