Delivery and DMARC Troubleshooting (closes #420)

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "RustyXML"

README.md

@@ -25,7 +25,7 @@
 <p align="center">
   <a href="https://discord.gg/jtgtCNj66U"><img src="https://img.shields.io/discord/923615863037390889?label=Join%20Discord&logo=discord&style=flat-square" alt="Discord"></a>
   &nbsp;
-  <a href="https://www.reddit.com/r/stalwartlabs/"><img src="https://img.shields.io/reddit/subreddit-subscribers/stalwartlabs?label=Follow%20%2Fr%2Fstalwartlabs&logo=reddit&style=flat-square" alt="Reddit"></a>
+  <a href="https://www.reddit.com/r/stalwartlabs/"><img src="https://img.shields.io/reddit/subreddit-subscribers/stalwartlabs?label=Join%20%2Fr%2Fstalwartlabs&logo=reddit&style=flat-square" alt="Reddit"></a>
 </p>
 
 ## Features

crates/common/src/auth/oauth/mod.rs

@@ -24,6 +24,7 @@ pub enum GrantType {
     RefreshToken,
     LiveTracing,
     LiveMetrics,
+    Troubleshoot,
 }
 
 impl GrantType {
@@ -33,6 +34,7 @@ impl GrantType {
             GrantType::RefreshToken => "refresh_token",
             GrantType::LiveTracing => "live_tracing",
             GrantType::LiveMetrics => "live_metrics",
+            GrantType::Troubleshoot => "troubleshoot",
         }
     }
 
@@ -42,6 +44,7 @@ impl GrantType {
             GrantType::RefreshToken => 1,
             GrantType::LiveTracing => 2,
             GrantType::LiveMetrics => 3,
+            GrantType::Troubleshoot => 4,
         }
     }
 
@@ -51,6 +54,7 @@ impl GrantType {
             1 => Some(GrantType::RefreshToken),
             2 => Some(GrantType::LiveTracing),
             3 => Some(GrantType::LiveMetrics),
+            4 => Some(GrantType::Troubleshoot),
             _ => None,
         }
     }

crates/common/src/config/smtp/resolver.rs

@@ -22,6 +22,7 @@ use mail_auth::{
     Resolver,
 };
 use parking_lot::Mutex;
+use serde::{Deserialize, Serialize};
 use utils::config::{utils::ParseValue, Config};
 
 use crate::Server;
@@ -42,7 +43,7 @@ pub struct DnsRecordCache {
     pub mta_sts: LruCache<String, Arc<Policy>>,
 }
 
-#[derive(Debug, Hash, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
 pub struct TlsaEntry {
     pub is_end_entity: bool,
     pub is_sha256: bool,
@@ -50,14 +51,15 @@ pub struct TlsaEntry {
     pub data: Vec<u8>,
 }
 
-#[derive(Debug, Hash, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)]
 pub struct Tlsa {
     pub entries: Vec<TlsaEntry>,
     pub has_end_entities: bool,
     pub has_intermediates: bool,
 }
 
-#[derive(Debug, PartialEq, Eq, Hash, Default, Clone, Copy)]
+#[derive(Debug, PartialEq, Eq, Hash, Default, Clone, Copy, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub enum Mode {
     Enforce,
     Testing,
@@ -65,13 +67,14 @@ pub enum Mode {
     None,
 }
 
-#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone)]
+#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub enum MxPattern {
     Equals(String),
     StartsWith(String),
 }
 
-#[derive(Debug, PartialEq, Eq, Hash, Clone)]
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)]
 pub struct Policy {
     pub id: String,
     pub mode: Mode,

crates/directory/src/core/mod.rs

@@ -198,6 +198,7 @@ impl Permission {
             Permission::OauthClientUpdate => "Modify OAuth clients",
             Permission::OauthClientDelete => "Remove OAuth clients",
             Permission::AiModelInteract => "Interact with AI models",
+            Permission::Troubleshoot => "Perform troubleshooting",
         }
     }
 }

crates/directory/src/lib.rs

@@ -264,7 +264,7 @@ pub enum Permission {
     OauthClientOverride,
 
     AiModelInteract,
-    // WARNING: add new ids at the end (TODO: use static ids)
+    Troubleshoot, // WARNING: add new ids at the end (TODO: use static ids)
 }
 
 pub type Permissions = Bitset<{ Permission::COUNT.div_ceil(std::mem::size_of::<usize>()) }>;

crates/jmap/src/api/http.rs

@@ -33,6 +33,7 @@ use jmap_proto::{
 };
 use std::future::Future;
 use trc::SecurityEvent;
+use utils::url_params::UrlParams;
 
 #[cfg(feature = "enterprise")]
 use crate::api::management::enterprise::telemetry::TelemetryApi;
@@ -54,7 +55,7 @@ use super::{
     autoconfig::Autoconfig,
     event_source::EventSourceHandler,
     form::FormHandler,
-    management::{ManagementApi, ManagementApiError},
+    management::{troubleshoot::TroubleshootApi, ManagementApi, ManagementApiError},
     request::RequestHandler,
     session::SessionHandler,
     HtmlResponse, HttpRequest, HttpResponse, HttpResponseBody, JmapSessionManager, JsonResponse,
@@ -335,45 +336,67 @@ impl ParseHttp for Server {
                             .await;
                     }
                     Err(err) => {
-                        #[cfg(feature = "enterprise")]
+                        if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) {
-                        {
+                            let params = UrlParams::new(req.uri().query());
+                            let path = req.uri().path().split('/').skip(2).collect::<Vec<_>>();
+
+                            let (grant_type, token) = match (
+                                path.first().copied(),
+                                path.get(1).copied(),
+                                params.get("token"),
+                            ) {
                                 // SPDX-SnippetBegin
                                 // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
                                 // SPDX-License-Identifier: LicenseRef-SEL
-
+                                #[cfg(feature = "enterprise")]
-                            // Eventsource does not support authentication, validate the token instead
+                                (Some("telemetry"), Some("traces"), Some(token))
-                            if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed))
+                                    if self.core.is_enterprise_edition() =>
-                                && self.core.is_enterprise_edition()
                                 {
-                                if let Some((live_path, grant_type, token)) = req
+                                    (GrantType::LiveTracing, token)
-                                    .uri()
+                                }
-                                    .path()
+                                #[cfg(feature = "enterprise")]
-                                    .strip_prefix("/api/telemetry/")
+                                (Some("telemetry"), Some("metrics"), Some(token))
-                                    .and_then(|p| {
+                                    if self.core.is_enterprise_edition() =>
-                                        p.strip_prefix("traces/live/")
-                                            .map(|t| ("traces", GrantType::LiveTracing, t))
-                                            .or_else(|| {
-                                                p.strip_prefix("metrics/live/")
-                                                    .map(|t| ("metrics", GrantType::LiveMetrics, t))
-                                            })
-                                    })
                                 {
-                                    let token_info = self
+                                    (GrantType::LiveMetrics, token)
-                                        .validate_access_token(grant_type.into(), token)
+                                }
-                                        .await?;
+                                // SPDX-SnippetEnd
+                                (Some("troubleshoot"), _, Some(token)) => {
+                                    (GrantType::Troubleshoot, token)
+                                }
+                                _ => return Err(err),
+                            };
+                            let token_info =
+                                self.validate_access_token(grant_type.into(), token).await?;
 
-                                    return self
+                            return match grant_type {
-                                        .handle_telemetry_api_request(
+                                // SPDX-SnippetBegin
+                                // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
+                                // SPDX-License-Identifier: LicenseRef-SEL
+                                #[cfg(feature = "enterprise")]
+                                GrantType::LiveTracing | GrantType::LiveMetrics => {
+                                    self.handle_telemetry_api_request(
                                         &req,
-                                            vec!["", live_path, "live"],
+                                        path,
                                         &AccessToken::from_id(token_info.account_id)
                                             .with_permission(Permission::MetricsLive)
                                             .with_permission(Permission::TracingLive),
                                     )
-                                        .await;
+                                    .await
-                                }
                                 }
                                 // SPDX-SnippetEnd
+                                GrantType::Troubleshoot => {
+                                    self.handle_troubleshoot_api_request(
+                                        &req,
+                                        path,
+                                        &AccessToken::from_id(token_info.account_id)
+                                            .with_permission(Permission::Troubleshoot),
+                                        None,
+                                    )
+                                    .await
+                                }
+                                _ => unreachable!(),
+                            };
                         }
 
                         return Err(err);

crates/jmap/src/api/management/enterprise/telemetry.rs

@@ -353,7 +353,6 @@ impl TelemetryApi for Server {
                 access_token.assert_has_permission(Permission::TracingLive)?;
 
                 // Issue a live telemetry token valid for 60 seconds
-
                 Ok(JsonResponse::new(json!({
                     "data": self.encode_access_token(GrantType::LiveTracing, account_id,  "web", 60).await?,
             }))
@@ -364,7 +363,6 @@ impl TelemetryApi for Server {
                 access_token.assert_has_permission(Permission::MetricsLive)?;
 
                 // Issue a live telemetry token valid for 60 seconds
-
                 Ok(JsonResponse::new(json!({
                     "data": self.encode_access_token(GrantType::LiveMetrics, account_id, "web", 60).await?,
             }))

crates/jmap/src/api/management/mod.rs

@@ -16,6 +16,7 @@ pub mod report;
 pub mod settings;
 pub mod sieve;
 pub mod stores;
+pub mod troubleshoot;
 
 use std::{borrow::Cow, str::FromStr, sync::Arc};
 
@@ -37,6 +38,7 @@ use settings::ManageSettings;
 use sieve::SieveHandler;
 use store::write::now;
 use stores::ManageStore;
+use troubleshoot::TroubleshootApi;
 
 use crate::{auth::oauth::auth::OAuthApiHandler, email::crypto::CryptoHandler};
 
@@ -155,6 +157,13 @@ impl ManagementApi for Server {
                 }
                 _ => Err(trc::ResourceEvent::NotFound.into_err()),
             },
+            "troubleshoot" => {
+                // Validate the access token
+                access_token.assert_has_permission(Permission::Troubleshoot)?;
+
+                self.handle_troubleshoot_api_request(req, path, &access_token, body)
+                    .await
+            }
             // SPDX-SnippetBegin
             // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
             // SPDX-License-Identifier: LicenseRef-SEL

crates/smtp/src/outbound/delivery.rs

@@ -360,6 +360,15 @@ impl DeliveryAttempt {
 
                             TlsRptOptions { record, interval }.into()
                         }
+                        Err(mail_auth::Error::DnsRecordNotFound(_)) => {
+                            trc::event!(
+                                TlsRpt(TlsRptEvent::RecordNotFound),
+                                SpanId = message.span_id,
+                                Domain = domain.domain.clone(),
+                                Elapsed = time.elapsed(),
+                            );
+                            None
+                        }
                         Err(err) => {
                             trc::event!(
                                 TlsRpt(TlsRptEvent::RecordFetchError),

crates/smtp/src/outbound/mod.rs

@@ -220,7 +220,7 @@ pub enum NextHop<'x> {
 
 impl NextHop<'_> {
     #[inline(always)]
-    fn hostname(&self) -> &str {
+    pub fn hostname(&self) -> &str {
         match self {
             NextHop::MX(host) => {
                 if let Some(host) = host.strip_suffix('.') {
@@ -234,7 +234,7 @@ impl NextHop<'_> {
     }
 
     #[inline(always)]
-    fn fqdn_hostname(&self) -> Cow<'_, str> {
+    pub fn fqdn_hostname(&self) -> Cow<'_, str> {
         match self {
             NextHop::MX(host) => {
                 if !host.ends_with('.') {

crates/trc/src/event/description.rs

@@ -881,6 +881,7 @@ impl TlsRptEvent {
         match self {
             TlsRptEvent::RecordFetch => "Fetched TLS-RPT record",
             TlsRptEvent::RecordFetchError => "Error fetching TLS-RPT record",
+            TlsRptEvent::RecordNotFound => "TLS-RPT record not found",
         }
     }
 
@@ -888,6 +889,7 @@ impl TlsRptEvent {
         match self {
             TlsRptEvent::RecordFetch => "The TLS-RPT record has been fetched",
             TlsRptEvent::RecordFetchError => "An error occurred while fetching the TLS-RPT record",
+            TlsRptEvent::RecordNotFound => "No TLS-RPT records were found",
         }
     }
 }

crates/trc/src/event/level.rs

@@ -478,7 +478,9 @@ impl EventType {
                 }
             },
             EventType::TlsRpt(event) => match event {
-                TlsRptEvent::RecordFetch | TlsRptEvent::RecordFetchError => Level::Info,
+                TlsRptEvent::RecordFetch
+                | TlsRptEvent::RecordFetchError
+                | TlsRptEvent::RecordNotFound => Level::Info,
             },
             EventType::MtaSts(event) => match event {
                 MtaStsEvent::PolicyFetch

crates/trc/src/lib.rs

@@ -545,6 +545,7 @@ pub enum MtaStsEvent {
 pub enum TlsRptEvent {
     RecordFetch,
     RecordFetchError,
+    RecordNotFound,
 }
 
 #[event_type]

crates/trc/src/serializers/binary.rs

@@ -865,6 +865,7 @@ impl EventType {
             EventType::Ai(AiEvent::ApiError) => 557,
             EventType::Security(SecurityEvent::ScanBan) => 558,
             EventType::Store(StoreEvent::AzureError) => 559,
+            EventType::TlsRpt(TlsRptEvent::RecordNotFound) => 560,
         }
     }
 
@@ -1470,6 +1471,7 @@ impl EventType {
             557 => Some(EventType::Ai(AiEvent::ApiError)),
             558 => Some(EventType::Security(SecurityEvent::ScanBan)),
             559 => Some(EventType::Store(StoreEvent::AzureError)),
+            560 => Some(EventType::TlsRpt(TlsRptEvent::RecordNotFound)),
             _ => None,
         }
     }