diff --git a/Cargo.lock b/Cargo.lock index c1225800..c056e31d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "RustyXML" diff --git a/README.md b/README.md index b6843544..7b2f0dff 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@

Discord   - Reddit + Reddit

## Features diff --git a/crates/common/src/auth/oauth/mod.rs b/crates/common/src/auth/oauth/mod.rs index 575b7591..0f255c01 100644 --- a/crates/common/src/auth/oauth/mod.rs +++ b/crates/common/src/auth/oauth/mod.rs @@ -24,6 +24,7 @@ pub enum GrantType { RefreshToken, LiveTracing, LiveMetrics, + Troubleshoot, } impl GrantType { @@ -33,6 +34,7 @@ impl GrantType { GrantType::RefreshToken => "refresh_token", GrantType::LiveTracing => "live_tracing", GrantType::LiveMetrics => "live_metrics", + GrantType::Troubleshoot => "troubleshoot", } } @@ -42,6 +44,7 @@ impl GrantType { GrantType::RefreshToken => 1, GrantType::LiveTracing => 2, GrantType::LiveMetrics => 3, + GrantType::Troubleshoot => 4, } } @@ -51,6 +54,7 @@ impl GrantType { 1 => Some(GrantType::RefreshToken), 2 => Some(GrantType::LiveTracing), 3 => Some(GrantType::LiveMetrics), + 4 => Some(GrantType::Troubleshoot), _ => None, } } diff --git a/crates/common/src/config/smtp/resolver.rs b/crates/common/src/config/smtp/resolver.rs index b6e6802f..7b7c39ab 100644 --- a/crates/common/src/config/smtp/resolver.rs +++ b/crates/common/src/config/smtp/resolver.rs @@ -22,6 +22,7 @@ use mail_auth::{ Resolver, }; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use utils::config::{utils::ParseValue, Config}; use crate::Server; @@ -42,7 +43,7 @@ pub struct DnsRecordCache { pub mta_sts: LruCache>, } -#[derive(Debug, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct TlsaEntry { pub is_end_entity: bool, pub is_sha256: bool, @@ -50,14 +51,15 @@ pub struct TlsaEntry { pub data: Vec, } -#[derive(Debug, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct Tlsa { pub entries: Vec, pub has_end_entities: bool, pub has_intermediates: bool, } -#[derive(Debug, PartialEq, Eq, Hash, Default, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Hash, Default, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub enum Mode { Enforce, Testing, @@ -65,13 +67,14 @@ pub enum Mode { None, } -#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone)] +#[derive(Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub enum MxPattern { Equals(String), StartsWith(String), } -#[derive(Debug, PartialEq, Eq, Hash, Clone)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] pub struct Policy { pub id: String, pub mode: Mode, diff --git a/crates/directory/src/core/mod.rs b/crates/directory/src/core/mod.rs index 89428ded..859ea631 100644 --- a/crates/directory/src/core/mod.rs +++ b/crates/directory/src/core/mod.rs @@ -198,6 +198,7 @@ impl Permission { Permission::OauthClientUpdate => "Modify OAuth clients", Permission::OauthClientDelete => "Remove OAuth clients", Permission::AiModelInteract => "Interact with AI models", + Permission::Troubleshoot => "Perform troubleshooting", } } } diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 69f5109f..cdfda42e 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -264,7 +264,7 @@ pub enum Permission { OauthClientOverride, AiModelInteract, - // WARNING: add new ids at the end (TODO: use static ids) + Troubleshoot, // WARNING: add new ids at the end (TODO: use static ids) } pub type Permissions = Bitset<{ Permission::COUNT.div_ceil(std::mem::size_of::()) }>; diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 50e2851b..5c5ce0eb 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -33,6 +33,7 @@ use jmap_proto::{ }; use std::future::Future; use trc::SecurityEvent; +use utils::url_params::UrlParams; #[cfg(feature = "enterprise")] use crate::api::management::enterprise::telemetry::TelemetryApi; @@ -54,7 +55,7 @@ use super::{ autoconfig::Autoconfig, event_source::EventSourceHandler, form::FormHandler, - management::{ManagementApi, ManagementApiError}, + management::{troubleshoot::TroubleshootApi, ManagementApi, ManagementApiError}, request::RequestHandler, session::SessionHandler, HtmlResponse, HttpRequest, HttpResponse, HttpResponseBody, JmapSessionManager, JsonResponse, @@ -335,45 +336,67 @@ impl ParseHttp for Server { .await; } Err(err) => { - #[cfg(feature = "enterprise")] - { - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - // SPDX-License-Identifier: LicenseRef-SEL + if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) { + let params = UrlParams::new(req.uri().query()); + let path = req.uri().path().split('/').skip(2).collect::>(); - // Eventsource does not support authentication, validate the token instead - if err.matches(trc::EventType::Auth(trc::AuthEvent::Failed)) - && self.core.is_enterprise_edition() - { - if let Some((live_path, grant_type, token)) = req - .uri() - .path() - .strip_prefix("/api/telemetry/") - .and_then(|p| { - p.strip_prefix("traces/live/") - .map(|t| ("traces", GrantType::LiveTracing, t)) - .or_else(|| { - p.strip_prefix("metrics/live/") - .map(|t| ("metrics", GrantType::LiveMetrics, t)) - }) - }) + let (grant_type, token) = match ( + path.first().copied(), + path.get(1).copied(), + params.get("token"), + ) { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + (Some("telemetry"), Some("traces"), Some(token)) + if self.core.is_enterprise_edition() => { - let token_info = self - .validate_access_token(grant_type.into(), token) - .await?; - - return self - .handle_telemetry_api_request( - &req, - vec!["", live_path, "live"], - &AccessToken::from_id(token_info.account_id) - .with_permission(Permission::MetricsLive) - .with_permission(Permission::TracingLive), - ) - .await; + (GrantType::LiveTracing, token) } - } - // SPDX-SnippetEnd + #[cfg(feature = "enterprise")] + (Some("telemetry"), Some("metrics"), Some(token)) + if self.core.is_enterprise_edition() => + { + (GrantType::LiveMetrics, token) + } + // SPDX-SnippetEnd + (Some("troubleshoot"), _, Some(token)) => { + (GrantType::Troubleshoot, token) + } + _ => return Err(err), + }; + let token_info = + self.validate_access_token(grant_type.into(), token).await?; + + return match grant_type { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + GrantType::LiveTracing | GrantType::LiveMetrics => { + self.handle_telemetry_api_request( + &req, + path, + &AccessToken::from_id(token_info.account_id) + .with_permission(Permission::MetricsLive) + .with_permission(Permission::TracingLive), + ) + .await + } + // SPDX-SnippetEnd + GrantType::Troubleshoot => { + self.handle_troubleshoot_api_request( + &req, + path, + &AccessToken::from_id(token_info.account_id) + .with_permission(Permission::Troubleshoot), + None, + ) + .await + } + _ => unreachable!(), + }; } return Err(err); diff --git a/crates/jmap/src/api/management/enterprise/telemetry.rs b/crates/jmap/src/api/management/enterprise/telemetry.rs index 0ae46fe9..78a6f3ac 100644 --- a/crates/jmap/src/api/management/enterprise/telemetry.rs +++ b/crates/jmap/src/api/management/enterprise/telemetry.rs @@ -353,7 +353,6 @@ impl TelemetryApi for Server { access_token.assert_has_permission(Permission::TracingLive)?; // Issue a live telemetry token valid for 60 seconds - Ok(JsonResponse::new(json!({ "data": self.encode_access_token(GrantType::LiveTracing, account_id, "web", 60).await?, })) @@ -364,7 +363,6 @@ impl TelemetryApi for Server { access_token.assert_has_permission(Permission::MetricsLive)?; // Issue a live telemetry token valid for 60 seconds - Ok(JsonResponse::new(json!({ "data": self.encode_access_token(GrantType::LiveMetrics, account_id, "web", 60).await?, })) diff --git a/crates/jmap/src/api/management/mod.rs b/crates/jmap/src/api/management/mod.rs index 776c723d..b3c96284 100644 --- a/crates/jmap/src/api/management/mod.rs +++ b/crates/jmap/src/api/management/mod.rs @@ -16,6 +16,7 @@ pub mod report; pub mod settings; pub mod sieve; pub mod stores; +pub mod troubleshoot; use std::{borrow::Cow, str::FromStr, sync::Arc}; @@ -37,6 +38,7 @@ use settings::ManageSettings; use sieve::SieveHandler; use store::write::now; use stores::ManageStore; +use troubleshoot::TroubleshootApi; use crate::{auth::oauth::auth::OAuthApiHandler, email::crypto::CryptoHandler}; @@ -155,6 +157,13 @@ impl ManagementApi for Server { } _ => Err(trc::ResourceEvent::NotFound.into_err()), }, + "troubleshoot" => { + // Validate the access token + access_token.assert_has_permission(Permission::Troubleshoot)?; + + self.handle_troubleshoot_api_request(req, path, &access_token, body) + .await + } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd // SPDX-License-Identifier: LicenseRef-SEL diff --git a/crates/jmap/src/api/management/troubleshoot.rs b/crates/jmap/src/api/management/troubleshoot.rs new file mode 100644 index 00000000..4a176f2a --- /dev/null +++ b/crates/jmap/src/api/management/troubleshoot.rs @@ -0,0 +1,1061 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{ + future::Future, + net::{IpAddr, SocketAddr}, + time::{Duration, Instant}, +}; + +use common::{ + auth::{oauth::GrantType, AccessToken}, + config::smtp::resolver::{Policy, Tlsa}, + psl, Server, +}; +use directory::backend::internal::manage; +use http_body_util::{combinators::BoxBody, StreamBody}; +use hyper::{ + body::{Bytes, Frame}, + Method, StatusCode, +}; +use mail_auth::{ + dmarc::{self}, + mta_sts::TlsRpt, + AuthenticatedMessage, DkimResult, DmarcResult, IpLookupStrategy, IprevOutput, IprevResult, + SpfOutput, SpfResult, +}; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use smtp::outbound::{ + client::{SmtpClient, StartTlsResult}, + dane::{dnssec::TlsaLookup, verify::TlsaVerify}, + lookup::{DnsLookup, ToNextHop}, + mta_sts::{lookup::MtaStsLookup, verify::VerifyPolicy}, +}; +use tokio::{io::AsyncWriteExt, sync::mpsc}; +use utils::url_params::UrlParams; + +use crate::api::{ + http::ToHttpResponse, management::decode_path_element, HttpRequest, HttpResponse, + HttpResponseBody, JsonResponse, +}; + +pub trait TroubleshootApi: Sync + Send { + fn handle_troubleshoot_api_request( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + body: Option>, + ) -> impl Future> + Send; +} + +impl TroubleshootApi for Server { + async fn handle_troubleshoot_api_request( + &self, + req: &HttpRequest, + path: Vec<&str>, + access_token: &AccessToken, + body: Option>, + ) -> trc::Result { + let params = UrlParams::new(req.uri().query()); + let account_id = access_token.primary_id(); + + match ( + path.get(1).copied().unwrap_or_default(), + path.get(2).copied(), + req.method(), + ) { + ("token", None, &Method::GET) => { + // Issue a live telemetry token valid for 60 seconds + Ok(JsonResponse::new(json!({ + "data": self.encode_access_token(GrantType::Troubleshoot, account_id, "web", 60).await?, + })) + .into_http_response()) + } + ("delivery", Some(target), &Method::GET) => { + let timeout = Duration::from_secs( + params + .parse::("timeout") + .filter(|interval| *interval >= 1) + .unwrap_or(30), + ); + + let mut rx = spawn_delivery_troubleshoot( + self.clone(), + decode_path_element(target).to_lowercase(), + timeout, + ); + + Ok(HttpResponse { + status: StatusCode::OK, + content_type: "text/event-stream".into(), + content_disposition: "".into(), + cache_control: "no-store".into(), + body: HttpResponseBody::Stream(BoxBody::new(StreamBody::new( + async_stream::stream! { + while let Some(stage) = rx.recv().await { + yield Ok(stage.to_frame()); + } + yield Ok(DeliveryStage::Completed.to_frame()); + }, + ))), + }) + } + ("dmarc", None, &Method::POST) => { + let request = serde_json::from_slice::( + body.as_deref().unwrap_or_default(), + ) + .map_err(|err| { + trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) + })?; + let response = dmarc_troubleshoot(self, request).await.ok_or_else(|| { + manage::error( + "Invalid message headers", + "Failed to parse message headers".into(), + ) + })?; + + Ok(JsonResponse::new(json!({ + "data": response, + })) + .into_http_response()) + } + _ => Err(trc::ResourceEvent::NotFound.into_err()), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(tag = "type")] +enum DeliveryStage { + MxLookupStart { + domain: String, + }, + MxLookupSuccess { + mxs: Vec, + elapsed: u64, + }, + MxLookupError { + reason: String, + elapsed: u64, + }, + MtaStsFetchStart, + MtaStsFetchSuccess { + policy: Policy, + elapsed: u64, + }, + MtaStsFetchError { + reason: String, + elapsed: u64, + }, + MtaStsNotFound { + elapsed: u64, + }, + TlsRptLookupStart, + TlsRptLookupSuccess { + rua: Vec, + elapsed: u64, + }, + TlsRptLookupError { + reason: String, + elapsed: u64, + }, + TlsRptNotFound { + elapsed: u64, + }, + DeliveryAttemptStart { + hostname: String, + }, + MtaStsVerifySuccess, + MtaStsVerifyError { + reason: String, + }, + TlsaLookupStart, + TlsaLookupSuccess { + record: Tlsa, + elapsed: u64, + }, + TlsaNotFound { + elapsed: u64, + reason: String, + }, + TlsaLookupError { + elapsed: u64, + reason: String, + }, + IpLookupStart, + IpLookupSuccess { + remote_ips: Vec, + elapsed: u64, + }, + IpLookupError { + reason: String, + elapsed: u64, + }, + ConnectionStart { + remote_ip: IpAddr, + }, + ConnectionSuccess { + elapsed: u64, + }, + ConnectionError { + elapsed: u64, + reason: String, + }, + ReadGreetingStart, + ReadGreetingSuccess { + elapsed: u64, + }, + ReadGreetingError { + elapsed: u64, + reason: String, + }, + EhloStart, + EhloSuccess { + elapsed: u64, + }, + EhloError { + elapsed: u64, + reason: String, + }, + StartTlsStart, + StartTlsSuccess { + elapsed: u64, + }, + StartTlsError { + elapsed: u64, + reason: String, + }, + DaneVerifySuccess, + DaneVerifyError { + reason: String, + }, + MailFromStart, + MailFromSuccess { + elapsed: u64, + }, + MailFromError { + reason: String, + elapsed: u64, + }, + RcptToStart, + RcptToSuccess { + elapsed: u64, + }, + RcptToError { + reason: String, + elapsed: u64, + }, + QuitStart, + QuitCompleted { + elapsed: u64, + }, + Completed, +} + +#[derive(Debug, Serialize, Deserialize)] +struct MX { + pub exchanges: Vec, + pub preference: u16, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(tag = "type")] +pub enum ReportUri { + Mail { email: String }, + Http { url: String }, +} + +impl DeliveryStage { + fn to_frame(&self) -> Frame { + let payload = format!( + "event: event\ndata: [{}]\n\n", + serde_json::to_string(self).unwrap_or_default() + ); + Frame::data(Bytes::from(payload)) + } +} + +trait ElapsedMs { + fn elapsed_ms(&self) -> u64; +} + +impl ElapsedMs for Instant { + fn elapsed_ms(&self) -> u64 { + self.elapsed().as_millis() as u64 + } +} +fn spawn_delivery_troubleshoot( + server: Server, + domain_or_email: String, + timeout: Duration, +) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(10); + + tokio::spawn(async move { + let _ = delivery_troubleshoot(tx, server, domain_or_email, timeout).await; + }); + + rx +} + +async fn delivery_troubleshoot( + tx: mpsc::Sender, + server: Server, + domain_or_email: String, + timeout: Duration, +) -> Result<(), mpsc::error::SendError> { + let (domain, email) = if let Some((_, domain)) = domain_or_email.rsplit_once('@') { + (domain.to_string(), Some(domain_or_email)) + } else { + (domain_or_email, None) + }; + + let local_host = server + .core + .storage + .config + .get("lookup.default.hostname") + .await + .unwrap_or_default() + .unwrap_or_else(|| "local.host".to_string()); + + tx.send(DeliveryStage::MxLookupStart { + domain: domain.to_string(), + }) + .await?; + + // Lookup MX + let now = Instant::now(); + let mxs = match server.core.smtp.resolvers.dns.mx_lookup(&domain).await { + Ok(mxs) => mxs, + Err(err) => { + tx.send(DeliveryStage::MxLookupError { + reason: err.to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + + return Ok(()); + } + }; + + // Obtain remote host list + let hosts = if let Some(hosts) = mxs.to_remote_hosts(&domain, mxs.len()) { + tx.send(DeliveryStage::MxLookupSuccess { + mxs: mxs + .iter() + .map(|mx| MX { + exchanges: mx.exchanges.clone(), + preference: mx.preference, + }) + .collect(), + elapsed: now.elapsed_ms(), + }) + .await?; + + hosts + } else { + tx.send(DeliveryStage::MxLookupError { + reason: "Null MX record".to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + + return Ok(()); + }; + + // Fetch MTA-STS policy + let now = Instant::now(); + tx.send(DeliveryStage::MtaStsFetchStart).await?; + let mta_sts_policy = match server.lookup_mta_sts_policy(&domain, timeout).await { + Ok(policy) => { + tx.send(DeliveryStage::MtaStsFetchSuccess { + policy: policy.as_ref().clone(), + elapsed: now.elapsed_ms(), + }) + .await?; + Some(policy) + } + Err(err) => { + if matches!( + &err, + smtp::outbound::mta_sts::Error::Dns(mail_auth::Error::DnsRecordNotFound(_)) + ) { + tx.send(DeliveryStage::MtaStsNotFound { + elapsed: now.elapsed_ms(), + }) + .await?; + } else { + tx.send(DeliveryStage::MtaStsFetchError { + reason: err.to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + None + } + }; + + // Fetch TLS-RPT settings + let now = Instant::now(); + tx.send(DeliveryStage::TlsRptLookupStart).await?; + match server + .core + .smtp + .resolvers + .dns + .txt_lookup::(format!("_smtp._tls.{domain}.")) + .await + { + Ok(record) => { + tx.send(DeliveryStage::TlsRptLookupSuccess { + rua: record + .rua + .iter() + .map(|r| match r { + mail_auth::mta_sts::ReportUri::Mail(email) => ReportUri::Mail { + email: email.clone(), + }, + mail_auth::mta_sts::ReportUri::Http(url) => { + ReportUri::Http { url: url.clone() } + } + }) + .collect(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + Err(err) => { + if matches!(&err, mail_auth::Error::DnsRecordNotFound(_)) { + tx.send(DeliveryStage::TlsRptNotFound { + elapsed: now.elapsed_ms(), + }) + .await?; + } else { + tx.send(DeliveryStage::TlsRptLookupError { + reason: err.to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + } + } + + // Try with each host + 'outer: for host in hosts { + let hostname = host.hostname(); + + tx.send(DeliveryStage::DeliveryAttemptStart { + hostname: hostname.to_string(), + }) + .await?; + + // Verify MTA-STS policy + if let Some(mta_sts_policy) = &mta_sts_policy { + if mta_sts_policy.verify(hostname) { + tx.send(DeliveryStage::MtaStsVerifySuccess).await?; + } else { + tx.send(DeliveryStage::MtaStsVerifyError { + reason: "Not authorized by policy".to_string(), + }) + .await?; + + continue; + } + } + + // Fetch TLSA record + tx.send(DeliveryStage::TlsaLookupStart).await?; + + let now = Instant::now(); + let dane_policy = match server.tlsa_lookup(format!("_25._tcp.{hostname}.")).await { + Ok(Some(tlsa)) if tlsa.has_end_entities => { + tx.send(DeliveryStage::TlsaLookupSuccess { + record: tlsa.as_ref().clone(), + elapsed: now.elapsed_ms(), + }) + .await?; + + Some(tlsa) + } + Ok(Some(_)) => { + tx.send(DeliveryStage::TlsaLookupError { + elapsed: now.elapsed_ms(), + reason: "TLSA record does not have end entities".to_string(), + }) + .await?; + + None + } + Ok(None) => { + tx.send(DeliveryStage::TlsaNotFound { + elapsed: now.elapsed_ms(), + reason: "No TLSA DNSSEC records found".to_string(), + }) + .await?; + + None + } + Err(err) => { + if matches!(&err, mail_auth::Error::DnsRecordNotFound(_)) { + tx.send(DeliveryStage::TlsaNotFound { + elapsed: now.elapsed_ms(), + reason: "No TLSA records found for MX".to_string(), + }) + .await?; + } else { + tx.send(DeliveryStage::TlsaLookupError { + elapsed: now.elapsed_ms(), + reason: err.to_string(), + }) + .await?; + } + None + } + }; + + tx.send(DeliveryStage::IpLookupStart).await?; + + let now = Instant::now(); + match server + .ip_lookup( + host.fqdn_hostname().as_ref(), + IpLookupStrategy::Ipv4thenIpv6, + usize::MAX, + ) + .await + { + Ok(remote_ips) if !remote_ips.is_empty() => { + tx.send(DeliveryStage::IpLookupSuccess { + remote_ips: remote_ips.clone(), + elapsed: now.elapsed_ms(), + }) + .await?; + + for remote_ip in remote_ips { + // Start connection + tx.send(DeliveryStage::ConnectionStart { remote_ip }) + .await?; + + let now = Instant::now(); + match SmtpClient::connect(SocketAddr::new(remote_ip, 25), timeout, 0).await { + Ok(mut client) => { + tx.send(DeliveryStage::ConnectionSuccess { + elapsed: now.elapsed_ms(), + }) + .await?; + + // Read greeting + tx.send(DeliveryStage::ReadGreetingStart).await?; + + let now = Instant::now(); + if let Err(status) = client.read_greeting(hostname).await { + tx.send(DeliveryStage::ReadGreetingError { + elapsed: now.elapsed_ms(), + reason: status.to_string(), + }) + .await?; + + continue; + } + tx.send(DeliveryStage::ReadGreetingSuccess { + elapsed: now.elapsed_ms(), + }) + .await?; + + // Say EHLO + tx.send(DeliveryStage::EhloStart).await?; + + let now = Instant::now(); + let capabilities = match tokio::time::timeout(timeout, async { + client + .stream + .write_all(format!("EHLO {local_host}\r\n",).as_bytes()) + .await?; + client.stream.flush().await?; + client.read_ehlo().await + }) + .await + { + Ok(Ok(capabilities)) => { + tx.send(DeliveryStage::EhloSuccess { + elapsed: now.elapsed_ms(), + }) + .await?; + + capabilities + } + Ok(Err(err)) => { + tx.send(DeliveryStage::EhloError { + elapsed: now.elapsed_ms(), + reason: err.to_string(), + }) + .await?; + + continue; + } + Err(_) => { + tx.send(DeliveryStage::EhloError { + elapsed: now.elapsed_ms(), + reason: "Timed out reading response".to_string(), + }) + .await?; + + continue; + } + }; + + // Start TLS + tx.send(DeliveryStage::StartTlsStart).await?; + + let now = Instant::now(); + let mut client = match client + .try_start_tls( + &server.inner.data.smtp_connectors.pki_verify, + hostname, + &capabilities, + ) + .await + { + StartTlsResult::Success { smtp_client } => { + tx.send(DeliveryStage::StartTlsSuccess { + elapsed: now.elapsed_ms(), + }) + .await?; + + smtp_client + } + StartTlsResult::Error { error } => { + tx.send(DeliveryStage::StartTlsError { + elapsed: now.elapsed_ms(), + reason: error.to_string(), + }) + .await?; + + continue; + } + StartTlsResult::Unavailable { response, .. } => { + tx.send(DeliveryStage::StartTlsError { + elapsed: now.elapsed_ms(), + reason: response.map(|r| r.to_string()).unwrap_or_else( + || "STARTTLS not advertised by host".to_string(), + ), + }) + .await?; + + continue; + } + }; + + // Verify DANE policy + if let Some(dane_policy) = &dane_policy { + if let Err(err) = dane_policy.verify( + 0, + hostname, + client.tls_connection().peer_certificates(), + ) { + tx.send(DeliveryStage::DaneVerifyError { + reason: err.to_string(), + }) + .await?; + } else { + tx.send(DeliveryStage::DaneVerifySuccess).await?; + } + } + + // Verify recipient + let mut is_success = email.is_none(); + if let Some(email) = &email { + // MAIL FROM + tx.send(DeliveryStage::MailFromStart).await?; + + let now = Instant::now(); + + match client.cmd(b"MAIL FROM:<>\r\n").await.and_then(|r| { + if r.is_positive_completion() { + Ok(r) + } else { + Err(mail_send::Error::UnexpectedReply(r)) + } + }) { + Ok(_) => { + tx.send(DeliveryStage::MailFromSuccess { + elapsed: now.elapsed_ms(), + }) + .await?; + + // RCPT TO + tx.send(DeliveryStage::RcptToStart).await?; + + let now = Instant::now(); + match client + .cmd(format!("RCPT TO:<{email}>\r\n").as_bytes()) + .await + .and_then(|r| { + if r.is_positive_completion() { + Ok(r) + } else { + Err(mail_send::Error::UnexpectedReply(r)) + } + }) { + Ok(_) => { + is_success = true; + tx.send(DeliveryStage::RcptToSuccess { + elapsed: now.elapsed_ms(), + }) + .await?; + } + Err(err) => { + tx.send(DeliveryStage::RcptToError { + reason: err.to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + } + } + Err(err) => { + tx.send(DeliveryStage::MailFromError { + reason: err.to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + } + } + + // QUIT + tx.send(DeliveryStage::QuitStart).await?; + + let now = Instant::now(); + client.quit().await; + tx.send(DeliveryStage::QuitCompleted { + elapsed: now.elapsed_ms(), + }) + .await?; + + if is_success { + break 'outer; + } + } + Err(err) => { + tx.send(DeliveryStage::ConnectionError { + elapsed: now.elapsed_ms(), + reason: err.to_string(), + }) + .await?; + } + } + } + } + Ok(_) => { + tx.send(DeliveryStage::IpLookupError { + reason: "No IP addresses found for host".to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + Err(err) => { + tx.send(DeliveryStage::IpLookupError { + reason: err.to_string(), + elapsed: now.elapsed_ms(), + }) + .await?; + } + } + } + + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize)] +struct DmarcTroubleshootRequest { + #[serde(rename = "remoteIp")] + remote_ip: IpAddr, + #[serde(rename = "ehloDomain")] + ehlo_domain: String, + #[serde(rename = "mailFrom")] + mail_from: String, + headers: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct DmarcTroubleshootResponse { + #[serde(rename = "spfEhloDomain")] + spf_ehlo_domain: String, + #[serde(rename = "spfEhloResult")] + spf_ehlo_result: AuthResult, + #[serde(rename = "spfMailFromDomain")] + spf_mail_from_domain: String, + #[serde(rename = "spfMailFromResult")] + spf_mail_from_result: AuthResult, + #[serde(rename = "ipRevResult")] + ip_rev_result: AuthResult, + #[serde(rename = "ipRevPtr")] + ip_rev_ptr: Vec, + #[serde(rename = "dkimResults")] + dkim_results: Vec, + #[serde(rename = "dkimPass")] + dkim_pass: bool, + #[serde(rename = "arcResult")] + arc_result: AuthResult, + #[serde(rename = "dmarcResult")] + dmarc_result: AuthResult, + #[serde(rename = "dmarcPass")] + dmarc_pass: bool, + #[serde(rename = "dmarcPolicy")] + dmarc_policy: DmarcPolicy, + elapsed: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(tag = "type")] +pub enum AuthResult { + Pass, + Fail { details: Option }, + SoftFail { details: Option }, + TempError { details: Option }, + PermError { details: Option }, + Neutral { details: Option }, + None, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DmarcPolicy { + None, + Quarantine, + Reject, + Unspecified, +} + +async fn dmarc_troubleshoot( + server: &Server, + request: DmarcTroubleshootRequest, +) -> Option { + let remote_ip = request.remote_ip; + let ehlo_domain = request.ehlo_domain.to_lowercase(); + let mail_from = request.mail_from.to_lowercase(); + let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); + + let local_host = server + .core + .storage + .config + .get("lookup.default.hostname") + .await + .unwrap_or_default() + .unwrap_or_else(|| "local.host".to_string()); + + let now = Instant::now(); + let ehlo_spf_output = server + .core + .smtp + .resolvers + .dns + .verify_spf_helo(remote_ip, &ehlo_domain, &local_host) + .await; + + let iprev = server.core.smtp.resolvers.dns.verify_iprev(remote_ip).await; + let mail_spf_output = if let Some(mail_from_domain) = mail_from_domain { + server + .core + .smtp + .resolvers + .dns + .check_host( + remote_ip, + mail_from_domain, + &ehlo_domain, + &local_host, + &mail_from, + ) + .await + } else { + server + .core + .smtp + .resolvers + .dns + .check_host( + remote_ip, + &ehlo_domain, + &ehlo_domain, + &local_host, + &format!("postmaster@{ehlo_domain}"), + ) + .await + }; + + let headers = request + .headers + .unwrap_or_else(|| format!("From: {mail_from}\r\nSubject: test\r\n\r\ntest")); + let auth_message = AuthenticatedMessage::parse_with_opts(headers.as_bytes(), true)?; + + let dkim_output = server + .core + .smtp + .resolvers + .dns + .verify_dkim(&auth_message) + .await; + let dkim_pass = dkim_output + .iter() + .any(|d| matches!(d.result(), DkimResult::Pass)); + + let arc_output = server + .core + .smtp + .resolvers + .dns + .verify_arc(&auth_message) + .await; + + let dmarc_output = server + .core + .smtp + .resolvers + .dns + .verify_dmarc( + &auth_message, + &dkim_output, + mail_from_domain.unwrap_or(ehlo_domain.as_str()), + &mail_spf_output, + |domain| psl::domain_str(domain).unwrap_or(domain), + ) + .await; + let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) + || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); + let dmarc_result = if dmarc_pass { + DmarcResult::Pass + } else if dmarc_output.spf_result() != &DmarcResult::None { + dmarc_output.spf_result().clone() + } else if dmarc_output.dkim_result() != &DmarcResult::None { + dmarc_output.dkim_result().clone() + } else { + DmarcResult::None + }; + + Some(DmarcTroubleshootResponse { + spf_ehlo_domain: ehlo_spf_output.domain().to_string(), + spf_ehlo_result: (&ehlo_spf_output).into(), + spf_mail_from_domain: mail_spf_output.domain().to_string(), + spf_mail_from_result: (&mail_spf_output).into(), + ip_rev_ptr: iprev + .ptr + .as_ref() + .map(|ptr| ptr.as_ref().clone()) + .unwrap_or_default(), + ip_rev_result: (&iprev).into(), + dkim_pass, + dkim_results: dkim_output + .iter() + .map(|result| result.result().into()) + .collect(), + arc_result: arc_output.result().into(), + dmarc_result: (&dmarc_result).into(), + dmarc_policy: (&dmarc_output.policy()).into(), + dmarc_pass, + elapsed: now.elapsed_ms(), + }) +} + +impl From<&SpfOutput> for AuthResult { + fn from(value: &SpfOutput) -> Self { + match value.result() { + SpfResult::Pass => AuthResult::Pass, + SpfResult::Fail => AuthResult::Fail { + details: value.explanation().map(|e| e.to_string()), + }, + SpfResult::SoftFail => AuthResult::SoftFail { + details: value.explanation().map(|e| e.to_string()), + }, + SpfResult::Neutral => AuthResult::Neutral { + details: value.explanation().map(|e| e.to_string()), + }, + SpfResult::TempError => AuthResult::TempError { + details: value.explanation().map(|e| e.to_string()), + }, + SpfResult::PermError => AuthResult::PermError { + details: value.explanation().map(|e| e.to_string()), + }, + SpfResult::None => AuthResult::None, + } + } +} + +impl From<&IprevOutput> for AuthResult { + fn from(value: &IprevOutput) -> Self { + match &value.result { + IprevResult::Pass => AuthResult::Pass, + IprevResult::Fail(error) => AuthResult::Fail { + details: error.to_string().into(), + }, + IprevResult::TempError(error) => AuthResult::TempError { + details: error.to_string().into(), + }, + IprevResult::PermError(error) => AuthResult::PermError { + details: error.to_string().into(), + }, + IprevResult::None => AuthResult::None, + } + } +} + +impl From<&DkimResult> for AuthResult { + fn from(value: &DkimResult) -> Self { + match value { + DkimResult::Pass => AuthResult::Pass, + DkimResult::Neutral(error) => AuthResult::Neutral { + details: error.to_string().into(), + }, + DkimResult::Fail(error) => AuthResult::Fail { + details: error.to_string().into(), + }, + DkimResult::PermError(error) => AuthResult::PermError { + details: error.to_string().into(), + }, + DkimResult::TempError(error) => AuthResult::TempError { + details: error.to_string().into(), + }, + DkimResult::None => AuthResult::None, + } + } +} + +impl From<&DmarcResult> for AuthResult { + fn from(value: &DmarcResult) -> Self { + match value { + DmarcResult::Pass => AuthResult::Pass, + DmarcResult::Fail(error) => AuthResult::Fail { + details: error.to_string().into(), + }, + DmarcResult::TempError(error) => AuthResult::TempError { + details: error.to_string().into(), + }, + DmarcResult::PermError(error) => AuthResult::PermError { + details: error.to_string().into(), + }, + DmarcResult::None => AuthResult::None, + } + } +} + +impl From<&dmarc::Policy> for DmarcPolicy { + fn from(value: &dmarc::Policy) -> Self { + match value { + dmarc::Policy::None => DmarcPolicy::None, + dmarc::Policy::Quarantine => DmarcPolicy::Quarantine, + dmarc::Policy::Reject => DmarcPolicy::Reject, + dmarc::Policy::Unspecified => DmarcPolicy::Unspecified, + } + } +} diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 2a8694f2..f8f00ad0 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -360,6 +360,15 @@ impl DeliveryAttempt { TlsRptOptions { record, interval }.into() } + Err(mail_auth::Error::DnsRecordNotFound(_)) => { + trc::event!( + TlsRpt(TlsRptEvent::RecordNotFound), + SpanId = message.span_id, + Domain = domain.domain.clone(), + Elapsed = time.elapsed(), + ); + None + } Err(err) => { trc::event!( TlsRpt(TlsRptEvent::RecordFetchError), diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index e3886e00..5461116c 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -220,7 +220,7 @@ pub enum NextHop<'x> { impl NextHop<'_> { #[inline(always)] - fn hostname(&self) -> &str { + pub fn hostname(&self) -> &str { match self { NextHop::MX(host) => { if let Some(host) = host.strip_suffix('.') { @@ -234,7 +234,7 @@ impl NextHop<'_> { } #[inline(always)] - fn fqdn_hostname(&self) -> Cow<'_, str> { + pub fn fqdn_hostname(&self) -> Cow<'_, str> { match self { NextHop::MX(host) => { if !host.ends_with('.') { diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index 91bd4f0a..6a2e83ad 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -881,6 +881,7 @@ impl TlsRptEvent { match self { TlsRptEvent::RecordFetch => "Fetched TLS-RPT record", TlsRptEvent::RecordFetchError => "Error fetching TLS-RPT record", + TlsRptEvent::RecordNotFound => "TLS-RPT record not found", } } @@ -888,6 +889,7 @@ impl TlsRptEvent { match self { TlsRptEvent::RecordFetch => "The TLS-RPT record has been fetched", TlsRptEvent::RecordFetchError => "An error occurred while fetching the TLS-RPT record", + TlsRptEvent::RecordNotFound => "No TLS-RPT records were found", } } } diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index d40b5649..927d2501 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -478,7 +478,9 @@ impl EventType { } }, EventType::TlsRpt(event) => match event { - TlsRptEvent::RecordFetch | TlsRptEvent::RecordFetchError => Level::Info, + TlsRptEvent::RecordFetch + | TlsRptEvent::RecordFetchError + | TlsRptEvent::RecordNotFound => Level::Info, }, EventType::MtaSts(event) => match event { MtaStsEvent::PolicyFetch diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index 62c809b8..19ffb5db 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -545,6 +545,7 @@ pub enum MtaStsEvent { pub enum TlsRptEvent { RecordFetch, RecordFetchError, + RecordNotFound, } #[event_type] diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index 7c635648..26c82e75 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -865,6 +865,7 @@ impl EventType { EventType::Ai(AiEvent::ApiError) => 557, EventType::Security(SecurityEvent::ScanBan) => 558, EventType::Store(StoreEvent::AzureError) => 559, + EventType::TlsRpt(TlsRptEvent::RecordNotFound) => 560, } } @@ -1470,6 +1471,7 @@ impl EventType { 557 => Some(EventType::Ai(AiEvent::ApiError)), 558 => Some(EventType::Security(SecurityEvent::ScanBan)), 559 => Some(EventType::Store(StoreEvent::AzureError)), + 560 => Some(EventType::TlsRpt(TlsRptEvent::RecordNotFound)), _ => None, } }