From f9b37a3c99b8d21ac122bee5fbc46cafc8f96bb9 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Fri, 20 Oct 2023 19:18:18 +0200 Subject: [PATCH] Spam filter combined tests passing --- Cargo.lock | 99 ++++---- crates/cli/Cargo.toml | 2 +- crates/imap/Cargo.toml | 2 +- crates/install/Cargo.toml | 2 +- crates/jmap/Cargo.toml | 2 +- crates/main/Cargo.toml | 2 +- crates/nlp/Cargo.toml | 2 +- crates/smtp/Cargo.toml | 2 +- crates/smtp/src/config/scripts.rs | 2 +- crates/smtp/src/inbound/data.rs | 12 +- crates/smtp/src/scripts/functions/misc.rs | 2 +- crates/utils/Cargo.toml | 2 +- crates/utils/src/config/mod.rs | 19 +- resources/config/directory/memory.toml | 8 + resources/config/smtp/remote.toml | 2 +- resources/config/smtp/sieve.toml | 1 + resources/config/smtp/spamfilter.toml | 58 +++-- .../config/spamfilter/scripts/dmarc.sieve | 2 +- .../config/spamfilter/scripts/epilogue.sieve | 6 +- .../config/spamfilter/scripts/replyto.sieve | 2 +- .../config/spamfilter/scripts/scores.sieve | 6 +- tests/resources/create_test_env.sh | 39 ++- tests/resources/smtp/antispam/combined.test | 10 +- tests/resources/test_config.toml | 236 ------------------ tests/src/smtp/inbound/antispam.rs | 66 ++++- 25 files changed, 229 insertions(+), 357 deletions(-) delete mode 100644 tests/resources/test_config.toml diff --git a/Cargo.lock b/Cargo.lock index ca45aca2..d02f3ae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -918,9 +918,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" dependencies = [ "libc", ] @@ -1104,7 +1104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lock_api", "once_cell", "parking_lot_core", @@ -1924,9 +1924,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash 0.8.3", "allocator-api2", @@ -1938,7 +1938,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -2147,7 +2147,7 @@ dependencies = [ [[package]] name = "hyper-util" version = "0.0.0" -source = "git+https://github.com/hyperium/hyper-util#9a17089cb5097d3617e30019e5202f181e058eb9" +source = "git+https://github.com/hyperium/hyper-util#1ed4c2ccdb23f576eb7024555f08b376b9d5c9eb" dependencies = [ "bytes", "futures-channel", @@ -2224,7 +2224,7 @@ checksum = "029d73f573d8e8d63e6d5020011d3255b28c3ba85d6cf870a07184ed23de9284" [[package]] name = "imap" -version = "0.3.10" +version = "0.4.0" dependencies = [ "ahash 0.8.3", "dashmap", @@ -2274,7 +2274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -2399,7 +2399,7 @@ checksum = "93f0c1347cd3ac8d7c6e3a2dc33ac496d365cf09fc0831aa61111e1a6738983e" dependencies = [ "cedarwood", "fxhash", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lazy_static", "phf", "phf_codegen", @@ -2408,7 +2408,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.3.10" +version = "0.4.0" dependencies = [ "aes", "aes-gcm", @@ -2685,9 +2685,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -2730,7 +2730,7 @@ dependencies = [ "mail-parser", "parking_lot", "quick-xml 0.30.0", - "ring 0.17.4", + "ring 0.17.5", "rustls-pemfile", "serde", "serde_json", @@ -2772,7 +2772,7 @@ dependencies = [ [[package]] name = "mail-server" -version = "0.3.10" +version = "0.4.0" dependencies = [ "directory", "imap", @@ -2979,7 +2979,7 @@ dependencies = [ [[package]] name = "nlp" -version = "0.3.10" +version = "0.4.0" dependencies = [ "ahash 0.8.3", "bincode", @@ -3338,13 +3338,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall 0.4.1", "smallvec", "windows-targets 0.48.5", ] @@ -3802,9 +3802,9 @@ dependencies = [ [[package]] name = "rasn" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb030bce0cb6578f484f4b2f9f0bd038d0562e235c856730cd845947c86fbd82" +checksum = "0ef6075807dc2ea24ddbfaf1546bf4b956ac1f8338eae4dffa35d69846c8b691" dependencies = [ "arrayvec", "bitvec", @@ -3824,9 +3824,9 @@ dependencies = [ [[package]] name = "rasn-cms" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97180cf43564a824df365b53339e2dc967db608b2aa2b7a247ccb619e723f990" +checksum = "8d8d8656a9e88bca3a726fc7c365e4c8a4455d1f95ecd58f7817cfc642674cbc" dependencies = [ "rasn", "rasn-pkix", @@ -3834,9 +3834,9 @@ dependencies = [ [[package]] name = "rasn-derive" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d71113bb47670ae208b1db7308aa35c676db8bb7b7a8f49b0c1e208af5ebf4" +checksum = "eda946c91626f958f853e47616cdfdd1fcd9c36a65a2388eb9a5d80dd644efd3" dependencies = [ "either", "itertools 0.10.5", @@ -3849,9 +3849,9 @@ dependencies = [ [[package]] name = "rasn-pkix" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf5c1f5213088c8552ed4dc61dc07dc8211cb4aa5e7c49eb9161b32d005a55" +checksum = "6c91ef69d71656ca94e119989fd2056679823852d2544f94cd96bd6912a49a06" dependencies = [ "rasn", ] @@ -3894,6 +3894,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -4035,9 +4044,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.4" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce3045ffa7c981a6ee93f640b538952e155f1ae3a1a02b84547fc7a56b7059a" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", "getrandom 0.2.10", @@ -4209,9 +4218,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" dependencies = [ "bitflags 2.4.1", "errno", @@ -4613,7 +4622,7 @@ checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" [[package]] name = "sieve-rs" version = "0.3.1" -source = "git+https://github.com/stalwartlabs/sieve#d6251ec246011d4bbd8ee9919b7f4864d626d915" +source = "git+https://github.com/stalwartlabs/sieve#b7b62b3b33d05b7c1e50ea5123a35ef461c66b8d" dependencies = [ "ahash 0.8.3", "bincode", @@ -4678,7 +4687,7 @@ checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "smtp" -version = "0.3.10" +version = "0.4.0" dependencies = [ "ahash 0.8.3", "blake3", @@ -5009,7 +5018,7 @@ dependencies = [ [[package]] name = "stalwart-cli" -version = "0.3.10" +version = "0.4.0" dependencies = [ "clap", "console", @@ -5031,7 +5040,7 @@ dependencies = [ [[package]] name = "stalwart-install" -version = "0.3.10" +version = "0.4.0" dependencies = [ "base64 0.21.4", "clap", @@ -5281,18 +5290,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", @@ -5536,9 +5545,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", @@ -5893,7 +5902,7 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utils" -version = "0.3.10" +version = "0.4.0" dependencies = [ "ahash 0.8.3", "chrono", @@ -5919,9 +5928,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom 0.2.10", ] @@ -6060,7 +6069,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.4", + "ring 0.17.5", "untrusted 0.9.0", ] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 415278a3..68f48ff2 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only" repository = "https://github.com/stalwartlabs/cli" homepage = "https://github.com/stalwartlabs/cli" -version = "0.3.10" +version = "0.4.0" edition = "2021" readme = "README.md" resolver = "2" diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 9ba74356..ae2649e0 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.3.10" +version = "0.4.0" edition = "2021" resolver = "2" diff --git a/crates/install/Cargo.toml b/crates/install/Cargo.toml index 67ee2ea4..51ca6520 100644 --- a/crates/install/Cargo.toml +++ b/crates/install/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Stalwart Labs Ltd. "] license = "AGPL-3.0-only" repository = "https://github.com/stalwartlabs/mail-server" homepage = "https://github.com/stalwartlabs/mail-server" -version = "0.3.10" +version = "0.4.0" edition = "2021" readme = "README.md" resolver = "2" diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 46b0e6ba..fb972099 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.3.10" +version = "0.4.0" edition = "2021" resolver = "2" diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 9feeb60f..53f2c0f9 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art" keywords = ["imap", "jmap", "smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only" -version = "0.3.10" +version = "0.4.0" edition = "2021" resolver = "2" diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index a9d8102e..e32041ba 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nlp" -version = "0.3.10" +version = "0.4.0" edition = "2021" resolver = "2" diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index b5cbb916..3a7d156b 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -7,7 +7,7 @@ homepage = "https://stalw.art/smtp" keywords = ["smtp", "email", "mail", "server"] categories = ["email"] license = "AGPL-3.0-only" -version = "0.3.10" +version = "0.4.0" edition = "2021" resolver = "2" diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs index 96c4f0b4..8f367d85 100644 --- a/crates/smtp/src/config/scripts.rs +++ b/crates/smtp/src/config/scripts.rs @@ -154,7 +154,7 @@ impl ConfigSieve for Config { let script = if !self.contains_key(key) { let mut script = Vec::new(); for sub_key in self.sub_keys(key) { - script.extend(self.file_contents(sub_key)?); + script.extend(self.file_contents(("sieve.smtp.scripts", id, sub_key))?); } script } else { diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 68c0928f..3e1ba1ee 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -291,7 +291,8 @@ impl Session { // Run Milter filters let mut edited_message = match self.run_milters(&auth_message).await { Ok(modifications) => { - tracing::debug!( + if !modifications.is_empty() { + tracing::debug!( parent: &self.span, context = "milter", event = "accept", @@ -305,9 +306,12 @@ impl Session { }), "Milter filter(s) accepted message."); - self.data - .apply_milter_modifications(modifications, &auth_message) - .map(Arc::new) + self.data + .apply_milter_modifications(modifications, &auth_message) + .map(Arc::new) + } else { + None + } } Err(response) => return response, }; diff --git a/crates/smtp/src/scripts/functions/misc.rs b/crates/smtp/src/scripts/functions/misc.rs index 0855d48a..d7bc229a 100644 --- a/crates/smtp/src/scripts/functions/misc.rs +++ b/crates/smtp/src/scripts/functions/misc.rs @@ -116,7 +116,7 @@ pub fn fn_hash<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variab pub fn fn_is_var_names<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec) -> Variable { Variable::Array( ctx.global_variable_names() - .map(|v| Variable::from(v.to_string())) + .map(|v| Variable::from(v.to_uppercase())) .collect::>() .into(), ) diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index f3ab38df..d7b64cc8 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "utils" -version = "0.3.10" +version = "0.4.0" edition = "2021" resolver = "2" diff --git a/crates/utils/src/config/mod.rs b/crates/utils/src/config/mod.rs index 5660f7d3..6b29b35d 100644 --- a/crates/utils/src/config/mod.rs +++ b/crates/utils/src/config/mod.rs @@ -181,9 +181,8 @@ impl Config { let mut includes = AHashSet::new(); let mut macros = AHashMap::new(); - for (key, mut value) in config.keys { - value.replace_macros(&key, ¯os); - if let Some(macro_name) = key.strip_prefix("macro.") { + for (key, value) in config.keys { + if let Some(macro_name) = key.strip_prefix("macros.") { macros.insert(macro_name.to_ascii_lowercase(), value); } else if key.starts_with("include.files.") { includes.insert(value); @@ -194,13 +193,13 @@ impl Config { // Include files config.keys = keys; - for include in includes { + for mut include in includes { + include.replace_macros("include.files", ¯os); config - .parse( - &std::fs::read_to_string(include) - .failed("Could not read included configuration file"), - ) - .failed("Invalid included configuration file"); + .parse(&std::fs::read_to_string(&include).failed(&format!( + "Could not read included configuration file {include:?}" + ))) + .failed(&format!("Invalid included configuration file {include:?}")); } // Replace macros @@ -227,7 +226,7 @@ impl ReplaceMacros for String { if !suffix.is_empty() { result.push_str(suffix); } - if let Some((macro_name, rest)) = macro_name.split_once('}') { + if let Some((macro_name, rest)) = macro_name.split_once("}%") { if let Some(macro_value) = macros.get(¯o_name.to_ascii_lowercase()) { result.push_str(macro_value); value = rest; diff --git a/resources/config/directory/memory.toml b/resources/config/directory/memory.toml index a5fe97ea..03b478c9 100644 --- a/resources/config/directory/memory.toml +++ b/resources/config/directory/memory.toml @@ -19,6 +19,14 @@ secret = "changeme" email = ["postmaster@%{DEFAULT_DOMAIN}%"] member-of = ["superusers"] +[[directory."default".users]] +name = "john" +description = "John Doe" +secret = "12345" +email = ["john@%{DEFAULT_DOMAIN}%", "jdoe@%{DEFAULT_DOMAIN}%", "john.doe@%{DEFAULT_DOMAIN}%"] +email-list = ["info@%{DEFAULT_DOMAIN}%"] +member-of = ["sales"] + [[directory."default".users]] name = "jane" description = "Jane Doe" diff --git a/resources/config/smtp/remote.toml b/resources/config/smtp/remote.toml index a159725f..27a658f5 100644 --- a/resources/config/smtp/remote.toml +++ b/resources/config/smtp/remote.toml @@ -5,7 +5,7 @@ [remote."local"] address = "127.0.0.1" port = 11200 -protocol = "local" +protocol = "lmtp" concurrency = 10 timeout = "1m" diff --git a/resources/config/smtp/sieve.toml b/resources/config/smtp/sieve.toml index 68664bf3..264a127e 100644 --- a/resources/config/smtp/sieve.toml +++ b/resources/config/smtp/sieve.toml @@ -7,6 +7,7 @@ from-name = "Automated Message" from-addr = "no-reply@%{DEFAULT_DOMAIN}%" return-path = "" #hostname = "%{HOST}%" +no-capability-check = true sign = ["rsa"] [sieve.smtp.limits] diff --git a/resources/config/smtp/spamfilter.toml b/resources/config/smtp/spamfilter.toml index 60c7e081..fcb89b66 100644 --- a/resources/config/smtp/spamfilter.toml +++ b/resources/config/smtp/spamfilter.toml @@ -28,6 +28,9 @@ query = ["DELETE FROM seen_ids WHERE ttl < CURRENT_TIMESTAMP", "DELETE FROM reputation WHERE ttl < CURRENT_TIMESTAMP"] frequency = "0 3 *" +[directory."spam"] +type = "memory" + [directory."spam".lookup."free-domains"] type = "glob" comment = '#' @@ -80,33 +83,34 @@ type = "map" values = "file://%{BASE_PATH}%/etc/spamfilter/maps/scores.map" [sieve.smtp.scripts] -spam-filter = ["file://%{BASE_PATH}%/etc/spamfilter/scripts/config", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/prelude", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/from", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/recipient", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/subject", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/replyto", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/date", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/messageid", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/received", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/headers", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/bounce", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/html", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/mime", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/dmarc", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/ip", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/helo", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/replies_in", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/spamtrap", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/bayes_classify", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/url", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/rbl", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/pyzor", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/composites", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/scores", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/reputation", - "file://%{BASE_PATH}%/etc/spamfilter/scripts/epilogue"] +spam-filter = ["file://%{BASE_PATH}%/etc/spamfilter/scripts/config.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/prelude.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/from.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/recipient.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/subject.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/replyto.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/date.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/messageid.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/received.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/headers.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/bounce.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/html.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/mime.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/dmarc.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/ip.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/helo.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/replies_in.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/spamtrap.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/bayes_classify.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/url.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/rbl.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/pyzor.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/composites.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/scores.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/reputation.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/epilogue.sieve"] -track-replies = "file://%{BASE_PATH}%/etc/spamfilter/scripts/replies_out.sieve" +track-replies = ["file://%{BASE_PATH}%/etc/spamfilter/scripts/config.sieve", + "file://%{BASE_PATH}%/etc/spamfilter/scripts/replies_out.sieve"] greylist = "file://%{BASE_PATH}%/etc/spamfilter/scripts/greylist.sieve" diff --git a/resources/config/spamfilter/scripts/dmarc.sieve b/resources/config/spamfilter/scripts/dmarc.sieve index dd0f3995..090c8323 100644 --- a/resources/config/spamfilter/scripts/dmarc.sieve +++ b/resources/config/spamfilter/scripts/dmarc.sieve @@ -52,7 +52,7 @@ if eval "env.dmarc.result == 'pass'" { } else { let "t.DMARC_POLICY_SOFTFAIL" "1"; } -} elsif eval "env.dmarc.result == ''" { +} else { let "t.DMARC_NA" "1"; } diff --git a/resources/config/spamfilter/scripts/epilogue.sieve b/resources/config/spamfilter/scripts/epilogue.sieve index b47a7f2a..1967bb84 100644 --- a/resources/config/spamfilter/scripts/epilogue.sieve +++ b/resources/config/spamfilter/scripts/epilogue.sieve @@ -7,13 +7,13 @@ if eval "AUTOLEARN_ENABLE && (score >= AUTOLEARN_SPAM_THRESHOLD || score <= AUTO } # Process score actions -if "SCORE_REJECT_THRESHOLD && score >= SCORE_REJECT_THRESHOLD" { +if eval "SCORE_REJECT_THRESHOLD && score >= SCORE_REJECT_THRESHOLD" { reject "Your message has been rejected because it has an excessive spam score. If you feel this is an error, please contact the postmaster."; stop; -} else if "SCORE_DISCARD_THRESHOLD && score >= SCORE_DISCARD_THRESHOLD" { +} elsif eval "SCORE_DISCARD_THRESHOLD && score >= SCORE_DISCARD_THRESHOLD" { discard; stop; -} else if "ADD_HEADER_SPAM" { +} elsif eval "ADD_HEADER_SPAM" { let "spam_status" ""; if eval "score >= SCORE_SPAM_THRESHOLD" { let "spam_status" "'Yes, score=' + score"; diff --git a/resources/config/spamfilter/scripts/replyto.sieve b/resources/config/spamfilter/scripts/replyto.sieve index d3293d8d..6d3265c8 100644 --- a/resources/config/spamfilter/scripts/replyto.sieve +++ b/resources/config/spamfilter/scripts/replyto.sieve @@ -46,7 +46,7 @@ if eval "!is_empty(rto_raw)" { } if eval "rto_addr == envelope.from" { - let "t.REPLYTO_ADDR_EQ_FROM" "1'; + let "t.REPLYTO_ADDR_EQ_FROM" "1"; } if eval "lookup('spam/free-domains', rto_domain_sld)" { diff --git a/resources/config/spamfilter/scripts/scores.sieve b/resources/config/spamfilter/scripts/scores.sieve index 745adfc2..95a12430 100644 --- a/resources/config/spamfilter/scripts/scores.sieve +++ b/resources/config/spamfilter/scripts/scores.sieve @@ -5,7 +5,7 @@ let "spam_result" ""; while "i > 0" { let "i" "i - 1"; let "tag" "tags[i]"; - let "tag_score" "map('spam/scores', tag)"; + let "tag_score" "lookup_map('spam/scores', tag)"; if eval "is_number(tag_score)" { let "score" "score + tag_score"; @@ -16,11 +16,11 @@ while "i > 0" { let "spam_result" "spam_result + tag + ' (' + tag_score + ')'"; } } - } elsif eval "tag_score == 'reject' { + } elsif eval "tag_score == 'reject'" { let "SCORE_REJECT_THRESHOLD" "1"; let "score" "2"; break; - } else if eval "tag_score == 'discard'" { + } elsif eval "tag_score == 'discard'" { discard; stop; } diff --git a/tests/resources/create_test_env.sh b/tests/resources/create_test_env.sh index 1f394ee3..1705ae26 100644 --- a/tests/resources/create_test_env.sh +++ b/tests/resources/create_test_env.sh @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/bash -BASE_DIR = "/tmp/stalwart-test" +BASE_DIR="/tmp/stalwart-test" +DOMAIN="example.org" # Delete previous tests rm -rf $BASE_DIR @@ -15,3 +16,37 @@ cp -r resources/config $BASE_DIR/etc cp -r tests/resources/tls_cert.pem $BASE_DIR/etc cp -r tests/resources/tls_privatekey.pem $BASE_DIR/etc +# Replace settings +sed -i '' -e "s/__DOMAIN__/$DOMAIN/g" -e "s/__HOST__/mail.$DOMAIN/g" -e 's/sql.toml/memory.toml/g' -e "s|__BASE_PATH__|$BASE_DIR|g" "$BASE_DIR/etc/config.toml" +sed -i '' -e "s|__CERT_PATH__|$BASE_DIR/etc/tls_cert.pem|g" -e "s|__PK_PATH__|$BASE_DIR/etc/tls_privatekey.pem|g" "$BASE_DIR/etc/common/tls.toml" +sed -i '' -e 's/method = "log"/method = "stdout"/g' -e 's/level = "info"/level = "trace"/g' "$BASE_DIR/etc/common/tracing.toml" +sed -i '' -e 's/user = "stalwart-mail"//g' -e 's/group = "stalwart-mail"//g' "$BASE_DIR/etc/common/server.toml" + +# Generate DKIM key +mkdir -p $BASE_DIR/etc/dkim +openssl genpkey -algorithm RSA -out $BASE_DIR/etc/dkim/$DOMAIN.key + +# Create antispam tables +sqlite3 $BASE_DIR/data/spamfilter.sqlite3 < To: licensing@stalw.art @@ -48,6 +50,8 @@ dkim.domains tenthrevolution.com dmarc.result pass remote_ip 185.58.86.181 tls.version TLSv1.3 +expect_header X-Spam-Status No, score=4. +expect_header X-Spam-Result expect from_eq_envfrom from_has_dn hfilter_helo_nores_a_or_mx forged_rcvd_trail date_in_past arc_na uri_count_odd dkim_signed has_attachment r_spf_allow rcvd_tls_last rcpt_count_one mime_good subject_ends_spaces hfilter_fromhost_nores_a_or_mx to_dn_eq_addr_all r_dkim_allow dmarc_policy_allow rcvd_count_three to_match_envrcpt_all DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tenthrevolution.com; @@ -621,7 +625,9 @@ dmarc.result fail dmarc.policy reject remote_ip 51.89.165.39 tls.version TLS1_2 -expect has_replyto once_received r_parts_differ mid_rhs_match_from hfilter_fromhost_nores_a_or_mx from_has_dn r_dkim_allow date_in_past to_match_envrcpt_all html_short_link_img_1 rcpt_count_one arc_na hfilter_helo_nores_a_or_mx r_spf_softfail rcvd_tls_last rcvd_count_zero replyto_dom_eq_from_dom to_dn_none has_list_unsub dkim_signed rdns_none from_eq_envfrom dmarc_policy_reject +expect_header X-Spam-Status Yes, score=13. +expect_header X-Spam-Result +expect has_replyto violated_direct_spf replyto_addr_eq_from once_received r_parts_differ mid_rhs_match_from hfilter_fromhost_nores_a_or_mx from_has_dn r_dkim_allow date_in_past to_match_envrcpt_all html_short_link_img_1 rcpt_count_one arc_na hfilter_helo_nores_a_or_mx r_spf_softfail rcvd_tls_last rcvd_count_zero replyto_dom_eq_from_dom to_dn_none has_list_unsub dkim_signed rdns_none from_eq_envfrom dmarc_policy_reject DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=sectionalism; d=grupokonecta.net; h=To:Subject:Message-ID:Date:From:Reply-To:MIME-Version:List-Unsubscribe: diff --git a/tests/resources/test_config.toml b/tests/resources/test_config.toml deleted file mode 100644 index c6fdfbf0..00000000 --- a/tests/resources/test_config.toml +++ /dev/null @@ -1,236 +0,0 @@ -[server] -hostname = "test.example.org" - -[server.listener.jmap] -bind = ["0.0.0.0:9990"] -url = "https://0.0.0.0:9990" -protocol = "jmap" -max-connections = 8192 - -[server.listener.imap] -bind = ["0.0.0.0:9991"] -protocol = "imap" -max-connections = 8192 - -[server.listener.imaptls] -bind = ["0.0.0.0:9992"] -protocol = "imap" -max-connections = 8192 -tls.implicit = true - -[server.listener.sieve] -bind = ["0.0.0.0:9993"] -protocol = "managesieve" -max-connections = 8192 -tls.implicit = true - -[server.listener.smtps] -bind = ['0.0.0.0:9994'] -greeting = 'Test SMTP instance' -protocol = 'smtp' -tls.implicit = true - -[server.listener.smtp] -bind = ['0.0.0.0:9995'] -greeting = 'Test SMTP instance' -protocol = 'smtp' -tls.implicit = false - -[server.socket] -reuse-addr = true - -[server.tls] -enable = true -implicit = false -certificate = "default" - -[global.tracing] -method = "stdout" -#level = "trace" -level = "info" - -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = [ { if = "authenticated-as", ne = "", then = true }, - { else = false } ] -directory = "local" - -[session.rcpt.errors] -total = 5 -wait = "1ms" - -[queue] -path = "/tmp/stalwart-test" -hash = 64 - -[report] -path = "/tmp/stalwart-test" -hash = 64 - -[resolver] -type = "system" - -[queue.outbound] -next-hop = [ { if = "rcpt-domain", in-list = "local/domains", then = "local" }, - { else = false } ] - -[remote."mock-smtp"] -address = "localhost" -port = 9999 -protocol = "smtp" - -[remote."mock-smtp".tls] -implicit = false -allow-invalid-certs = true - -[session.extensions] -future-release = [ { if = "authenticated-as", ne = "", then = "99999999d"}, - { else = false } ] - -[store] -db.path = "/tmp/stalwart-test/sqlite.db" - -[store.blob] -type = "local" - -[store.blob.local] -path = "/tmp/stalwart-test" - -[certificate.default] -cert = "file://./tests/resources/tls_cert.pem" -private-key = "file://./tests/resources/tls_privatekey.pem" - -[jmap] -directory = "local" - -[jmap.http] -headers = ["Access-Control-Allow-Origin: *", - "Access-Control-Allow-Methods: POST, GET, HEAD, OPTIONS", - "Access-Control-Allow-Headers: *"] - -[jmap.protocol] -set.max-objects = 100000 - -[jmap.protocol.request] -max-concurrent = 8 - -[jmap.protocol.upload] -max-size = 5000000 -max-concurrent = 4 -ttl = "1m" - -[jmap.protocol.upload.quota] -files = 3 -size = 50000 - -[jmap.rate-limit] -account = "1000/1m" -authentication = "100/2s" -anonymous = "100/1m" - -[jmap.event-source] -throttle = "500ms" - -[jmap.web-sockets] -throttle = "500ms" - -[jmap.push] -throttle = "500ms" -attempts.interval = "500ms" - -[directory."local"] -type = "memory" - -[directory."local".options] -catch-all = true -subaddressing = true - -[directory."local".lookup] -domains = ["example.org"] - -[[directory."local".users]] -name = "admin" -description = "Superadmin" -secret = "secret" -member-of = ["superusers"] - -[[directory."local".users]] -name = "john" -description = "John Doe" -secret = "12345" -#secret = "$argon2id$v=19$m=16,t=2,p=1$Ym1GMmMwd210YXpGWUF2Ng$MKUyI28a4OADfd7r2iHulQ" -email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] -email-list = ["info@example.org"] -member-of = ["sales"] - -[[directory."local".users]] -name = "jane" -description = "Jane Doe" -secret = "12345" -email = "jane@example.org" -email-list = ["info@example.org"] -member-of = ["sales", "support"] - -[[directory."local".users]] -name = "bill" -description = "Bill Foobar" -secret = "12345" -quota = 500000 -email = "bill@example.org" -email-list = ["info@example.org"] - -[[directory."local".groups]] -name = "sales" -email = "sales@example.org" -description = "Sales Team" - -[[directory."local".groups]] -name = "support" -email = "support@example.org" -description = "Support Team" - -[oauth] -key = "parerga_und_paralipomena" -[oauth.auth] -max-attempts = 1 - -[oauth.expiry] -user-code = "1s" -token = "1s" -refresh-token = "3s" -refresh-token-renew = "2s" - -[imap.auth] -allow-plain-text = true - -[imap.rate-limit] -requests = "90000/1s" -concurrent = 9000 - -[signature."ed25519"] -public-key = "-----BEGIN PUBLIC KEY----- -MCowBQYDK2VwAyEAUA3S0BdVG7LeA1agv7ZtqLkQMn+/AoYx1VPyGmLWEIM= ------END PUBLIC KEY-----" -private-key = "-----BEGIN PRIVATE KEY----- -MC4CAQAwBQYDK2VwBCIEIPdRszAzLvx4JaSIE4dQdZtN9y2XW+55K+YkCJI6lcn8 ------END PRIVATE KEY-----" -domain = "example.org" -selector = "stalwart_ed" -headers = ["From", "To", "Date", "Subject", "Message-ID"] -algorithm = "ed25519-sha256" -canonicalization = "simple/simple" -set-body-length = true -report = true - -[signature."ed25519-2"] -public-key = "11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo=" -private-key = "nWGxne/9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A=" -domain = "example.org" -selector = "stalwart_ed2" -headers = ["From", "To", "Date", "Subject", "Message-ID"] -algorithm = "ed25519-sha256" -canonicalization = "simple/simple" -set-body-length = true -report = true diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index fd0c273d..418a0e24 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -18,7 +18,7 @@ use smtp::{ inbound::AuthResult, scripts::{ functions::html::{get_attribute, html_attr_tokens, html_img_area, html_to_tokens}, - ScriptResult, + ScriptModification, ScriptResult, }, }; use tokio::runtime::Handle; @@ -122,7 +122,7 @@ values = ["spamtrap@*"] [directory."spam".lookup."scores"] type = "map" -values = ["SPAM_TRAP discard"] +values = "file://%CFG_PATH%/maps/scores.map" [resolver] public-suffix = "file://%LIST_PATH%/public-suffix.dat" @@ -192,6 +192,13 @@ async fn antispam() { ]; let mut core = SMTP::test(); let qr = core.init_test_queue("smtp_antispam_test"); + let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf() + .join("resources") + .join("config") + .join("spamfilter"); let mut config = CONFIG .replace("%PATH%", qr._temp_dir.temp_dir.as_path().to_str().unwrap()) .replace( @@ -202,15 +209,9 @@ async fn antispam() { .join("lists") .to_str() .unwrap(), - ); - let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .to_path_buf() - .join("resources") - .join("config") - .join("spamfilter") - .join("scripts"); + ) + .replace("%CFG_PATH%", base_path.as_path().to_str().unwrap()); + let base_path = base_path.join("scripts"); let script_config = fs::read_to_string(base_path.join("config.sieve")).unwrap(); let script_prelude = fs::read_to_string(base_path.join("prelude.sieve")).unwrap(); let mut all_scripts = script_config.clone() + "\n" + script_prelude.as_str(); @@ -235,6 +236,13 @@ async fn antispam() { "{test_name} = '''{script_config}\n{script_prelude}\n{script}\n'''\n" )); } + for test_name in ["composites", "scores", "epilogue"] { + all_scripts = all_scripts + + "\n" + + fs::read_to_string(base_path.join(format!("{test_name}.sieve"))) + .unwrap() + .as_str(); + } config.push_str(&format!("combined = '''{all_scripts}\n'''\n")); @@ -331,6 +339,7 @@ async fn antispam() { let mut in_params = true; let mut variables: HashMap = HashMap::new(); let mut expected_variables = AHashMap::new(); + let mut expected_headers = AHashMap::new(); // Build session let mut session = Session::test(core.clone()); @@ -420,6 +429,14 @@ async fn antispam() { .unwrap_or((v.to_lowercase(), Variable::Integer(1))) })); } + "expect_header" => { + if let Some((header, value)) = value.split_once(' ') { + expected_headers + .insert(header.to_string(), value.trim().to_string()); + } else { + expected_headers.insert(value.to_string(), String::new()); + } + } "score" | "final_score" => { variables .insert(param.to_string(), value.parse::().unwrap().into()); @@ -466,7 +483,32 @@ async fn antispam() { .await .unwrap() { - ScriptResult::Accept { .. } => {} + ScriptResult::Accept { modifications } => { + if modifications.len() != expected_headers.len() { + panic!( + "Expected {:?} headers, got {:?}", + expected_headers, modifications + ); + } + for modification in modifications { + if let ScriptModification::AddHeader { name, value } = modification { + if let Some(expected_value) = expected_headers.remove(name.as_str()) { + if !expected_value.is_empty() + && !value.starts_with(expected_value.as_str()) + { + panic!( + "Expected header {:?} to be {:?}, got {:?}", + name, expected_value, value + ); + } + } else { + panic!("Unexpected header {:?}", name); + } + } else { + panic!("Unexpected modification {:?}", modification); + } + } + } ScriptResult::Reject(message) => panic!("{}", message), ScriptResult::Replace { message,