diff --git a/crates/imap-proto/src/protocol/mod.rs b/crates/imap-proto/src/protocol/mod.rs index 568bec97..da498ac6 100644 --- a/crates/imap-proto/src/protocol/mod.rs +++ b/crates/imap-proto/src/protocol/mod.rs @@ -24,7 +24,7 @@ use std::{cmp::Ordering, fmt::Display}; use ahash::AHashSet; -use chrono::{DateTime, NaiveDateTime, Utc}; +use chrono::{DateTime, Utc}; use jmap_proto::types::keyword::Keyword; use crate::{Command, ResponseCode, ResponseType, StatusResponse}; @@ -218,13 +218,11 @@ pub fn literal_string(buf: &mut Vec, text: &[u8]) { pub fn quoted_timestamp(buf: &mut Vec, timestamp: i64) { buf.push(b'"'); buf.extend_from_slice( - DateTime::::from_naive_utc_and_offset( - NaiveDateTime::from_timestamp_opt(timestamp, 0).unwrap_or_default(), - Utc, - ) - .format("%d-%b-%Y %H:%M:%S %z") - .to_string() - .as_bytes(), + DateTime::::from_timestamp(timestamp, 0) + .unwrap_or_default() + .format("%d-%b-%Y %H:%M:%S %z") + .to_string() + .as_bytes(), ); buf.push(b'"'); } diff --git a/crates/imap/src/lib.rs b/crates/imap/src/lib.rs index a573a2ea..3eb47ded 100644 --- a/crates/imap/src/lib.rs +++ b/crates/imap/src/lib.rs @@ -44,9 +44,10 @@ static SERVER_GREETING: &str = concat!( impl IMAP { pub async fn init(config: &Config) -> utils::config::Result> { let shard_amount = config - .property::("global.shared-map.shard")? + .property::("cache.shard")? .unwrap_or(32) .next_power_of_two() as usize; + let capacity = config.property("cache.capacity")?.unwrap_or(100); Ok(Arc::new(IMAP { max_request_size: config.property_or_static("imap.request.max-size", "52428800")?, @@ -69,7 +70,7 @@ impl IMAP { }) .into_bytes(), rate_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( - config.property("cache.rate-limit.size")?.unwrap_or(2048), + capacity, RandomState::default(), shard_amount, ), @@ -77,10 +78,10 @@ impl IMAP { rate_concurrent: config.property("imap.rate-limit.concurrent")?.unwrap_or(4), allow_plain_auth: config.property_or_static("imap.auth.allow-plain-text", "false")?, cache_account: LruCache::with_capacity( - config.property("cache.messages.size")?.unwrap_or(2048), + config.property("cache.account.size")?.unwrap_or(2048), ), cache_mailbox: LruCache::with_capacity( - config.property("cache.messages.size")?.unwrap_or(2048), + config.property("cache.mailbox.size")?.unwrap_or(2048), ), })) } diff --git a/crates/jmap/src/api/config.rs b/crates/jmap/src/api/config.rs index 2b9f3a3b..8d0cc5e8 100644 --- a/crates/jmap/src/api/config.rs +++ b/crates/jmap/src/api/config.rs @@ -33,7 +33,7 @@ impl crate::Config { let mut config = Self { default_language: Language::from_iso_639( settings - .value("storage.fts.default-language") + .value("storage.full-text.default-language") .unwrap_or("en"), ) .unwrap_or(Language::English), @@ -102,7 +102,7 @@ impl crate::Config { rate_authenticated: settings .property_or_static("jmap.rate-limit.account", "1000/1m")?, rate_authenticate_req: settings - .property_or_static("jmap.rate-limit.authentication", "10/1m")?, + .property_or_static("authentication.rate-limit", "10/1m")?, rate_anonymous: settings.property_or_static("jmap.rate-limit.anonymous", "100/1m")?, rate_use_forwarded: settings .property("jmap.rate-limit.use-forwarded")? @@ -143,7 +143,7 @@ impl crate::Config { .unwrap_or(true), encrypt: settings.property_or_static("storage.encryption.enable", "true")?, encrypt_append: settings.property_or_static("storage.encryption.append", "false")?, - spam_header: settings.value("storage.spam.header").and_then(|v| { + spam_header: settings.value("spam.header.is-spam").and_then(|v| { v.split_once(':').map(|(k, v)| { ( mail_parser::HeaderName::parse(k.trim().to_string()).unwrap(), @@ -152,26 +152,26 @@ impl crate::Config { }) }), http_headers: settings - .values("jmap.http.headers") + .values("server.http.headers") .map(|(_, v)| { if let Some((k, v)) = v.split_once(':') { Ok(( hyper::header::HeaderName::from_str(k.trim()).map_err(|err| { format!( - "Invalid header found in property \"jmap.http.headers\": {}", + "Invalid header found in property \"server.http.headers\": {}", err ) })?, hyper::header::HeaderValue::from_str(v.trim()).map_err(|err| { format!( - "Invalid header found in property \"jmap.http.headers\": {}", + "Invalid header found in property \"server.http.headers\": {}", err ) })?, )) } else { Err(format!( - "Invalid header found in property \"jmap.http.headers\": {}", + "Invalid header found in property \"server.http.headers\": {}", v )) } diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 966dd48f..aea61960 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -191,9 +191,10 @@ impl JMAP { let (state_tx, state_rx) = init_state_manager(); let (housekeeper_tx, housekeeper_rx) = init_housekeeper(); let shard_amount = config - .property::("global.shared-map.shard")? + .property::("cache.shard")? .unwrap_or(32) .next_power_of_two() as usize; + let capacity = config.property("cache.capacity")?.unwrap_or(100); let jmap_server = Arc::new(JMAP { directory: directories @@ -213,25 +214,16 @@ impl JMAP { blob_store: stores.get_blob_store(config, "storage.blob")?, lookup_store: stores.get_lookup_store(config, "storage.lookup")?, config: Config::new(config).failed("Invalid configuration file"), - sessions: TtlDashMap::with_capacity( - config.property("cache.session.size")?.unwrap_or(100), - shard_amount, - ), - access_tokens: TtlDashMap::with_capacity( - config.property("cache.session.size")?.unwrap_or(100), - shard_amount, - ), + sessions: TtlDashMap::with_capacity(capacity, shard_amount), + access_tokens: TtlDashMap::with_capacity(capacity, shard_amount), concurrency_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( - config.property("cache.rate-limit.size")?.unwrap_or(1024), + capacity, RandomState::default(), shard_amount, ), - oauth_codes: TtlDashMap::with_capacity( - config.property("cache.oauth.size")?.unwrap_or(128), - shard_amount, - ), + oauth_codes: TtlDashMap::with_capacity(capacity, shard_amount), cache_threads: LruCache::with_capacity( - config.property("cache.messages.size")?.unwrap_or(2048), + config.property("cache.thread.size")?.unwrap_or(2048), ), state_tx, housekeeper_tx, diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index 8c11b1bb..e0a32a30 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -67,7 +67,12 @@ async fn main() -> std::io::Result<()> { .failed("Invalid configuration"); // Update configuration - config.update(data_store.config_list("").await.failed("Storage error")); + config.update( + data_store + .config_list("", false) + .await + .failed("Storage error"), + ); // Parse directories let directory = config diff --git a/crates/smtp/src/config/resolver.rs b/crates/smtp/src/config/resolver.rs index bc2490a0..2f1fb1da 100644 --- a/crates/smtp/src/config/resolver.rs +++ b/crates/smtp/src/config/resolver.rs @@ -119,7 +119,7 @@ impl ConfigResolver for Config { let mut capacities = [1024usize; 5]; for (pos, key) in ["txt", "mx", "ipv4", "ipv6", "ptr"].into_iter().enumerate() { - if let Some(capacity) = self.property(("resolver.cache", key))? { + if let Some(capacity) = self.property(("cache.resolver", key))? { capacities[pos] = capacity; } } diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index bdb63e51..48421fa8 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -96,6 +96,11 @@ impl SMTP { } // Build core + let capacity = config.property("cache.capacity")?.unwrap_or(2); + let shard = config + .property::("cache.shard")? + .unwrap_or(32) + .next_power_of_two() as usize; let (queue_tx, queue_rx) = mpsc::channel(1024); let (report_tx, report_rx) = mpsc::channel(1024); let core = Arc::new(SMTP { @@ -112,23 +117,17 @@ impl SMTP { session: SessionCore { config: session_config, throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - config.property("global.shared-map.capacity")?.unwrap_or(2), + capacity, ThrottleKeyHasherBuilder::default(), - config - .property::("global.shared-map.shard")? - .unwrap_or(32) - .next_power_of_two() as usize, + shard, ), }, queue: QueueCore { config: queue_config, throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - config.property("global.shared-map.capacity")?.unwrap_or(2), + capacity, ThrottleKeyHasherBuilder::default(), - config - .property::("global.shared-map.shard")? - .unwrap_or(32) - .next_power_of_two() as usize, + shard, ), snowflake_id: config .property::("storage.cluster.node-id")? diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index 36bf5dce..eb4fa00e 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -55,6 +55,7 @@ impl From for crate::Error { } } +#[allow(dead_code)] fn deserialize_i64_le(bytes: &[u8]) -> crate::Result { Ok(i64::from_le_bytes(bytes[..].try_into().map_err(|_| { crate::Error::InternalError("Failed to deserialize i64 value.".to_string()) diff --git a/crates/store/src/dispatch/blocked.rs b/crates/store/src/dispatch/blocked.rs index 8716016a..1b8bfea3 100644 --- a/crates/store/src/dispatch/blocked.rs +++ b/crates/store/src/dispatch/blocked.rs @@ -62,7 +62,7 @@ impl BlockedIps { pub fn reload(&self, config: &Config) -> utils::config::Result<()> { self.limiter_rate.store( config - .property::("server.security.fail2ban")? + .property::("authentication.fail2ban")? .map(Arc::new), ); self.reload_blocked_ips(config.set_values(BLOCKED_IP_KEY)) diff --git a/resources/config/common/cache.toml b/resources/config/common/cache.toml new file mode 100644 index 00000000..f08eaa74 --- /dev/null +++ b/resources/config/common/cache.toml @@ -0,0 +1,35 @@ +############################################# +# Cache configuration +############################################# + +[cache] +capacity = 512 +shard = 32 + +[cache.session] +ttl = "1h" + +[cache.account] +size = 2048 + +[cache.mailbox] +size = 2048 + +[cache.thread] +size = 2048 + +[cache.bayes] +capacity = 8192 + +[cache.bayes.ttl] +positive = "1h" +negative = "1h" + +[cache.resolver] +txt = 2048 +mx = 1024 +ipv4 = 1024 +ipv6 = 1024 +ptr = 1024 +tlsa = 1024 +mta-sts = 1024 diff --git a/resources/config/common/server.toml b/resources/config/common/server.toml index ad6824a9..c80c226c 100644 --- a/resources/config/common/server.toml +++ b/resources/config/common/server.toml @@ -9,9 +9,9 @@ max-connections = 8192 #[server.proxy] #trusted-networks = ["127.0.0.0/8", "::1", "10.0.0.0/8"] -[server.security] -blocked-networks = {} +[authentication] fail2ban = "100/1d" +rate-limit = "10/1m" [server.run-as] user = "stalwart-mail" @@ -29,5 +29,9 @@ backlog = 1024 #tos = 1 [global] -shared-map = {shard = 32, capacity = 10} #thread-pool = 8 + +[server.http] +#headers = ["Access-Control-Allow-Origin: *", +# "Access-Control-Allow-Methods: POST, GET, PATCH, PUT, DELETE, HEAD, OPTIONS", +# "Access-Control-Allow-Headers: Authorization, Content-Type, Accept, X-Requested-With"] diff --git a/resources/config/common/store.toml b/resources/config/common/store.toml index 6924769f..ea96531c 100644 --- a/resources/config/common/store.toml +++ b/resources/config/common/store.toml @@ -13,10 +13,7 @@ directory = "%{DEFAULT_DIRECTORY}%" enable = true append = false -[storage.spam] -header = "X-Spam-Status: Yes" - -[storage.fts] +[storage.full-text] default-language = "en" [storage.cluster] diff --git a/resources/config/config.toml b/resources/config/config.toml index a2b14cd5..63b23694 100644 --- a/resources/config/config.toml +++ b/resources/config/config.toml @@ -15,6 +15,7 @@ files = [ "%{BASE_PATH}%/etc/common/server.toml", "%{BASE_PATH}%/etc/common/store.toml", "%{BASE_PATH}%/etc/common/tracing.toml", "%{BASE_PATH}%/etc/common/sieve.toml", + "%{BASE_PATH}%/etc/common/cache.toml", "%{BASE_PATH}%/etc/directory/imap.toml", "%{BASE_PATH}%/etc/directory/internal.toml", "%{BASE_PATH}%/etc/directory/ldap.toml", diff --git a/resources/config/jmap/auth.toml b/resources/config/jmap/auth.toml index a222d5e6..eaf68d83 100644 --- a/resources/config/jmap/auth.toml +++ b/resources/config/jmap/auth.toml @@ -2,9 +2,5 @@ # JMAP authentication & session configuration ############################################# -[jmap.session.cache] -ttl = "1h" -size = 100 - [jmap.session.purge] frequency = "15 * *" diff --git a/resources/config/jmap/oauth.toml b/resources/config/jmap/oauth.toml index 62c932b0..acb2e594 100644 --- a/resources/config/jmap/oauth.toml +++ b/resources/config/jmap/oauth.toml @@ -14,6 +14,3 @@ auth-code = "10m" token = "1h" refresh-token = "30d" refresh-token-renew = "4d" - -[oauth.cache] -size = 128 diff --git a/resources/config/jmap/protocol.toml b/resources/config/jmap/protocol.toml index b965fc65..336bc4a1 100644 --- a/resources/config/jmap/protocol.toml +++ b/resources/config/jmap/protocol.toml @@ -41,8 +41,3 @@ max-items = 10 [jmap.principal] allow-lookups = true - -[jmap.http] -#headers = ["Access-Control-Allow-Origin: *", -# "Access-Control-Allow-Methods: POST, GET, PATCH, PUT, DELETE, HEAD, OPTIONS", -# "Access-Control-Allow-Headers: Authorization, Content-Type, Accept, X-Requested-With"] diff --git a/resources/config/jmap/ratelimit.toml b/resources/config/jmap/ratelimit.toml index 96ee0b6e..4fef99f1 100644 --- a/resources/config/jmap/ratelimit.toml +++ b/resources/config/jmap/ratelimit.toml @@ -5,9 +5,5 @@ [jmap.rate-limit] account = "1000/1m" -authentication = "10/1m" anonymous = "100/1m" use-forwarded = false - -[jmap.rate-limit.cache] -size = 1024 diff --git a/resources/config/smtp/resolver.toml b/resources/config/smtp/resolver.toml index af2bdb37..397b22e6 100644 --- a/resources/config/smtp/resolver.toml +++ b/resources/config/smtp/resolver.toml @@ -11,12 +11,3 @@ attempts = 2 try-tcp-on-error = true public-suffix = ["https://publicsuffix.org/list/public_suffix_list.dat", "file://%{BASE_PATH}%/etc/spamfilter/maps/suffix_list.dat.gz"] - -[resolver.cache] -txt = 2048 -mx = 1024 -ipv4 = 1024 -ipv6 = 1024 -ptr = 1024 -tlsa = 1024 -mta-sts = 1024 diff --git a/resources/config/smtp/spamfilter.toml b/resources/config/smtp/spamfilter.toml index e43326f6..965ef8ff 100644 --- a/resources/config/smtp/spamfilter.toml +++ b/resources/config/smtp/spamfilter.toml @@ -2,6 +2,31 @@ # SMTP Spam & Phishing filter configuration ############################################# +[spam.header] +add-spam = true +add-spam-result = true +is-spam = "X-Spam-Status: Yes" + +[spam.autolearn] +enable = true +balance = 0.9 + +[spam.autolearn.ham] +replies = true +threshold = -0.5 + +[spam.autolearn.spam] +threshold = 6.0 + +[spam.threshold] +spam = 5.0 +discard = 0 +reject = 0 + +[spam.data] +directory = "" +lookup = "" + [store."spam/free-domains"] type = "memory" format = "glob" diff --git a/resources/config/spamfilter/scripts/config.sieve b/resources/config/spamfilter/scripts/config.sieve index 45779a55..29467958 100644 --- a/resources/config/spamfilter/scripts/config.sieve +++ b/resources/config/spamfilter/scripts/config.sieve @@ -1,35 +1,35 @@ # Whether to add an X-Spam-Status header -let "ADD_HEADER_SPAM" "true"; +let "ADD_HEADER_SPAM" "%{cfg:spam.header.add-spam}%"; # Whether to add an X-Spam-Result header -let "ADD_HEADER_SPAM_RESULT" "true"; +let "ADD_HEADER_SPAM_RESULT" "%{cfg:spam.header.add-spam-result}%"; # Whether message replies from authenticated users should be learned as ham -let "AUTOLEARN_REPLIES_HAM" "true"; +let "AUTOLEARN_REPLIES_HAM" "%{cfg:spam.autolearn.ham.replies}%"; # Whether the bayes classifier should be trained automatically -let "AUTOLEARN_ENABLE" "true"; +let "AUTOLEARN_ENABLE" "%{cfg:spam.autolearn.enable}%"; # When to learn ham (score >= threshold) -let "AUTOLEARN_HAM_THRESHOLD" "-0.5"; +let "AUTOLEARN_HAM_THRESHOLD" "%{cfg:spam.autolearn.ham.threshold}%"; # When to learn spam (score <= threshold) -let "AUTOLEARN_SPAM_THRESHOLD" "6.0"; +let "AUTOLEARN_SPAM_THRESHOLD" "%{cfg:spam.autolearn.spam.threshold}%"; # Keep difference for spam/ham learns for at least this value -let "AUTOLEARN_SPAM_HAM_BALANCE" "0.9"; +let "AUTOLEARN_SPAM_HAM_BALANCE" "%{cfg:spam.autolearn.balance}%"; # If ADD_HEADER_SPAM is enabled, mark as SPAM messages with a score above this threshold -let "SCORE_SPAM_THRESHOLD" "5.0"; +let "SCORE_SPAM_THRESHOLD" "%{cfg:spam.threshold.spam}%"; # Discard messages with a score above this threshold -let "SCORE_DISCARD_THRESHOLD" "0"; +let "SCORE_DISCARD_THRESHOLD" "%{cfg:spam.threshold.discard}%"; # Reject messages with a score above this threshold -let "SCORE_REJECT_THRESHOLD" "0"; +let "SCORE_REJECT_THRESHOLD" "%{cfg:spam.threshold.reject}%"; # Directory name to use for local domain lookups (leave empty for default) -let "DOMAIN_DIRECTORY" ""; +let "DOMAIN_DIRECTORY" "%{cfg:spam.data.directory}%"; # Store to use for Bayes tokens and ids (leave empty for default) -let "SPAM_DB" ""; +let "SPAM_DB" "%{cfg:spam.data.lookup}%"; diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 8e89d5b8..62ea7648 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -188,8 +188,8 @@ blob = "{STORE}" lookup = "{STORE}" directory = "auth" -[storage.spam] -header = "X-Spam-Status: Yes" +[spam.header] +is-spam = "X-Spam-Status: Yes" [jmap.protocol.get] max-objects = 100000