diff --git a/crates/directory/src/backend/sql/config.rs b/crates/directory/src/backend/sql/config.rs index 709c593c..0c351a6b 100644 --- a/crates/directory/src/backend/sql/config.rs +++ b/crates/directory/src/backend/sql/config.rs @@ -72,9 +72,10 @@ impl SqlDirectory { ("expand", &mut mappings.query_expand), ("domains", &mut mappings.query_domains), ] { - if let Some(query_) = stores.lookups.get(&format!("{}/{}", store_id, query_id)) { - *query = query_.query.to_string(); - } + *query = config + .value(("store", store_id, "query", query_id)) + .unwrap_or_default() + .to_string(); } Ok(SqlDirectory { diff --git a/crates/smtp/src/config/condition.rs b/crates/smtp/src/config/condition.rs index 3d61e0b9..9708d984 100644 --- a/crates/smtp/src/config/condition.rs +++ b/crates/smtp/src/config/condition.rs @@ -217,7 +217,7 @@ impl ConfigCondition for Config { MatchType::Lookup => { if let Some(lookup) = ctx.directory.lookups.get(value_str) { ConditionMatch::Lookup(lookup.clone().into()) - } else if let Some(lookup) = ctx.stores.lookups.get(value_str) { + } else if let Some(lookup) = ctx.stores.lookup_stores.get(value_str) { ConditionMatch::Lookup(lookup.clone().into()) } else { return Err(format!( diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs index 92eec865..1ddbf1a2 100644 --- a/crates/smtp/src/config/scripts.rs +++ b/crates/smtp/src/config/scripts.rs @@ -121,7 +121,7 @@ impl ConfigSieve for Config { .with_valid_notification_uri("mailto") .with_valid_ext_lists( ctx.stores - .lookups + .lookup_stores .keys() .chain(ctx.directory.lookups.keys()) .map(|k| k.to_string()), @@ -193,18 +193,6 @@ impl ConfigSieve for Config { Ok(SieveCore { runtime, scripts: ctx.scripts.clone(), - lookup: ctx - .stores - .lookups - .iter() - .map(|(k, v)| (k.to_string(), v.clone().into())) - .chain( - ctx.directory - .lookups - .iter() - .map(|(k, v)| (k.to_string(), v.clone().into())), - ) - .collect(), lookup_stores: ctx.stores.lookup_stores.clone(), directories: ctx.directory.directories.clone(), default_directory: self diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index a17d1dcb..8fff4c64 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -22,7 +22,6 @@ */ use std::{ - cmp::Ordering, hash::Hash, net::IpAddr, sync::{atomic::AtomicU32, Arc}, @@ -41,7 +40,7 @@ use smtp_proto::{ }, IntoString, }; -use store::{LookupStore, Row, Value}; +use store::{LookupKey, LookupStore, LookupValue, Value}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, @@ -65,6 +64,7 @@ use crate::{ }, queue::{self, DomainPart, QueueId, QuotaLimiter}, reporting, + scripts::plugins::lookup::VariableExists, }; use self::throttle::{Limiter, ThrottleKey, ThrottleKeyHasherBuilder}; @@ -119,7 +119,6 @@ pub struct SieveCore { pub sign: Vec>, pub directories: AHashMap>, pub lookup_stores: AHashMap, - pub lookup: AHashMap, pub default_lookup_store: Option, pub default_directory: Option>, } @@ -161,7 +160,7 @@ pub struct TlsConnectors { #[derive(Clone)] pub enum Lookup { - Store(Arc), + Store(LookupStore), Directory(directory::Lookup), } @@ -283,68 +282,35 @@ impl SessionData { } impl Lookup { - pub async fn contains(&self, item: impl Into>) -> Option { + pub async fn contains(&self, item: &str) -> Option { match self { - Lookup::Store(lookup) => lookup + Lookup::Store(LookupStore::Query(lookup)) => lookup .store .query::(&lookup.query, vec![item.into()]) .await .ok(), + Lookup::Store(store) => store + .key_get::(LookupKey::Key(item.to_string().into_bytes())) + .await + .ok() + .map(|v| !matches!(v, LookupValue::None)), Lookup::Directory(lookup) => match lookup { - directory::Lookup::DomainExists(directory) => directory - .is_local_domain(item.into().to_str().as_ref()) - .await - .ok(), - directory::Lookup::EmailExists(directory) => { - directory.rcpt(item.into().to_str().as_ref()).await.ok() + directory::Lookup::DomainExists(directory) => { + directory.is_local_domain(item).await.ok() } + directory::Lookup::EmailExists(directory) => directory.rcpt(item).await.ok(), }, } } - - pub async fn lookup(&self, items: Vec>) -> Option { - match self { - Lookup::Store(lookup) => lookup - .store - .query::>(&lookup.query, items) - .await - .ok() - .map(|row| { - let mut row = row.map(|row| row.values).unwrap_or_default(); - match row.len().cmp(&1) { - Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { - row.pop().map(into_sieve_value).unwrap() - } - Ordering::Less => Variable::default(), - _ => Variable::Array( - row.into_iter() - .map(into_sieve_value) - .collect::>() - .into(), - ), - } - }), - Lookup::Directory(_) => None, - } - } - - pub async fn query(&self, items: Vec>) -> Option>> { - match self { - Lookup::Store(lookup) => lookup - .store - .query::>(&lookup.query, items) - .await - .ok() - .map(|row| row.map(|row| row.values).unwrap_or_default()), - Lookup::Directory(_) => None, - } - } } impl PartialEq for Lookup { fn eq(&self, other: &Self) -> bool { match (self, other) { - (Lookup::Store(a), Lookup::Store(b)) => a.query == b.query, + (Lookup::Store(LookupStore::Query(a)), Lookup::Store(LookupStore::Query(b))) => { + a.query == b.query + } + (Lookup::Store(LookupStore::Store(_)), Lookup::Store(LookupStore::Store(_))) => true, (Lookup::Directory(a), Lookup::Directory(b)) => matches!( (a, b), ( @@ -389,8 +355,8 @@ pub fn to_store_value(value: &Variable) -> Value<'static> { } } -impl From> for Lookup { - fn from(lookup: Arc) -> Self { +impl From for Lookup { + fn from(lookup: LookupStore) -> Self { Lookup::Store(lookup) } } diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 8e543742..e2db65ab 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -32,15 +32,18 @@ use smtp_proto::{ MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; -use store::backend::memory::MemoryStore; +use store::{backend::memory::MemoryStore, LookupKey, LookupStore, LookupValue}; use tokio::runtime::Handle; use crate::{ - core::{Lookup, SMTP}, + core::SMTP, queue::{DomainPart, InstantFromTimestamp, Message}, }; -use super::{plugins::PluginContext, ScriptModification, ScriptParameters, ScriptResult}; +use super::{ + plugins::{lookup::VariableExists, PluginContext}, + ScriptModification, ScriptParameters, ScriptResult, +}; impl SMTP { pub fn run_script_blocking( @@ -92,14 +95,18 @@ impl SMTP { } => { input = false.into(); 'outer: for list in lists { - if let Some(list) = self.sieve.lookup.get(&list) { + if let Some(store) = self.sieve.lookup_stores.get(&list) { for value in &values { - let result = if !matches!(match_as, MatchAs::Lowercase) { - handle.block_on(list.contains(value)) - } else { - handle.block_on(list.contains(&value.to_lowercase())) - }; - if let Some(true) = result { + if let Ok(LookupValue::Value { .. }) = handle.block_on( + store.key_get::(LookupKey::Key( + if !matches!(match_as, MatchAs::Lowercase) { + value.clone() + } else { + value.to_lowercase() + } + .into_bytes(), + )), + ) { input = true.into(); break 'outer; } @@ -165,18 +172,13 @@ impl SMTP { } } Recipient::List(list) => { - if let Some(list) = self.sieve.lookup.get(&list) { - if let Lookup::Store(list) = list { - if let store::LookupStore::Memory(list) = &list.store { - if let MemoryStore::List(list) = list.as_ref() { - for rcpt in &list.set { - handle.block_on( - message.add_recipient( - rcpt, - &self.queue.config, - ), - ); - } + if let Some(list) = self.sieve.lookup_stores.get(&list) { + if let LookupStore::Memory(list) = list { + if let MemoryStore::List(list) = list.as_ref() { + for rcpt in &list.set { + handle.block_on( + message.add_recipient(rcpt, &self.queue.config), + ); } } } diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs index b96879b3..afd832e6 100644 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ b/crates/smtp/src/scripts/plugins/lookup.rs @@ -29,11 +29,11 @@ use std::{ use mail_auth::flate2; use sieve::{runtime::Variable, FunctionMap}; -use store::{Deserialize, LookupKey, LookupValue}; +use store::{Deserialize, LookupKey, LookupValue, Value}; use crate::{ config::scripts::{RemoteList, SieveContext}, - core::to_store_value, + core::into_sieve_value, USER_AGENT, }; @@ -61,32 +61,6 @@ pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap) -> Variable { let store = match &ctx.arguments[0] { - Variable::String(v) if v.contains('/') => { - if let Some(lookup) = ctx.core.sieve.lookup.get(v.as_ref()) { - return match &ctx.arguments[1] { - Variable::Array(items) => { - for item in items.iter() { - if !item.is_empty() - && ctx - .handle - .block_on(lookup.contains(to_store_value(item))) - .unwrap_or(false) - { - return true.into(); - } - } - false - } - v if !v.is_empty() => ctx - .handle - .block_on(lookup.contains(to_store_value(v))) - .unwrap_or(false), - _ => false, - } - .into(); - } - None - } Variable::String(v) if !v.is_empty() => ctx.core.sieve.lookup_stores.get(v.as_ref()), _ => ctx.core.sieve.default_lookup_store.as_ref(), }; @@ -133,23 +107,6 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { pub fn exec_get(ctx: PluginContext<'_>) -> Variable { let store = match &ctx.arguments[0] { - Variable::String(v) if v.contains('/') => { - if let Some(lookup) = ctx.core.sieve.lookup.get(v.as_ref()) { - let items = match &ctx.arguments[1] { - Variable::Array(l) => l.iter().map(to_store_value).collect(), - v if !v.is_empty() => vec![to_store_value(v)], - _ => vec![], - }; - return if !items.is_empty() { - ctx.handle - .block_on(lookup.lookup(items)) - .unwrap_or_default() - } else { - Variable::default() - }; - } - None - } Variable::String(v) if !v.is_empty() => ctx.core.sieve.lookup_stores.get(v.as_ref()), _ => ctx.core.sieve.default_lookup_store.as_ref(), }; @@ -467,10 +424,10 @@ pub fn exec_local_domain(ctx: PluginContext<'_>) -> Variable { } #[derive(Debug, PartialEq, Eq)] -pub(super) struct VariableWrapper(Variable); +pub struct VariableWrapper(Variable); #[derive(Debug, PartialEq, Eq)] -pub(super) struct VariableExists; +pub struct VariableExists; impl Deserialize for VariableWrapper { fn deserialize(bytes: &[u8]) -> store::Result { @@ -493,3 +450,15 @@ impl VariableWrapper { self.0 } } + +impl From> for VariableExists { + fn from(_: Value<'static>) -> Self { + VariableExists + } +} + +impl From> for VariableWrapper { + fn from(value: Value<'static>) -> Self { + VariableWrapper(into_sieve_value(value)) + } +} diff --git a/crates/store/src/backend/memory/lookup.rs b/crates/store/src/backend/memory/lookup.rs index 41a4dc08..4538fd21 100644 --- a/crates/store/src/backend/memory/lookup.rs +++ b/crates/store/src/backend/memory/lookup.rs @@ -21,57 +21,9 @@ * for more details. */ -use crate::{IntoRows, QueryResult, QueryType, Row, Value}; +use crate::{IntoRows, Row}; -use super::{LookupList, MatchType, MemoryStore}; - -impl MemoryStore { - pub(crate) fn query( - &self, - _: &str, - params: Vec>, - ) -> crate::Result { - let exists = match T::query_type() { - QueryType::Exists => true, - QueryType::QueryOne => false, - QueryType::QueryAll | QueryType::Execute => { - return Err(crate::Error::InternalError( - "Unsupported query type".to_string(), - )) - } - }; - - let needle = params.first().map(|v| v.to_str()).unwrap_or_default(); - - match self { - MemoryStore::List(list) => { - let found = list.contains(needle.as_ref()); - if exists { - Ok(T::from_exists(found)) - } else { - Ok(T::from_query_one(Some(Row { - values: vec![Value::Bool(found)], - }))) - } - } - MemoryStore::Map(map) => { - if let Some(value) = map.get(needle.as_ref()) { - if exists { - Ok(T::from_exists(true)) - } else { - Ok(T::from_query_one(Some(Row { - values: vec![value.clone()], - }))) - } - } else if exists { - Ok(T::from_exists(false)) - } else { - Ok(T::from_query_one(None::)) - } - } - } - } -} +use super::{LookupList, MatchType}; impl IntoRows for Option { fn into_row(self) -> Option { diff --git a/crates/store/src/backend/memory/main.rs b/crates/store/src/backend/memory/main.rs index 9274331f..ec56b2d3 100644 --- a/crates/store/src/backend/memory/main.rs +++ b/crates/store/src/backend/memory/main.rs @@ -54,7 +54,7 @@ impl MemoryStore { pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { let prefix = prefix.as_key(); - let lookup_type = config.property_require::((&prefix, "type"))?; + let lookup_type = config.property_require::((&prefix, "format"))?; let format = LookupFormat { lookup_type, comment: config.value((&prefix, "comment")).map(|s| s.to_string()), diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 355e4106..7d66a3e4 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -24,12 +24,12 @@ use std::sync::Arc; use async_trait::async_trait; -use utils::config::{cron::SimpleCron, utils::AsKey, Config}; +use utils::config::{cron::SimpleCron, Config}; use crate::{ backend::{fs::FsStore, memory::MemoryStore}, write::purge::{PurgeSchedule, PurgeStore}, - Lookup, LookupStore, Store, Stores, + LookupStore, QueryStore, Store, Stores, }; #[cfg(feature = "s3")] @@ -178,21 +178,9 @@ impl ConfigStore for Config { continue; } "memory" => { - let prefix = prefix.as_key(); - for lookup_id in self.sub_keys((&prefix, "lookup")) { - config.lookups.insert( - format!("{store_id}/{lookup_id}"), - Arc::new(Lookup { - store: MemoryStore::open( - self, - (prefix.as_str(), "lookup", lookup_id), - ) - .await? - .into(), - query: String::new(), - }), - ); - } + config + .lookup_stores + .insert(store_id, MemoryStore::open(self, prefix).await?.into()); continue; } @@ -202,15 +190,15 @@ impl ConfigStore for Config { } }; - // Add queries + // Add queries as lookup stores let lookup_store: LookupStore = lookup_store.into(); for lookup_id in self.sub_keys(("store", id, "query")) { - config.lookups.insert( + config.lookup_stores.insert( format!("{store_id}/{lookup_id}"), - Arc::new(Lookup { + LookupStore::Query(Arc::new(QueryStore { store: lookup_store.clone(), query: self.property_require(("store", id, "query", lookup_id))?, - }), + })), ); } config.lookup_stores.insert(store_id, lookup_store.clone()); diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 580f9f43..a1893778 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -21,6 +21,7 @@ * for more details. */ +use crate::{backend::memory::MemoryStore, Row}; #[allow(unused_imports)] use crate::{ write::{ @@ -39,21 +40,14 @@ impl LookupStore { params: Vec>, ) -> crate::Result { let result = match self { - LookupStore::Store(store) => match store { - #[cfg(feature = "sqlite")] - Store::SQLite(store) => store.query(query, params).await, - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.query(query, params).await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.query(query, params).await, - _ => Err(crate::Error::InternalError( - "Store does not support queries".into(), - )), - }, - LookupStore::Memory(store) => store.query(query, params), - #[cfg(feature = "redis")] - LookupStore::Redis(_) => Err(crate::Error::InternalError( - "Redis does not support queries".into(), + #[cfg(feature = "sqlite")] + LookupStore::Store(Store::SQLite(store)) => store.query(query, params).await, + #[cfg(feature = "postgres")] + LookupStore::Store(Store::PostgreSQL(store)) => store.query(query, params).await, + #[cfg(feature = "mysql")] + LookupStore::Store(Store::MySQL(store)) => store.query(query, params).await, + _ => Err(crate::Error::InternalError( + "Store does not support queries".into(), )), }; @@ -89,11 +83,21 @@ impl LookupStore { } #[cfg(feature = "redis")] LookupStore::Redis(store) => store.key_set(key, value).await, - LookupStore::Memory(_) => unimplemented!(), + LookupStore::Query(lookup) => lookup + .store + .query::( + &lookup.query, + vec![String::from_utf8(key).unwrap_or_default().into()], + ) + .await + .map(|_| ()), + LookupStore::Memory(_) => Err(crate::Error::InternalError( + "This store does not support key_set".into(), + )), } } - pub async fn key_get( + pub async fn key_get> + std::fmt::Debug + 'static>( &self, key: LookupKey, ) -> crate::Result> { @@ -120,7 +124,38 @@ impl LookupStore { }, #[cfg(feature = "redis")] LookupStore::Redis(store) => store.key_get(key).await, - LookupStore::Memory(_) => unimplemented!(), + LookupStore::Memory(store) => { + let key = String::from(key); + match store.as_ref() { + MemoryStore::List(list) => Ok(if list.contains(&key) { + LookupValue::Value { + value: T::from(Value::Bool(true)), + expires: 0, + } + } else { + LookupValue::None + }), + MemoryStore::Map(map) => Ok(map + .get(&key) + .map(|value| LookupValue::Value { + value: T::from(value.to_owned()), + expires: 0, + }) + .unwrap_or(LookupValue::None)), + } + } + LookupStore::Query(lookup) => lookup + .store + .query::>(&lookup.query, vec![String::from(key).into()]) + .await + .map(|row| { + row.and_then(|row| row.values.into_iter().next()) + .map(|value| LookupValue::Value { + value: T::from(value), + expires: 0, + }) + .unwrap_or(LookupValue::None) + }), } } @@ -169,7 +204,7 @@ impl LookupStore { } #[cfg(feature = "redis")] LookupStore::Redis(_) => {} - LookupStore::Memory(_) => {} + LookupStore::Memory(_) | LookupStore::Query(_) => {} } Ok(()) @@ -190,3 +225,16 @@ impl Deserialize for LookupValue { }) } } + +impl From> for String { + fn from(value: Value<'static>) -> Self { + match value { + Value::Text(string) => string.into_owned(), + Value::Blob(bytes) => String::from_utf8_lossy(bytes.as_ref()).into_owned(), + Value::Bool(boolean) => boolean.to_string(), + Value::Null => String::new(), + Value::Integer(num) => num.to_string(), + Value::Float(num) => num.to_string(), + } + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index f780b6fb..06f2aca0 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -194,13 +194,6 @@ pub struct Stores { pub blob_stores: AHashMap, pub fts_stores: AHashMap, pub lookup_stores: AHashMap, - pub lookups: AHashMap>, -} - -#[derive(Clone)] -pub struct Lookup { - pub store: LookupStore, - pub query: String, } #[derive(Clone)] @@ -235,11 +228,17 @@ pub enum FtsStore { #[derive(Clone)] pub enum LookupStore { Store(Store), + Query(Arc), Memory(Arc), #[cfg(feature = "redis")] Redis(Arc), } +pub struct QueryStore { + pub store: LookupStore, + pub query: String, +} + #[cfg(feature = "sqlite")] impl From for Store { fn from(store: SqliteStore) -> Self { @@ -364,6 +363,16 @@ impl<'x> Value<'x> { } } +impl From for String { + fn from(value: LookupKey) -> Self { + let key = match value { + LookupKey::Key(key) | LookupKey::Counter(key) => key, + }; + String::from_utf8(key) + .unwrap_or_else(|err| String::from_utf8_lossy(&err.into_bytes()).into_owned()) + } +} + #[derive(Clone, Debug)] pub struct Row { pub values: Vec>, diff --git a/resources/config.zip b/resources/config.zip index 2b1276b0..b6b723a8 100644 Binary files a/resources/config.zip and b/resources/config.zip differ diff --git a/resources/config/smtp/spamfilter.toml b/resources/config/smtp/spamfilter.toml index e4960cf3..e43326f6 100644 --- a/resources/config/smtp/spamfilter.toml +++ b/resources/config/smtp/spamfilter.toml @@ -2,58 +2,64 @@ # SMTP Spam & Phishing filter configuration ############################################# -[store."spam"] +[store."spam/free-domains"] type = "memory" - -[store."spam".lookup."free-domains"] -type = "glob" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/domains_free.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/domains_free.list"] -[store."spam".lookup."disposable-domains"] -type = "glob" +[store."spam/disposable-domains"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/domains_disposable.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/domains_disposable.list"] -[store."spam".lookup."redirectors"] -type = "glob" +[store."spam/redirectors"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/url_redirectors.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/url_redirectors.list"] -[store."spam".lookup."domains-allow"] -type = "glob" +[store."spam/domains-allow"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_domains.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_domains.list"] -[store."spam".lookup."dmarc-allow"] -type = "glob" +[store."spam/dmarc-allow"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_dmarc.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_dmarc.list"] -[store."spam".lookup."spf-dkim-allow"] -type = "glob" +[store."spam/spf-dkim-allow"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_spf_dkim.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_spf_dkim.list"] -[store."spam".lookup."mime-types"] -type = "map" +[store."spam/mime-types"] +type = "memory" +format = "map" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/mime_types.map", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/mime_types.map"] -[store."spam".lookup."trap-address"] -type = "glob" +[store."spam/trap-address"] +type = "memory" +format = "glob" comment = '#' values = "file://%{BASE_PATH}%/etc/spamfilter/maps/spam_trap.list" -[store."spam".lookup."scores"] -type = "map" +[store."spam/scores"] +type = "memory" +format = "map" values = "file://%{BASE_PATH}%/etc/spamfilter/maps/scores.map" [sieve.trusted.scripts] diff --git a/resources/config/store/mysql.toml b/resources/config/store/mysql.toml index bd95dd9c..356f4e18 100644 --- a/resources/config/store/mysql.toml +++ b/resources/config/store/mysql.toml @@ -14,7 +14,7 @@ disable = true [store."mysql".timeout] wait = "15s" -#[store."postgresql".pool] +#[store."mysql".pool] #max-connections = 10 #min-connections = 5 diff --git a/tests/resources/smtp/config/rules-dynvalue.toml b/tests/resources/smtp/config/rules-dynvalue.toml index 7e750fbe..1f83e6c6 100644 --- a/tests/resources/smtp/config/rules-dynvalue.toml +++ b/tests/resources/smtp/config/rules-dynvalue.toml @@ -61,22 +61,19 @@ test = [ ] expect = false -[store."list_mx"] +[store."list_mx/domains"] type = "memory" -[store."list_mx".lookup."domains"] -type = "list" +format = "list" values = ["mx"] -[store."list_foo"] +[store."list_foo/domains"] type = "memory" -[store."list_foo".lookup."domains"] -type = "list" +format = "list" values = ["foo"] -[store."list_123"] +[store."list_123/domains"] type = "memory" -[store."list_123".lookup."domains"] -type = "list" +format = "list" values = ["123"] [maybe-eval."dyn_mx"] diff --git a/tests/resources/smtp/config/rules-eval.toml b/tests/resources/smtp/config/rules-eval.toml index 3133c327..1a658753 100644 --- a/tests/resources/smtp/config/rules-eval.toml +++ b/tests/resources/smtp/config/rules-eval.toml @@ -164,9 +164,7 @@ nested-none-of-false = { none-of = [ ]} ]} -[store."list"] +[store."list/domains"] type = "memory" - -[store."list".lookup."domains"] -type = "list" +format = "list" values = ["mydomain1.org", "foo.net", "otherdomain.net"] diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index af6a19ff..36a50114 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -523,24 +523,25 @@ impl core::fmt::Debug for Item { #[ignore] async fn lookup_local() { const LOOKUP_CONFIG: &str = r#" - [store."local"] + [store."local/regex"] type = "memory" - - [store."local".lookup."regex"] - type = "regex" + format = "regex" values = ["^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"] - [store."local".lookup."glob"] - type = "glob" + [store."local/glob"] + type = "memory" + format = "glob" values = ["*@example.org", "test@*", "localhost", "*+*@*.domain.net"] - [store."local".lookup."list"] - type = "list" + [store."local/list"] + type = "memory" + format = "list" values = ["abc", "xyz", "123"] - [store."local".lookup."suffix"] - type = "glob" + [store."local/suffix"] + type = "memory" + format = "glob" comment = "//" values = ["https://publicsuffix.org/list/public_suffix_list.dat", "fallback+file://%PATH%/public_suffix_list.dat.gz"] "#; @@ -570,7 +571,7 @@ async fn lookup_local() { .parse_stores() .await .unwrap() - .lookups; + .lookup_stores; for (lookup, item, expect) in [ ("glob", "user@example.org", true), diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 3166851f..e3dd5990 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -47,9 +47,9 @@ async fn sql_directory() { let mut config = DirectoryTest::new(directory_id.into()).await; let lookups = config .stores - .lookups - .into_iter() - .map(|(k, v)| (k, Lookup::from(v))) + .lookup_stores + .iter() + .map(|(k, v)| (k.clone(), Lookup::from(v.clone()))) .collect::>(); println!("Testing SQL directory {:?}", directory_id); diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index d0b0c368..c403a291 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -233,15 +233,14 @@ email = "address" quota = "quota" type = "type" -[store."local"] +[store."local/domains"] type = "memory" - -[store."local".lookup."domains"] -type = "list" +format = "list" values = ["example.com"] -[store."local".lookup."remote-domains"] -type = "list" +[store."local/remote-domains"] +type = "memory" +format = "list" values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] [oauth] diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 9ecb89bd..8abb8929 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -239,15 +239,14 @@ email = "address" quota = "quota" type = "type" -[store."local"] +[store."local/domains"] type = "memory" - -[store."local".lookup."domains"] -type = "list" +format = "list" values = ["example.com"] -[store."local".lookup."remote-domains"] -type = "list" +[store."local/remote-domains"] +type = "memory" +format = "list" values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] [oauth] diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 9e2d1c00..0fd2f1d3 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -33,6 +33,7 @@ use std::{ use store::{ backend::memory::{LookupList, MemoryStore}, config::ConfigStore, + LookupStore, }; use tokio::net::TcpSocket; @@ -80,13 +81,13 @@ fn parse_conditions() { ..Default::default() }]; let mut context = ConfigContext::new(&servers); - let list = Arc::new(store::Lookup { + let list = LookupStore::Query(Arc::new(store::QueryStore { store: MemoryStore::List(LookupList::default()).into(), query: "abc".into(), - }); + })); context .stores - .lookups + .lookup_stores .insert("test-list".to_string(), list.clone()); let mut conditions = config.parse_conditions(&context).unwrap(); @@ -648,6 +649,12 @@ async fn eval_dynvalue() { "failed for test {test_name:?}" ); } + let wrapped_stores = context + .stores + .lookup_stores + .iter() + .map(|(k, v)| (k.clone(), Arc::new(v.clone()))) + .collect::>(); for test_name in config.sub_keys("maybe-eval") { //println!("============= Testing {:?} ==================", key); @@ -670,11 +677,7 @@ async fn eval_dynvalue() { ) .unwrap() .unwrap() - .map_if_block( - &context.stores.lookups, - ("maybe-eval", test_name, "test"), - "test", - ) + .map_if_block(&wrapped_stores, ("maybe-eval", test_name, "test"), "test") .unwrap(); let expected = config .value_require(("maybe-eval", test_name, "expect")) @@ -685,6 +688,8 @@ async fn eval_dynvalue() { .await .into_value(&envelope) .unwrap() + .as_ref() + .clone() .into(); assert!(lookup.contains(expected).await.unwrap()); diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 6613f71e..84362f79 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -54,48 +54,50 @@ path = "%PATH%/test_antispam.db" #type = "redis" #url = "redis://127.0.0.1" -[store."default"] +[store."default/domains"] type = "memory" - -[store."default".lookup."domains"] -type = "list" +format = "list" values = ["local-domain.org"] -[store."spam"] +[store."spam/free-domains"] type = "memory" - -[store."spam".lookup."free-domains"] -type = "glob" +format = "glob" comment = '#' values = ["gmail.com", "googlemail.com", "yahoomail.com", "*.freemail.org"] -[store."spam".lookup."disposable-domains"] -type = "glob" +[store."spam/disposable-domains"] +type = "memory" +format = "glob" comment = '#' values = ["guerrillamail.com", "*.disposable.org"] -[store."spam".lookup."redirectors"] -type = "glob" +[store."spam/redirectors"] +type = "memory" +format = "glob" comment = '#' values = ["bit.ly", "redirect.io", "redirect.me", "redirect.org", "redirect.com", "redirect.net", "t.ly", "tinyurl.com"] -[store."spam".lookup."dmarc-allow"] -type = "glob" +[store."spam/dmarc-allow"] +type = "memory" +format = "glob" comment = '#' values = ["dmarc-allow.org"] -[store."spam".lookup."spf-dkim-allow"] -type = "glob" +[store."spam/spf-dkim-allow"] +type = "memory" +format = "glob" comment = '#' values = ["spf-dkim-allow.org"] -[store."spam".lookup."domains-allow"] -type = "glob" +[store."spam/domains-allow"] +type = "memory" +format = "glob" values = [] -[store."spam".lookup."mime-types"] -type = "map" +[store."spam/mime-types"] +type = "memory" +format = "map" comment = '#' values = ["html text/html|BAD", "pdf application/pdf|NZ", @@ -104,13 +106,15 @@ values = ["html text/html|BAD", "js BAD|NZ", "hta BAD|NZ"] -[store."spam".lookup."trap-address"] -type = "glob" +[store."spam/trap-address"] +type = "memory" +format = "glob" comment = '#' values = ["spamtrap@*"] -[store."spam".lookup."scores"] -type = "map" +[store."spam/scores"] +type = "memory" +format = "map" values = "file://%CFG_PATH%/maps/scores.map" [resolver] diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 2d27cf30..5bb99c2a 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -49,11 +49,9 @@ max-connections = 10 min-connections = 0 idle-timeout = "5m" -[store."local"] +[store."local/invalid-ehlos"] type = "memory" - -[store."local".lookup."invalid-ehlos"] -type = "list" +format = "list" values = ["spammer.org", "spammer.net"] [session.data.pipe."test"] diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 0121c343..e170053d 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -433,7 +433,6 @@ impl TestConfig for SieveCore { SieveCore { runtime: Runtime::new_with_context(SieveContext::default()), scripts: AHashMap::new(), - lookup: AHashMap::new(), from_addr: "MAILER-DAEMON@example.org".to_string(), from_name: "Mailer Daemon".to_string(), return_path: "".to_string(), diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index 53c9036e..b37f6b29 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -37,6 +37,18 @@ pub async fn lookup_tests() { println!("Testing lookup store {}...", store_id); if let LookupStore::Store(store) = &store { store.destroy().await; + } else { + // Reset redis counter + store + .key_set( + "abc".as_bytes().to_vec(), + LookupValue::Value { + value: "0".as_bytes().to_vec(), + expires: 0, + }, + ) + .await + .unwrap(); } // Test key