diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index a02c3930..ce3f939d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -31,9 +31,9 @@ body: label: Version description: What version of our software are you running? options: + - v0.5.x - v0.4.x - - v0.3.x - - v0.2.x or lower + - v0.3.x or lower validations: required: true - type: dropdown @@ -41,23 +41,30 @@ body: attributes: label: What database are you using? options: - - SQLite + - RocksDB - FoundationDB + - PostgreSQL + - mySQL + - SQLite - type: dropdown id: blob attributes: label: What blob storage are you using? options: - - Local + - RocksDB + - FoundationDB + - PostgreSQL + - mySQL + - SQLite + - Filesystem - S3-compatible - type: dropdown id: directory attributes: label: Where is your directory located? options: - - SQLite - - mySQL - - PostgreSQL + - Internal + - SQL - LDAP - type: dropdown id: os diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d019949..fd34acd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file. This projec ## [0.5.1] - 2024-01-xx ## Added +- SMTP smuggling protection: Sanitization of outgoing messages that do not use `CRLF` as line endings. ### Changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec659418..f0495070 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,35 +1,26 @@ # Contributing -When contributing to this repository, please first discuss the change you wish to make via issue, -email, or any other method with the owners of this repository before making a change. +Thank you for your interest in contributing to the Stalwart Mail Server project! We appreciate your support and enthusiasm. This document provides guidelines for contributing to the project. -Please note we have a code of conduct, please follow it in all your interactions with the project. +## Not accepting Pull Requests at this time +As of now, the Stalwart Mail Server project is in a phase of rapid development and evolution. Until we reach version 1.0, **we are not open to direct code contributions** in the form of pull requests. This is due to the following reasons: +- The project is undergoing significant changes which might make integrating external contributions challenging. +- Our current resources do not allow for the efficient evaluation and testing of submitted pull requests. -## Any contributions you make will be under AGPL +We understand that this might be disappointing, but rest assured, this measure is temporary and is aimed at maintaining the integrity and consistency of the project during its critical development phase. -This software is licensed under the Affero General Public License (AGPL). Any contributions made to -this project will be under this license. Before any contributions can be made, contributors are -required to sign a Contributor License Agreement (CLA). The purpose of the agreement is to clarify -and document the rights granted by contributors to us. +## Other Ways to Contribute +While we're not accepting pull requests at this moment, there are several other valuable ways you can contribute to the Stalwart Mail Server project: -## Pull Request Process +- **Enhancement Requests**: If you have an idea for a new feature or an improvement, we encourage you to file an enhancement request on GitHub. This way, your idea can be evaluated and potentially included in future versions of the project. To do this, create a new issue on our GitHub repository and label it as an 'enhancement'. +- **Extensive Testing**: One of the most helpful contributions is to rigorously test the software in various environments and use-cases. Your testing can uncover important issues and help improve the stability of the project. +- **Writing Unit Tests**: Contributing new unit tests or improving existing ones is a great way to ensure the reliability and robustness of our codebase. +- **Reviewing Documentation**: As the project evolves, so does the need for accurate and up-to-date documentation. Reviewing, updating, or writing new documentation is a critical contribution that helps new and existing users understand and effectively use Stalwart Mail Server. -1. Ensure any install or build dependencies are removed before the end of the layer when doing a - build. -2. Update the README.md with details of changes to the interface, this includes new environment - variables, exposed ports, useful file locations and container parameters. -3. Increase the version numbers in any examples files and the README.md to the new version that this - Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). -4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you - do not have permission to do that, you may request the second reviewer to merge it for you. +## Stay Updated +We anticipate opening up for code contributions once version 1.0 is launched. We recommend keeping an eye on our project announcements for any updates regarding contribution opportunities. -## Code of Conduct +## Questions or Suggestions? +If you have any questions or suggestions regarding the contribution process, feel free to open a discussion on our GitHub repository. -We as members, contributors, and leaders pledge to make participation in our community a harassment-free -experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex -characteristics, gender identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity and orientation. -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, -and healthy community. - -You can read the full Code of Conduct [here](https://github.com/stalwartlabs/.github/blob/main/CODE_OF_CONDUCT.md). +Thank you for your support and understanding. Together, we are building a robust and reliable Stalwart Mail Server! diff --git a/Cargo.lock b/Cargo.lock index 59b9910f..c760e273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -874,9 +874,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" dependencies = [ "clap_builder", "clap_derive", @@ -884,9 +884,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" dependencies = [ "anstream", "anstyle", @@ -2661,13 +2661,13 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2775,9 +2775,9 @@ dependencies = [ [[package]] name = "jmap-client" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bafb96896c352dd68f3a32f0a2e0450a589366a76c0120bd4ae585f68ad381f5" +checksum = "12c697483ad894a8184d0fd61848e057f86b16642049993b3e6a80c959dbc90a" dependencies = [ "ahash 0.8.6", "async-stream", @@ -2787,7 +2787,8 @@ dependencies = [ "maybe-async", "parking_lot", "reqwest", - "rustls 0.21.10", + "rustls 0.22.1", + "rustls-pki-types", "serde", "serde_json", "tokio", @@ -3238,9 +3239,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -3983,9 +3984,9 @@ checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" [[package]] name = "platforms" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" +checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" [[package]] name = "polyval" @@ -6936,9 +6937,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" dependencies = [ "libc", "linux-raw-sys", diff --git a/crates/cli/src/modules/import.rs b/crates/cli/src/modules/import.rs index d468eb30..f8f0c237 100644 --- a/crates/cli/src/modules/import.rs +++ b/crates/cli/src/modules/import.rs @@ -165,7 +165,7 @@ impl ImportCommands { } children .entry(mailbox.parent_id()) - .or_insert_with(Vec::new) + .or_default() .push(mailbox_id); mailbox_ids.insert(mailbox_id, mailbox.name().unwrap_or("Untitled")); } diff --git a/crates/imap/src/lib.rs b/crates/imap/src/lib.rs index cafde2f3..d7d444b2 100644 --- a/crates/imap/src/lib.rs +++ b/crates/imap/src/lib.rs @@ -73,7 +73,7 @@ impl IMAP { rate_requests: config.property_or_static("imap.rate-limit.requests", "2000/1m")?, rate_concurrent: config.property("imap.rate-limit.concurrent")?.unwrap_or(4), allow_plain_auth: config.property_or_static("imap.auth.allow-plain-text", "false")?, - enable_uidplus: config.property_or_static("imap.protocol.uidplus", "true")?, + enable_uidplus: config.property_or_static("imap.protocol.uidplus", "false")?, })) } } diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index 9c5d6721..b31138d8 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -117,10 +117,7 @@ impl SessionData { if let (Some(thread_id), Some((imap_id, _))) = (thread_id, state.map_result_id(document_id, is_uid)) { - threads - .entry(thread_id) - .or_insert_with(Vec::new) - .push(imap_id); + threads.entry(thread_id).or_default().push(imap_id); } } diff --git a/crates/jmap-proto/src/response/references.rs b/crates/jmap-proto/src/response/references.rs index ed35729d..24f6754c 100644 --- a/crates/jmap-proto/src/response/references.rs +++ b/crates/jmap-proto/src/response/references.rs @@ -199,7 +199,7 @@ impl Response { match &response.method { ResponseMethod::Get(response) => { return match rr.path.item_subquery() { - Some((root, property)) if root == "list" => { + Some(("list", property)) => { let property = Property::parse(property); EvalResult::Values( diff --git a/crates/jmap-proto/src/types/pointer.rs b/crates/jmap-proto/src/types/pointer.rs index e2286dcf..0ad1d9ee 100644 --- a/crates/jmap-proto/src/types/pointer.rs +++ b/crates/jmap-proto/src/types/pointer.rs @@ -154,7 +154,7 @@ impl JSONPointer { JSONPointer::String(property) => property.as_str().into(), JSONPointer::Path(path) if path.len() == 2 => { if let (Some(JSONPointer::String(property)), Some(JSONPointer::Wildcard)) = - (path.get(0), path.get(1)) + (path.first(), path.get(1)) { property.as_str().into() } else { @@ -168,7 +168,7 @@ impl JSONPointer { pub fn item_subquery(&self) -> Option<(&str, &str)> { match self { JSONPointer::Path(path) if path.len() == 3 => { - match (path.get(0), path.get(1), path.get(2)) { + match (path.first(), path.get(1), path.get(2)) { ( Some(JSONPointer::String(root)), Some(JSONPointer::Wildcard), diff --git a/crates/jmap/src/services/state.rs b/crates/jmap/src/services/state.rs index b294c8cc..e51b0c0a 100644 --- a/crates/jmap/src/services/state.rs +++ b/crates/jmap/src/services/state.rs @@ -87,6 +87,7 @@ pub fn init_state_manager() -> (mpsc::Sender, mpsc::Receiver) { mpsc::channel::(IPC_CHANNEL_BUFFER) } +#[allow(clippy::unwrap_or_default)] pub fn spawn_state_manager( core: Arc, settings: &Config, diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/jmap/src/sieve/ingest.rs index c81f0340..f2176295 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/jmap/src/sieve/ingest.rs @@ -480,6 +480,7 @@ impl JMAP { Ok(ingested_message) } else { // There were problems during delivery + #[allow(clippy::unnecessary_unwrap)] Err(last_temp_error.unwrap()) } } diff --git a/crates/jmap/src/sieve/mod.rs b/crates/jmap/src/sieve/mod.rs index c7ae3b2e..e806a3cc 100644 --- a/crates/jmap/src/sieve/mod.rs +++ b/crates/jmap/src/sieve/mod.rs @@ -65,7 +65,7 @@ impl SeenIdHash { impl PartialOrd for SeenIdHash { fn partial_cmp(&self, other: &Self) -> Option { - self.expiry.partial_cmp(&other.expiry) + Some(self.cmp(other)) } } diff --git a/crates/jmap/src/websocket/upgrade.rs b/crates/jmap/src/websocket/upgrade.rs index afa53198..5516ebf1 100644 --- a/crates/jmap/src/websocket/upgrade.rs +++ b/crates/jmap/src/websocket/upgrade.rs @@ -68,7 +68,7 @@ pub async fn upgrade_websocket_connection( .get("Sec-WebSocket-Version") .and_then(|h| h.to_str().ok()), ) { - (Some(key), Some(version)) if version == "13" => derive_accept_key(key.as_bytes()), + (Some(key), Some("13")) => derive_accept_key(key.as_bytes()), _ => { return RequestError::blank( StatusCode::BAD_REQUEST.as_u16(), diff --git a/crates/nlp/src/language/search_snippet.rs b/crates/nlp/src/language/search_snippet.rs index 9c428fef..676191ea 100644 --- a/crates/nlp/src/language/search_snippet.rs +++ b/crates/nlp/src/language/search_snippet.rs @@ -91,7 +91,7 @@ pub fn generate_snippet( } let mut snippet = String::with_capacity(text.len()); - let start_offset = terms.get(0)?.offset; + let start_offset = terms.first()?.offset; if start_offset > 0 { let mut word_count = 0; diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 8fff4c64..50a00d8c 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -410,10 +410,7 @@ impl Ord for SessionAddress { impl PartialOrd for SessionAddress { fn partial_cmp(&self, other: &Self) -> Option { - match self.domain.partial_cmp(&other.domain) { - Some(std::cmp::Ordering::Equal) => self.address_lcase.partial_cmp(&other.address_lcase), - order => order, - } + Some(self.cmp(other)) } } diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 9a0b8434..32e4135b 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -193,7 +193,7 @@ impl Ord for Schedule { impl PartialOrd for Schedule { fn partial_cmp(&self, other: &Self) -> Option { - other.due.partial_cmp(&self.due) + Some(self.cmp(other)) } } diff --git a/crates/smtp/src/scripts/plugins/bayes.rs b/crates/smtp/src/scripts/plugins/bayes.rs index c4ea2622..ddf04c1c 100644 --- a/crates/smtp/src/scripts/plugins/bayes.rs +++ b/crates/smtp/src/scripts/plugins/bayes.rs @@ -185,7 +185,7 @@ pub fn exec_classify(ctx: PluginContext<'_>) -> Variable { // Create classifier from defaults let mut classifier = BayesClassifier::default(); if let Some(params) = ctx.arguments[2].as_array() { - if let Some(Variable::Integer(value)) = params.get(0) { + if let Some(Variable::Integer(value)) = params.first() { classifier.min_token_hits = *value as u32; } if let Some(Variable::Integer(value)) = params.get(1) { diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs index afd832e6..8b5c0792 100644 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ b/crates/smtp/src/scripts/plugins/lookup.rs @@ -226,7 +226,7 @@ pub fn exec_remote(ctx: PluginContext<'_>) -> Variable { if let Some(arr) = ctx.arguments[2].as_array() { // Obtain expiration - match arr.get(0) { + match arr.first() { Some(Variable::Integer(v)) if *v > 0 => { expires = Duration::from_secs(*v as u64); } diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 954b6d42..2944c920 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -86,6 +86,11 @@ impl RocksDbStore { db_opts.create_missing_column_families(true); db_opts.create_if_missing(true); db_opts.set_max_background_jobs(std::cmp::max(num_cpus::get() as i32, 3)); + db_opts.increase_parallelism(std::cmp::max(num_cpus::get() as i32, 3)); + db_opts.set_level_zero_file_num_compaction_trigger(1); + db_opts.set_level_compaction_dynamic_level_bytes(true); + //db_opts.set_keep_log_file_num(100); + //db_opts.set_max_successive_merges(100); db_opts.set_write_buffer_size( config.property_or_static((&prefix, "write-buffer-size"), "134217728")?, ); diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 27b94d44..5e93426b 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -50,33 +50,28 @@ impl RocksDbStore { let db = self.db.clone(); self.spawn_worker(move || { - let start = Instant::now(); - let mut retry_count = 0; - - let mut txn_opts = OptimisticTransactionOptions::default(); - txn_opts.set_snapshot(true); - - let txn = RocksDBTransaction { + let mut txn = RocksDBTransaction { db: &db, cf_bitmaps: db.cf_handle(CF_BITMAPS).unwrap(), cf_values: db.cf_handle(CF_VALUES).unwrap(), cf_indexes: db.cf_handle(CF_INDEXES).unwrap(), cf_logs: db.cf_handle(CF_LOGS).unwrap(), cf_counters: db.cf_handle(CF_COUNTERS).unwrap(), - txn_opts, + txn_opts: OptimisticTransactionOptions::default(), batch: &batch, }; + txn.txn_opts.set_snapshot(true); + // Begin write + let mut retry_count = 0; + let start = Instant::now(); loop { match txn.commit() { - Ok(success) => { - return if success { - Ok(()) - } else { - Err(crate::Error::AssertValueFailed) - }; + Ok(_) => { + return Ok(()); } - Err(err) => match err.kind() { + Err(CommitError::Internal(err)) => return Err(err), + Err(CommitError::RocksDB(err)) => match err.kind() { ErrorKind::Busy | ErrorKind::MergeInProgress | ErrorKind::TryAgain if retry_count < MAX_COMMIT_ATTEMPTS && start.elapsed() < MAX_COMMIT_TIME => @@ -140,8 +135,13 @@ struct RocksDBTransaction<'x> { batch: &'x Batch, } +enum CommitError { + Internal(crate::Error), + RocksDB(rocksdb::Error), +} + impl<'x> RocksDBTransaction<'x> { - fn commit(&self) -> Result { + fn commit(&self) -> Result<(), CommitError> { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; @@ -188,15 +188,15 @@ impl<'x> RocksDBTransaction<'x> { collection, document_id, class, - }; - let key = key.serialize(0); + } + .serialize(0); if let ValueOp::Set(value) = op { txn.put_cf(&self.cf_values, &key, value)?; if matches!(class, ValueClass::ReservedId) { if let Some(bitmap) = txn - .get_pinned_cf( + .get_pinned_for_update_cf( &self.cf_bitmaps, &BitmapKey { account_id, @@ -205,12 +205,24 @@ impl<'x> RocksDBTransaction<'x> { block_num: 0, } .serialize(WITHOUT_BLOCK_NUM), - )? - .and_then(|bytes| RoaringBitmap::deserialize(&bytes).ok()) + true, + ) + .map_err(CommitError::from) + .and_then(|bytes| { + if let Some(bytes) = bytes { + RoaringBitmap::deserialize(&bytes) + .map(Some) + .map_err(CommitError::from) + } else { + Ok(None) + } + })? { if bitmap.contains(document_id) { txn.rollback()?; - return Ok(false); + return Err(CommitError::Internal( + crate::Error::AssertValueFailed, + )); } } } @@ -274,8 +286,8 @@ impl<'x> RocksDBTransaction<'x> { collection, document_id, class, - }; - let key = key.serialize(0); + } + .serialize(0); let matches = txn .get_pinned_for_update_cf(&self.cf_values, &key, true)? .map(|value| assert_value.matches(&value)) @@ -283,13 +295,13 @@ impl<'x> RocksDBTransaction<'x> { if !matches { txn.rollback()?; - return Ok(false); + return Err(CommitError::Internal(crate::Error::AssertValueFailed)); } } } } - txn.commit().map(|_| true) + txn.commit().map_err(Into::into) } else { let mut wb = txn.get_writebatch(); for op in &self.batch.ops { @@ -389,7 +401,19 @@ impl<'x> RocksDBTransaction<'x> { } } - self.db.write(wb).map(|_| true) + self.db.write(wb).map_err(Into::into) } } } + +impl From for CommitError { + fn from(err: rocksdb::Error) -> Self { + CommitError::RocksDB(err) + } +} + +impl From for CommitError { + fn from(err: crate::Error) -> Self { + CommitError::Internal(err) + } +} diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs index 53e2e4e9..135004b0 100644 --- a/crates/store/src/query/filter.rs +++ b/crates/store/src/query/filter.rs @@ -177,7 +177,7 @@ impl Store { Ok(ResultSet { account_id, collection, - results: state.bm.unwrap_or_else(RoaringBitmap::new), + results: state.bm.unwrap_or_default(), }) } diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 4eb5fe51..e191de6c 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -215,6 +215,13 @@ impl Batch { ) }) } + + pub fn first_account_id(&self) -> Option { + self.ops.iter().find_map(|op| match op { + Operation::AccountId { account_id } => Some(*account_id), + _ => None, + }) + } } impl Default for BatchBuilder { diff --git a/resources/config/imap/settings.toml b/resources/config/imap/settings.toml index 9f5d85c4..a8573d49 100644 --- a/resources/config/imap/settings.toml +++ b/resources/config/imap/settings.toml @@ -20,3 +20,6 @@ idle = "30m" [imap.rate-limit] requests = "2000/1m" concurrent = 4 + +[imap.protocol] +uidplus = false diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 7eb54c0d..40a7a97e 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ resolver = "2" [features] #default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] -default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] +default = ["sqlite", "foundationdb", "postgres", "mysql", "rocks", "elastic", "s3", "redis"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] postgres = ["store/postgres"] diff --git a/tests/resources/create_test_env.sh b/tests/resources/create_test_env.sh deleted file mode 100644 index b58fabf8..00000000 --- a/tests/resources/create_test_env.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -BASE_DIR="/tmp/stalwart-test" -DOMAIN="example.org" - -# Stores -STORE="rocksdb" -FTS_STORE="rocksdb" -BLOB_STORE="rocksdb" - -# Directories -DIRECTORY="internal" -SQL_STORE="sqlite" - -# Delete previous tests -rm -rf $BASE_DIR - -# Create directories -mkdir -p $BASE_DIR $BASE_DIR/data $BASE_DIR/data/blobs $BASE_DIR/logs $BASE_DIR/reports $BASE_DIR/queue - -# Copy config files -cp -r resources/config $BASE_DIR/etc - -# Copy self-signed certs -cp -r tests/resources/tls_cert.pem $BASE_DIR/etc -cp -r tests/resources/tls_privatekey.pem $BASE_DIR/etc - -# Replace stores and directories -sed -i '' -e "s|__SQL_STORE__|$SQL_STORE|g" "$BASE_DIR/etc/directory/sql.toml" -sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/directory/$DIRECTORY.toml" -sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$STORE.toml" -sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$FTS_STORE.toml" -sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$BLOB_STORE.toml" -sed -i '' -e "s/__FTS_STORE__/$FTS_STORE/g" \ - -e "s/__BLOB_STORE__/$BLOB_STORE/g" "$BASE_DIR/etc/jmap/store.toml" - -# Replace settings -sed -i '' -e "s/__STORE__/$STORE/g" \ - -e "s/__DIRECTORY__/$DIRECTORY/g" \ - -e "s/__DOMAIN__/$DOMAIN/g" \ - -e "s/__HOST__/mail.$DOMAIN/g" \ - -e "s|__BASE_PATH__|$BASE_DIR|g" "$BASE_DIR/etc/config.toml" -sed -i '' -e "s|__CERT_PATH__|$BASE_DIR/etc/tls_cert.pem|g" \ - -e "s|__PK_PATH__|$BASE_DIR/etc/tls_privatekey.pem|g" "$BASE_DIR/etc/common/tls.toml" -sed -i '' -e 's/method = "log"/method = "stdout"/g' \ - -e 's/level = "info"/level = "trace"/g' "$BASE_DIR/etc/common/tracing.toml" -sed -i '' -e 's/%{HOST}%/127.0.0.1/g' "$BASE_DIR/etc/jmap/listener.toml" -sed -i '' -e 's/allow-plain-text = false/allow-plain-text = true/g' "$BASE_DIR/etc/imap/settings.toml" -sed -i '' -e 's/user = "stalwart-mail"//g' \ - -e 's/group = "stalwart-mail"//g' "$BASE_DIR/etc/common/server.toml" - -# Generate DKIM key -mkdir -p $BASE_DIR/etc/dkim -openssl genpkey -algorithm RSA -out $BASE_DIR/etc/dkim/$DOMAIN.key - -: ' -SET_ADMIN_USER="admin" SET_ADMIN_PASS="secret" cargo run --manifest-path=crates/main/Cargo.toml -- --config=/tmp/stalwart-test/etc/config.toml -cargo run --manifest-path=crates/main/Cargo.toml -- --config=/tmp/stalwart-test/etc/config.toml -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret domain create example.org -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account create john 12345 -d "John Doe" -a john@example.org -a john.doe@example.org -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account create jane abcde -d "Jane Doe" -a jane@example.org -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account create bill xyz12 -d "Bill Foobar" -a bill@example.org -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret group create sales -d "Sales Department" -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret group create support -d "Technical Support" -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account add-to-group john sales support -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account remove-from-group john support -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account add-email jane jane.doe@example.org -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret list create everyone everyone@example.org -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret list add-members everyone jane john bill -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret account list -cargo run --manifest-path=crates/cli/Cargo.toml -- -u https://127.0.0.1:8080 -c admin:secret import messages --format mbox john _ignore/dovecot-crlf -' diff --git a/tests/resources/scripts/create_test_env.sh b/tests/resources/scripts/create_test_env.sh new file mode 100644 index 00000000..c8b2302b --- /dev/null +++ b/tests/resources/scripts/create_test_env.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +BASE_DIR="/tmp/stalwart-test" +DOMAIN="example.org" + +# Stores +#STORE="foundationdb" +#FTS_STORE="foundationdb" +#BLOB_STORE="foundationdb" +STORE="rocksdb" +FTS_STORE="rocksdb" +BLOB_STORE="rocksdb" + +# Directories +DIRECTORY="internal" +SQL_STORE="sqlite" + +# Delete previous tests +rm -rf $BASE_DIR + +# Create directories +mkdir -p $BASE_DIR $BASE_DIR/data $BASE_DIR/data/blobs $BASE_DIR/logs $BASE_DIR/reports $BASE_DIR/queue + +# Copy config files +cp -r resources/config $BASE_DIR/etc + +# Copy self-signed certs +cp -r tests/resources/tls_cert.pem $BASE_DIR/etc +cp -r tests/resources/tls_privatekey.pem $BASE_DIR/etc + +# Replace stores and directories +sed -i '' -e "s|__SQL_STORE__|$SQL_STORE|g" "$BASE_DIR/etc/directory/sql.toml" +sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/directory/$DIRECTORY.toml" +sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$STORE.toml" +sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$FTS_STORE.toml" +sed -i '' -e 's/disable = true//g' "$BASE_DIR/etc/store/$BLOB_STORE.toml" +sed -i '' -e "s/__FTS_STORE__/$FTS_STORE/g" \ + -e "s/__BLOB_STORE__/$BLOB_STORE/g" "$BASE_DIR/etc/jmap/store.toml" + +# Replace settings +sed -i '' -e "s/__STORE__/$STORE/g" \ + -e "s/__DIRECTORY__/$DIRECTORY/g" \ + -e "s/__DOMAIN__/$DOMAIN/g" \ + -e "s/__HOST__/mail.$DOMAIN/g" \ + -e "s|__BASE_PATH__|$BASE_DIR|g" "$BASE_DIR/etc/config.toml" +sed -i '' -e "s|__CERT_PATH__|$BASE_DIR/etc/tls_cert.pem|g" \ + -e "s|__PK_PATH__|$BASE_DIR/etc/tls_privatekey.pem|g" "$BASE_DIR/etc/common/tls.toml" +sed -i '' -e 's/method = "log"/method = "stdout"/g' \ + -e 's/level = "info"/level = "info"/g' "$BASE_DIR/etc/common/tracing.toml" +sed -i '' -e 's/%{HOST}%/127.0.0.1/g' "$BASE_DIR/etc/jmap/listener.toml" +sed -i '' -e 's/allow-plain-text = false/allow-plain-text = true/g' \ + -e 's/2000\/1m/9999999\/100m/g' \ + -e 's/concurrent = 4/concurrent = 90000/g' "$BASE_DIR/etc/imap/settings.toml" +sed -i '' -e 's/user = "stalwart-mail"//g' \ + -e 's/group = "stalwart-mail"//g' "$BASE_DIR/etc/common/server.toml" + +# Generate DKIM key +mkdir -p $BASE_DIR/etc/dkim +openssl genpkey -algorithm RSA -out $BASE_DIR/etc/dkim/$DOMAIN.key + +# Create admin user +SET_ADMIN_USER="admin" SET_ADMIN_PASS="secret" cargo run -p mail-server --no-default-features --features "foundationdb postgres mysql rocks elastic s3 redis" -- --config=/tmp/stalwart-test/etc/config.toml +cargo run -p mail-server --no-default-features --features "foundationdb postgres mysql rocks elastic s3 redis" -- --config=/tmp/stalwart-test/etc/config.toml diff --git a/tests/resources/scripts/create_test_users.sh b/tests/resources/scripts/create_test_users.sh new file mode 100644 index 00000000..5e0f295c --- /dev/null +++ b/tests/resources/scripts/create_test_users.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret domain create example.org +cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account create john 12345 -d "John Doe" -a john@example.org -a john.doe@example.org +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account create jane abcde -d "Jane Doe" -a jane@example.org +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account create bill xyz12 -d "Bill Foobar" -a bill@example.org +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret group create sales -d "Sales Department" +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret group create support -d "Technical Support" +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account add-to-group john sales support +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account remove-from-group john support +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account add-email jane jane.doe@example.org +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret list create everyone everyone@example.org +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret list add-members everyone jane john bill +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret account list +#cargo run -p stalwart-cli -- -u https://127.0.0.1:8080 -c admin:secret import messages --format mbox john _ignore/dovecot-crlf + diff --git a/tests/resources/scripts/imap_import.py b/tests/resources/scripts/imap_import.py new file mode 100644 index 00000000..481b152b --- /dev/null +++ b/tests/resources/scripts/imap_import.py @@ -0,0 +1,48 @@ +import imaplib +import socket +import time +import threading +from email.message import Message + +def append_message(thread_id, start, end): + conn = imaplib.IMAP4('localhost') + conn.login('john', '12345') + conn.socket().setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + start_time = time.time() + + for n in range(start, end): + msg = Message() + msg['From'] = 'somebody@some.where' + msg['To'] = 'john@example.org' + msg['Message-Id'] = f'unique.message.id.{n}@nowhere' + msg['Subject'] = f"This is message #{n}" + msg.set_payload('...nothing...') + + response_code, response_details = conn.append('INBOX', '', None, str(msg).encode('utf-8')) + if response_code != 'OK': + print(f'Thread {thread_id}: Error while appending message #{n}: {response_code} {response_details}') + break + if n != 0 and n % 100 == 0: + elapsed_time = (time.time() - start_time) * 1000 + print(f'Thread {thread_id}: Inserting batch {n} took {elapsed_time} ms.', flush=True) + start_time = time.time() + + conn.logout() + +num_threads = 5 +num_messages = 10000 +messages_per_thread = num_messages // num_threads + +threads = [] +for i in range(num_threads): + start = i * messages_per_thread + end = start + messages_per_thread + thread = threading.Thread(target=append_message, args=(i, start, end)) + threads.append(thread) + thread.start() + +for thread in threads: + thread.join() + +print("All messages appended.") + diff --git a/tests/src/imap/append.rs b/tests/src/imap/append.rs index 02ebe9e5..f503d358 100644 --- a/tests/src/imap/append.rs +++ b/tests/src/imap/append.rs @@ -37,7 +37,7 @@ pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, h .assert_response_code("TRYCREATE"); // Import test messages - let mut entries = fs::read_dir(&resources_dir()) + let mut entries = fs::read_dir(resources_dir()) .unwrap() .map(|res| res.map(|e| e.path())) .collect::, io::Error>>() diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index 11516521..e26b3efb 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -34,7 +34,7 @@ use super::resources_dir; #[test] fn body_structure() { - for file_name in fs::read_dir(&resources_dir()).unwrap() { + for file_name in fs::read_dir(resources_dir()).unwrap() { let mut file_name = file_name.as_ref().unwrap().path(); if file_name.extension().map_or(true, |e| e != "txt") { continue;