diff --git a/crates/jmap-proto/src/method/query.rs b/crates/jmap-proto/src/method/query.rs index 1fd097b5..f247ef0c 100644 --- a/crates/jmap-proto/src/method/query.rs +++ b/crates/jmap-proto/src/method/query.rs @@ -185,9 +185,15 @@ impl JsonObjectParser for QueryRequest { return Err(token.error("filter", "object or null")); } }, - 0x7472_6f73 => { - request.sort = >>::parse(parser)?; - } + 0x7472_6f73 => match parser.next_token::()? { + Token::ArrayStart => { + request.sort = parse_sort(parser)?.into(); + } + Token::Null => (), + token => { + return Err(token.error("sort", "array or null")); + } + }, 0x6e6f_6974_6973_6f70 => { request.position = parser .next_token::()? @@ -444,53 +450,59 @@ pub fn parse_filter(parser: &mut Parser) -> crate::parser::Result> { Ok(filter) } -impl JsonObjectParser for Comparator { - fn parse(parser: &mut Parser<'_>) -> crate::parser::Result - where - Self: Sized, - { - let mut comp = Comparator { - is_ascending: true, - collation: None, - property: SortProperty::Type, - keyword: None, - }; +pub fn parse_sort(parser: &mut Parser) -> crate::parser::Result> { + let mut sort = vec![]; - parser - .next_token::()? - .assert_jmap(Token::DictStart)?; - - while let Some(key) = parser.next_dict_key::()? { - match key { - 0x676e_6964_6e65_6373_4173_69 => { - comp.is_ascending = parser - .next_token::()? - .unwrap_bool_or_null("isAscending")? - .unwrap_or_default(); - } - 0x6e6f_6974_616c_6c6f_63 => { - comp.collation = parser - .next_token::()? - .unwrap_string_or_null("collation")?; - } - 0x7974_7265_706f_7270 => { - comp.property = parser - .next_token::()? - .unwrap_string("property")?; - } - 0x6472_6f77_7965_6b => { - comp.keyword = parser - .next_token::()? - .unwrap_string_or_null("keyword")?; - } - _ => { - parser.skip_token(parser.depth_array, parser.depth_dict)?; + loop { + match parser.next_token::()? { + Token::DictStart => { + let mut comp = Comparator { + is_ascending: true, + collation: None, + property: SortProperty::Type, + keyword: None, + }; + while let Some(key) = parser.next_dict_key::()? { + match key { + 0x676e_6964_6e65_6373_4173_69 => { + comp.is_ascending = parser + .next_token::()? + .unwrap_bool_or_null("isAscending")? + .unwrap_or_default(); + } + 0x6e6f_6974_616c_6c6f_63 => { + comp.collation = parser + .next_token::()? + .unwrap_string_or_null("collation")?; + } + 0x7974_7265_706f_7270 => { + comp.property = parser + .next_token::()? + .unwrap_string("property")?; + } + 0x6472_6f77_7965_6b => { + comp.keyword = parser + .next_token::()? + .unwrap_string_or_null("keyword")?; + } + _ => { + parser.skip_token(parser.depth_array, parser.depth_dict)?; + } + } } + sort.push(comp); + } + Token::Comma => (), + Token::ArrayEnd => { + break; + } + token => { + return Err(token.error("sort", "object")); } } - - Ok(comp) } + + Ok(sort) } impl JsonObjectParser for SortProperty { @@ -681,3 +693,15 @@ impl QueryRequest { } } } + +impl From for store::query::Filter { + fn from(value: Filter) -> Self { + match value { + Filter::And => Self::And, + Filter::Or => Self::Or, + Filter::Not => Self::Not, + Filter::Close => Self::End, + _ => unreachable!(), + } + } +} diff --git a/crates/jmap-proto/src/method/query_changes.rs b/crates/jmap-proto/src/method/query_changes.rs index 9c7576e9..6d012839 100644 --- a/crates/jmap-proto/src/method/query_changes.rs +++ b/crates/jmap-proto/src/method/query_changes.rs @@ -5,7 +5,7 @@ use crate::{ types::{id::Id, state::State}, }; -use super::query::{parse_filter, Comparator, Filter, RequestArguments}; +use super::query::{parse_filter, parse_sort, Comparator, Filter, RequestArguments}; #[derive(Debug, Clone)] pub struct QueryChangesRequest { @@ -97,9 +97,15 @@ impl JsonObjectParser for QueryChangesRequest { return Err(token.error("filter", "object or null")); } }, - 0x7472_6f73 => { - request.sort = >>::parse(parser)?; - } + 0x7472_6f73 => match parser.next_token::()? { + Token::ArrayStart => { + request.sort = parse_sort(parser)?.into(); + } + Token::Null => (), + token => { + return Err(token.error("sort", "array or null")); + } + }, 0x6574_6174_5379_7265_7551_6563_6e69_73 => { request.since_query_state = parser .next_token::()? diff --git a/crates/jmap-proto/src/parser/impls.rs b/crates/jmap-proto/src/parser/impls.rs index 4e10e4ea..4a622d17 100644 --- a/crates/jmap-proto/src/parser/impls.rs +++ b/crates/jmap-proto/src/parser/impls.rs @@ -200,13 +200,13 @@ impl JsonObjectParser for Option> { Token::String(item) => vec.push(item), Token::Comma => (), Token::ArrayEnd => break, - token => return Err(token.error("", &token.to_string())), + token => return Err(token.error("", "string")), } } Ok(Some(vec)) } Token::Null => Ok(None), - token => Err(token.error("", &token.to_string())), + token => Err(token.error("", "array or null")), } } } diff --git a/crates/jmap-proto/src/request/parser.rs b/crates/jmap-proto/src/request/parser.rs index acfe17fc..ce7c28c0 100644 --- a/crates/jmap-proto/src/request/parser.rs +++ b/crates/jmap-proto/src/request/parser.rs @@ -69,7 +69,7 @@ impl Request { Token::ArrayStart => (), Token::Comma => continue, Token::ArrayEnd => break, - token => { + _ => { return Err(RequestError::not_request("Invalid JMAP request")); } }; diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 2ad49dbf..155f01fd 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -38,12 +38,17 @@ impl JMAP { "jmap" => match (path.next().unwrap_or(""), req.method()) { ("", &Method::POST) => { return match fetch_body(req, self.config.request_max_size).await { - Ok(bytes) => match self.handle_request(&bytes).await { - Ok(response) => response.into_http_response(), - Err(err) => err.into_http_response(), - }, + Ok(bytes) => { + let delete = "fd"; + //println!("<- {}", String::from_utf8_lossy(&bytes)); + + match self.handle_request(&bytes).await { + Ok(response) => response.into_http_response(), + Err(err) => err.into_http_response(), + } + } Err(err) => err.into_http_response(), - } + }; } ("download", &Method::GET) => { if let (Some(account_id), Some(blob_id), Some(name)) = ( @@ -258,8 +263,8 @@ trait ToHttpResponse { impl ToHttpResponse for Response { fn into_http_response(self) -> hyper::Response> { - let delete = ""; - println!("-> {}", serde_json::to_string_pretty(&self).unwrap()); + //let delete = ""; + //println!("-> {}", serde_json::to_string_pretty(&self).unwrap()); hyper::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/json; charset=utf-8") @@ -274,8 +279,6 @@ impl ToHttpResponse for Response { impl ToHttpResponse for Session { fn into_http_response(self) -> hyper::Response> { - let delete = ""; - println!("-> {}", serde_json::to_string_pretty(&self).unwrap()); hyper::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/json; charset=utf-8") @@ -315,9 +318,6 @@ impl ToHttpResponse for DownloadResponse { impl ToHttpResponse for UploadResponse { fn into_http_response(self) -> hyper::Response> { - let delete = ""; - println!("-> {}", serde_json::to_string_pretty(&self).unwrap()); - hyper::Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/json; charset=utf-8") @@ -332,9 +332,6 @@ impl ToHttpResponse for UploadResponse { impl ToHttpResponse for RequestError { fn into_http_response(self) -> hyper::Response> { - let delete = ""; - println!("-> {}", serde_json::to_string_pretty(&self).unwrap()); - hyper::Response::builder() .status(self.status) .header(header::CONTENT_TYPE, "application/json; charset=utf-8") diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index eccbc3ab..ceac851a 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -9,7 +9,6 @@ use crate::JMAP; impl JMAP { pub async fn handle_request(&self, bytes: &[u8]) -> Result { - println!("<- {}", String::from_utf8_lossy(bytes)); let request = Request::parse( bytes, self.config.request_max_calls, diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index c03e1a87..98022cbc 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -4,7 +4,6 @@ use jmap_proto::{ object::Object, types::{ date::UTCDate, - id::Id, keyword::Keyword, property::{HeaderForm, Property}, value::Value, @@ -16,7 +15,10 @@ use mail_parser::{ Addr, GetHeader, Group, HeaderName, HeaderValue, Message, MessagePart, PartType, RfcHeader, }; use store::{ - fts::{builder::FtsIndexBuilder, Language}, + fts::{ + builder::{FtsIndexBuilder, MAX_TOKEN_LENGTH}, + Language, + }, write::{BatchBuilder, F_BITMAP, F_INDEX, F_VALUE}, }; @@ -89,28 +91,30 @@ impl IndexMessage for BatchBuilder { language = part_language; for header in part.headers.into_iter().rev() { if let HeaderName::Rfc(rfc_header) = header.name { + // Index hasHeader property + let header_num = (rfc_header as u8).to_string(); + fts.index_raw_token(Property::Headers, &header_num); + match rfc_header { RfcHeader::MessageId | RfcHeader::InReplyTo | RfcHeader::References | RfcHeader::ResentMessageId => { - match &header.value { - HeaderValue::Text(id) if id.len() < MAX_ID_LENGTH => { - self.value(Property::MessageId, id.as_ref(), F_INDEX); + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + println!("indexing {}: {}", rfc_header.as_str(), id); + self.value(Property::MessageId, id, F_INDEX); } - HeaderValue::TextList(ids) => { - for id in ids { - if id.len() < MAX_ID_LENGTH { - self.value( - Property::MessageId, - id.as_ref(), - F_INDEX, - ); - } - } + + // Index ids without stemming + if id.len() < MAX_TOKEN_LENGTH { + fts.index_raw_token( + Property::Headers, + format!("{header_num}{id}"), + ); } - _ => (), - } + }); if matches!( rfc_header, @@ -135,6 +139,7 @@ impl IndexMessage for BatchBuilder { | RfcHeader::Bcc | RfcHeader::ReplyTo | RfcHeader::Sender => { + let property = Property::from(rfc_header); let seen_header = seen_headers[rfc_header as usize]; if matches!( rfc_header, @@ -172,13 +177,13 @@ impl IndexMessage for BatchBuilder { } // Index an address name or email without stemming - fts.index_raw(rfc_header, value); + fts.index_raw(u8::from(&property), value); }); if !seen_header { // Add address to inverted index self.value( - rfc_header, + u8::from(&property), if !sort_text.is_empty() { &sort_text } else { @@ -192,7 +197,7 @@ impl IndexMessage for BatchBuilder { if !seen_header { // Add address to object object.append( - rfc_header.into(), + property, header .value .trim_text(MAX_STORED_FIELD_LENGTH) @@ -255,6 +260,20 @@ impl IndexMessage for BatchBuilder { // Index subject for FTS fts.index(Property::Subject, subject, language); } + + RfcHeader::Comments | RfcHeader::Keywords | RfcHeader::ListId => { + // Index headers + header.value.visit_text(|text| { + for token in text.split_ascii_whitespace() { + if token.len() < MAX_TOKEN_LENGTH { + fts.index_raw_token( + Property::Headers, + format!("{header_num}{}", token.to_lowercase()), + ); + } + } + }); + } _ => (), } } @@ -370,11 +389,12 @@ impl GetContentLanguage for MessagePart<'_> { } } -trait VisitAddresses { +trait VisitValues { fn visit_addresses(&self, visitor: impl FnMut(&str, bool)); + fn visit_text(&self, visitor: impl FnMut(&str)); } -impl VisitAddresses for HeaderValue<'_> { +impl VisitValues for HeaderValue<'_> { fn visit_addresses(&self, mut visitor: impl FnMut(&str, bool)) { match self { HeaderValue::Address(addr) => { @@ -426,6 +446,19 @@ impl VisitAddresses for HeaderValue<'_> { _ => (), } } + fn visit_text(&self, mut visitor: impl FnMut(&str)) { + match &self { + HeaderValue::Text(text) => { + visitor(text.as_ref()); + } + HeaderValue::TextList(texts) => { + for text in texts { + visitor(text.as_ref()); + } + } + _ => (), + } + } } pub trait TrimTextValue { diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 1467b889..79e630fc 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -9,9 +9,10 @@ use mail_parser::{ parsers::fields::thread::thread_name, HeaderName, HeaderValue, Message, RfcHeader, }; use store::{ + ahash::AHashSet, query::Filter, write::{log::ChangeLogBuilder, now, BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE}, - ValueKey, + BitmapKey, ValueKey, }; use utils::map::vec_map::VecMap; @@ -210,6 +211,7 @@ impl JMAP { ) -> Result, MaybeError> { let mut try_count = 0; + println!("-----------\nthread name: {:?}", thread_name); loop { // Find messages with matching references let mut filters = Vec::with_capacity(references.len() + 3); @@ -232,6 +234,9 @@ impl JMAP { MaybeError::Temporary })? .results; + + println!("found messages {:?}", results); + if results.is_empty() { return Ok(None); } @@ -261,6 +266,7 @@ impl JMAP { "Failed to obtain threadIds."); MaybeError::Temporary })?; + println!("found thread ids {:?}", thread_ids); if thread_ids.len() == 1 { return Ok(thread_ids.into_iter().next().unwrap()); } @@ -277,6 +283,7 @@ impl JMAP { thread_id = *thread_id_; } } + println!("common thread id {:?}", thread_id); if thread_id == u32::MAX { return Ok(None); // This should never happen } else if thread_counts.len() == 1 { @@ -310,19 +317,38 @@ impl JMAP { // Move messages to the new threadId batch.with_collection(Collection::Email); - for (document_id, old_thread_id) in results.iter().zip(thread_ids.into_iter()) { - let old_thread_id = old_thread_id.unwrap_or(u32::MAX); + for old_thread_id in thread_ids.into_iter().flatten().collect::>() { if thread_id != old_thread_id { - batch - .update_document(document_id) - .assert_value(Property::ThreadId, old_thread_id) - .value(Property::ThreadId, old_thread_id, F_BITMAP | F_CLEAR) - .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP); - changes.log_move( - Collection::Email, - Id::from_parts(old_thread_id, document_id), - Id::from_parts(thread_id, document_id), - ) + for document_id in self + .store + .get_bitmap(BitmapKey::value( + account_id, + Collection::Email, + Property::ThreadId, + old_thread_id, + )) + .await + .map_err(|err| { + tracing::error!( + event = "error", + context = "find_or_merge_thread", + error = ?err, + "Failed to obtain threadId bitmap."); + MaybeError::Temporary + })? + .unwrap_or_default() + { + batch + .update_document(document_id) + .assert_value(Property::ThreadId, old_thread_id) + .value(Property::ThreadId, old_thread_id, F_BITMAP | F_CLEAR) + .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP); + changes.log_move( + Collection::Email, + Id::from_parts(old_thread_id, document_id), + Id::from_parts(thread_id, document_id), + ); + } } } batch.custom(changes).map_err(|err| { diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 52583fa4..7ae0e751 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,8 +4,9 @@ use jmap_proto::{ object::email::QueryArguments, types::{collection::Collection, keyword::Keyword, property::Property}, }; +use mail_parser::{HeaderName, RfcHeader}; use store::{ - fts::Language, + fts::{builder::MAX_TOKEN_LENGTH, Language}, query::{self, sort::Pagination}, roaring::RoaringBitmap, ValueKey, @@ -87,20 +88,20 @@ impl JMAP { &text, Language::None, )); - filters.push(query::Filter::has_text( + filters.push(query::Filter::has_text_detect( Property::Subject, &text, - Language::Unknown, + self.config.default_language, )); - filters.push(query::Filter::has_text( + filters.push(query::Filter::has_text_detect( Property::TextBody, &text, - Language::Unknown, + self.config.default_language, )); - filters.push(query::Filter::has_text( + filters.push(query::Filter::has_text_detect( Property::Attachments, text, - Language::Unknown, + self.config.default_language, )); filters.push(query::Filter::End); } @@ -118,21 +119,78 @@ impl JMAP { Filter::Bcc(text) => { filters.push(query::Filter::has_text(Property::Bcc, text, Language::None)) } - Filter::Subject(text) => filters.push(query::Filter::has_text( + Filter::Subject(text) => filters.push(query::Filter::has_text_detect( Property::Subject, text, - Language::Unknown, + self.config.default_language, )), - Filter::Body(text) => filters.push(query::Filter::has_text( + Filter::Body(text) => filters.push(query::Filter::has_text_detect( Property::TextBody, text, - Language::Unknown, + self.config.default_language, )), Filter::Header(header) => { - return Err(MethodError::InvalidArguments(format!( - "Querying headers '{}' is not supported.", - header.join(":") - ))); + let mut header = header.into_iter(); + let header_name = header.next().ok_or_else(|| { + MethodError::InvalidArguments("Header name is missing.".to_string()) + })?; + if let Some(HeaderName::Rfc(header_name)) = HeaderName::parse(&header_name) { + let is_id = matches!( + header_name, + RfcHeader::MessageId + | RfcHeader::InReplyTo + | RfcHeader::References + | RfcHeader::ResentMessageId + ); + let tokens = if let Some(header_value) = header.next() { + let header_num = u8::from(header_name).to_string(); + header_value + .split_ascii_whitespace() + .filter_map(|token| { + if token.len() < MAX_TOKEN_LENGTH { + if is_id { + format!("{header_num}{token}") + } else { + format!("{header_num}{}", token.to_lowercase()) + } + .into() + } else { + None + } + }) + .collect::>() + } else { + vec![] + }; + match tokens.len() { + 0 => { + filters.push(query::Filter::has_raw_text( + Property::Headers, + u8::from(header_name).to_string(), + )); + } + 1 => { + filters.push(query::Filter::has_raw_text( + Property::Headers, + tokens.into_iter().next().unwrap(), + )); + } + _ => { + filters.push(query::Filter::And); + for token in tokens { + filters.push(query::Filter::has_raw_text( + Property::Headers, + token, + )); + } + filters.push(query::Filter::End); + } + } + } else { + return Err(MethodError::InvalidArguments(format!( + "Querying non-RFC header '{header_name}' is not allowed.", + ))); + }; } // Non-standard @@ -149,6 +207,9 @@ impl JMAP { Property::ThreadId, id.document_id(), )), + Filter::And | Filter::Or | Filter::Not | Filter::Close => { + filters.push(cond.into()); + } other => return Err(MethodError::UnsupportedFilter(other.to_string())), } diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index 5fa30089..676b55ea 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -42,6 +42,7 @@ impl ReadTransaction<'_> { let begin = key.serialize(); key.block_num = u32::MAX; let end = key.serialize(); + let key_len = begin.len(); let mut values = self.trx.get_ranges( RangeOption { begin: KeySelector::first_greater_or_equal(begin), @@ -56,10 +57,12 @@ impl ReadTransaction<'_> { while let Some(values) = values.next().await { for value in values? { let key = value.key(); - bm.deserialize_block( - value.value(), - key.deserialize_be_u32(key.len() - std::mem::size_of::())?, - ); + if key.len() == key_len { + bm.deserialize_block( + value.value(), + key.deserialize_be_u32(key.len() - std::mem::size_of::())?, + ); + } } } diff --git a/crates/store/src/backend/sqlite/id_assign.rs b/crates/store/src/backend/sqlite/id_assign.rs index 73a7a209..69e78966 100644 --- a/crates/store/src/backend/sqlite/id_assign.rs +++ b/crates/store/src/backend/sqlite/id_assign.rs @@ -42,28 +42,46 @@ impl IdCacheKey { #[derive(Clone)] pub struct IdAssigner { - pub available_document_ids: RoaringBitmap, + pub freed_document_ids: Option, + pub next_document_id: u32, pub next_change_id: u64, } impl IdAssigner { pub fn new(used_ids: Option, next_change_id: u64) -> Self { let mut assigner = IdAssigner { - available_document_ids: RoaringBitmap::full(), + freed_document_ids: None, + next_document_id: 0, next_change_id, }; - if let Some(used_ids) = used_ids { - assigner.available_document_ids ^= &used_ids; + if let Some(max) = used_ids.max() { + assigner.next_document_id = max + 1; + let mut freed_ids = + RoaringBitmap::from_sorted_iter(0..assigner.next_document_id).unwrap(); + freed_ids ^= used_ids; + if !freed_ids.is_empty() { + assigner.freed_document_ids = Some(freed_ids); + } + } } assigner } pub fn assign_document_id(&mut self) -> u32 { - let id = self.available_document_ids.min().unwrap(); - self.available_document_ids.remove(id); - id + if let Some(freed_ids) = &mut self.freed_document_ids { + let id = freed_ids.min().unwrap(); + freed_ids.remove(id); + if freed_ids.is_empty() { + self.freed_document_ids = None; + } + id + } else { + let id = self.next_document_id; + self.next_document_id += 1; + id + } } pub fn assign_change_id(&mut self) -> u64 { diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index 404e719c..b726836a 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -38,6 +38,7 @@ impl ReadTransaction<'_> { ) -> crate::Result<()> { let begin = key.serialize(); key.block_num = u32::MAX; + let key_len = begin.len(); let end = key.serialize(); let mut query = self .conn @@ -46,27 +47,29 @@ impl ReadTransaction<'_> { while let Some(row) = rows.next()? { let key = row.get_ref(0)?.as_bytes()?; - let block_num = key.deserialize_be_u32(key.len() - std::mem::size_of::())?; + if key.len() == key_len { + let block_num = key.deserialize_be_u32(key.len() - std::mem::size_of::())?; - for word_num in 0..WORDS_PER_BLOCK { - match row.get::<_, i64>((word_num + 1) as usize)? as u64 { - 0 => (), - u64::MAX => { - bm.insert_range( - block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS - ..(block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS) - + WORD_SIZE_BITS, - ); - } - mut word => { - while word != 0 { - let trailing_zeros = word.trailing_zeros(); - bm.insert( - block_num * BITS_PER_BLOCK - + word_num * WORD_SIZE_BITS - + trailing_zeros, + for word_num in 0..WORDS_PER_BLOCK { + match row.get::<_, i64>((word_num + 1) as usize)? as u64 { + 0 => (), + u64::MAX => { + bm.insert_range( + block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS + ..(block_num * BITS_PER_BLOCK + word_num * WORD_SIZE_BITS) + + WORD_SIZE_BITS, ); - word ^= 1 << trailing_zeros; + } + mut word => { + while word != 0 { + let trailing_zeros = word.trailing_zeros(); + bm.insert( + block_num * BITS_PER_BLOCK + + word_num * WORD_SIZE_BITS + + trailing_zeros, + ); + word ^= 1 << trailing_zeros; + } } } } diff --git a/crates/store/src/fts/builder.rs b/crates/store/src/fts/builder.rs index 92474d4f..b583be7d 100644 --- a/crates/store/src/fts/builder.rs +++ b/crates/store/src/fts/builder.rs @@ -65,6 +65,10 @@ impl<'x> FtsIndexBuilder<'x> { self.tokens.insert((field, token)); } } + + pub fn index_raw_token(&mut self, field: impl Into, token: impl Into) { + self.tokens.insert((field.into(), token.into())); + } } impl<'x> IntoOperations for FtsIndexBuilder<'x> { diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs index 5f044423..32803532 100644 --- a/crates/store/src/query/filter.rs +++ b/crates/store/src/query/filter.rs @@ -69,6 +69,10 @@ impl ReadTransaction<'_> { ) .await? } + TextMatch::Raw => { + self.get_bitmap(BitmapKey::hash(&text, account_id, collection, 0, field)) + .await? + } }, Filter::InBitmap { family, field, key } => { self.get_bitmap(BitmapKey { diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 2f2cce6f..508d2158 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -49,6 +49,7 @@ pub enum TextMatch { Exact(Language), Stemmed(Language), Tokenized, + Raw, } #[derive(Debug)] @@ -129,27 +130,32 @@ impl Filter { } } - pub fn has_text(field: impl Into, text: impl Into, mut language: Language) -> Self { + pub fn has_text_detect( + field: impl Into, + text: impl Into, + default_language: Language, + ) -> Self { let mut text = text.into(); + let language = if let Some((l, t)) = text + .split_once(':') + .and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into()) + { + text = t; + l + } else { + LanguageDetector::detect_single(&text) + .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) + .unwrap_or(default_language) + }; + Self::has_text(field, text, language) + } + + pub fn has_text(field: impl Into, text: impl Into, language: Language) -> Self { + let text = text.into(); let op = if !matches!(language, Language::None) { - let match_phrase = (text.starts_with('"') && text.ends_with('"')) - || (text.starts_with('\'') && text.ends_with('\'')); - - if !match_phrase && language == Language::Unknown { - language = if let Some((l, t)) = text - .split_once(':') - .and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into()) - { - text = t; - l - } else { - LanguageDetector::detect_single(&text) - .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) - .unwrap_or(Language::Unknown) - }; - } - - if match_phrase { + if (text.starts_with('"') && text.ends_with('"')) + || (text.starts_with('\'') && text.ends_with('\'')) + { TextMatch::Exact(language) } else { TextMatch::Stemmed(language) @@ -165,6 +171,14 @@ impl Filter { } } + pub fn has_raw_text(field: impl Into, text: impl Into) -> Self { + Filter::HasText { + field: field.into(), + text: text.into(), + op: TextMatch::Raw, + } + } + pub fn has_english_text(field: impl Into, text: impl Into) -> Self { Self::has_text(field, text, Language::English) } diff --git a/crates/store/src/query/sort.rs b/crates/store/src/query/sort.rs index c9cde6fe..70a314cd 100644 --- a/crates/store/src/query/sort.rs +++ b/crates/store/src/query/sort.rs @@ -1,3 +1,5 @@ +use std::cmp::Ordering; + use ahash::{AHashMap, AHashSet}; use crate::{ReadTransaction, Store, ValueKey}; @@ -155,7 +157,10 @@ impl ReadTransaction<'_> { let mut seen_prefixes = AHashSet::new(); let mut sorted_ids = sorted_ids.into_iter().collect::>(); - sorted_ids.sort_by(|a, b| a.1.cmp(&b.1)); + sorted_ids.sort_by(|a, b| match a.1.cmp(&b.1) { + Ordering::Equal => a.0.cmp(&b.0), + other => other, + }); for (document_id, _) in sorted_ids { // Obtain document prefixId let prefix_id = if let Some(prefix_key) = &paginate.prefix_key { @@ -252,13 +257,17 @@ impl Pagination { // Pagination if !self.has_anchor { - if self.position > 0 { - self.position -= 1; + if self.position >= 0 { + if self.position > 0 { + self.position -= 1; + } else { + self.ids.push(id); + if self.ids.len() == self.limit { + return false; + } + } } else { self.ids.push(id); - if self.ids.len() == self.limit { - return false; - } } } else if self.anchor_offset >= 0 { if !self.anchor_found { diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 2af89ee4..80d0a129 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -347,7 +347,14 @@ impl AssertValue { pub fn matches(&self, bytes: &[u8]) -> bool { match self { AssertValue::U32(v) => { - bytes.len() == std::mem::size_of::() && u32::deserialize(bytes).unwrap() == *v + let coco = "fd"; + let a = u32::deserialize(bytes).unwrap(); + let b = *v; + if a != b { + println!("has {} != expected {}", a, b); + } + a == b + //bytes.len() == std::mem::size_of::() && u32::deserialize(bytes).unwrap() == *v } AssertValue::U64(v) => { bytes.len() == std::mem::size_of::() && u64::deserialize(bytes).unwrap() == *v diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 2e405cf6..c128f5a2 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -16,58 +16,60 @@ const MAX_THREADS: usize = 100; const MAX_MESSAGES: usize = 1000; const MAX_MESSAGES_PER_THREAD: usize = 100; -pub async fn test(server: Arc, client: &mut Client) { +pub async fn test(server: Arc, client: &mut Client, insert: bool) { println!("Running Email Query tests..."); - // Add some "virtual" mailbox ids so create doesn't fail - let mut batch = BatchBuilder::new(); - let account_id = Id::from_bytes(client.default_account_id().as_bytes()) - .unwrap() - .document_id(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox); - for mailbox_id in 0..99999 { - batch.create_document(mailbox_id); - } - server.store.write(batch.build()).await.unwrap(); + if insert { + // Add some "virtual" mailbox ids so create doesn't fail + let mut batch = BatchBuilder::new(); + let account_id = Id::from_bytes(client.default_account_id().as_bytes()) + .unwrap() + .document_id(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox); + for mailbox_id in 0..99999 { + batch.create_document(mailbox_id); + } + server.store.write(batch.build()).await.unwrap(); - // Create test messages - println!("Inserting JMAP Mail query test messages..."); - create(client).await; + // Create test messages + println!("Inserting JMAP Mail query test messages..."); + create(client).await; - // Remove mailboxes - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox); - for mailbox_id in 0..99999 { - batch.delete_document(mailbox_id); - } - server.store.write(batch.build()).await.unwrap(); + // Remove mailboxes + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox); + for mailbox_id in 0..99999 { + batch.delete_document(mailbox_id); + } + server.store.write(batch.build()).await.unwrap(); + + for thread_id in 0..MAX_THREADS { + assert!( + client + .thread_get(&Id::new(thread_id as u64).to_string()) + .await + .unwrap() + .is_some(), + "thread {} not found", + thread_id + ); + } - for thread_id in 0..MAX_THREADS { assert!( client - .thread_get(&Id::new(thread_id as u64).to_string()) + .thread_get(&Id::new(MAX_THREADS as u64).to_string()) .await .unwrap() - .is_some(), - "thread {} not found", - thread_id + .is_none(), + "thread {} found", + MAX_THREADS ); } - assert!( - client - .thread_get(&Id::new(MAX_THREADS as u64).to_string()) - .await - .unwrap() - .is_none(), - "thread {} found", - MAX_THREADS - ); - println!("Running JMAP Mail query tests..."); query(client).await; @@ -305,8 +307,8 @@ pub async fn query(client: &mut Client) { let mut request = client.build(); let query_request = request .query_email() - .filter(filter) - .sort(sort) + .filter(filter.clone()) + .sort(sort.clone()) .calculate_total(true); query_request.arguments().collapse_threads(false); let query_result_ref = query_request.result_reference(); @@ -314,22 +316,36 @@ pub async fn query(client: &mut Client) { .get_email() .ids_ref(query_result_ref) .properties([email::Property::MessageId]); + let results = request + .send() + .await + .unwrap_or_else(|_| panic!("invalid response for {filter:?}")) + .unwrap_method_responses() + .pop() + .unwrap_or_else(|| panic!("invalid response for {filter:?}")) + .unwrap_get_email() + .unwrap_or_else(|_| panic!("invalid response for {filter:?}")) + .take_list() + .into_iter() + .map(|e| e.message_id().unwrap().first().unwrap().to_string()) + .collect::>(); + + let mut missing = Vec::new(); + let mut extra = Vec::new(); + for &expected in &expected_results { + if !results.iter().any(|r| r.as_str() == expected) { + missing.push(expected); + } + } + for result in &results { + if !expected_results.contains(&result.as_str()) { + extra.push(result.as_str()); + } + } assert_eq!( - request - .send() - .await - .unwrap() - .unwrap_method_responses() - .pop() - .unwrap() - .unwrap_get_email() - .unwrap() - .take_list() - .into_iter() - .map(|e| e.message_id().unwrap().first().unwrap().to_string()) - .collect::>(), - expected_results + results, expected_results, + "failed test!\nfilter: {filter:?}\nsort: {sort:?}\nmissing: {missing:?}\nextra: {extra:?}" ); } } diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index cba56828..1e512061 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -10,6 +10,7 @@ use crate::{add_test_certs, store::TempDir}; pub mod email_get; pub mod email_query; pub mod thread_get; +pub mod thread_merge; const SERVER: &str = " [server] @@ -48,9 +49,10 @@ pub async fn jmap_tests() { let delete = true; let mut params = init_jmap_tests(delete).await; - email_get::test(params.server.clone(), &mut params.client).await; - //email_query::test(params.server.clone(), &mut params.client).await; + //email_get::test(params.server.clone(), &mut params.client).await; + //email_query::test(params.server.clone(), &mut params.client, delete).await; //thread_get::test(params.server.clone(), &mut params.client).await; + thread_merge::test(params.server.clone(), &mut params.client).await; if delete { params.temp_dir.delete(); } @@ -82,6 +84,7 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { // Create client let mut client = Client::new() .credentials(Credentials::bearer("DO_NOT_ATTEMPT_THIS_AT_HOME")) + .timeout(360000) .accept_invalid_certs(true) .connect("https://127.0.0.1:8899") .await diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs new file mode 100644 index 00000000..cc2206ae --- /dev/null +++ b/tests/src/jmap/thread_merge.rs @@ -0,0 +1,942 @@ +use std::sync::Arc; + +use jmap::JMAP; +use jmap_client::{client::Client, email, mailbox::Role}; +use jmap_proto::types::id::Id; +use store::ahash::{AHashMap, AHashSet}; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Merge Threads tests..."); + + simple_test(client).await; + + let mut all_mailboxes = AHashMap::default(); + + for (base_test_num, test) in [test_1(), test_2(), test_3()].iter().enumerate() { + let base_test_num = ((base_test_num * 6) as u32) + 1; + let mut messages = Vec::new(); + let mut total_messages = 0; + let mut messages_per_thread = + build_messages(test, &mut messages, &mut total_messages, None, 0); + messages_per_thread.sort_unstable(); + + let mut mailbox_ids = Vec::with_capacity(6); + + for test_num in 0..=5 { + mailbox_ids.push(char::from(b'a' + test_num as u8).to_string()); + let coco = "fd"; + /*mailbox_ids.push( + client + .set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()) + .mailbox_create("Thread nightmare", None::, Role::None) + .await + .unwrap() + .take_id(), + );*/ + } + + for message in &messages { + client + .set_default_account_id(Id::new(base_test_num as u64).to_string()) + .email_import( + message.to_string().into_bytes(), + [mailbox_ids[0].clone()], + None::>, + None, + ) + .await + .unwrap(); + } + + for message in messages.iter().rev() { + client + .set_default_account_id(Id::new((base_test_num + 1) as u64).to_string()) + .email_import( + message.to_string().into_bytes(), + [mailbox_ids[1].clone()], + None::>, + None, + ) + .await + .unwrap(); + } + + for chunk in messages.chunks(5) { + client.set_default_account_id(Id::new((base_test_num + 2) as u64).to_string()); + + for message in chunk { + client + .email_import( + message.to_string().into_bytes(), + [mailbox_ids[2].clone()], + None::>, + None, + ) + .await + .unwrap(); + } + + client.set_default_account_id(Id::new((base_test_num + 3) as u64).to_string()); + + for message in chunk.iter().rev() { + client + .email_import( + message.to_string().into_bytes(), + [mailbox_ids[3].clone()], + None::>, + None, + ) + .await + .unwrap(); + } + } + + for chunk in messages.chunks(5).rev() { + client.set_default_account_id(Id::new((base_test_num + 4) as u64).to_string()); + + for message in chunk { + client + .email_import( + message.to_string().into_bytes(), + [mailbox_ids[4].clone()], + None::>, + None, + ) + .await + .unwrap(); + } + + client.set_default_account_id(Id::new((base_test_num + 5) as u64).to_string()); + + for message in chunk.iter().rev() { + client + .email_import( + message.to_string().into_bytes(), + [mailbox_ids[5].clone()], + None::>, + None, + ) + .await + .unwrap(); + } + } + + for test_num in 0..=5 { + let result = client + .set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()) + .email_query( + email::query::Filter::in_mailbox(mailbox_ids[test_num as usize].clone()).into(), + None::>, + ) + .await + .unwrap(); + + assert_eq!( + result.ids().len(), + total_messages, + "test# {}/{}", + base_test_num, + test_num + ); + + let thread_ids: AHashSet = result + .ids() + .iter() + .map(|id| Id::from_bytes(id.as_bytes()).unwrap().prefix_id()) + .collect(); + + assert_eq!( + thread_ids.len(), + messages_per_thread.len(), + "{:?}: test# {}/{}", + thread_ids, + base_test_num, + test_num + ); + + let mut messages_per_thread_db = Vec::new(); + + for thread_id in thread_ids { + messages_per_thread_db.push( + client + .thread_get(&Id::new(thread_id as u64).to_string()) + .await + .unwrap() + .unwrap() + .email_ids() + .len(), + ); + } + messages_per_thread_db.sort_unstable(); + + assert_eq!(messages_per_thread_db, messages_per_thread); + println!("passed test# {}/{}", base_test_num, test_num); + } + + all_mailboxes.insert(base_test_num as usize, mailbox_ids); + } + + // Delete all messages and make sure no keys are left in the store. + let implement = "fdf"; + /*for (base_test_num, mailbox_ids) in all_mailboxes { + for (test_num, mailbox_id) in mailbox_ids.into_iter().enumerate() { + client + .set_default_account_id(Id::new((base_test_num + test_num) as u64).to_string()) + .mailbox_destroy(&mailbox_id, true) + .await + .unwrap(); + } + } + + server.store.assert_is_empty();*/ +} + +async fn simple_test(client: &mut Client) { + let coco = "fdf"; + let mailbox_id = "a".to_string(); + /* + let mailbox_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("JMAP Get", None::, Role::None) + .await + .unwrap() + .take_id();*/ + + // A simple thread that uses in-reply-to to link messages together + let thread_1 = vec![ + client + .email_import( + "Message-ID: +From: test1@example.com +To: test2@example.com +Subject: my thread + +message here!" + .into(), + [&mailbox_id], + None::>, + Some(1), + ) + .await + .unwrap(), + client + .email_import( + "Message-ID: +From: test2@example.com +To: test1@example.com +In-Reply-To: +Subject: Re: my thread + +reply here!" + .into(), + [&mailbox_id], + None::>, + Some(2), + ) + .await + .unwrap(), + client + .email_import( + "Message-ID: +From: test1@example.com +To: test2@example.com +In-Reply-To: +Subject: Re: my thread + +last reply" + .into(), + [&mailbox_id], + None::>, + Some(3), + ) + .await + .unwrap(), + ]; + + // Another simple thread, but this time with a shared reference header instead + let thread_2 = vec![ + client + .email_import( + "Message-ID: +From: test1@example.com +To: test2@example.com +Subject: my thread + +message here!" + .into(), + [&mailbox_id], + None::>, + Some(1), + ) + .await + .unwrap(), + client + .email_import( + "Message-ID: +References: +From: test2@example.com +To: test1@example.com +Subject: my thread + +reply here!" + .into(), + [&mailbox_id], + None::>, + Some(2), + ) + .await + .unwrap(), + client + .email_import( + "Message-ID: +References: +From: test1@example.com +To: test2@example.com +Subject: my thread + +reply here!" + .into(), + [&mailbox_id], + None::>, + Some(3), + ) + .await + .unwrap(), + ]; + + // Make sure none of the separate threads end up with the same thread ID + assert_ne!( + thread_1.first().unwrap().thread_id().unwrap(), + thread_2.first().unwrap().thread_id().unwrap(), + "Making sure thread 1 and thread 2 have different thread IDs" + ); + + // Make sure each message in each thread ends up with the right thread ID + assert_thread_ids_match(client, &thread_1, "thread chained with In-Reply-To header").await; + assert_thread_ids_match(client, &thread_2, "thread with References header").await; + + //client.mailbox_destroy(&mailbox_id, true).await.unwrap(); +} + +async fn assert_thread_ids_match( + client: &mut Client, + emails: &Vec, + description: &str, +) { + let thread_id = emails.first().unwrap().thread_id().unwrap(); + + // First, make sure the thread ID is the same for all messages in the thread + for email in emails { + assert_eq!( + email.thread_id().unwrap(), + thread_id, + "Comparing thread IDs of messages in: {}", + description + ); + } + + // Next, make sure querying the thread yields the same messages + let full_thread = client.thread_get(thread_id).await.unwrap().unwrap(); + let mut email_ids_in_fetched_thread = full_thread.email_ids().to_vec(); + email_ids_in_fetched_thread.sort(); + + let mut expected_email_ids = emails + .iter() + .map(|email| email.id().unwrap()) + .collect::>(); + expected_email_ids.sort(); + + assert_eq!( + email_ids_in_fetched_thread, expected_email_ids, + "Comparing email IDs in: {}", + description + ); +} + +fn build_message(message: usize, in_reply_to: Option, thread_num: usize) -> String { + if let Some(in_reply_to) = in_reply_to { + format!( + "Message-ID: <{}>\nReferences: <{}>\nSubject: re: T{}\n\nreply\n", + message, in_reply_to, thread_num + ) + } else { + format!( + "Message-ID: <{}>\nSubject: T{}\n\nmsg\n", + message, thread_num + ) + } +} + +fn build_messages( + three: &ThreadTest, + messages: &mut Vec, + total_messages: &mut usize, + in_reply_to: Option, + thread_num: usize, +) -> Vec { + let mut messages_per_thread = Vec::new(); + match three { + ThreadTest::Message => { + *total_messages += 1; + messages.push(build_message(*total_messages, in_reply_to, thread_num)); + } + ThreadTest::MessageWithReplies(replies) => { + *total_messages += 1; + messages.push(build_message(*total_messages, in_reply_to, thread_num)); + let in_reply_to = Some(*total_messages); + for reply in replies { + build_messages(reply, messages, total_messages, in_reply_to, thread_num); + } + } + ThreadTest::Root(items) => { + for (thread_num, item) in items.iter().enumerate() { + let count_start = *total_messages; + build_messages(item, messages, total_messages, None, thread_num); + messages_per_thread.push(*total_messages - count_start); + } + } + } + messages_per_thread +} + +pub fn build_thread_test_messages() -> Vec { + let mut messages = Vec::new(); + let mut total_messages = 0; + build_messages(&test_3(), &mut messages, &mut total_messages, None, 0); + messages +} + +pub enum ThreadTest { + Message, + MessageWithReplies(Vec), + Root(Vec), +} + +fn test_1() -> ThreadTest { + ThreadTest::Root(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ]), + ]), + ]), + ]), + ]) +} + +fn test_2() -> ThreadTest { + ThreadTest::Root(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( + vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ], + )]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ]), + ]), + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ]), + ]), + ThreadTest::Message, + ThreadTest::Message, + ])]), + ]), + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message, ThreadTest::Message]), + ]) +} + +fn test_3() -> ThreadTest { + ThreadTest::Root(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::Message, ThreadTest::Message]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( + vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ])], + )]), + ThreadTest::Message, + ThreadTest::Message, + ])]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ]), + ]), + ]), + ]), + ThreadTest::Message, + ThreadTest::Message, + ])]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ])]), + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message, ThreadTest::Message]), + ThreadTest::Message, + ThreadTest::Message, + ])]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( + vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ], + )]), + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( + vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ], + )]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ])]), + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ThreadTest::Message]), + ThreadTest::Message, + ]), + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ]), + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ + ThreadTest::Message, + ThreadTest::MessageWithReplies(vec![ThreadTest::MessageWithReplies( + vec![ThreadTest::Message, ThreadTest::Message], + )]), + ThreadTest::Message, + ]), + ThreadTest::Message, + ]), + ]), + ]), + ]), + ]) +}