From ba537a43dc2652ed339cdac9c56ceea261c0c94f Mon Sep 17 00:00:00 2001 From: Mauro D Date: Fri, 5 May 2023 15:59:30 +0000 Subject: [PATCH] Email/copy passing tests --- crates/jmap-proto/src/method/copy.rs | 6 +- crates/jmap-proto/src/object/mod.rs | 12 + crates/jmap-proto/src/request/method.rs | 4 + crates/jmap-proto/src/request/mod.rs | 4 +- crates/jmap-proto/src/types/value.rs | 7 + crates/jmap/src/api/request.rs | 126 +++++---- crates/jmap/src/email/copy.rs | 346 ++++++++++++++++++++++++ crates/jmap/src/email/index.rs | 100 ++++++- crates/jmap/src/email/ingest.rs | 9 +- crates/jmap/src/email/mod.rs | 1 + crates/jmap/src/email/set.rs | 79 +----- crates/jmap/src/email/snippet.rs | 4 +- crates/jmap/src/lib.rs | 6 +- crates/store/src/fts/builder.rs | 23 +- crates/store/src/query/mod.rs | 19 +- tests/src/jmap/email_copy.rs | 98 +++++++ tests/src/jmap/mod.rs | 4 +- 17 files changed, 696 insertions(+), 152 deletions(-) create mode 100644 crates/jmap/src/email/copy.rs create mode 100644 tests/src/jmap/email_copy.rs diff --git a/crates/jmap-proto/src/method/copy.rs b/crates/jmap-proto/src/method/copy.rs index 83205423..50f7d860 100644 --- a/crates/jmap-proto/src/method/copy.rs +++ b/crates/jmap-proto/src/method/copy.rs @@ -15,7 +15,7 @@ use crate::{ }; #[derive(Debug, Clone)] -pub struct CopyRequest { +pub struct CopyRequest { pub from_account_id: Id, pub if_from_in_state: Option, pub account_id: Id, @@ -23,7 +23,7 @@ pub struct CopyRequest { pub create: VecMap, Object>, pub on_success_destroy_original: Option, pub destroy_from_if_in_state: Option, - pub arguments: RequestArguments, + pub arguments: T, } #[derive(Debug, Clone, serde::Serialize)] @@ -78,7 +78,7 @@ pub enum RequestArguments { Email, } -impl JsonObjectParser for CopyRequest { +impl JsonObjectParser for CopyRequest { fn parse(parser: &mut Parser) -> crate::parser::Result where Self: Sized, diff --git a/crates/jmap-proto/src/object/mod.rs b/crates/jmap-proto/src/object/mod.rs index 42060b28..74957898 100644 --- a/crates/jmap-proto/src/object/mod.rs +++ b/crates/jmap-proto/src/object/mod.rs @@ -82,6 +82,12 @@ impl ToBitmaps for Object { } } +impl ToBitmaps for &Object { + fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { + unreachable!() + } +} + const TEXT: u8 = 0; const UNSIGNED_INT: u8 = 1; const BOOL_TRUE: u8 = 2; @@ -112,6 +118,12 @@ impl Deserialize for Value { } impl Serialize for Object { + fn serialize(self) -> Vec { + (&self).serialize() + } +} + +impl Serialize for &Object { fn serialize(self) -> Vec { let mut buf = Vec::with_capacity(1024); self.serialize_into(&mut buf); diff --git a/crates/jmap-proto/src/request/method.rs b/crates/jmap-proto/src/request/method.rs index 2c6674ed..f44b24c3 100644 --- a/crates/jmap-proto/src/request/method.rs +++ b/crates/jmap-proto/src/request/method.rs @@ -113,6 +113,10 @@ impl Display for MethodName { } impl MethodName { + pub fn new(obj: MethodObject, fnc: MethodFunction) -> Self { + Self { obj, fnc } + } + pub fn error() -> Self { Self { obj: MethodObject::Thread, diff --git a/crates/jmap-proto/src/request/mod.rs b/crates/jmap-proto/src/request/mod.rs index f03ab288..f3fc84a1 100644 --- a/crates/jmap-proto/src/request/mod.rs +++ b/crates/jmap-proto/src/request/mod.rs @@ -13,7 +13,7 @@ use crate::{ error::method::MethodError, method::{ changes::ChangesRequest, - copy::{CopyBlobRequest, CopyRequest}, + copy::{self, CopyBlobRequest, CopyRequest}, get::{self, GetRequest}, import::ImportEmailRequest, parse::ParseEmailRequest, @@ -54,7 +54,7 @@ pub enum RequestMethod { Get(GetRequest), Set(SetRequest), Changes(ChangesRequest), - Copy(CopyRequest), + Copy(CopyRequest), CopyBlob(CopyBlobRequest), ImportEmail(ImportEmailRequest), ParseEmail(ParseEmailRequest), diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index 0a9b517b..780950d9 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -262,6 +262,13 @@ impl Value { } } + pub fn as_uint(&self) -> Option { + match self { + Value::UnsignedInt(u) => Some(*u), + _ => None, + } + } + pub fn try_cast_uint(&self) -> Option { match self { Value::UnsignedInt(u) => Some(*u), diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 187fa1bf..83990991 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -26,65 +26,79 @@ impl JMAP { continue; } - let method_response: ResponseMethod = match call.method { - RequestMethod::Get(mut req) => match req.take_arguments() { - get::RequestArguments::Email(arguments) => { - self.email_get(req.with_arguments(arguments)).await.into() + loop { + let mut next_call = None; + let method_response: ResponseMethod = match call.method { + RequestMethod::Get(mut req) => match req.take_arguments() { + get::RequestArguments::Email(arguments) => { + self.email_get(req.with_arguments(arguments)).await.into() + } + get::RequestArguments::Mailbox => self.mailbox_get(req).await.into(), + get::RequestArguments::Thread => self.thread_get(req).await.into(), + get::RequestArguments::Identity => todo!(), + get::RequestArguments::EmailSubmission => todo!(), + get::RequestArguments::PushSubscription => todo!(), + get::RequestArguments::SieveScript => todo!(), + get::RequestArguments::VacationResponse => todo!(), + get::RequestArguments::Principal => todo!(), + }, + RequestMethod::Query(mut req) => match req.take_arguments() { + query::RequestArguments::Email(arguments) => { + self.email_query(req.with_arguments(arguments)).await.into() + } + query::RequestArguments::Mailbox(arguments) => self + .mailbox_query(req.with_arguments(arguments)) + .await + .into(), + query::RequestArguments::EmailSubmission => todo!(), + query::RequestArguments::SieveScript => todo!(), + query::RequestArguments::Principal => todo!(), + }, + RequestMethod::Set(mut req) => match req.take_arguments() { + set::RequestArguments::Email => self.email_set(req).await.into(), + set::RequestArguments::Mailbox(arguments) => { + self.mailbox_set(req.with_arguments(arguments)).await.into() + } + set::RequestArguments::Identity => todo!(), + set::RequestArguments::EmailSubmission(_) => todo!(), + set::RequestArguments::PushSubscription => todo!(), + set::RequestArguments::SieveScript(_) => todo!(), + set::RequestArguments::VacationResponse => todo!(), + set::RequestArguments::Principal => todo!(), + }, + RequestMethod::Changes(req) => self.changes(req).await.into(), + RequestMethod::Copy(req) => self.email_copy(req, &mut next_call).await.into(), + RequestMethod::CopyBlob(_) => todo!(), + RequestMethod::ImportEmail(req) => self.email_import(req).await.into(), + RequestMethod::ParseEmail(req) => self.email_parse(req).await.into(), + RequestMethod::QueryChanges(req) => self.query_changes(req).await.into(), + RequestMethod::SearchSnippet(req) => { + self.email_search_snippet(req).await.into() } - get::RequestArguments::Mailbox => self.mailbox_get(req).await.into(), - get::RequestArguments::Thread => self.thread_get(req).await.into(), - get::RequestArguments::Identity => todo!(), - get::RequestArguments::EmailSubmission => todo!(), - get::RequestArguments::PushSubscription => todo!(), - get::RequestArguments::SieveScript => todo!(), - get::RequestArguments::VacationResponse => todo!(), - get::RequestArguments::Principal => todo!(), - }, - RequestMethod::Query(mut req) => match req.take_arguments() { - query::RequestArguments::Email(arguments) => { - self.email_query(req.with_arguments(arguments)).await.into() - } - query::RequestArguments::Mailbox(arguments) => self - .mailbox_query(req.with_arguments(arguments)) - .await - .into(), - query::RequestArguments::EmailSubmission => todo!(), - query::RequestArguments::SieveScript => todo!(), - query::RequestArguments::Principal => todo!(), - }, - RequestMethod::Set(mut req) => match req.take_arguments() { - set::RequestArguments::Email => self.email_set(req).await.into(), - set::RequestArguments::Mailbox(arguments) => { - self.mailbox_set(req.with_arguments(arguments)).await.into() - } - set::RequestArguments::Identity => todo!(), - set::RequestArguments::EmailSubmission(_) => todo!(), - set::RequestArguments::PushSubscription => todo!(), - set::RequestArguments::SieveScript(_) => todo!(), - set::RequestArguments::VacationResponse => todo!(), - set::RequestArguments::Principal => todo!(), - }, - RequestMethod::Changes(req) => self.changes(req).await.into(), - RequestMethod::Copy(_) => todo!(), - RequestMethod::CopyBlob(_) => todo!(), - RequestMethod::ImportEmail(req) => self.email_import(req).await.into(), - RequestMethod::ParseEmail(req) => self.email_parse(req).await.into(), - RequestMethod::QueryChanges(req) => self.query_changes(req).await.into(), - RequestMethod::SearchSnippet(req) => self.email_search_snippet(req).await.into(), - RequestMethod::ValidateScript(_) => todo!(), - RequestMethod::Echo(req) => req.into(), - RequestMethod::Error(error) => error.into(), - }; + RequestMethod::ValidateScript(_) => todo!(), + RequestMethod::Echo(req) => req.into(), + RequestMethod::Error(error) => error.into(), + }; - response.push_response( - call.id, - if !matches!(method_response, ResponseMethod::Error(_)) { - call.name + // Add response + response.push_response( + call.id, + if !matches!(method_response, ResponseMethod::Error(_)) { + call.name + } else { + MethodName::error() + }, + method_response, + ); + + // Process next call + if let Some(next_call) = next_call { + call = next_call; + call.id = response.method_responses.last().unwrap().id.clone(); } else { - MethodName::error() - }, - method_response, - ); + break; + } + } } Ok(response) diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs new file mode 100644 index 00000000..1dbaca58 --- /dev/null +++ b/crates/jmap/src/email/copy.rs @@ -0,0 +1,346 @@ +use jmap_proto::{ + error::{method::MethodError, set::SetError}, + method::{ + copy::{CopyRequest, CopyResponse, RequestArguments}, + set::{self, SetRequest}, + }, + object::Object, + request::{ + method::{MethodFunction, MethodName, MethodObject}, + reference::MaybeReference, + Call, RequestMethod, + }, + types::{ + blob::BlobId, + collection::Collection, + id::Id, + property::Property, + value::{SetValue, Value}, + }, +}; +use mail_parser::parsers::fields::thread::thread_name; +use store::{ + fts::term_index::TokenIndex, + query::RawValue, + write::{BatchBuilder, F_BITMAP, F_VALUE}, + BlobKind, +}; +use utils::map::vec_map::VecMap; + +use crate::JMAP; + +use super::{ + index::{EmailIndexBuilder, TrimTextValue, MAX_SORT_FIELD_LENGTH}, + ingest::IngestedEmail, +}; + +impl JMAP { + pub async fn email_copy( + &self, + request: CopyRequest, + next_call: &mut Option>, + ) -> Result { + let account_id = request.account_id.document_id(); + let from_account_id = request.from_account_id.document_id(); + + if account_id == from_account_id { + return Err(MethodError::InvalidArguments( + "From accountId is equal to fromAccountId".to_string(), + )); + } + let old_state = self + .assert_state(account_id, Collection::Email, &request.if_in_state) + .await?; + let mut response = CopyResponse { + from_account_id: request.from_account_id, + account_id: request.account_id, + new_state: old_state.clone(), + old_state, + created: VecMap::with_capacity(request.create.len()), + not_created: VecMap::new(), + }; + + let from_message_ids = self + .get_document_ids(from_account_id, Collection::Email) + .await? + .unwrap_or_default(); + let mailbox_ids = self + .get_document_ids(account_id, Collection::Mailbox) + .await? + .unwrap_or_default(); + let on_success_delete = request.on_success_destroy_original.unwrap_or(false); + let mut destroy_ids = Vec::new(); + + 'create: for (id, create) in request.create { + let id = id.unwrap(); + let from_message_id = id.document_id(); + if !from_message_ids.contains(from_message_id) { + response.not_created.append( + id, + SetError::not_found().with_description(format!( + "Item {} not found not found in account {}.", + id, response.from_account_id + )), + ); + continue; + } + + let mut mailboxes = Vec::new(); + let mut keywords = Vec::new(); + let mut received_at = None; + + for (property, value) in create.properties { + match (property, value) { + (Property::MailboxIds, SetValue::Value(Value::List(ids))) => { + mailboxes = ids + .into_iter() + .map(|id| id.unwrap_id().document_id()) + .collect(); + } + + (Property::MailboxIds, SetValue::Patch(patch)) => { + let mut patch = patch.into_iter(); + let document_id = patch.next().unwrap().unwrap_id().document_id(); + if patch.next().unwrap().unwrap_bool() { + if !mailboxes.contains(&document_id) { + mailboxes.push(document_id); + } + } else { + mailboxes.retain(|id| id != &document_id); + } + } + + (Property::Keywords, SetValue::Value(Value::List(keywords_))) => { + keywords = keywords_ + .into_iter() + .map(|keyword| keyword.unwrap_keyword()) + .collect(); + } + + (Property::Keywords, SetValue::Patch(patch)) => { + let mut patch = patch.into_iter(); + let keyword = patch.next().unwrap().unwrap_keyword(); + if patch.next().unwrap().unwrap_bool() { + if !keywords.contains(&keyword) { + keywords.push(keyword); + } + } else { + keywords.retain(|k| k != &keyword); + } + } + (Property::ReceivedAt, SetValue::Value(Value::Date(value))) => { + received_at = value.into(); + } + (property, _) => { + response.not_created.append( + id, + SetError::invalid_properties() + .with_property(property) + .with_description("Invalid property or value.".to_string()), + ); + continue 'create; + } + } + } + + // Make sure message belongs to at least one mailbox + if mailboxes.is_empty() { + response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::MailboxIds) + .with_description("Message has to belong to at least one mailbox."), + ); + continue 'create; + } + + // Verify that the mailboxIds are valid + for mailbox_id in &mailboxes { + if !mailbox_ids.contains(*mailbox_id) { + response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::MailboxIds) + .with_description(format!("mailboxId {mailbox_id} does not exist.")), + ); + continue 'create; + } + } + + let validate_acl = "true"; + + // Obtain term index and metadata + let (mut metadata, token_index) = if let (Some(metadata), Some(token_index)) = ( + self.get_property::>( + from_account_id, + Collection::Email, + from_message_id, + Property::BodyStructure, + ) + .await?, + self.get_term_index::>( + from_account_id, + Collection::Email, + from_message_id, + ) + .await?, + ) { + (metadata, token_index) + } else { + response.not_created.append( + id, + SetError::not_found().with_description(format!( + "Item {} not found not found in account {}.", + id, response.from_account_id + )), + ); + continue; + }; + + // Set receivedAt + if let Some(received_at) = received_at { + metadata.set(Property::ReceivedAt, Value::Date(received_at)); + } + + // Obtain threadId + let mut references = vec![]; + let mut subject = ""; + for (property, value) in &metadata.properties { + match property { + Property::MessageId + | Property::InReplyTo + | Property::References + | Property::EmailIds => match value { + Value::Text(text) => { + references.push(text.as_str()); + } + Value::List(list) => { + references.extend(list.iter().filter_map(|v| v.as_string())); + } + _ => (), + }, + Property::Subject => { + if let Some(value) = value.as_string() { + subject = thread_name(value).trim_text(MAX_SORT_FIELD_LENGTH); + } + if subject.is_empty() { + subject = "!"; + } + } + _ => (), + } + } + let thread_id = if !references.is_empty() { + self.find_or_merge_thread(account_id, subject, &references) + .await + .map_err(|_| MethodError::ServerPartialFail)? + } else { + None + }; + + // Copy blob + let message_id = self + .assign_document_id(account_id, Collection::Email) + .await?; + let mut email = IngestedEmail { + blob_id: BlobId::new(BlobKind::LinkedMaildir { + account_id, + document_id: message_id, + }), + size: metadata.get(&Property::Size).as_uint().unwrap_or(0) as usize, + ..Default::default() + }; + self.store + .copy_blob( + &BlobKind::LinkedMaildir { + account_id: from_account_id, + document_id: from_message_id, + }, + &email.blob_id.kind, + ) + .await + .map_err(|err| { + tracing::error!( + event = "error", + context = "email_copy", + from_account_id = from_account_id, + from_message_id = from_message_id, + account_id = account_id, + message_id = message_id, + error = ?err, + "Failed to copy blob."); + MethodError::ServerPartialFail + })?; + + // Build change log + let mut changes = self.begin_changes(account_id).await?; + let thread_id = if let Some(thread_id) = thread_id { + changes.log_child_update(Collection::Thread, thread_id); + thread_id + } else { + let thread_id = self + .assign_document_id(account_id, Collection::Thread) + .await?; + changes.log_insert(Collection::Thread, thread_id); + thread_id + }; + email.id = Id::from_parts(thread_id, message_id); + email.change_id = changes.change_id; + changes.log_insert(Collection::Email, email.id); + for mailbox_id in &mailboxes { + changes.log_child_update(Collection::Mailbox, *mailbox_id); + } + + // Build batch + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .create_document(message_id) + .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP) + .value(Property::MailboxIds, mailboxes, F_VALUE | F_BITMAP) + .value(Property::Keywords, keywords, F_VALUE | F_BITMAP) + .custom(EmailIndexBuilder::set(metadata)) + .custom(token_index) + .custom(changes); + self.store.write(batch.build()).await.map_err(|err| { + tracing::error!( + event = "error", + context = "email_copy", + error = ?err, + "Failed to write message to database."); + MethodError::ServerPartialFail + })?; + + // Update state + response.new_state = email.change_id.into(); + + // Add response + response.created.append(id, email.into()); + + // Add to destroy list + if on_success_delete { + destroy_ids.push(id); + } + } + + // Destroy ids + if on_success_delete && !destroy_ids.is_empty() { + *next_call = Call { + id: String::new(), + name: MethodName::new(MethodObject::Email, MethodFunction::Set), + method: RequestMethod::Set(SetRequest { + account_id: request.from_account_id, + if_in_state: request.destroy_from_if_in_state, + create: None, + update: None, + destroy: MaybeReference::Value(destroy_ids).into(), + arguments: set::RequestArguments::Email, + }), + } + .into(); + } + + Ok(response) + } +} diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index 5e730fbb..3e4da4bf 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -19,7 +19,7 @@ use store::{ builder::{FtsIndexBuilder, MAX_TOKEN_LENGTH}, Language, }, - write::{BatchBuilder, F_BITMAP, F_INDEX, F_VALUE}, + write::{BatchBuilder, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE}, }; use crate::email::headers::IntoForm; @@ -43,7 +43,7 @@ pub(super) trait IndexMessage { mailbox_ids: Vec, received_at: u64, default_language: Language, - ) -> store::Result<()>; + ) -> store::Result<&mut Self>; } impl IndexMessage for BatchBuilder { @@ -54,7 +54,7 @@ impl IndexMessage for BatchBuilder { mailbox_ids: Vec, received_at: u64, default_language: Language, - ) -> store::Result<()> { + ) -> store::Result<&mut Self> { let mut metadata = Object::with_capacity(15); // Index keywords @@ -360,7 +360,99 @@ impl IndexMessage for BatchBuilder { // Store full text index self.custom(fts); - Ok(()) + Ok(self) + } +} + +pub struct EmailIndexBuilder { + inner: Object, + set: bool, +} + +impl EmailIndexBuilder { + pub fn set(inner: Object) -> Self { + Self { inner, set: true } + } + + pub fn clear(inner: Object) -> Self { + Self { inner, set: false } + } +} + +impl IntoOperations for EmailIndexBuilder { + fn build(self, batch: &mut BatchBuilder) { + let options = if self.set { + // Serialize metadata + batch.value(Property::BodyStructure, &self.inner, F_VALUE); + 0 + } else { + // Delete metadata + batch.value(Property::BodyStructure, (), F_VALUE | F_CLEAR); + F_CLEAR + }; + + // Remove properties from index + for (property, value) in self.inner.properties { + match (&property, value) { + (Property::Size, Value::UnsignedInt(size)) => { + batch.value(Property::Size, size as u32, F_INDEX | options); + } + (Property::ReceivedAt | Property::SentAt, Value::Date(date)) => { + batch.value(property, date.timestamp() as u64, F_INDEX | options); + } + ( + Property::MessageId + | Property::InReplyTo + | Property::References + | Property::EmailIds, + Value::List(ids), + ) => { + // Remove messageIds from index + for id in ids { + match id { + Value::Text(id) if id.len() < MAX_ID_LENGTH => { + batch.value(Property::MessageId, id, F_INDEX | options); + } + _ => {} + } + } + } + ( + Property::From | Property::To | Property::Cc | Property::Bcc, + Value::List(addresses), + ) => { + let mut sort_text = SortedAddressBuilder::new(); + 'outer: for addr in addresses { + if let Some(addr) = addr.try_unwrap_object() { + for part in [Property::Name, Property::Email] { + if let Some(Value::Text(value)) = addr.properties.get(&part) { + if !sort_text.push(value) || part == Property::Email { + break 'outer; + } + } + } + } + } + batch.value(property, sort_text.build(), F_INDEX | options); + } + (Property::Subject, Value::Text(value)) => { + let thread_name = thread_name(&value); + batch.value( + Property::Subject, + if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + }, + F_INDEX | options, + ); + } + (Property::HasAttachment, Value::Bool(true)) => { + batch.bitmap(Property::HasAttachment, (), options); + } + _ => {} + } + } } } diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 29065647..f71d8665 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -23,6 +23,7 @@ use crate::{ use super::index::{TrimTextValue, MAX_SORT_FIELD_LENGTH}; +#[derive(Default)] pub struct IngestedEmail { pub id: Id, pub change_id: u64, @@ -176,9 +177,9 @@ impl JMAP { error = ?err, "Failed to index message."); MaybeError::Temporary - })?; - batch.value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP); - batch.custom(changes); + })? + .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP) + .custom(changes); self.store.write(batch.build()).await.map_err(|err| { tracing::error!( event = "error", @@ -196,7 +197,7 @@ impl JMAP { }) } - async fn find_or_merge_thread( + pub async fn find_or_merge_thread( &self, account_id: u32, thread_name: &str, diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index 0b643f28..6fd84c11 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -1,4 +1,5 @@ pub mod body; +pub mod copy; pub mod get; pub mod headers; pub mod import; diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index e17f3847..a1b9b627 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -23,13 +23,12 @@ use mail_builder::{ mime::{BodyPart, MimePart}, MessageBuilder, }; -use mail_parser::parsers::fields::thread::thread_name; use store::{ ahash::AHashSet, fts::term_index::TokenIndex, write::{ assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, DeserializeFrom, SerializeInto, - ToBitmaps, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + ToBitmaps, F_BITMAP, F_CLEAR, F_VALUE, }, BlobKind, Serialize, ValueKey, }; @@ -38,7 +37,7 @@ use crate::JMAP; use super::{ headers::{BuildHeader, ValueToHeader}, - index::{SortedAddressBuilder, TrimTextValue, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, + index::EmailIndexBuilder, }; impl JMAP { @@ -53,7 +52,7 @@ impl JMAP { .await?; // Obtain mailboxIds - let mut mailbox_ids = self + let mailbox_ids = self .get_document_ids(account_id, Collection::Mailbox) .await? .unwrap_or_default(); @@ -1037,8 +1036,8 @@ impl JMAP { return Ok(Err(SetError::not_found())); } - // Obtain message metadata - let metadata = if let Some(metadata) = self + // Remove message metadata + if let Some(metadata) = self .get_property::>( account_id, Collection::Email, @@ -1047,7 +1046,7 @@ impl JMAP { ) .await? { - metadata + batch.custom(EmailIndexBuilder::clear(metadata)); } else { tracing::debug!( event = "error", @@ -1059,72 +1058,6 @@ impl JMAP { return Ok(Err(SetError::not_found())); }; - // Delete metadata - batch.value(Property::BodyStructure, (), F_VALUE | F_CLEAR); - - // Remove properties from index - for (property, value) in metadata.properties { - match (&property, value) { - (Property::Size, Value::UnsignedInt(size)) => { - batch.value(Property::Size, size as u32, F_INDEX | F_CLEAR); - } - (Property::ReceivedAt | Property::SentAt, Value::Date(date)) => { - batch.value(property, date.timestamp() as u64, F_INDEX | F_CLEAR); - } - ( - Property::MessageId - | Property::InReplyTo - | Property::References - | Property::EmailIds, - Value::List(ids), - ) => { - // Remove messageIds from index - for id in ids { - match id { - Value::Text(id) if id.len() < MAX_ID_LENGTH => { - batch.value(Property::MessageId, id, F_INDEX | F_CLEAR); - } - _ => {} - } - } - } - ( - Property::From | Property::To | Property::Cc | Property::Bcc, - Value::List(addresses), - ) => { - let mut sort_text = SortedAddressBuilder::new(); - 'outer: for addr in addresses { - if let Some(addr) = addr.try_unwrap_object() { - for part in [Property::Name, Property::Email] { - if let Some(Value::Text(value)) = addr.properties.get(&part) { - if !sort_text.push(value) || part == Property::Email { - break 'outer; - } - } - } - } - } - batch.value(property, sort_text.build(), F_INDEX | F_CLEAR); - } - (Property::Subject, Value::Text(value)) => { - let thread_name = thread_name(&value); - batch.value( - Property::Subject, - if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" - }, - F_INDEX | F_CLEAR, - ); - } - (Property::HasAttachment, Value::Bool(true)) => { - batch.bitmap(Property::HasAttachment, (), F_CLEAR); - } - _ => {} - } - } - // Delete term index if let Some(token_index) = self .store diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index f2bf342b..03b15ec9 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -12,7 +12,7 @@ use store::{ builder::MAX_TOKEN_LENGTH, search_snippet::generate_snippet, stemmer::Stemmer, - term_index::{self}, + term_index::{self, TermIndex}, tokenizers::Tokenizer, Language, }, @@ -110,7 +110,7 @@ impl JMAP { // Obtain the term index and raw message let (term_index, raw_message) = if let (Some(term_index), Some(raw_message)) = ( - self.get_term_index(account_id, Collection::Email, document_id) + self.get_term_index::(account_id, Collection::Email, document_id) .await?, self.get_blob( &BlobKind::LinkedMaildir { diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index f549c4fe..b21eea64 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -153,15 +153,15 @@ impl JMAP { } } - pub async fn get_term_index( + pub async fn get_term_index( &self, account_id: u32, collection: Collection, document_id: u32, - ) -> Result, MethodError> { + ) -> Result, MethodError> { match self .store - .get_value::(ValueKey { + .get_value::(ValueKey { account_id, collection: collection.into(), document_id, diff --git a/crates/store/src/fts/builder.rs b/crates/store/src/fts/builder.rs index fbe04d44..4fde2673 100644 --- a/crates/store/src/fts/builder.rs +++ b/crates/store/src/fts/builder.rs @@ -4,6 +4,7 @@ use ahash::AHashSet; use utils::map::vec_map::VecMap; use crate::{ + query::RawValue, write::{BatchBuilder, IntoOperations, Operation}, Serialize, HASH_EXACT, HASH_STEMMED, }; @@ -132,8 +133,8 @@ impl<'x> IntoOperations for FtsIndexBuilder<'x> { } } -impl IntoOperations for TokenIndex { - fn build(self, batch: &mut BatchBuilder) { +impl TokenIndex { + fn build_index(self, batch: &mut BatchBuilder, set: bool) { for term in self.terms { for (term_ids, is_exact) in [(term.exact_terms, true), (term.stemmed_terms, false)] { for term_id in term_ids { @@ -142,13 +143,18 @@ impl IntoOperations for TokenIndex { word, if is_exact { HASH_EXACT } else { HASH_STEMMED }, term.field_id, - false, + set, )); } } } } + } +} +impl IntoOperations for TokenIndex { + fn build(self, batch: &mut BatchBuilder) { + self.build_index(batch, false); batch.ops.push(Operation::Value { field: u8::MAX, family: u8::MAX, @@ -157,6 +163,17 @@ impl IntoOperations for TokenIndex { } } +impl IntoOperations for RawValue { + fn build(self, batch: &mut BatchBuilder) { + self.inner.build_index(batch, true); + batch.ops.push(Operation::Value { + field: u8::MAX, + family: u8::MAX, + set: self.raw.into(), + }); + } +} + pub trait ToTokens { fn to_tokens(&self) -> HashSet; } diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 7896c8af..2d6b31d0 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -5,7 +5,9 @@ pub mod sort; use roaring::RoaringBitmap; -use crate::{fts::Language, write::BitmapFamily, BitmapKey, Serialize, BM_DOCUMENT_IDS}; +use crate::{ + fts::Language, write::BitmapFamily, BitmapKey, Deserialize, Serialize, BM_DOCUMENT_IDS, +}; #[derive(Debug, Clone, Copy)] pub enum Operator { @@ -220,3 +222,18 @@ impl BitmapKey<&'static [u8]> { } } } + +#[derive(Debug)] +pub struct RawValue { + pub raw: Vec, + pub inner: T, +} + +impl Deserialize for RawValue { + fn deserialize(bytes: &[u8]) -> crate::Result { + Ok(RawValue { + inner: T::deserialize(bytes)?, + raw: bytes.to_vec(), + }) + } +} diff --git a/tests/src/jmap/email_copy.rs b/tests/src/jmap/email_copy.rs new file mode 100644 index 00000000..3cd4b21c --- /dev/null +++ b/tests/src/jmap/email_copy.rs @@ -0,0 +1,98 @@ +use std::sync::Arc; + +use jmap::JMAP; +use jmap_client::{client::Client, mailbox::Role}; +use jmap_proto::types::id::Id; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Copy tests..."); + + // Create a mailbox on account 1 + let ac1_mailbox_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("Copy Test Ac# 1", None::, Role::None) + .await + .unwrap() + .take_id(); + + // Insert a message on account 1 + let ac1_email_id = client + .email_import( + concat!( + "From: bill@example.com\r\n", + "To: jdoe@example.com\r\n", + "Subject: TPS Report\r\n", + "\r\n", + "I'm going to need those TPS reports ASAP. ", + "So, if you could do that, that'd be great." + ) + .as_bytes() + .to_vec(), + [&ac1_mailbox_id], + None::>, + None, + ) + .await + .unwrap() + .take_id(); + + // Create a mailbox on account 2 + let ac2_mailbox_id = client + .set_default_account_id(Id::new(2).to_string()) + .mailbox_create("Copy Test Ac# 2", None::, Role::None) + .await + .unwrap() + .take_id(); + + // Copy the email and delete it from the first account + let mut request = client.build(); + request + .copy_email(Id::new(1).to_string()) + .on_success_destroy_original(true) + .create(&ac1_email_id) + .mailbox_id(&ac2_mailbox_id, true) + .keyword("$draft", true) + .received_at(311923920); + let ac2_email_id = request + .send() + .await + .unwrap() + .method_response_by_pos(0) + .unwrap_copy_email() + .unwrap() + .created(&ac1_email_id) + .unwrap() + .take_id(); + + // Check that the email was copied + let email = client + .email_get(&ac2_email_id, None::>) + .await + .unwrap() + .unwrap(); + assert_eq!( + email.preview().unwrap(), + "I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great." + ); + assert_eq!(email.subject().unwrap(), "TPS Report"); + assert_eq!(email.mailbox_ids(), &[&ac2_mailbox_id]); + assert_eq!(email.keywords(), &["$draft"]); + assert_eq!(email.received_at().unwrap(), 311923920); + + // Check that the email was deleted + assert!(client + .set_default_account_id(Id::new(1).to_string()) + .email_get(&ac1_email_id, None::>) + .await + .unwrap() + .is_none()); + + // Empty store + client.mailbox_destroy(&ac1_mailbox_id, true).await.unwrap(); + client + .set_default_account_id(Id::new(2).to_string()) + .mailbox_destroy(&ac2_mailbox_id, true) + .await + .unwrap(); + server.store.assert_is_empty().await; +} diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 40553f57..5a3ccff5 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -8,6 +8,7 @@ use tokio::sync::watch; use crate::{add_test_certs, store::TempDir}; pub mod email_changes; +pub mod email_copy; pub mod email_get; pub mod email_parse; pub mod email_query; @@ -65,7 +66,8 @@ pub async fn jmap_tests() { //email_parse::test(params.server.clone(), &mut params.client).await; //email_search_snippet::test(params.server.clone(), &mut params.client).await; //email_changes::test(params.server.clone(), &mut params.client).await; - email_query_changes::test(params.server.clone(), &mut params.client).await; + //email_query_changes::test(params.server.clone(), &mut params.client).await; + email_copy::test(params.server.clone(), &mut params.client).await; //thread_get::test(params.server.clone(), &mut params.client).await; //thread_merge::test(params.server.clone(), &mut params.client).await; //mailbox::test(params.server.clone(), &mut params.client).await;