diff --git a/.gitignore b/.gitignore index d936efaa..abd0eab2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /Cargo.lock .vscode *.failed +*_failed diff --git a/crates/jmap-proto/src/error/set.rs b/crates/jmap-proto/src/error/set.rs index d7a9e267..8959079b 100644 --- a/crates/jmap-proto/src/error/set.rs +++ b/crates/jmap-proto/src/error/set.rs @@ -34,13 +34,19 @@ pub struct SetError { description: Option>, #[serde(skip_serializing_if = "Option::is_none")] - properties: Option>, + properties: Option>, #[serde(rename = "existingId")] #[serde(skip_serializing_if = "Option::is_none")] existing_id: Option, } +#[derive(Debug, Clone)] +pub enum InvalidProperty { + Property(Property), + Path(Vec), +} + #[derive(Debug, Clone, serde::Serialize)] pub enum SetErrorType { #[serde(rename = "forbidden")] @@ -142,13 +148,20 @@ impl SetError { self } - pub fn with_property(mut self, property: Property) -> Self { - self.properties = vec![property].into(); + pub fn with_property(mut self, property: impl Into) -> Self { + self.properties = vec![property.into()].into(); self } - pub fn with_properties(mut self, properties: impl IntoIterator) -> Self { - self.properties = properties.into_iter().collect::>().into(); + pub fn with_properties( + mut self, + properties: impl IntoIterator>, + ) -> Self { + self.properties = properties + .into_iter() + .map(Into::into) + .collect::>() + .into(); self } @@ -165,9 +178,49 @@ impl SetError { Self::new(SetErrorType::Forbidden) } + pub fn not_found() -> Self { + Self::new(SetErrorType::NotFound) + } + pub fn already_exists() -> Self { Self::new(SetErrorType::AlreadyExists) } + + pub fn will_destroy() -> Self { + Self::new(SetErrorType::WillDestroy).with_description("ID will be destroyed.") + } } -pub type Result = std::result::Result; +impl From for InvalidProperty { + fn from(property: Property) -> Self { + InvalidProperty::Property(property) + } +} + +impl From<(Property, Property)> for InvalidProperty { + fn from((a, b): (Property, Property)) -> Self { + InvalidProperty::Path(vec![a, b]) + } +} + +impl serde::Serialize for InvalidProperty { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + InvalidProperty::Property(p) => p.serialize(serializer), + InvalidProperty::Path(p) => { + use std::fmt::Write; + let mut path = String::with_capacity(64); + for (i, p) in p.iter().enumerate() { + if i > 0 { + path.push('/'); + } + let _ = write!(path, "{}", p); + } + path.serialize(serializer) + } + } + } +} diff --git a/crates/jmap-proto/src/lib.rs b/crates/jmap-proto/src/lib.rs index b34bcb30..de61eab0 100644 --- a/crates/jmap-proto/src/lib.rs +++ b/crates/jmap-proto/src/lib.rs @@ -5,108 +5,3 @@ pub mod parser; pub mod request; pub mod response; pub mod types; - -/* -#[cfg(test)] -mod tests { - use std::{collections::BTreeMap, sync::Arc}; - - #[test] - fn gen_hash() { - //let mut table = BTreeMap::new(); - for value in ["blobIds", "ifInState", "emails"] { - let mut hash = 0; - let mut shift = 0; - let lower_first = false; - - for (pos, &ch) in value.as_bytes().iter().take(16).enumerate() { - if pos == 0 && lower_first { - hash |= (ch.to_ascii_lowercase() as u128) << shift; - } else { - hash |= (ch as u128) << shift; - } - shift += 8; - } - - shift = 0; - let mut hash2 = 0; - for &ch in value.as_bytes().iter().skip(16).take(16) { - hash2 |= (ch as u128) << shift; - shift += 8; - } - - println!( - "0x{} => {{}} // {}", - format!("{hash:x}") - .as_bytes() - .chunks(4) - .into_iter() - .map(|s| std::str::from_utf8(s).unwrap()) - .collect::>() - .join("_"), - value - ); - /*println!( - "(0x{}, 0x{}) => Filter::{}(),", - format!("{hash:x}") - .as_bytes() - .chunks(4) - .into_iter() - .map(|s| std::str::from_utf8(s).unwrap()) - .collect::>() - .join("_"), - format!("{hash2:x}") - .as_bytes() - .chunks(4) - .into_iter() - .map(|s| std::str::from_utf8(s).unwrap()) - .collect::>() - .join("_"), - value - );*/ - - /*let mut hash = 0; - let mut shift = 0; - let mut first_ch = 0; - let mut name = Vec::new(); - - for (pos, &ch) in value.as_bytes().iter().take(16).enumerate() { - if pos == 0 { - first_ch = ch.to_ascii_lowercase(); - name.push(ch.to_ascii_uppercase()); - } else { - hash |= (ch as u128) << shift; - shift += 8; - name.push(ch); - } - } - - //println!("Property::{} => {{}}", std::str::from_utf8(&name).unwrap()); - - table - .entry(first_ch) - .or_insert_with(|| vec![]) - .push((hash, name));*/ - } - - /*for (k, v) in table { - println!("b'{}' => match hash {{", k as char); - for (hash, value) in v { - println!( - " 0x{} => Property::{},", - format!("{hash:x}") - .as_bytes() - .chunks(4) - .into_iter() - .map(|s| std::str::from_utf8(s).unwrap()) - .collect::>() - .join("_"), - std::str::from_utf8(&value).unwrap() - ); - } - println!(" _ => parser.invalid_property()?,"); - println!("}}"); - }*/ - } -} -*/ diff --git a/crates/jmap-proto/src/method/set.rs b/crates/jmap-proto/src/method/set.rs index 226545c0..e88c2a52 100644 --- a/crates/jmap-proto/src/method/set.rs +++ b/crates/jmap-proto/src/method/set.rs @@ -2,7 +2,10 @@ use ahash::AHashMap; use utils::map::vec_map::VecMap; use crate::{ - error::{method::MethodError, set::SetError}, + error::{ + method::MethodError, + set::{InvalidProperty, SetError}, + }, object::{email_submission, mailbox, sieve, Object}, parser::{json::Parser, Error, JsonObjectParser, Token}, request::{ @@ -342,3 +345,58 @@ impl RequestPropertyParser for RequestArguments { } } } + +impl SetRequest { + pub fn validate(&self, max_objects_in_set: usize) -> Result<(), MethodError> { + if self.create.as_ref().map_or(0, |objs| objs.len()) + + self.update.as_ref().map_or(0, |objs| objs.len()) + + self.destroy.as_ref().map_or(0, |objs| { + if let MaybeReference::Value(ids) = objs { + ids.len() + } else { + 0 + } + }) + > max_objects_in_set + { + Err(MethodError::RequestTooLarge) + } else { + Ok(()) + } + } + + pub fn unwrap_create(&mut self) -> VecMap> { + self.create.take().unwrap_or_default() + } + + pub fn unwrap_update(&mut self) -> VecMap> { + self.update.take().unwrap_or_default() + } + + pub fn unwrap_destroy(&mut self) -> Vec { + self.destroy + .take() + .map(|ids| ids.unwrap()) + .unwrap_or_default() + } +} + +impl SetResponse { + pub fn invalid_property_create(&mut self, id: String, property: impl Into) { + self.not_created.append( + id, + SetError::invalid_properties() + .with_property(property) + .with_description("Invalid property or value.".to_string()), + ); + } + + pub fn invalid_property_update(&mut self, id: Id, property: impl Into) { + self.not_updated.append( + id, + SetError::invalid_properties() + .with_property(property) + .with_description("Invalid property or value.".to_string()), + ); + } +} diff --git a/crates/jmap-proto/src/response/references.rs b/crates/jmap-proto/src/response/references.rs index 04f38212..96a4d481 100644 --- a/crates/jmap-proto/src/response/references.rs +++ b/crates/jmap-proto/src/response/references.rs @@ -12,7 +12,7 @@ use crate::{ types::{ id::Id, property::Property, - value::{SetValue, Value}, + value::{MaybePatchValue, SetValue, Value}, }, }; @@ -294,6 +294,53 @@ impl Response { } } +impl Object { + pub fn iterate_and_eval_references( + self, + response: &Response, + ) -> impl Iterator> + '_ { + self.properties + .into_iter() + .map(|(property, set_value)| match set_value { + SetValue::Value(value) => Ok((property, MaybePatchValue::Value(value))), + SetValue::Patch(patch) => Ok((property, MaybePatchValue::Patch(patch))), + SetValue::IdReference(MaybeReference::Reference(id_ref)) => { + if let Some(id) = response.created_ids.get(&id_ref) { + Ok((property, MaybePatchValue::Value(Value::Id(*id)))) + } else { + Err(MethodError::InvalidResultReference(format!( + "Id reference {id_ref:?} not found." + ))) + } + } + SetValue::IdReference(MaybeReference::Value(id)) => { + Ok((property, MaybePatchValue::Value(Value::Id(id)))) + } + SetValue::IdReferences(id_refs) => { + let mut ids = Vec::with_capacity(id_refs.len()); + for id_ref in id_refs { + match id_ref { + MaybeReference::Value(id) => { + ids.push(Value::Id(id)); + } + MaybeReference::Reference(id_ref) => { + if let Some(id) = response.created_ids.get(&id_ref) { + ids.push(Value::Id(*id)); + } else { + return Err(MethodError::InvalidResultReference(format!( + "Id reference {id_ref:?} not found." + ))); + } + } + } + } + Ok((property, MaybePatchValue::Value(Value::List(ids)))) + } + _ => unreachable!(), + }) + } +} + impl EvalResult { pub fn unwrap_ids(self, rr: &ResultReference) -> Result, MethodError> { if let EvalResult::Values(values) = self { diff --git a/crates/jmap-proto/src/types/blob.rs b/crates/jmap-proto/src/types/blob.rs index 9656cfe1..c9c40ff0 100644 --- a/crates/jmap-proto/src/types/blob.rs +++ b/crates/jmap-proto/src/types/blob.rs @@ -29,7 +29,7 @@ use store::{ BlobKind, }; use utils::codec::{ - base32_custom::Base32Writer, + base32_custom::{Base32Reader, Base32Writer}, leb128::{Leb128Iterator, Leb128Writer}, }; @@ -109,6 +109,10 @@ impl BlobId { } } + pub fn from_base32(value: &str) -> Option { + BlobId::from_iter(&mut Base32Reader::new(value.as_bytes())) + } + #[allow(clippy::should_implement_trait)] pub fn from_iter(it: &mut T) -> Option where diff --git a/crates/jmap-proto/src/types/property.rs b/crates/jmap-proto/src/types/property.rs index fd19d005..bbcf3421 100644 --- a/crates/jmap-proto/src/types/property.rs +++ b/crates/jmap-proto/src/types/property.rs @@ -668,6 +668,23 @@ impl Property { Property::_T(value.to_string()) } } + + pub fn as_rfc_header(&self) -> RfcHeader { + match self { + Property::MessageId => RfcHeader::MessageId, + Property::InReplyTo => RfcHeader::InReplyTo, + Property::References => RfcHeader::References, + Property::Sender => RfcHeader::Sender, + Property::From => RfcHeader::From, + Property::To => RfcHeader::To, + Property::Cc => RfcHeader::Cc, + Property::Bcc => RfcHeader::Bcc, + Property::ReplyTo => RfcHeader::ReplyTo, + Property::Subject => RfcHeader::Subject, + Property::SentAt => RfcHeader::Date, + _ => unreachable!(), + } + } } impl Display for Property { @@ -968,6 +985,7 @@ impl From for Property { RfcHeader::InReplyTo => Property::InReplyTo, RfcHeader::MessageId => Property::MessageId, RfcHeader::References => Property::References, + RfcHeader::ResentMessageId => Property::EmailIds, _ => unreachable!(), } } diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index e3690068..90fdcb6e 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -47,6 +47,12 @@ pub enum SetValue { ResultReference(ResultReference), } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MaybePatchValue { + Value(Value), + Patch(Vec), +} + #[derive(Debug, Clone)] pub struct SetValueMap { pub values: Vec, @@ -161,7 +167,70 @@ impl Value { .unwrap_bool_or_null("")? .map(Value::Bool) .unwrap_or(Value::Null)), - _ => Value::parse::(parser.next_token()?, parser), + _ => Value::parse::(parser.next_token()?, parser), + } + } + + pub fn unwrap_id(self) -> Id { + match self { + Value::Id(id) => id, + _ => panic!("Expected id"), + } + } + + pub fn unwrap_bool(self) -> bool { + match self { + Value::Bool(b) => b, + _ => panic!("Expected bool"), + } + } + + pub fn unwrap_keyword(self) -> Keyword { + match self { + Value::Keyword(k) => k, + _ => panic!("Expected keyword"), + } + } + + pub fn try_unwrap_string(self) -> Option { + match self { + Value::Text(s) => Some(s), + _ => None, + } + } + + pub fn try_unwrap_object(self) -> Option> { + match self { + Value::Object(o) => Some(o), + _ => None, + } + } + + pub fn try_unwrap_list(self) -> Option> { + match self { + Value::List(l) => Some(l), + _ => None, + } + } + + pub fn try_unwrap_date(self) -> Option { + match self { + Value::Date(d) => Some(d), + _ => None, + } + } + + pub fn try_unwrap_blob_id(self) -> Option { + match self { + Value::BlobId(b) => Some(b), + _ => None, + } + } + + pub fn try_unwrap_uint(self) -> Option { + match self { + Value::UnsignedInt(u) => Some(u), + _ => None, } } } diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 34a3ae2e..a4b194ea 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -19,3 +19,6 @@ http-body-util = "0.1.0-rc.2" form_urlencoded = "1.1.0" tracing = "0.1" tokio = { version = "1.23", features = ["rt"] } + +[features] +test_mode = [] diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 155f01fd..50c398ee 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -53,8 +53,7 @@ impl JMAP { ("download", &Method::GET) => { if let (Some(account_id), Some(blob_id), Some(name)) = ( path.next().and_then(|p| Id::from_bytes(p.as_bytes())), - path.next() - .and_then(|p| BlobId::from_iter(&mut p.as_bytes().iter())), + path.next().and_then(BlobId::from_base32), path.next(), ) { return match self.blob_download(&blob_id, account_id.document_id()).await { diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index ceac851a..94152e0c 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -1,7 +1,10 @@ use jmap_proto::{ - error::request::RequestError, + error::{method::MethodError, request::RequestError}, method::{get, query}, - request::{method::MethodName, Request, RequestMethod}, + request::{ + method::{MethodName, MethodObject}, + Request, RequestMethod, + }, response::{Response, ResponseMethod}, }; @@ -27,12 +30,12 @@ impl JMAP { } let method_response: ResponseMethod = match call.method { - RequestMethod::Get(mut call) => match call.take_arguments() { + RequestMethod::Get(mut req) => match req.take_arguments() { get::RequestArguments::Email(arguments) => { - self.email_get(call.with_arguments(arguments)).await.into() + self.email_get(req.with_arguments(arguments)).await.into() } get::RequestArguments::Mailbox => todo!(), - get::RequestArguments::Thread => self.thread_get(call).await.into(), + get::RequestArguments::Thread => self.thread_get(req).await.into(), get::RequestArguments::Identity => todo!(), get::RequestArguments::EmailSubmission => todo!(), get::RequestArguments::PushSubscription => todo!(), @@ -40,26 +43,28 @@ impl JMAP { get::RequestArguments::VacationResponse => todo!(), get::RequestArguments::Principal => todo!(), }, - RequestMethod::Query(mut call) => match call.take_arguments() { - query::RequestArguments::Email(arguments) => self - .email_query(call.with_arguments(arguments)) - .await - .into(), + RequestMethod::Query(mut req) => match req.take_arguments() { + query::RequestArguments::Email(arguments) => { + self.email_query(req.with_arguments(arguments)).await.into() + } query::RequestArguments::Mailbox(_) => todo!(), query::RequestArguments::EmailSubmission => todo!(), query::RequestArguments::SieveScript => todo!(), query::RequestArguments::Principal => todo!(), }, - RequestMethod::Set(_) => todo!(), + RequestMethod::Set(req) => match call.name.obj { + MethodObject::Email => self.email_set(req, &response).await.into(), + _ => MethodError::UnknownMethod(format!("{}/set", call.name.obj)).into(), + }, RequestMethod::Changes(_) => todo!(), RequestMethod::Copy(_) => todo!(), RequestMethod::CopyBlob(_) => todo!(), - RequestMethod::ImportEmail(call) => self.email_import(call).await.into(), + RequestMethod::ImportEmail(req) => self.email_import(req).await.into(), RequestMethod::ParseEmail(_) => todo!(), RequestMethod::QueryChanges(_) => todo!(), RequestMethod::SearchSnippet(_) => todo!(), RequestMethod::ValidateScript(_) => todo!(), - RequestMethod::Echo(call) => call.into(), + RequestMethod::Echo(req) => req.into(), RequestMethod::Error(error) => error.into(), }; diff --git a/crates/jmap/src/changes/mod.rs b/crates/jmap/src/changes/mod.rs index 85bd08f9..16de3093 100644 --- a/crates/jmap/src/changes/mod.rs +++ b/crates/jmap/src/changes/mod.rs @@ -24,4 +24,20 @@ impl JMAP { } } } + + pub async fn assert_state( + &self, + account_id: u32, + collection: Collection, + if_in_state: &Option, + ) -> Result { + let old_state: State = self.get_state(account_id, collection).await?; + if let Some(if_in_state) = if_in_state { + if &old_state != if_in_state { + return Err(MethodError::StateMismatch); + } + } + + Ok(old_state) + } } diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index c7fc30c9..666e86e9 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -1,10 +1,23 @@ +use std::borrow::Cow; + use jmap_proto::{ object::Object, types::{ - property::{HeaderForm, Property}, + property::{HeaderForm, HeaderProperty, Property}, value::Value, }, }; +use mail_builder::{ + headers::{ + address::{Address, EmailAddress, GroupedAddresses}, + date::Date, + message_id::MessageId, + raw::Raw, + text::Text, + url::URL, + }, + MessageBuilder, +}; use mail_parser::{parsers::MessageStream, Addr, HeaderName, HeaderValue, MessagePart, RfcHeader}; pub trait IntoForm { @@ -16,6 +29,16 @@ pub trait HeaderToValue { fn headers_to_value(&self, raw_message: &[u8]) -> Value; } +pub trait ValueToHeader<'x> { + fn try_into_grouped_addresses(self) -> Option>; + fn try_into_address_list(self) -> Option>>; + fn try_into_address(self) -> Option>; +} + +pub trait BuildHeader: Sized { + fn build_header(self, header: HeaderProperty, value: Value) -> Result; +} + impl HeaderToValue for MessagePart<'_> { fn header_to_value(&self, property: &Property, raw_message: &[u8]) -> Value { let (header_name, form, all) = match property { @@ -209,6 +232,165 @@ impl IntoForm for HeaderValue<'_> { } } +impl<'x> ValueToHeader<'x> for Value { + fn try_into_grouped_addresses(self) -> Option> { + let mut obj = self.try_unwrap_object()?; + Some(GroupedAddresses { + name: obj + .properties + .remove(&Property::Name) + .and_then(|n| n.try_unwrap_string()) + .map(|n| n.into()), + addresses: obj + .properties + .remove(&Property::Addresses)? + .try_into_address_list()?, + }) + } + + fn try_into_address_list(self) -> Option>> { + let list = self.try_unwrap_list()?; + let mut addresses = Vec::with_capacity(list.len()); + for value in list { + addresses.push(Address::Address(value.try_into_address()?)); + } + Some(addresses) + } + + fn try_into_address(self) -> Option> { + let mut obj = self.try_unwrap_object()?; + Some(EmailAddress { + name: obj + .properties + .remove(&Property::Name) + .and_then(|n| n.try_unwrap_string()) + .map(|n| n.into()), + email: obj + .properties + .remove(&Property::Email)? + .try_unwrap_string()? + .into(), + }) + } +} + +impl BuildHeader for MessageBuilder<'_> { + fn build_header(self, header: HeaderProperty, value: Value) -> Result { + Ok(match (&header.form, header.all, value) { + (HeaderForm::Raw, false, Value::Text(value)) => { + self.header(header.header, Raw::from(value)) + } + (HeaderForm::Raw, true, Value::List(value)) => self.headers( + header.header, + value + .into_iter() + .filter_map(|v| Raw::from(v.try_unwrap_string()?).into()), + ), + (HeaderForm::Date, false, Value::Date(value)) => { + self.header(header.header, Date::new(value.timestamp())) + } + (HeaderForm::Date, true, Value::List(value)) => self.headers( + header.header, + value + .into_iter() + .filter_map(|v| Date::new(v.try_unwrap_date()?.timestamp()).into()), + ), + (HeaderForm::Text, false, Value::Text(value)) => { + self.header(header.header, Text::from(value)) + } + (HeaderForm::Text, true, Value::List(value)) => self.headers( + header.header, + value + .into_iter() + .filter_map(|v| Text::from(v.try_unwrap_string()?).into()), + ), + (HeaderForm::URLs, false, Value::List(value)) => self.header( + header.header, + URL { + url: value + .into_iter() + .filter_map(|v| Cow::from(v.try_unwrap_string()?).into()) + .collect(), + }, + ), + (HeaderForm::URLs, true, Value::List(value)) => self.headers( + header.header, + value.into_iter().filter_map(|value| { + URL { + url: value + .try_unwrap_list()? + .into_iter() + .filter_map(|v| Cow::from(v.try_unwrap_string()?).into()) + .collect(), + } + .into() + }), + ), + (HeaderForm::MessageIds, false, Value::List(value)) => self.header( + header.header, + MessageId { + id: value + .into_iter() + .filter_map(|v| Cow::from(v.try_unwrap_string()?).into()) + .collect(), + }, + ), + (HeaderForm::MessageIds, true, Value::List(value)) => self.headers( + header.header, + value.into_iter().filter_map(|value| { + MessageId { + id: value + .try_unwrap_list()? + .into_iter() + .filter_map(|v| Cow::from(v.try_unwrap_string()?).into()) + .collect(), + } + .into() + }), + ), + (HeaderForm::Addresses, false, Value::List(value)) => self.header( + header.header, + Address::new_list( + value + .into_iter() + .filter_map(|v| Address::Address(v.try_into_address()?).into()) + .collect(), + ), + ), + (HeaderForm::Addresses, true, Value::List(value)) => self.headers( + header.header, + value + .into_iter() + .filter_map(|v| Address::new_list(v.try_into_address_list()?).into()), + ), + (HeaderForm::GroupedAddresses, false, Value::List(value)) => self.header( + header.header, + Address::new_list( + value + .into_iter() + .filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into()) + .collect(), + ), + ), + (HeaderForm::GroupedAddresses, true, Value::List(value)) => self.headers( + header.header, + value.into_iter().filter_map(|v| { + Address::new_list( + v.try_unwrap_list()? + .into_iter() + .filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into()) + .collect::>(), + ) + .into() + }), + ), + _ => { + return Err(header); + } + }) + } +} + trait ByteTrim { fn trim_end(&self) -> Self; } diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index eccd38b7..5d2ad2d4 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -17,12 +17,9 @@ impl JMAP { ) -> Result { // Validate state let account_id = request.account_id.document_id(); - let old_state: State = self.get_state(account_id, Collection::Email).await?; - if let Some(if_in_state) = request.if_in_state { - if old_state != if_in_state { - return Err(MethodError::StateMismatch); - } - } + let old_state: State = self + .assert_state(account_id, Collection::Email, &request.if_in_state) + .await?; let cococ = "implement ACLS"; let valid_mailbox_ids = self diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index 98022cbc..27355dc1 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -30,6 +30,11 @@ pub const MAX_SORT_FIELD_LENGTH: usize = 255; pub const MAX_STORED_FIELD_LENGTH: usize = 512; pub const PREVIEW_LENGTH: usize = 256; +pub struct SortedAddressBuilder { + last_is_space: bool, + buf: String, +} + pub(super) trait IndexMessage { fn index_message( &mut self, @@ -50,7 +55,7 @@ impl IndexMessage for BatchBuilder { received_at: u64, default_language: Language, ) -> store::Result<()> { - let mut object = Object::with_capacity(15); + let mut metadata = Object::with_capacity(15); // Index keywords self.value(Property::Keywords, keywords, F_VALUE | F_BITMAP); @@ -59,11 +64,11 @@ impl IndexMessage for BatchBuilder { self.value(Property::MailboxIds, mailbox_ids, F_VALUE | F_BITMAP); // Index size - object.append(Property::Size, message.raw_message.len()); + metadata.append(Property::Size, message.raw_message.len()); self.value(Property::Size, message.raw_message.len() as u32, F_INDEX); // Index receivedAt - object.append( + metadata.append( Property::ReceivedAt, Value::Date(UTCDate::from_timestamp(received_at as i64)), ); @@ -89,6 +94,7 @@ impl IndexMessage for BatchBuilder { let part_language = part.language().unwrap_or(language); if part_id == 0 { language = part_language; + let mut extra_ids = Vec::new(); for header in part.headers.into_iter().rev() { if let HeaderName::Rfc(rfc_header) = header.name { // Index hasHeader property @@ -103,7 +109,6 @@ impl IndexMessage for BatchBuilder { header.value.visit_text(|id| { // Add ids to inverted index if id.len() < MAX_ID_LENGTH { - println!("indexing {}: {}", rfc_header.as_str(), id); self.value(Property::MessageId, id, F_INDEX); } @@ -123,7 +128,7 @@ impl IndexMessage for BatchBuilder { | RfcHeader::References ) && !seen_headers[rfc_header as usize] { - object.append( + metadata.append( rfc_header.into(), header .value @@ -131,6 +136,10 @@ impl IndexMessage for BatchBuilder { .into_form(&HeaderForm::MessageIds), ); seen_headers[rfc_header as usize] = true; + } else { + header.value.into_visit_text(|id| { + extra_ids.push(Value::Text(id)); + }); } } RfcHeader::From @@ -148,31 +157,20 @@ impl IndexMessage for BatchBuilder { | RfcHeader::Cc | RfcHeader::Bcc ) { - let mut sort_text = - String::with_capacity(MAX_SORT_FIELD_LENGTH); + let mut sort_text = SortedAddressBuilder::new(); let mut found_addr = seen_header; - let mut last_is_space = true; - header.value.visit_addresses(|value, is_addr| { + header.value.visit_addresses(|element, value| { if !found_addr { - if !sort_text.is_empty() { - sort_text.push(' '); - last_is_space = true; - } - found_addr = is_addr; - 'outer: for ch in value.chars() { - for ch in ch.to_lowercase() { - if sort_text.len() < MAX_SORT_FIELD_LENGTH { - let is_space = ch.is_whitespace(); - if !is_space || !last_is_space { - sort_text.push(ch); - last_is_space = is_space; - } - } else { - found_addr = true; - break 'outer; - } + match element { + AddressElement::Name => { + found_addr = sort_text.push(value); } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), } } @@ -182,21 +180,13 @@ impl IndexMessage for BatchBuilder { if !seen_header { // Add address to inverted index - self.value( - u8::from(&property), - if !sort_text.is_empty() { - &sort_text - } else { - "!" - }, - F_INDEX, - ); + self.value(u8::from(&property), sort_text.build(), F_INDEX); } } if !seen_header { - // Add address to object - object.append( + // Add address to metadata + metadata.append( property, header .value @@ -215,7 +205,7 @@ impl IndexMessage for BatchBuilder { F_INDEX, ); } - object.append( + metadata.append( Property::SentAt, header.value.into_form(&HeaderForm::Date), ); @@ -233,8 +223,8 @@ impl IndexMessage for BatchBuilder { }; if !seen_headers[rfc_header as usize] { - // Add to object - object.append( + // Add to metadata + metadata.append( Property::Subject, header .value @@ -278,14 +268,19 @@ impl IndexMessage for BatchBuilder { } } } + + // Add any extra Ids to metadata + if !extra_ids.is_empty() { + metadata.append(Property::EmailIds, Value::List(extra_ids)); + } } match part.body { PartType::Text(text) => { if part_id == preview_part_id { - object.append( + metadata.append( Property::Preview, - preview_text(text.clone(), PREVIEW_LENGTH), + preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH), ); } @@ -300,9 +295,9 @@ impl IndexMessage for BatchBuilder { PartType::Html(html) => { let text = html_to_text(&html); if part_id == preview_part_id { - object.append( + metadata.append( Property::Preview, - preview_text(text.clone().into(), PREVIEW_LENGTH), + preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH), ); } @@ -354,21 +349,67 @@ impl IndexMessage for BatchBuilder { } // Store and index hasAttachment property - object.append(Property::HasAttachment, has_attachments); + metadata.append(Property::HasAttachment, has_attachments); if has_attachments { self.bitmap(Property::HasAttachment, (), 0); } // Store properties - self.value(Property::BodyStructure, object, F_VALUE); + self.value(Property::BodyStructure, metadata, F_VALUE); // Store full text index - self.custom(fts)?; + self.custom(fts); Ok(()) } } +impl SortedAddressBuilder { + pub fn new() -> Self { + Self { + last_is_space: true, + buf: String::with_capacity(32), + } + } + + pub fn push(&mut self, text: &str) -> bool { + if !text.is_empty() { + if !self.buf.is_empty() { + self.buf.push(' '); + self.last_is_space = true; + } + for ch in text.chars() { + for ch in ch.to_lowercase() { + if self.buf.len() < MAX_SORT_FIELD_LENGTH { + let is_space = ch.is_whitespace(); + if !is_space || !self.last_is_space { + self.buf.push(ch); + self.last_is_space = is_space; + } + } else { + return false; + } + } + } + } + true + } + + pub fn build(self) -> String { + if !self.buf.is_empty() { + self.buf + } else { + "!".to_string() + } + } +} + +impl Default for SortedAddressBuilder { + fn default() -> Self { + Self::new() + } +} + trait GetContentLanguage { fn language(&self) -> Option; } @@ -390,55 +431,64 @@ impl GetContentLanguage for MessagePart<'_> { } trait VisitValues { - fn visit_addresses(&self, visitor: impl FnMut(&str, bool)); + fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str)); fn visit_text(&self, visitor: impl FnMut(&str)); + fn into_visit_text(self, visitor: impl FnMut(String)); +} + +enum AddressElement { + Name, + Address, + GroupName, } impl VisitValues for HeaderValue<'_> { - fn visit_addresses(&self, mut visitor: impl FnMut(&str, bool)) { + fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { HeaderValue::Address(addr) => { if let Some(name) = &addr.name { - visitor(name.as_ref(), false); + visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { - visitor(addr.as_ref(), true); + visitor(AddressElement::Address, addr); } } HeaderValue::AddressList(addr_list) => { for addr in addr_list { if let Some(name) = &addr.name { - visitor(name.as_ref(), false); + visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { - visitor(addr.as_ref(), true); + visitor(AddressElement::Address, addr); } } } HeaderValue::Group(group) => { if let Some(name) = &group.name { - visitor(name.as_ref(), false); + visitor(AddressElement::GroupName, name); } + for addr in &group.addresses { if let Some(name) = &addr.name { - visitor(name.as_ref(), false); + visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { - visitor(addr.as_ref(), true); + visitor(AddressElement::Address, addr); } } } HeaderValue::GroupList(groups) => { for group in groups { if let Some(name) = &group.name { - visitor(name.as_ref(), false); + visitor(AddressElement::GroupName, name); } + for addr in &group.addresses { if let Some(name) = &addr.name { - visitor(name.as_ref(), false); + visitor(AddressElement::Name, name); } if let Some(addr) = &addr.address { - visitor(addr.as_ref(), true); + visitor(AddressElement::Address, addr); } } } @@ -446,6 +496,7 @@ impl VisitValues for HeaderValue<'_> { _ => (), } } + fn visit_text(&self, mut visitor: impl FnMut(&str)) { match &self { HeaderValue::Text(text) => { @@ -459,6 +510,20 @@ impl VisitValues for HeaderValue<'_> { _ => (), } } + + fn into_visit_text(self, mut visitor: impl FnMut(String)) { + match self { + HeaderValue::Text(text) => { + visitor(text.into_owned()); + } + HeaderValue::TextList(texts) => { + for text in texts { + visitor(text.into_owned()); + } + } + _ => (), + } + } } pub trait TrimTextValue { diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 79e630fc..29065647 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -104,7 +104,7 @@ impl JMAP { })?; let change_id = self .store - .assign_change_id(account_id, Collection::Email) + .assign_change_id(account_id) .await .map_err(|err| { tracing::error!( @@ -178,14 +178,7 @@ impl JMAP { MaybeError::Temporary })?; batch.value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP); - batch.custom(changes).map_err(|err| { - tracing::error!( - event = "error", - context = "email_ingest", - error = ?err, - "Failed to add changelog to write batch."); - MaybeError::Temporary - })?; + batch.custom(changes); self.store.write(batch.build()).await.map_err(|err| { tracing::error!( event = "error", @@ -211,7 +204,6 @@ impl JMAP { ) -> Result, MaybeError> { let mut try_count = 0; - println!("-----------\nthread name: {:?}", thread_name); loop { // Find messages with matching references let mut filters = Vec::with_capacity(references.len() + 3); @@ -235,8 +227,6 @@ impl JMAP { })? .results; - println!("found messages {:?}", results); - if results.is_empty() { return Ok(None); } @@ -266,7 +256,7 @@ impl JMAP { "Failed to obtain threadIds."); MaybeError::Temporary })?; - println!("found thread ids {:?}", thread_ids); + if thread_ids.len() == 1 { return Ok(thread_ids.into_iter().next().unwrap()); } @@ -283,7 +273,7 @@ impl JMAP { thread_id = *thread_id_; } } - println!("common thread id {:?}", thread_id); + if thread_id == u32::MAX { return Ok(None); // This should never happen } else if thread_counts.len() == 1 { @@ -294,7 +284,7 @@ impl JMAP { let mut batch = BatchBuilder::new(); let change_id = self .store - .assign_change_id(account_id, Collection::Thread) + .assign_change_id(account_id) .await .map_err(|err| { tracing::error!( @@ -351,14 +341,7 @@ impl JMAP { } } } - batch.custom(changes).map_err(|err| { - tracing::error!( - event = "error", - context = "find_or_merge_thread", - error = ?err, - "Failed to add changelog to write batch."); - MaybeError::Temporary - })?; + batch.custom(changes); match self.store.write(batch.build()).await { Ok(_) => return Ok(Some(thread_id)), diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index 5e9f53f4..1fef3c22 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -5,3 +5,4 @@ pub mod import; pub mod index; pub mod ingest; pub mod query; +pub mod set; diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs new file mode 100644 index 00000000..190f671a --- /dev/null +++ b/crates/jmap/src/email/set.rs @@ -0,0 +1,1164 @@ +use std::{borrow::Cow, collections::HashMap}; + +use jmap_proto::{ + error::{ + method::MethodError, + set::{SetError, SetErrorType}, + }, + method::set::{SetRequest, SetResponse}, + object::Object, + response::Response, + types::{ + collection::Collection, + keyword::Keyword, + property::Property, + value::{MaybePatchValue, SetValue, Value}, + }, +}; +use mail_builder::{ + headers::{ + address::Address, content_type::ContentType, date::Date, message_id::MessageId, raw::Raw, + text::Text, HeaderType, + }, + mime::{BodyPart, MimePart}, + MessageBuilder, +}; +use mail_parser::parsers::fields::thread::thread_name; +use store::{ + ahash::AHashSet, + write::{ + assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, DeserializeFrom, SerializeInto, + ToBitmaps, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + }, + BlobKind, Serialize, +}; + +use crate::JMAP; + +use super::{ + headers::{BuildHeader, ValueToHeader}, + index::{SortedAddressBuilder, TrimTextValue, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, +}; + +impl JMAP { + pub async fn email_set( + &self, + mut request: SetRequest, + response: &Response, + ) -> Result { + // Prepare response + let account_id = request.account_id.document_id(); + let mut set_response = self + .prepare_set_response(&request, Collection::Email) + .await?; + + // Obtain mailboxIds + let mut mailbox_ids = self + .get_document_ids(account_id, Collection::Mailbox) + .await? + .unwrap_or_default(); + + let remove = "fdf"; + mailbox_ids.insert(0); + mailbox_ids.insert(1); + + let will_destroy = request.unwrap_destroy(); + + // Process creates + 'create: for (id, mut object) in request.unwrap_create() { + let has_body_structure = object + .properties + .keys() + .any(|key| matches!(key, Property::BodyStructure)); + let mut builder = MessageBuilder::new(); + let mut mailboxes = Vec::new(); + let mut keywords = Vec::new(); + let mut received_at = None; + + // Parse body values + let body_values = object + .properties + .remove(&Property::BodyValues) + .and_then(|obj| { + if let SetValue::Value(Value::Object(obj)) = obj { + let mut values = HashMap::with_capacity(obj.properties.len()); + for (key, value) in obj.properties { + if let (Property::_T(id), Value::Object(mut bv)) = (key, value) { + values.insert( + id, + bv.properties + .remove(&Property::Value)? + .try_unwrap_string()?, + ); + } else { + return None; + } + } + Some(values) + } else { + None + } + }); + let mut size_attachments = 0; + + // Parse properties + for item in object.iterate_and_eval_references(response) { + match item? { + (Property::MailboxIds, MaybePatchValue::Value(Value::List(ids))) => { + mailboxes = ids + .into_iter() + .map(|id| id.unwrap_id().document_id()) + .collect(); + } + + (Property::MailboxIds, MaybePatchValue::Patch(patch)) => { + let mut patch = patch.into_iter(); + let document_id = patch.next().unwrap().unwrap_id().document_id(); + if patch.next().unwrap().unwrap_bool() { + if !mailboxes.contains(&document_id) { + mailboxes.push(document_id); + } + } else { + mailboxes.retain(|id| id != &document_id); + } + } + + (Property::Keywords, MaybePatchValue::Value(Value::List(keywords_))) => { + keywords = keywords_ + .into_iter() + .map(|keyword| keyword.unwrap_keyword()) + .collect(); + } + + (Property::Keywords, MaybePatchValue::Patch(patch)) => { + let mut patch = patch.into_iter(); + let keyword = patch.next().unwrap().unwrap_keyword(); + if patch.next().unwrap().unwrap_bool() { + if !keywords.contains(&keyword) { + keywords.push(keyword); + } + } else { + keywords.retain(|k| k != &keyword); + } + } + + ( + header @ (Property::MessageId | Property::InReplyTo | Property::References), + MaybePatchValue::Value(Value::List(values)), + ) => { + builder = builder.header( + header.as_rfc_header(), + MessageId { + id: values + .into_iter() + .filter_map(|value| value.try_unwrap_string()) + .map(|value| value.into()) + .collect(), + }, + ); + } + + ( + header @ (Property::Sender + | Property::From + | Property::To + | Property::Cc + | Property::Bcc + | Property::ReplyTo), + MaybePatchValue::Value(value), + ) => { + if let Some(addresses) = value.try_into_address_list() { + builder = + builder.header(header.as_rfc_header(), Address::List(addresses)); + } else { + set_response.invalid_property_create(id, header); + continue 'create; + } + } + (Property::Subject, MaybePatchValue::Value(Value::Text(value))) => { + builder = builder.subject(value); + } + + (Property::ReceivedAt, MaybePatchValue::Value(Value::Date(value))) => { + received_at = (value.timestamp() as u64).into(); + } + + (Property::SentAt, MaybePatchValue::Value(Value::Date(value))) => { + builder = builder.date(Date::new(value.timestamp())); + } + + ( + property @ (Property::TextBody + | Property::HtmlBody + | Property::Attachments + | Property::BodyStructure), + MaybePatchValue::Value(value), + ) => { + // Validate request + let (values, expected_content_type) = match property { + Property::BodyStructure => (vec![value], None), + Property::TextBody | Property::HtmlBody if !has_body_structure => { + let values = value.try_unwrap_list().unwrap_or_default(); + if values.len() <= 1 { + ( + values, + Some(match property { + Property::TextBody => "text/plain", + Property::HtmlBody => "text/html", + _ => unreachable!(), + }), + ) + } else { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property(property) + .with_description("Only one part is allowed."), + ); + continue 'create; + } + } + Property::Attachments if !has_body_structure => { + (value.try_unwrap_list().unwrap_or_default(), None) + } + _ => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_properties([property, Property::BodyStructure]) + .with_description( + "Cannot set both properties on a same request.", + ), + ); + continue 'create; + } + }; + + // Iterate parts + let mut values_stack = Vec::new(); + let mut values = values.into_iter(); + let mut parts = Vec::new(); + loop { + while let Some(value) = values.next() { + let mut blob_id = None; + let mut part_id = None; + let mut content_type = None; + let mut content_disposition = None; + let mut name = None; + let mut charset = None; + let mut subparts = None; + let mut has_size = false; + let mut headers: Vec<(Cow, HeaderType)> = Vec::new(); + + if let Some(obj) = value.try_unwrap_object() { + for (body_property, value) in obj.properties { + match (body_property, value) { + (Property::Type, Value::Text(value)) => { + content_type = value.into(); + } + (Property::PartId, Value::Text(value)) => { + part_id = value.into(); + } + (Property::BlobId, Value::BlobId(value)) => { + blob_id = value.into(); + } + (Property::Disposition, Value::Text(value)) => { + content_disposition = value.into(); + } + (Property::Name, Value::Text(value)) => { + name = value.into(); + } + (Property::Charset, Value::Text(value)) => { + charset = value.into(); + } + (Property::Language, Value::List(values)) => { + headers.push(( + "Content-Language".into(), + Text::new( + values + .into_iter() + .filter_map(|v| v.try_unwrap_string()) + .fold( + String::with_capacity(64), + |mut h, v| { + if !h.is_empty() { + h.push_str(", "); + } + h.push_str(&v); + h + }, + ), + ) + .into(), + )); + } + (Property::Cid, Value::Text(value)) => { + headers.push(( + "Content-ID".into(), + MessageId::new(value).into(), + )); + } + (Property::Location, Value::Text(value)) => { + headers.push(( + "Content-Location".into(), + Text::new(value).into(), + )); + } + (Property::Header(header), Value::Text(value)) + if !header.header.eq_ignore_ascii_case( + "content-transfer-encoding", + ) => + { + headers.push(( + header.header.into(), + Raw::from(value).into(), + )); + } + (Property::Header(header), Value::List(values)) + if !header.header.eq_ignore_ascii_case( + "content-transfer-encoding", + ) => + { + for value in values { + if let Some(value) = value.try_unwrap_string() { + headers.push(( + header.header.clone().into(), + Raw::from(value).into(), + )); + } + } + } + (Property::Headers, _) => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property(( + property, + Property::Headers, + )) + .with_description( + "Headers have to be set individually.", + ), + ); + continue 'create; + } + (Property::Size, _) => { + has_size = true; + } + (Property::SubParts, Value::List(values)) => { + subparts = values.into(); + } + (body_property, value) if value != Value::Null => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property((property, body_property)) + .with_description("Cannot set property."), + ); + continue 'create; + } + _ => {} + } + } + } + + // Validate content-type + let content_type = + content_type.unwrap_or_else(|| "text/plain".to_string()); + let is_multipart = content_type.starts_with("multipart/"); + if is_multipart { + if !matches!(property, Property::BodyStructure) { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property((property, Property::Type)) + .with_description("Multiparts can only be set with bodyStructure."), + ); + continue 'create; + } + } else if expected_content_type + .as_ref() + .map_or(false, |v| v != &content_type) + { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property((property, Property::Type)) + .with_description(format!( + "Expected one body part of type \"{}\"", + expected_content_type.unwrap() + )), + ); + continue 'create; + } + + // Validate partId/blobId + match (blob_id.is_some(), part_id.is_some()) { + (true, true) if !is_multipart => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_properties([(property.clone(), Property::BlobId), (property, Property::PartId)]) + .with_description( + "Cannot specify both \"partId\" and \"blobId\".", + ), + ); + continue 'create; + } + (false, false) if !is_multipart => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_description("Expected a \"partId\" or \"blobId\" field in body part."), + ); + continue 'create; + } + (false, true) if !is_multipart && has_size => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property((property, Property::Size)) + .with_description( + "Cannot specify \"size\" when providing a \"partId\".", + ), + ); + continue 'create; + } + (true, _) | (_, true) if is_multipart => { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_properties([(property.clone(), Property::BlobId), (property, Property::PartId)]) + .with_description( + "Cannot specify \"partId\" or \"blobId\" in multipart body parts.", + ), + ); + continue 'create; + } + _ => (), + } + + // Set Content-Type and Content-Disposition + let mut content_type = ContentType::new(content_type); + if !is_multipart { + if let Some(charset) = charset { + if part_id.is_none() { + content_type + .attributes + .push(("charset".into(), charset.into())); + } else { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property((property, Property::Charset)) + .with_description( + "Cannot specify a character set when providing a \"partId\".", + ), + ); + continue 'create; + } + } else if part_id.is_some() { + content_type + .attributes + .push(("charset".into(), "utf-8".into())); + } + match (content_disposition, name) { + (Some(disposition), Some(filename)) => { + headers.push(( + "Content-Disposition".into(), + ContentType::new(disposition) + .attribute("filename", filename) + .into(), + )); + } + (Some(disposition), None) => { + headers.push(( + "Content-Disposition".into(), + ContentType::new(disposition).into(), + )); + } + (None, Some(filename)) => { + content_type + .attributes + .push(("name".into(), filename.into())); + } + (None, None) => (), + }; + } + headers.push(("Content-Type".into(), content_type.into())); + + // In test, sort headers to avoid randomness + #[cfg(feature = "test_mode")] + { + headers.sort_unstable_by(|a, b| match a.0.cmp(&b.0) { + std::cmp::Ordering::Equal => a.1.cmp(&b.1), + ord => ord, + }); + } + // Retrieve contents + parts.push(MimePart { + headers, + contents: if !is_multipart { + if let Some(blob_id) = blob_id { + match self.blob_download(&blob_id, account_id).await { + Ok(Some(contents)) => { + BodyPart::Binary(contents.into()) + } + Ok(None) => { + set_response.not_created.append( + id, + SetError::new(SetErrorType::BlobNotFound).with_description( + format!("blobId {blob_id} does not exist on this server.") + ), + ); + continue 'create; + } + Err(err) => { + tracing::error!(event = "error", + context = "email_set", + account_id = account_id, + blob_id = ?blob_id, + error = ?err, + "Failed to retrieve blob while creating message"); + return Err(MethodError::ServerPartialFail); + } + } + } else if let Some(part_id) = part_id { + if let Some(contents) = + body_values.as_ref().and_then(|bv| bv.get(&part_id)) + { + BodyPart::Text(contents.as_str().into()) + } else { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property((property, Property::PartId)) + .with_description(format!( + "Missing body value for partId {part_id:?}" + )), + ); + continue 'create; + } + } else { + unreachable!() + } + } else { + BodyPart::Multipart(vec![]) + }, + }); + + // Check attachment sizes + if !is_multipart { + size_attachments += parts.last().unwrap().size(); + if self.config.mail_attachments_max_size > 0 + && size_attachments > self.config.mail_attachments_max_size + { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property(property) + .with_description(format!( + "Message exceeds maximum size of {} bytes.", + self.config.mail_attachments_max_size + )), + ); + continue 'create; + } + } else if let Some(subparts) = subparts { + values_stack.push((values, parts)); + parts = Vec::with_capacity(subparts.len()); + values = subparts.into_iter(); + continue; + } + } + + if let Some((prev_values, mut prev_parts)) = values_stack.pop() { + values = prev_values; + prev_parts.last_mut().unwrap().contents = + BodyPart::Multipart(parts); + parts = prev_parts; + } else { + break; + } + } + + match property { + Property::TextBody => { + builder.text_body = parts.pop(); + } + Property::HtmlBody => { + builder.html_body = parts.pop(); + } + Property::Attachments => { + builder.attachments = parts.into(); + } + _ => { + builder.body = parts.pop(); + } + } + } + + (Property::Header(header), MaybePatchValue::Value(value)) => { + match builder.build_header(header, value) { + Ok(builder_) => { + builder = builder_; + } + Err(header) => { + set_response.invalid_property_create(id, Property::Header(header)); + continue 'create; + } + } + } + + (_, MaybePatchValue::Value(Value::Null)) => (), + + (property, _) => { + set_response.invalid_property_create(id, property); + continue 'create; + } + } + } + + // Make sure message belongs to at least one mailbox + if mailboxes.is_empty() { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::MailboxIds) + .with_description("Message has to belong to at least one mailbox."), + ); + continue 'create; + } + + // Verify that the mailboxIds are valid + for mailbox_id in &mailboxes { + if !mailbox_ids.contains(*mailbox_id) { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::MailboxIds) + .with_description(format!("mailboxId {mailbox_id} does not exist.")), + ); + continue 'create; + } + } + + // Make sure the message is not empty + if builder.headers.is_empty() + && builder.body.is_none() + && builder.html_body.is_none() + && builder.text_body.is_none() + && builder.attachments.is_none() + { + set_response.not_created.append( + id, + SetError::invalid_properties() + .with_description("Message has to have at least one header or body part."), + ); + continue 'create; + } + + // In test, sort headers to avoid randomness + #[cfg(feature = "test_mode")] + { + builder + .headers + .sort_unstable_by(|a, b| match a.0.cmp(&b.0) { + std::cmp::Ordering::Equal => a.1.cmp(&b.1), + ord => ord, + }); + } + + // Build message + let mut raw_message = Vec::with_capacity((4 * size_attachments / 3) + 1024); + builder.write_to(&mut raw_message).unwrap_or_default(); + + // Ingest message + set_response.created.insert( + id, + self.email_ingest(&raw_message, account_id, mailboxes, keywords, received_at) + .await + .map_err(|_| MethodError::ServerPartialFail)? + .into(), + ); + } + + // Process updates + 'update: for (id, object) in request.unwrap_update() { + // Make sure id won't be destroyed + if will_destroy.contains(&id) { + set_response + .not_updated + .append(id, SetError::will_destroy()); + continue 'update; + } + + // Obtain current keywords and mailboxes + let document_id = id.document_id(); + let (mut mailboxes, mut keywords) = if let (Some(mailboxes), Some(keywords)) = ( + self.get_property::>>( + account_id, + Collection::Email, + document_id, + Property::MailboxIds, + ) + .await?, + self.get_property::>>( + account_id, + Collection::Email, + document_id, + Property::MailboxIds, + ) + .await?, + ) { + (TagManager::new(mailboxes), TagManager::new(keywords)) + } else { + set_response.not_updated.append(id, SetError::not_found()); + continue 'update; + }; + + for item in object.iterate_and_eval_references(response) { + match item? { + (Property::MailboxIds, MaybePatchValue::Value(Value::List(ids))) => { + mailboxes.set( + ids.into_iter() + .map(|id| id.unwrap_id().document_id()) + .collect(), + ); + } + (Property::MailboxIds, MaybePatchValue::Patch(patch)) => { + let mut patch = patch.into_iter(); + mailboxes.update( + patch.next().unwrap().unwrap_id().document_id(), + patch.next().unwrap().unwrap_bool(), + ); + } + (Property::Keywords, MaybePatchValue::Value(Value::List(keywords_))) => { + keywords.set( + keywords_ + .into_iter() + .map(|keyword| keyword.unwrap_keyword()) + .collect(), + ); + } + (Property::Keywords, MaybePatchValue::Patch(patch)) => { + let mut patch = patch.into_iter(); + keywords.update( + patch.next().unwrap().unwrap_keyword(), + patch.next().unwrap().unwrap_bool(), + ); + } + (property, _) => { + set_response.invalid_property_update(id, property); + continue 'update; + } + } + } + + if !mailboxes.has_changes() && !keywords.has_changes() { + set_response.not_updated.append( + id, + SetError::invalid_properties() + .with_description("No changes found in request.".to_string()), + ); + continue 'update; + } + + // Prepare write batch + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .update_document(document_id); + + // Log change + let mut changed_mailboxes = AHashSet::new(); + let mut changes = ChangeLogBuilder::with_change_id( + self.store + .assign_change_id(account_id) + .await + .map_err(|err| { + tracing::error!( + event = "error", + context = "email_set", + error = ?err, + "Failed to assign changeId for email update."); + MethodError::ServerPartialFail + })?, + ); + changes.log_update(Collection::Email, id); + + // Process keywords + if keywords.has_changes() { + // Set all current mailboxes as changed if the Seen tag changed + if keywords + .changed_tags() + .any(|keyword| keyword == &Keyword::Seen) + { + for mailbox_id in mailboxes.current() { + changed_mailboxes.insert(*mailbox_id); + } + } + + // Update keywords property + keywords.update_batch(&mut batch, Property::Keywords); + } + + // Process mailboxes + if mailboxes.has_changes() { + // Make sure the message is at least in one mailbox + if !mailboxes.has_tags() { + set_response.not_updated.append( + id, + SetError::invalid_properties() + .with_property(Property::MailboxIds) + .with_description("Message has to belong to at least one mailbox."), + ); + continue 'update; + } + + // Make sure all new mailboxIds are valid + for mailbox_id in mailboxes.added() { + if mailbox_ids.contains(*mailbox_id) { + changed_mailboxes.insert(*mailbox_id); + } else { + set_response.not_updated.append( + id, + SetError::invalid_properties() + .with_property(Property::MailboxIds) + .with_description(format!( + "mailboxId {mailbox_id} does not exist." + )), + ); + continue 'update; + } + } + + // Add all removed mailboxes to change list + for mailbox_id in mailboxes.removed() { + changed_mailboxes.insert(*mailbox_id); + } + + // Update mailboxIds property + mailboxes.update_batch(&mut batch, Property::MailboxIds); + } + + // Log mailbox changes + for mailbox_id in changed_mailboxes { + changes.log_child_update(Collection::Mailbox, mailbox_id); + } + + // Write changes + batch.custom(changes); + self.store.write(batch.build()).await.map_err(|err| { + tracing::error!( + event = "error", + context = "email_set", + error = ?err, + "Failed to write message changes to database."); + MethodError::ServerPartialFail + })?; + } + + for destroy_id in will_destroy { + // Todo + } + + Ok(set_response) + } + + pub async fn email_delete( + &self, + account_id: u32, + document_id: u32, + batch: &mut BatchBuilder, + changes: &mut ChangeLogBuilder, + ) -> Result { + // Delete document + batch + .with_collection(Collection::Email) + .delete_document(document_id); + + // Remove mailboxes + let mailboxes = if let Some(mailboxes) = self + .get_property::>>( + account_id, + Collection::Email, + document_id, + Property::MailboxIds, + ) + .await? + { + mailboxes + } else { + return Ok(false); + }; + for mailbox_id in &mailboxes.inner { + changes.log_child_update(Collection::Mailbox, *mailbox_id); + } + batch.assert_value(Property::MailboxIds, &mailboxes).value( + Property::MailboxIds, + mailboxes.inner, + F_VALUE | F_BITMAP | F_CLEAR, + ); + + // Remove keywords + if let Some(keywords) = self + .get_property::>>( + account_id, + Collection::Email, + document_id, + Property::Keywords, + ) + .await? + { + batch.assert_value(Property::Keywords, &keywords).value( + Property::Keywords, + keywords.inner, + F_VALUE | F_BITMAP | F_CLEAR, + ); + } else { + return Ok(false); + }; + + // Remove threadIds + if let Some(thread_id) = self + .get_property::( + account_id, + Collection::Email, + document_id, + Property::ThreadId, + ) + .await? + { + // Obtain all documentIds in thread + if let Some(thread_tags) = self + .get_tag(account_id, Collection::Email, Property::ThreadId, thread_id) + .await? + { + if thread_tags.len() > 1 { + // Thread has other document ids, remove this one + changes.log_child_update(Collection::Thread, thread_id); + } else { + // Thread is empty, delete it + batch + .with_collection(Collection::Thread) + .delete_document(thread_id) + .with_collection(Collection::Email); + changes.log_delete(Collection::Thread, thread_id); + } + + // Remove threadId value and tag + batch.assert_value(Property::ThreadId, thread_id).value( + Property::ThreadId, + thread_id, + F_VALUE | F_BITMAP | F_CLEAR, + ); + } else { + return Ok(false); + } + } else { + return Ok(false); + } + + // Obtain message metadata + let metadata = if let Some(metadata) = self + .get_property::>( + account_id, + Collection::Email, + document_id, + Property::ThreadId, + ) + .await? + { + metadata + } else { + return Ok(false); + }; + + // Remove properties from index + for (property, value) in metadata.properties { + match (&property, value) { + (Property::Size, Value::UnsignedInt(size)) => { + batch.value(Property::Size, size as u32, F_INDEX | F_CLEAR); + } + (Property::ReceivedAt | Property::SentAt, Value::Date(date)) => { + batch.value(property, date.timestamp() as u64, F_INDEX | F_CLEAR); + } + ( + Property::MessageId + | Property::InReplyTo + | Property::References + | Property::EmailIds, + Value::List(ids), + ) => { + // Remove messageIds from index + for id in ids { + match id { + Value::Text(id) if id.len() < MAX_ID_LENGTH => { + batch.value(Property::MessageId, id, F_INDEX | F_CLEAR); + } + _ => {} + } + } + } + (Property::From | Property::To | Property::Bcc, Value::List(addresses)) => { + let mut sort_text = SortedAddressBuilder::new(); + 'outer: for addr in addresses { + if let Some(addr) = addr.try_unwrap_object() { + for part in [Property::Name, Property::Email] { + if let Some(Value::Text(value)) = addr.properties.get(&part) { + if !sort_text.push(value) || part == Property::Email { + break 'outer; + } + } + } + } + } + batch.value(property, sort_text.build(), F_INDEX | F_CLEAR); + } + (Property::Subject, Value::Text(value)) => { + let thread_name = thread_name(&value); + batch.value( + Property::Subject, + if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + }, + F_INDEX | F_CLEAR, + ); + } + (Property::HasAttachment, Value::Bool(true)) => { + batch.bitmap(Property::HasAttachment, (), F_CLEAR); + } + _ => {} + } + } + + // Delete metadata + batch.value(Property::BodyStructure, (), F_VALUE | F_CLEAR); + + // Delete blob + self.store + .delete_blob(&BlobKind::LinkedMaildir { + account_id, + document_id, + }) + .await + .map_err(|err| { + tracing::error!( + event = "error", + context = "email_delete", + error = ?err, + "Failed to delete blob."); + MethodError::ServerPartialFail + })?; + + Ok(true) + } +} + +struct TagManager< + T: PartialEq + Clone + ToBitmaps + SerializeInto + Serialize + DeserializeFrom + Sync + Send, +> { + current: HashedValue>, + added: Vec, + removed: Vec, + last: LastTag, +} + +enum LastTag { + Set, + Update, + None, +} + +impl< + T: PartialEq + Clone + ToBitmaps + SerializeInto + Serialize + DeserializeFrom + Sync + Send, + > TagManager +{ + pub fn new(current: HashedValue>) -> Self { + Self { + current, + added: Vec::new(), + removed: Vec::new(), + last: LastTag::None, + } + } + + pub fn set(&mut self, tags: Vec) { + if matches!(self.last, LastTag::None) { + self.added.clear(); + self.removed.clear(); + + for tag in &tags { + if !self.current.inner.contains(tag) { + self.added.push(tag.clone()); + } + } + + for tag in &self.current.inner { + if !tags.contains(tag) { + self.removed.push(tag.clone()); + } + } + + self.current.inner = tags; + self.last = LastTag::Set; + } + } + + pub fn update(&mut self, tag: T, add: bool) { + if matches!(self.last, LastTag::None | LastTag::Update) { + if add { + if !self.current.inner.contains(&tag) { + self.added.push(tag.clone()); + self.current.inner.push(tag); + } + } else if let Some(index) = self.current.inner.iter().position(|t| t == &tag) { + self.current.inner.swap_remove(index); + self.removed.push(tag); + } + self.last = LastTag::Update; + } + } + + pub fn added(&self) -> &[T] { + &self.added + } + + pub fn removed(&self) -> &[T] { + &self.removed + } + + pub fn current(&self) -> &[T] { + &self.current.inner + } + + pub fn changed_tags(&self) -> impl Iterator { + self.added.iter().chain(self.removed.iter()) + } + + pub fn has_tags(&self) -> bool { + !self.current.inner.is_empty() + } + + pub fn has_changes(&self) -> bool { + !self.added.is_empty() || !self.removed.is_empty() + } + + pub fn update_batch(self, batch: &mut BatchBuilder, property: Property) { + let property = u8::from(property); + + batch + .assert_value(property, &self.current) + .value(property, self.current.inner, F_VALUE); + for added in self.added { + batch.value(property, added, F_BITMAP); + } + for removed in self.removed { + batch.value(property, removed, F_BITMAP | F_CLEAR); + } + } +} diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 53657e07..a87e30e8 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -1,13 +1,15 @@ use api::session::BaseCapabilities; use jmap_proto::{ error::method::MethodError, + method::set::{SetRequest, SetResponse}, + request::reference::MaybeReference, types::{collection::Collection, property::Property}, }; use store::{ - fts::Language, roaring::RoaringBitmap, write::BitmapFamily, BitmapKey, Deserialize, Serialize, - Store, ValueKey, + ahash::AHashMap, fts::Language, roaring::RoaringBitmap, write::BitmapFamily, BitmapKey, + Deserialize, Serialize, Store, ValueKey, }; -use utils::UnwrapFailure; +use utils::{map::vec_map::VecMap, UnwrapFailure}; pub mod api; pub mod blob; @@ -176,4 +178,42 @@ impl JMAP { } } } + + pub async fn prepare_set_response( + &self, + request: &SetRequest, + collection: Collection, + ) -> Result { + let n_create = request.create.as_ref().map_or(0, |objs| objs.len()); + let n_update = request.update.as_ref().map_or(0, |objs| objs.len()); + let n_destroy = request.destroy.as_ref().map_or(0, |objs| { + if let MaybeReference::Value(ids) = objs { + ids.len() + } else { + 0 + } + }); + if n_create + n_update + n_destroy > self.config.set_max_objects { + return Err(MethodError::RequestTooLarge); + } + let old_state = self + .assert_state( + request.account_id.document_id(), + collection, + &request.if_in_state, + ) + .await?; + + Ok(SetResponse { + account_id: request.account_id.into(), + new_state: old_state.clone().into(), + old_state: old_state.into(), + created: AHashMap::with_capacity(n_create), + updated: VecMap::with_capacity(n_update), + destroyed: Vec::with_capacity(n_destroy), + not_created: VecMap::new(), + not_updated: VecMap::new(), + not_destroyed: VecMap::new(), + }) + } } diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 2bb4703b..5331e402 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -242,14 +242,7 @@ impl Store { } } else { // Find the next available id - let mut key = BitmapKey { - account_id, - collection, - family: BM_DOCUMENT_IDS, - field: u8::MAX, - key: b"", - block_num: 0, - }; + let mut key = BitmapKey::document_ids(account_id, collection); let begin = key.serialize(); key.block_num = u32::MAX; let end = key.serialize(); diff --git a/crates/store/src/backend/sqlite/id_assign.rs b/crates/store/src/backend/sqlite/id_assign.rs index 69e78966..38b00160 100644 --- a/crates/store/src/backend/sqlite/id_assign.rs +++ b/crates/store/src/backend/sqlite/id_assign.rs @@ -108,12 +108,9 @@ impl Store { unreachable!() } - pub async fn assign_change_id( - &self, - account_id: u32, - collection: impl Into, - ) -> crate::Result { - let key = IdCacheKey::new(account_id, collection.into()); + pub async fn assign_change_id(&self, account_id: u32) -> crate::Result { + let collection = u8::MAX; + let key = IdCacheKey::new(account_id, collection); for _ in 0..2 { if let Some(assigner) = self.id_assigner.lock().get_mut(&key) { return Ok(assigner.assign_change_id()); diff --git a/crates/store/src/fts/builder.rs b/crates/store/src/fts/builder.rs index b583be7d..6a8e8c04 100644 --- a/crates/store/src/fts/builder.rs +++ b/crates/store/src/fts/builder.rs @@ -72,7 +72,7 @@ impl<'x> FtsIndexBuilder<'x> { } impl<'x> IntoOperations for FtsIndexBuilder<'x> { - fn build(self, batch: &mut BatchBuilder) -> crate::Result<()> { + fn build(self, batch: &mut BatchBuilder) { let default_language = self .detect .most_frequent_language() @@ -122,7 +122,5 @@ impl<'x> IntoOperations for FtsIndexBuilder<'x> { .ops .push(Operation::hash(&token, HASH_EXACT, field, true)); } - - Ok(()) } } diff --git a/crates/store/src/write/assert.rs b/crates/store/src/write/assert.rs new file mode 100644 index 00000000..d367b096 --- /dev/null +++ b/crates/store/src/write/assert.rs @@ -0,0 +1,65 @@ +use crate::Deserialize; + +#[derive(Debug)] +pub struct HashedValue { + pub hash: u64, + pub inner: T, +} + +#[derive(Debug)] +pub enum AssertValue { + U32(u32), + U64(u64), + Hash(u64), +} + +pub trait ToAssertValue { + fn to_assert_value(&self) -> AssertValue; +} + +impl ToAssertValue for u64 { + fn to_assert_value(&self) -> AssertValue { + AssertValue::U64(*self) + } +} + +impl ToAssertValue for u32 { + fn to_assert_value(&self) -> AssertValue { + AssertValue::U32(*self) + } +} + +impl ToAssertValue for HashedValue { + fn to_assert_value(&self) -> AssertValue { + AssertValue::Hash(self.hash) + } +} + +impl ToAssertValue for &HashedValue { + fn to_assert_value(&self) -> AssertValue { + AssertValue::Hash(self.hash) + } +} + +impl AssertValue { + pub fn matches(&self, bytes: &[u8]) -> bool { + match self { + AssertValue::U32(v) => { + bytes.len() == std::mem::size_of::() && u32::deserialize(bytes).unwrap() == *v + } + AssertValue::U64(v) => { + bytes.len() == std::mem::size_of::() && u64::deserialize(bytes).unwrap() == *v + } + AssertValue::Hash(v) => xxhash_rust::xxh3::xxh3_64(bytes) == *v, + } + } +} + +impl Deserialize for HashedValue { + fn deserialize(bytes: &[u8]) -> crate::Result { + Ok(HashedValue { + hash: xxhash_rust::xxh3::xxh3_64(bytes), + inner: T::deserialize(bytes)?, + }) + } +} diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 5b9d9b6a..373ddd46 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -1,8 +1,8 @@ use crate::BM_DOCUMENT_IDS; use super::{ - Batch, BatchBuilder, BitmapFamily, HasFlag, IntoOperations, Operation, Serialize, - ToAssertValue, ToBitmaps, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, + assert::ToAssertValue, Batch, BatchBuilder, BitmapFamily, HasFlag, IntoOperations, Operation, + Serialize, ToBitmaps, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE, }; impl BatchBuilder { @@ -124,7 +124,7 @@ impl BatchBuilder { self } - pub fn custom(&mut self, value: impl IntoOperations) -> crate::Result<()> { + pub fn custom(&mut self, value: impl IntoOperations) { value.build(self) } diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index 7d09ca25..486fabf4 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -68,7 +68,7 @@ impl ChangeLogBuilder { } impl IntoOperations for ChangeLogBuilder { - fn build(self, batch: &mut super::BatchBuilder) -> crate::Result<()> { + fn build(self, batch: &mut super::BatchBuilder) { for (collection, changes) in self.changes { batch.ops.push(Operation::Log { change_id: self.change_id, @@ -76,8 +76,6 @@ impl IntoOperations for ChangeLogBuilder { set: changes.serialize(), }); } - - Ok(()) } } diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 80d0a129..d27118fe 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -7,6 +7,9 @@ use crate::{ Deserialize, Serialize, BM_TAG, HASH_EXACT, TAG_ID, TAG_STATIC, }; +use self::assert::AssertValue; + +pub mod assert; pub mod batch; pub mod key; pub mod log; @@ -68,13 +71,6 @@ pub enum Operation { }, } -#[derive(Debug)] -pub enum AssertValue { - U32(u32), - U64(u64), - Hash(u64), -} - impl Serialize for u32 { fn serialize(self) -> Vec { self.to_be_bytes().to_vec() @@ -311,57 +307,14 @@ impl Serialize for () { } } +impl ToBitmaps for () { + fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { + unreachable!() + } +} + pub trait IntoOperations { - fn build(self, batch: &mut BatchBuilder) -> crate::Result<()>; -} - -pub trait ToAssertValue { - fn to_assert_value(&self) -> AssertValue; -} - -impl ToAssertValue for u64 { - fn to_assert_value(&self) -> AssertValue { - AssertValue::U64(*self) - } -} - -impl ToAssertValue for u32 { - fn to_assert_value(&self) -> AssertValue { - AssertValue::U32(*self) - } -} - -impl ToAssertValue for &[u8] { - fn to_assert_value(&self) -> AssertValue { - AssertValue::Hash(xxhash_rust::xxh3::xxh3_64(self)) - } -} - -impl ToAssertValue for Vec { - fn to_assert_value(&self) -> AssertValue { - self.as_slice().to_assert_value() - } -} - -impl AssertValue { - pub fn matches(&self, bytes: &[u8]) -> bool { - match self { - AssertValue::U32(v) => { - let coco = "fd"; - let a = u32::deserialize(bytes).unwrap(); - let b = *v; - if a != b { - println!("has {} != expected {}", a, b); - } - a == b - //bytes.len() == std::mem::size_of::() && u32::deserialize(bytes).unwrap() == *v - } - AssertValue::U64(v) => { - bytes.len() == std::mem::size_of::() && u64::deserialize(bytes).unwrap() == *v - } - AssertValue::Hash(v) => xxhash_rust::xxh3::xxh3_64(bytes) == *v, - } - } + fn build(self, batch: &mut BatchBuilder); } #[inline(always)] diff --git a/tests/Cargo.toml b/tests/Cargo.toml index c6bfab8a..117646ee 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,10 +6,11 @@ resolver = "2" [dependencies] store = { path = "../crates/store", features = ["test_mode"] } -jmap = { path = "../crates/jmap" } +jmap = { path = "../crates/jmap", features = ["test_mode"] } jmap_proto = { path = "../crates/jmap-proto" } utils = { path = "../crates/utils" } -jmap-client = { git = "https://github.com/stalwartlabs/jmap-client", features = ["websockets", "debug", "async"] } +#jmap-client = { git = "https://github.com/stalwartlabs/jmap-client", features = ["websockets", "debug", "async"] } +jmap-client = { path = "/home/vagrant/code/jmap-client", features = ["websockets", "debug", "async"] } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "serde_support", "ludicrous_mode"] } tokio = { version = "1.23", features = ["full"] } csv = "1.1" diff --git a/tests/src/jmap/email_set.rs b/tests/src/jmap/email_set.rs new file mode 100644 index 00000000..e312eba4 --- /dev/null +++ b/tests/src/jmap/email_set.rs @@ -0,0 +1,341 @@ +/* + * Copyright (c) 2020-2022, Stalwart Labs Ltd. + * + * This file is part of the Stalwart JMAP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{fs, path::PathBuf, sync::Arc}; + +use jmap::JMAP; +use jmap_client::{ + client::Client, + core::set::{SetError, SetErrorType}, + email::{self, Email}, + mailbox::Role, + Error, Set, +}; +use jmap_proto::types::id::Id; + +use super::{find_values, replace_blob_ids, replace_boundaries, replace_values}; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Set tests..."); + + let mailbox_id = "a"; + let coco = "fix"; + /*client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("JMAP Set", None::, Role::None) + .await + .unwrap() + .take_id();*/ + + create(client, &mailbox_id).await; + //update(client, &mailbox_id).await; + + let coco = "fd"; + //client.mailbox_destroy(&mailbox_id, true).await.unwrap(); + + //server.store.assert_is_empty(); +} + +async fn create(client: &mut Client, mailbox_id: &str) { + let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_dir.push("resources"); + test_dir.push("jmap_mail_set"); + + for file_name in fs::read_dir(&test_dir).unwrap() { + let mut file_name = file_name.as_ref().unwrap().path(); + if file_name.extension().map_or(true, |e| e != "json") { + continue; + } + println!("Creating email from {:?}", file_name); + + // Upload blobs + let mut json_request = String::from_utf8(fs::read(&file_name).unwrap()).unwrap(); + let blob_values = find_values(&json_request, "\"blobId\""); + if !blob_values.is_empty() { + let mut blob_ids = Vec::with_capacity(blob_values.len()); + for blob_value in &blob_values { + let blob_value = blob_value.replace("\\r", "\r").replace("\\n", "\n"); + blob_ids.push( + client + .upload(None, blob_value.into_bytes(), None) + .await + .unwrap() + .take_blob_id(), + ); + } + json_request = replace_values(json_request, &blob_values, &blob_ids); + } + + // Create message and obtain its blobId + let mut request = client.build(); + let mut create_item = + serde_json::from_slice::>(json_request.as_bytes()).unwrap(); + create_item.mailbox_ids([mailbox_id]); + let create_id = request.set_email().create_item(create_item); + let created_email = request + .send_set_email() + .await + .unwrap() + .created(&create_id) + .unwrap(); + + // Download raw message + let raw_message = client + .download(created_email.blob_id().unwrap()) + .await + .unwrap(); + + // Fetch message + let mut request = client.build(); + request + .get_email() + .ids([created_email.id().unwrap()]) + .properties([ + email::Property::Id, + email::Property::BlobId, + email::Property::ThreadId, + email::Property::MailboxIds, + email::Property::Keywords, + email::Property::ReceivedAt, + email::Property::MessageId, + email::Property::InReplyTo, + email::Property::References, + email::Property::Sender, + email::Property::From, + email::Property::To, + email::Property::Cc, + email::Property::Bcc, + email::Property::ReplyTo, + email::Property::Subject, + email::Property::SentAt, + email::Property::HasAttachment, + email::Property::Preview, + email::Property::BodyValues, + email::Property::TextBody, + email::Property::HtmlBody, + email::Property::Attachments, + email::Property::BodyStructure, + ]) + .arguments() + .body_properties([ + email::BodyProperty::PartId, + email::BodyProperty::BlobId, + email::BodyProperty::Size, + email::BodyProperty::Name, + email::BodyProperty::Type, + email::BodyProperty::Charset, + email::BodyProperty::Headers, + email::BodyProperty::Disposition, + email::BodyProperty::Cid, + email::BodyProperty::Language, + email::BodyProperty::Location, + ]) + .fetch_all_body_values(true) + .max_body_value_bytes(100); + let email = request + .send_get_email() + .await + .unwrap() + .pop() + .unwrap() + .into_test(); + + // Compare raw message + file_name.set_extension("eml"); + let result = replace_boundaries(String::from_utf8(raw_message).unwrap()); + + if fs::read(&file_name).unwrap() != result.as_bytes() { + file_name.set_extension("eml_failed"); + fs::write(&file_name, result.as_bytes()).unwrap(); + panic!("Test failed, output saved to {}", file_name.display()); + } + + // Compare response + file_name.set_extension("jmap"); + let result = replace_blob_ids(replace_boundaries( + serde_json::to_string_pretty(&email).unwrap(), + )); + if fs::read(&file_name).unwrap() != result.as_bytes() { + file_name.set_extension("jmap_failed"); + fs::write(&file_name, result.as_bytes()).unwrap(); + panic!("Test failed, output saved to {}", file_name.display()); + } + } +} + +async fn update(client: &mut Client, root_mailbox_id: &str) { + // Obtain all messageIds previously created + let mailbox = client + .email_query( + email::query::Filter::in_mailbox(root_mailbox_id).into(), + None::>, + ) + .await + .unwrap(); + + // Create two test mailboxes + let test_mailbox1_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("Test 1", None::, Role::None) + .await + .unwrap() + .take_id(); + let test_mailbox2_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("Test 2", None::, Role::None) + .await + .unwrap() + .take_id(); + + // Set keywords and mailboxes + let mut request = client.build(); + request + .set_email() + .update(mailbox.id(0)) + .mailbox_ids([&test_mailbox1_id, &test_mailbox2_id]) + .keywords(["test1", "test2"]); + request + .send_set_email() + .await + .unwrap() + .updated(mailbox.id(0)) + .unwrap(); + assert_email_properties( + client, + mailbox.id(0), + &[&test_mailbox1_id, &test_mailbox2_id], + &["test1", "test2"], + ) + .await; + + // Patch keywords and mailboxes + let mut request = client.build(); + request + .set_email() + .update(mailbox.id(0)) + .mailbox_id(&test_mailbox1_id, false) + .keyword("test1", true) + .keyword("test2", false) + .keyword("test3", true); + request + .send_set_email() + .await + .unwrap() + .updated(mailbox.id(0)) + .unwrap(); + assert_email_properties( + client, + mailbox.id(0), + &[&test_mailbox2_id], + &["test1", "test3"], + ) + .await; + + // Orphan messages should not be permitted + let mut request = client.build(); + request + .set_email() + .update(mailbox.id(0)) + .mailbox_id(&test_mailbox2_id, false); + assert!(matches!( + request + .send_set_email() + .await + .unwrap() + .updated(mailbox.id(0)), + Err(Error::Set(SetError { + type_: SetErrorType::InvalidProperties, + .. + })) + )); + + // Updating and destroying the same item should not be allowed + let mut request = client.build(); + let set_email_request = request.set_email(); + set_email_request + .update(mailbox.id(0)) + .mailbox_id(&test_mailbox2_id, false); + set_email_request.destroy([mailbox.id(0)]); + assert!(matches!( + request + .send_set_email() + .await + .unwrap() + .updated(mailbox.id(0)), + Err(Error::Set(SetError { + type_: SetErrorType::WillDestroy, + .. + })) + )); + + // Delete some messages + let mut request = client.build(); + request.set_email().destroy([mailbox.id(1), mailbox.id(2)]); + assert_eq!( + request + .send_set_email() + .await + .unwrap() + .destroyed_ids() + .unwrap() + .count(), + 2 + ); + let mut request = client.build(); + request.get_email().ids([mailbox.id(1), mailbox.id(2)]); + assert_eq!(request.send_get_email().await.unwrap().not_found().len(), 2); + + // Destroy test mailboxes + client + .mailbox_destroy(&test_mailbox1_id, true) + .await + .unwrap(); + client + .mailbox_destroy(&test_mailbox2_id, true) + .await + .unwrap(); +} + +pub async fn assert_email_properties( + client: &mut Client, + message_id: &str, + mailbox_ids: &[&str], + keywords: &[&str], +) { + let result = client + .email_get( + message_id, + [email::Property::MailboxIds, email::Property::Keywords].into(), + ) + .await + .unwrap() + .unwrap(); + + let mut mailbox_ids_ = result.mailbox_ids().to_vec(); + let mut keywords_ = result.keywords().to_vec(); + mailbox_ids_.sort_unstable(); + keywords_.sort_unstable(); + + assert_eq!(mailbox_ids_, mailbox_ids); + assert_eq!(keywords_, keywords); +} diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 1e512061..389eea16 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeSet, sync::Arc}; +use std::{sync::Arc, time::Duration}; use jmap::{api::SessionManager, JMAP}; use jmap_client::client::{Client, Credentials}; @@ -9,6 +9,7 @@ use crate::{add_test_certs, store::TempDir}; pub mod email_get; pub mod email_query; +pub mod email_set; pub mod thread_get; pub mod thread_merge; @@ -50,9 +51,10 @@ pub async fn jmap_tests() { let delete = true; let mut params = init_jmap_tests(delete).await; //email_get::test(params.server.clone(), &mut params.client).await; + email_set::test(params.server.clone(), &mut params.client).await; //email_query::test(params.server.clone(), &mut params.client, delete).await; //thread_get::test(params.server.clone(), &mut params.client).await; - thread_merge::test(params.server.clone(), &mut params.client).await; + //thread_merge::test(params.server.clone(), &mut params.client).await; if delete { params.temp_dir.delete(); } @@ -84,7 +86,7 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { // Create client let mut client = Client::new() .credentials(Credentials::bearer("DO_NOT_ATTEMPT_THIS_AT_HOME")) - .timeout(360000) + .timeout(Duration::from_secs(60)) .accept_invalid_certs(true) .connect("https://127.0.0.1:8899") .await @@ -143,7 +145,6 @@ pub fn replace_boundaries(string: String) -> String { pub fn replace_blob_ids(string: String) -> String { let values = find_values(&string, "blobId\":"); if !values.is_empty() { - //let values = BTreeSet::from_iter(values).into_iter().collect::>(); replace_values( string, &values, diff --git a/tests/src/store/assign_id.rs b/tests/src/store/assign_id.rs index 0031e540..a986d7c3 100644 --- a/tests/src/store/assign_id.rs +++ b/tests/src/store/assign_id.rs @@ -20,7 +20,7 @@ async fn test_1(db: Arc) { for id in 0..100 { handles.push({ let db = db.clone(); - tokio::spawn(async move { db.assign_change_id(0, 0).await }) + tokio::spawn(async move { db.assign_change_id(0).await }) }); expected_ids.insert(id); } diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index 71b4e157..24a443a1 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -163,7 +163,7 @@ pub async fn test(db: Arc, do_insert: bool) { } } - builder.custom(fts_builder).unwrap(); + builder.custom(fts_builder); documents.lock().unwrap().push(builder.build()); }); }