From 0568456d29ec5b9f55cd802bbaf3bf8017c9d4d0 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sun, 13 Apr 2025 08:40:40 +0200 Subject: [PATCH] Improved cache memory layout --- crates/common/src/config/inner.rs | 6 +- crates/common/src/core.rs | 21 +- crates/common/src/lib.rs | 39 ++- crates/common/src/storage/index.rs | 10 +- crates/email/src/mailbox/cache.rs | 153 +++++++---- crates/email/src/mailbox/destroy.rs | 6 +- crates/email/src/mailbox/manage.rs | 6 +- crates/email/src/message/cache.rs | 301 +++++++++++++++------- crates/email/src/message/copy.rs | 2 +- crates/email/src/message/delete.rs | 6 +- crates/email/src/message/index.rs | 8 +- crates/email/src/message/ingest.rs | 14 +- crates/email/src/sieve/ingest.rs | 21 +- crates/groupware/src/contact/index.rs | 14 +- crates/imap/src/core/mailbox.rs | 28 +- crates/imap/src/core/message.rs | 6 +- crates/imap/src/op/expunge.rs | 4 +- crates/imap/src/op/fetch.rs | 27 +- crates/imap/src/op/search.rs | 91 +++---- crates/imap/src/op/status.rs | 6 +- crates/imap/src/op/store.rs | 7 +- crates/imap/src/op/thread.rs | 8 +- crates/jmap-proto/src/method/copy.rs | 5 +- crates/jmap-proto/src/method/import.rs | 5 +- crates/jmap-proto/src/types/collection.rs | 29 +-- crates/jmap-proto/src/types/keyword.rs | 18 ++ crates/jmap-proto/src/types/type_state.rs | 10 +- crates/jmap/src/api/request.rs | 11 - crates/jmap/src/blob/download.rs | 2 +- crates/jmap/src/changes/get.rs | 115 ++++++--- crates/jmap/src/email/copy.rs | 14 +- crates/jmap/src/email/get.rs | 24 +- crates/jmap/src/email/import.rs | 19 +- crates/jmap/src/email/query.rs | 38 +-- crates/jmap/src/email/set.rs | 8 +- crates/jmap/src/email/snippet.rs | 2 +- crates/jmap/src/mailbox/get.rs | 10 +- crates/jmap/src/mailbox/query.rs | 38 ++- crates/jmap/src/mailbox/set.rs | 4 +- crates/jmap/src/thread/get.rs | 6 +- crates/pop3/src/mailbox.rs | 13 +- crates/store/src/query/log.rs | 27 +- crates/store/src/write/batch.rs | 10 +- crates/store/src/write/log.rs | 38 +-- crates/utils/src/map/bitmap.rs | 1 + tests/src/jmap/delivery.rs | 2 +- tests/src/jmap/email_changes.rs | 2 +- tests/src/jmap/email_query.rs | 57 ++-- tests/src/jmap/mod.rs | 8 +- tests/src/jmap/purge.rs | 2 +- tests/src/jmap/stress_test.rs | 4 +- 51 files changed, 716 insertions(+), 590 deletions(-) diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 1c22fa1f..82f5a505 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -23,7 +23,7 @@ use utils::{ }; use crate::{ - CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MessageItemCache, + CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MailboxStoreCache, MessageStoreCache, MessageUidCache, TlsConnectors, auth::{AccessToken, roles::RolePermissions}, config::smtp::resolver::{Policy, Tlsa}, @@ -112,7 +112,7 @@ impl Caches { "mailbox", MB_10, (std::mem::size_of::() - + std::mem::size_of::>>() + + std::mem::size_of::>() + (15 * (std::mem::size_of::() + 60))) as u64, ), messages: Cache::from_config( @@ -120,7 +120,7 @@ impl Caches { "message", MB_10, (std::mem::size_of::() - + std::mem::size_of::>>() + + std::mem::size_of::>() + (1024 * std::mem::size_of::())) as u64, ), dav: Cache::from_config( diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 46d078ab..11e1fc63 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -531,11 +531,18 @@ impl Server { })?; for collection in [ - Collection::Email, - Collection::Mailbox, - Collection::Thread, - Collection::Identity, - Collection::EmailSubmission, + Collection::Email.into(), + Collection::Mailbox.into(), + Collection::Mailbox.as_child_update(), + Collection::Thread.into(), + Collection::Identity.into(), + Collection::EmailSubmission.into(), + Collection::SieveScript.into(), + Collection::FileNode.into(), + Collection::AddressBook.into(), + Collection::ContactCard.into(), + Collection::Calendar.into(), + Collection::CalendarEvent.into(), ] { self.core .storage @@ -543,12 +550,12 @@ impl Server { .delete_range( LogKey { account_id, - collection: collection.into(), + collection, change_id: 0, }, LogKey { account_id, - collection: collection.into(), + collection, change_id: reference_cid, }, ) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 67a3f1de..a2bbbdee 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -34,7 +34,7 @@ use config::{ }; use ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; -use jmap_proto::types::{keyword::Keyword, value::AclGrant}; +use jmap_proto::types::value::AclGrant; use listener::{asn::AsnGeoLookupData, blocked::Security, tls::AcmeProviders}; use mail_auth::{MX, Txt}; @@ -145,8 +145,8 @@ pub struct Caches { pub http_auth: Cache, pub permissions: Cache>, - pub messages: Cache>>, - pub mailboxes: Cache>>, + pub messages: Cache>, + pub mailboxes: Cache>, pub dav: Cache>, pub bayes: CacheWithTtl, @@ -165,17 +165,29 @@ pub struct Caches { pub struct CacheSwap(pub Arc>); #[derive(Debug, Clone)] -pub struct MessageStoreCache { +pub struct MailboxStoreCache { pub change_id: u64, - pub items: AHashMap, + pub index: AHashMap, + pub items: Vec, pub update_lock: Arc, pub size: u64, } #[derive(Debug, Clone)] -pub struct MessageItemCache { +pub struct MessageStoreCache { + pub change_id: u64, + pub items: Vec, + pub index: AHashMap, + pub keywords: Vec, + pub update_lock: Arc, + pub size: u64, +} + +#[derive(Debug, Clone)] +pub struct MessageCache { + pub document_id: u32, pub mailboxes: TinyVec<[MessageUidCache; 2]>, - pub keywords: TinyVec<[Keyword; 2]>, + pub keywords: u128, pub thread_id: u32, pub change_id: u64, } @@ -188,6 +200,7 @@ pub struct MessageUidCache { #[derive(Debug, Clone)] pub struct MailboxCache { + pub document_id: u32, pub name: CompactString, pub path: CompactString, pub role: SpecialUse, @@ -272,7 +285,13 @@ impl CacheItemWeight for CacheSwap { } } -impl CacheItemWeight for MessageStoreCache { +impl CacheItemWeight for MessageStoreCache { + fn weight(&self) -> u64 { + self.size + } +} + +impl CacheItemWeight for MailboxStoreCache { fn weight(&self) -> u64 { self.size } @@ -515,7 +534,7 @@ impl std::borrow::Borrow for DavResource { } } -impl MessageStoreCache { +impl MessageStoreCache { pub fn assign_thread_id(&self, thread_name: &[u8], message_id: &[u8]) -> u32 { let mut bytes = Vec::with_capacity(thread_name.len() + message_id.len()); bytes.extend_from_slice(thread_name); @@ -529,7 +548,7 @@ impl MessageStoreCache { // Naive pass, assume hash is unique let mut threads_ids = RoaringBitmap::new(); let mut is_unique_hash = true; - for item in self.items.values() { + for item in self.items.iter() { if is_unique_hash && item.thread_id != hash { is_unique_hash = false; } diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 67dfa18a..fb92877d 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -5,7 +5,7 @@ */ use ahash::AHashSet; -use jmap_proto::types::{collection::Collection, property::Property, value::AclGrant}; +use jmap_proto::types::{property::Property, value::AclGrant}; use rkyv::{ option::ArchivedOption, primitive::{ArchivedU32, ArchivedU64}, @@ -44,7 +44,7 @@ pub enum IndexValue<'x> { prefix: Option, }, LogParent { - collection: Collection, + collection: u8, ids: Vec, }, Acl { @@ -380,7 +380,7 @@ fn build_index(batch: &mut BatchBuilder, item: IndexValue<'_>, tenant_id: Option } IndexValue::LogParent { collection, ids } => { for parent_id in ids { - batch.log_child_update(collection, parent_id); + batch.log_parent_update(collection, parent_id); } } } @@ -526,12 +526,12 @@ fn merge_index( ) => { for parent_id in &old_ids { if !new_ids.contains(parent_id) { - batch.log_child_update(collection, *parent_id); + batch.log_parent_update(collection, *parent_id); } } for parent_id in new_ids { if !old_ids.contains(&parent_id) { - batch.log_child_update(collection, parent_id); + batch.log_parent_update(collection, parent_id); } } } diff --git a/crates/email/src/mailbox/cache.rs b/crates/email/src/mailbox/cache.rs index c8d9e1d6..60d30038 100644 --- a/crates/email/src/mailbox/cache.rs +++ b/crates/email/src/mailbox/cache.rs @@ -7,14 +7,14 @@ use std::sync::Arc; use common::{ - CacheSwap, MailboxCache, MessageStoreCache, Server, auth::AccessToken, + CacheSwap, MailboxCache, MailboxStoreCache, Server, auth::AccessToken, config::jmap::settings::SpecialUse, sharing::EffectiveAcl, }; use compact_str::CompactString; use jmap_proto::types::{acl::Acl, collection::Collection, value::AclGrant}; use std::future::Future; use store::{ - ahash::AHashMap, + ahash::{AHashMap, AHashSet}, query::log::{Change, Query}, roaring::RoaringBitmap, }; @@ -28,14 +28,11 @@ pub trait MessageMailboxCache: Sync + Send { fn get_cached_mailboxes( &self, account_id: u32, - ) -> impl Future>>> + Send; + ) -> impl Future>> + Send; } impl MessageMailboxCache for Server { - async fn get_cached_mailboxes( - &self, - account_id: u32, - ) -> trc::Result>> { + async fn get_cached_mailboxes(&self, account_id: u32) -> trc::Result> { let cache_ = match self .inner .cache @@ -98,9 +95,14 @@ impl MessageMailboxCache for Server { return Ok(cache); } - let mut has_changes = false; - let mut cache = cache.as_ref().clone(); - cache.change_id = changes.to_change_id; + let mut changed_ids = AHashSet::with_capacity(changes.changes.len()); + let mut new_cache = MailboxStoreCache { + items: Vec::with_capacity(cache.items.len()), + index: AHashMap::with_capacity(cache.items.len()), + size: 0, + change_id: changes.to_change_id, + update_lock: cache.update_lock.clone(), + }; for change in changes.changes { match change { @@ -111,24 +113,30 @@ impl MessageMailboxCache for Server { .await .caused_by(trc::location!())? { - insert_item(&mut cache, document_id, archive.unarchive::()?); - has_changes = true; + insert_item(&mut new_cache, document_id, archive.unarchive::()?); + changed_ids.insert(document_id); } } Change::Delete(id) => { - if cache.items.remove(&(id as u32)).is_some() { - has_changes = true; - } + changed_ids.insert(id as u32); } - Change::ChildUpdate(_) => {} } } - if has_changes { - build_tree(&mut cache); + for item in cache.items.iter() { + if !changed_ids.contains(&item.document_id) { + new_cache.insert(item.clone()); + } } - let cache = Arc::new(cache); + build_tree(&mut new_cache); + + if cache.items.len() > new_cache.items.len() { + new_cache.items.shrink_to_fit(); + new_cache.index.shrink_to_fit(); + } + + let cache = Arc::new(new_cache); cache_.update(cache.clone()); Ok(cache) @@ -139,10 +147,11 @@ async fn full_cache_build( server: &Server, account_id: u32, update_lock: Arc, -) -> trc::Result>> { +) -> trc::Result> { // Build cache - let mut cache = MessageStoreCache { - items: AHashMap::with_capacity(16), + let mut cache = MailboxStoreCache { + items: Default::default(), + index: Default::default(), size: 0, change_id: 0, update_lock, @@ -185,13 +194,10 @@ async fn full_cache_build( Ok(Arc::new(cache)) } -fn insert_item( - cache: &mut MessageStoreCache, - document_id: u32, - mailbox: &ArchivedMailbox, -) { +fn insert_item(cache: &mut MailboxStoreCache, document_id: u32, mailbox: &ArchivedMailbox) { let parent_id = mailbox.parent_id.to_native(); let item = MailboxCache { + document_id, name: mailbox.name.as_str().into(), path: "".into(), role: (&mailbox.role).into(), @@ -217,21 +223,21 @@ fn insert_item( .collect(), }; - cache.items.insert(document_id, item); + cache.insert(item); } -fn build_tree(cache: &mut MessageStoreCache) { +fn build_tree(cache: &mut MailboxStoreCache) { cache.size = 0; let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); - for (idx, (&document_id, mailbox)) in cache.items.iter_mut().enumerate() { + for (idx, mailbox) in cache.items.iter_mut().enumerate() { topological_sort.insert( if mailbox.parent_id == u32::MAX { 0 } else { mailbox.parent_id + 1 }, - document_id + 1, + mailbox.document_id + 1, ); mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) { "INBOX".into() @@ -241,35 +247,28 @@ fn build_tree(cache: &mut MessageStoreCache) { mailbox.name.clone() }; - cache.size += (std::mem::size_of::() - + std::mem::size_of::() - + mailbox.name.len() - + mailbox.path.len()) as u64; + cache.size += item_size(mailbox); } for folder_id in topological_sort.into_iterator() { if folder_id != 0 { let folder_id = folder_id - 1; if let Some((path, parent_path)) = cache - .items - .get(&folder_id) + .by_id(&folder_id) .and_then(|folder| { folder .parent_id() .map(|parent_id| (&folder.path, parent_id)) }) .and_then(|(path, parent_id)| { - cache - .items - .get(&parent_id) - .map(|folder| (path, &folder.path)) + cache.by_id(&parent_id).map(|folder| (path, &folder.path)) }) { let mut new_path = CompactString::with_capacity(parent_path.len() + path.len() + 1); new_path.push_str(parent_path.as_str()); new_path.push('/'); new_path.push_str(path.as_str()); - let folder = cache.items.get_mut(&folder_id).unwrap(); + let folder = cache.by_id_mut(&folder_id).unwrap(); folder.path = new_path; } } @@ -277,31 +276,35 @@ fn build_tree(cache: &mut MessageStoreCache) { } pub trait MailboxCacheAccess { - fn by_name(&self, name: &str) -> Option<(&u32, &MailboxCache)>; - fn by_path(&self, name: &str) -> Option<(&u32, &MailboxCache)>; - fn by_role(&self, role: &SpecialUse) -> Option<(&u32, &MailboxCache)>; + fn by_id(&self, id: &u32) -> Option<&MailboxCache>; + fn by_id_mut(&mut self, id: &u32) -> Option<&mut MailboxCache>; + fn insert(&mut self, item: MailboxCache); + fn by_name(&self, name: &str) -> Option<&MailboxCache>; + fn by_path(&self, name: &str) -> Option<&MailboxCache>; + fn by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>; fn shared_mailboxes( &self, access_token: &AccessToken, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap; + fn has_id(&self, id: &u32) -> bool; } -impl MailboxCacheAccess for MessageStoreCache { - fn by_name(&self, name: &str) -> Option<(&u32, &MailboxCache)> { +impl MailboxCacheAccess for MailboxStoreCache { + fn by_name(&self, name: &str) -> Option<&MailboxCache> { self.items .iter() - .find(|(_, m)| m.name.eq_ignore_ascii_case(name)) + .find(|m| m.name.eq_ignore_ascii_case(name)) } - fn by_path(&self, path: &str) -> Option<(&u32, &MailboxCache)> { + fn by_path(&self, path: &str) -> Option<&MailboxCache> { self.items .iter() - .find(|(_, m)| m.path.eq_ignore_ascii_case(path)) + .find(|m| m.path.eq_ignore_ascii_case(path)) } - fn by_role(&self, role: &SpecialUse) -> Option<(&u32, &MailboxCache)> { - self.items.iter().find(|(_, m)| &m.role == role) + fn by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> { + self.items.iter().find(|m| &m.role == role) } fn shared_mailboxes( @@ -314,13 +317,55 @@ impl MailboxCacheAccess for MessageStoreCache { RoaringBitmap::from_iter( self.items .iter() - .filter(|(_, m)| { + .filter(|m| { m.acls .as_slice() .effective_acl(access_token) .contains_all(check_acls) }) - .map(|(id, _)| *id), + .map(|m| m.document_id), ) } + + fn by_id(&self, id: &u32) -> Option<&MailboxCache> { + self.index + .get(id) + .and_then(|idx| self.items.get(*idx as usize)) + } + + fn by_id_mut(&mut self, id: &u32) -> Option<&mut MailboxCache> { + self.index + .get(id) + .and_then(|idx| self.items.get_mut(*idx as usize)) + } + + fn insert(&mut self, item: MailboxCache) { + let id = item.document_id; + if let Some(idx) = self.index.get(&id) { + self.items[*idx as usize] = item; + } else { + let idx = self.items.len() as u32; + self.items.push(item); + self.index.insert(id, idx); + } + } + + fn has_id(&self, id: &u32) -> bool { + self.index.contains_key(id) + } +} + +#[inline(always)] +fn item_size(item: &MailboxCache) -> u64 { + (std::mem::size_of::() + + (if item.name.len() > std::mem::size_of::() { + item.name.len() + } else { + 0 + }) + + (if item.path.len() > std::mem::size_of::() { + item.path.len() + } else { + 0 + })) as u64 } diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 0701c979..3201b00e 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -16,7 +16,7 @@ use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use crate::message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheAccess, MessageCacheFetch}, delete::EmailDeletion, metadata::MessageData, }; @@ -66,7 +66,7 @@ impl MailboxDestroy for Server { .await? .items .iter() - .any(|(_, m)| m.parent_id == document_id) + .any(|item| item.parent_id == document_id) { return Ok(Err(SetError::new(SetErrorType::MailboxHasChild) .with_description("Mailbox has at least one children."))); @@ -82,7 +82,7 @@ impl MailboxDestroy for Server { .await .caused_by(trc::location!())? .in_mailbox(document_id) - .map(|(id, _)| id), + .map(|m| m.document_id), ); if !message_ids.is_empty() { diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index f39c9578..22623dcd 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -101,12 +101,12 @@ impl MailboxFnc for Server { } } - if let Some((document_id, _)) = folders + if let Some(item) = folders .items .iter() - .find(|(_, item)| item.path.to_lowercase() == found_path) + .find(|item| item.path.to_lowercase() == found_path) { - next_parent_id = *document_id + 1; + next_parent_id = item.document_id + 1; } else { create_paths.push(name.to_string()); create_paths.extend(path.map(|v| v.to_string())); diff --git a/crates/email/src/message/cache.rs b/crates/email/src/message/cache.rs index 8a4a825c..3cd2e3a3 100644 --- a/crates/email/src/message/cache.rs +++ b/crates/email/src/message/cache.rs @@ -4,16 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - +use super::metadata::{ArchivedMessageData, MessageData}; use common::{ - CacheSwap, MailboxCache, MessageItemCache, MessageStoreCache, MessageUidCache, Server, + CacheSwap, MailboxStoreCache, MessageCache, MessageStoreCache, MessageUidCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; -use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; -use std::future::Future; +use compact_str::CompactString; +use jmap_proto::types::{ + acl::Acl, + collection::Collection, + keyword::{Keyword, OTHER}, +}; +use std::sync::Arc; +use std::{collections::hash_map::Entry, future::Future}; use store::{ - ahash::{AHashMap, AHashSet}, + ahash::AHashMap, query::log::{Change, Query}, roaring::RoaringBitmap, }; @@ -21,20 +26,15 @@ use tokio::sync::Semaphore; use trc::AddContext; use utils::map::bitmap::Bitmap; -use super::metadata::{ArchivedMessageData, MessageData}; - -pub trait MessageCache: Sync + Send { +pub trait MessageCacheFetch: Sync + Send { fn get_cached_messages( &self, account_id: u32, - ) -> impl Future>>> + Send; + ) -> impl Future>> + Send; } -impl MessageCache for Server { - async fn get_cached_messages( - &self, - account_id: u32, - ) -> trc::Result>> { +impl MessageCacheFetch for Server { + async fn get_cached_messages(&self, account_id: u32) -> trc::Result> { let cache_ = match self .inner .cache @@ -91,51 +91,74 @@ impl MessageCache for Server { return Ok(cache); } - let mut cache = cache.as_ref().clone(); - cache.change_id = changes.to_change_id; - let mut delete = AHashSet::with_capacity(changes.changes.len() / 2); - let mut update = AHashMap::with_capacity(changes.changes.len()); + let mut new_cache = MessageStoreCache { + index: AHashMap::with_capacity(cache.items.len()), + items: Vec::with_capacity(cache.items.len()), + size: 0, + change_id: changes.to_change_id, + update_lock: cache.update_lock.clone(), + keywords: cache.keywords.clone(), + }; + let mut changed_ids: AHashMap = AHashMap::with_capacity(changes.changes.len()); for change in changes.changes { match change { - Change::Insert(id) => { - if let Some(item) = cache.items.get_mut(&(id as u32)) { - item.thread_id = (id >> 32) as u32; + Change::Insert(id) => match changed_ids.entry(id as u32) { + Entry::Occupied(mut entry) => { + *entry.get_mut() = true; } - update.insert(id as u32, true); - } - Change::Update(id) | Change::ChildUpdate(id) => { - update.insert(id as u32, false); + Entry::Vacant(entry) => { + entry.insert(true); + } + }, + Change::Update(id) => { + changed_ids.insert(id as u32, true); } Change::Delete(id) => { - delete.insert(id as u32); + match changed_ids.entry(id as u32) { + Entry::Occupied(mut entry) => { + // Thread reassignment + *entry.get_mut() = true; + } + Entry::Vacant(entry) => { + entry.insert(false); + } + } } } } - for document_id in delete { - if update.remove(&document_id).is_none() { - if let Some(item) = cache.items.remove(&document_id) { - cache.size -= (std::mem::size_of::() - + std::mem::size_of::() - + (item.mailboxes.len() * std::mem::size_of::())) - as u64; + for (document_id, is_update) in &changed_ids { + if *is_update { + if let Some(archive) = self + .get_archive(account_id, Collection::Email, *document_id) + .await + .caused_by(trc::location!())? + { + insert_item( + &mut new_cache, + *document_id, + archive.unarchive::()?, + ); } } } - for (document_id, is_insert) in update { - if let Some(archive) = self - .get_archive(account_id, Collection::Email, document_id) - .await - .caused_by(trc::location!())? - { - let message = archive.unarchive::()?; - insert_item(&mut cache, document_id, message, is_insert); + for item in &cache.items { + if !changed_ids.contains_key(&item.document_id) { + new_cache.insert(item.clone()); } } - let cache = Arc::new(cache); + if cache.items.len() > new_cache.items.len() { + new_cache.items.shrink_to_fit(); + new_cache.index.shrink_to_fit(); + } + if cache.keywords.len() > new_cache.keywords.len() { + new_cache.keywords.shrink_to_fit(); + } + + let cache = Arc::new(new_cache); cache_.update(cache.clone()); Ok(cache) @@ -146,10 +169,12 @@ async fn full_cache_build( server: &Server, account_id: u32, update_lock: Arc, -) -> trc::Result>> { +) -> trc::Result> { // Build cache let mut cache = MessageStoreCache { - items: AHashMap::with_capacity(16), + items: Vec::with_capacity(16), + index: AHashMap::with_capacity(16), + keywords: Vec::new(), size: 0, change_id: 0, update_lock, @@ -164,7 +189,7 @@ async fn full_cache_build( let message = archive.unarchive::()?; cache.change_id = std::cmp::max(cache.change_id, message.change_id.to_native()); - insert_item(&mut cache, document_id, message, true); + insert_item(&mut cache, document_id, message); Ok(true) }, @@ -175,13 +200,8 @@ async fn full_cache_build( Ok(Arc::new(cache)) } -fn insert_item( - cache: &mut MessageStoreCache, - document_id: u32, - message: &ArchivedMessageData, - update_size: bool, -) { - let item = MessageItemCache { +fn insert_item(cache: &mut MessageStoreCache, document_id: u32, message: &ArchivedMessageData) { + let mut item = MessageCache { mailboxes: message .mailboxes .iter() @@ -190,75 +210,107 @@ fn insert_item( uid: m.uid.to_native(), }) .collect(), - keywords: message.keywords.iter().map(Into::into).collect(), + keywords: 0, thread_id: message.thread_id.to_native(), change_id: message.change_id.to_native(), + document_id, }; - - if update_size { - cache.size += (std::mem::size_of::() - + std::mem::size_of::() - + (item.mailboxes.len() * std::mem::size_of::())) - as u64; + for keyword in message.keywords.iter() { + match keyword.id() { + Ok(id) => { + item.keywords |= 1 << id; + } + Err(custom) => { + if let Some(idx) = cache.keywords.iter().position(|k| k == custom) { + item.keywords |= 1 << (OTHER + idx); + } else if cache.keywords.len() < (128 - OTHER) { + cache.keywords.push(CompactString::from(custom)); + item.keywords |= 1 << (OTHER + cache.keywords.len() - 1); + } + } + } } - cache.items.insert(document_id, item); + + cache.insert(item); } pub trait MessageCacheAccess { - fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; + fn by_id(&self, id: &u32) -> Option<&MessageCache>; - fn in_thread(&self, thread_id: u32) -> impl Iterator; + fn has_id(&self, id: &u32) -> bool; - fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; + fn by_id_mut(&mut self, id: &u32) -> Option<&mut MessageCache>; + + fn insert(&mut self, item: MessageCache); + + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; + + fn in_thread(&self, thread_id: u32) -> impl Iterator; + + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; + + fn without_keyword(&self, keyword: &Keyword) -> impl Iterator; fn in_mailbox_with_keyword( &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator; + ) -> impl Iterator; fn in_mailbox_without_keyword( &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator; + ) -> impl Iterator; fn document_ids(&self) -> RoaringBitmap; fn shared_messages( &self, access_token: &AccessToken, - mailboxes: &MessageStoreCache, + mailboxes: &MailboxStoreCache, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap; + + fn expand_keywords(&self, message: &MessageCache) -> impl Iterator; + + fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; } -impl MessageCacheAccess for MessageStoreCache { - fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { +impl MessageCacheAccess for MessageStoreCache { + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { self.items .iter() - .filter(move |(_, m)| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) + .filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) } - fn in_thread(&self, thread_id: u32) -> impl Iterator { - self.items - .iter() - .filter(move |(_, m)| m.thread_id == thread_id) + fn in_thread(&self, thread_id: u32) -> impl Iterator { + self.items.iter().filter(move |m| m.thread_id == thread_id) } - fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); self.items .iter() - .filter(move |(_, m)| m.keywords.contains(keyword)) + .filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)) + } + + fn without_keyword(&self, keyword: &Keyword) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.items + .iter() + .filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)) } fn in_mailbox_with_keyword( &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator { - self.items.iter().filter(move |(_, m)| { - m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) && m.keywords.contains(keyword) + ) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.items.iter().filter(move |m| { + m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) + && keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0) }) } @@ -266,34 +318,111 @@ impl MessageCacheAccess for MessageStoreCache { &self, mailbox_id: u32, keyword: &Keyword, - ) -> impl Iterator { - self.items.iter().filter(move |(_, m)| { - m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) && !m.keywords.contains(keyword) + ) -> impl Iterator { + let keyword_id = keyword_to_id(self, keyword); + self.items.iter().filter(move |m| { + m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) + && keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0) }) } fn shared_messages( &self, access_token: &AccessToken, - mailboxes: &MessageStoreCache, + mailboxes: &MailboxStoreCache, check_acls: impl Into> + Sync + Send, ) -> RoaringBitmap { let check_acls = check_acls.into(); let mut shared_messages = RoaringBitmap::new(); - for (mailbox_id, mailbox) in &mailboxes.items { + for mailbox in &mailboxes.items { if mailbox .acls .as_slice() .effective_acl(access_token) .contains_all(check_acls) { - shared_messages.extend(self.in_mailbox(*mailbox_id).map(|(id, _)| *id)); + shared_messages.extend( + self.in_mailbox(mailbox.document_id) + .map(|item| item.document_id), + ); } } shared_messages } fn document_ids(&self) -> RoaringBitmap { - RoaringBitmap::from_iter(self.items.keys()) + RoaringBitmap::from_iter(self.index.keys()) + } + + fn by_id(&self, id: &u32) -> Option<&MessageCache> { + self.index + .get(id) + .and_then(|idx| self.items.get(*idx as usize)) + } + + fn by_id_mut(&mut self, id: &u32) -> Option<&mut MessageCache> { + self.index + .get(id) + .and_then(|idx| self.items.get_mut(*idx as usize)) + } + + fn insert(&mut self, item: MessageCache) { + let id = item.document_id; + if let Some(idx) = self.index.get(&id) { + self.items[*idx as usize] = item; + } else { + self.size += (std::mem::size_of::() + + (std::mem::size_of::() * 2) + + (item.mailboxes.len() * std::mem::size_of::())) + as u64; + + let idx = self.items.len() as u32; + self.items.push(item); + self.index.insert(id, idx); + } + } + + fn has_id(&self, id: &u32) -> bool { + self.index.contains_key(id) + } + + fn expand_keywords(&self, message: &MessageCache) -> impl Iterator { + KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) { + Ok(keyword) => keyword, + Err(id) => Keyword::Other(self.keywords[id - OTHER].clone()), + }) + } + + fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool { + keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0) + } +} + +#[inline] +fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option { + match keyword.id() { + Ok(id) => Some(id), + Err(name) => cache + .keywords + .iter() + .position(|k| k == name) + .map(|idx| (OTHER + idx) as u32), + } +} + +#[derive(Clone, Copy, Debug)] +struct KeywordsIter(u128); + +impl Iterator for KeywordsIter { + type Item = usize; + + fn next(&mut self) -> Option { + if self.0 != 0 { + let item = 127 - self.0.leading_zeros(); + self.0 ^= 1 << item; + Some(item as usize) + } else { + None + } } } diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 589d13e3..8a6abe56 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -22,7 +22,7 @@ use trc::AddContext; use crate::mailbox::UidMailbox; use super::{ - cache::MessageCache, + cache::MessageCacheFetch, index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue}, ingest::{EmailIngest, IngestedEmail, ThreadResult}, metadata::{HeaderName, HeaderValue, MessageData, MessageMetadata}, diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index b1a3fbd1..0c8c167d 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -21,7 +21,7 @@ use store::rand::prelude::SliceRandom; use crate::{ mailbox::*, - message::{cache::MessageCache, metadata::MessageMetadata}, + message::{cache::MessageCacheFetch, metadata::MessageMetadata}, }; use super::metadata::MessageData; @@ -186,14 +186,14 @@ impl EmailDeletion for Server { .caused_by(trc::location!())? .items .iter() - .filter(|(_, item)| { + .filter(|item| { item.change_id < reference_cid && item .mailboxes .iter() .any(|id| id.mailbox_id == TRASH_ID || id.mailbox_id == JUNK_ID) }) - .map(|(id, _)| id), + .map(|item| item.document_id), ); if destroy_ids.is_empty() { return Ok(()); diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index 155a92aa..3050d7bb 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -581,11 +581,11 @@ impl IndexableObject for MessageData { prefix: self.thread_id.into(), }, IndexValue::LogParent { - collection: Collection::Thread, + collection: Collection::Thread.into(), ids: vec![self.thread_id], }, IndexValue::LogParent { - collection: Collection::Mailbox, + collection: Collection::Mailbox.as_child_update(), ids: self.mailboxes.iter().map(|m| m.mailbox_id).collect(), }, ] @@ -600,11 +600,11 @@ impl IndexableObject for &ArchivedMessageData { prefix: self.thread_id.to_native().into(), }, IndexValue::LogParent { - collection: Collection::Thread, + collection: Collection::Thread.into(), ids: vec![self.thread_id.to_native()], }, IndexValue::LogParent { - collection: Collection::Mailbox, + collection: Collection::Mailbox.as_child_update(), ids: self .mailboxes .iter() diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 0bac4be0..d60ef16e 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -45,7 +45,7 @@ use trc::{AddContext, MessageIngestEvent}; use crate::{ mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, message::{ - cache::MessageCache, + cache::MessageCacheFetch, crypto::EncryptionParams, index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, metadata::MessageData, @@ -733,7 +733,7 @@ impl EmailIngest for Server { if !found_message_id.is_empty() && cache .in_mailbox(skip_duplicate.unwrap().1) - .any(|(id, _)| found_message_id.contains(id)) + .any(|m| found_message_id.contains(&m.document_id)) { return Ok(ThreadResult::Skip); } @@ -742,8 +742,8 @@ impl EmailIngest for Server { let mut thread_counts = AHashMap::::with_capacity(16); let mut thread_id = u32::MAX; let mut thread_count = 0; - for (document_id, item) in &cache.items { - if results.contains(*document_id) { + for item in &cache.items { + if results.contains(item.document_id) { let tc = thread_counts.entry(item.thread_id).or_default(); *tc += 1; if *tc > thread_count { @@ -773,12 +773,12 @@ impl EmailIngest for Server { // Move messages to the new threadId batch.with_collection(Collection::Email); - for (&document_id, item) in &cache.items { + for item in &cache.items { if thread_id == item.thread_id || !thread_counts.contains_key(&item.thread_id) { continue; } if let Some(data_) = self - .get_archive(account_id, Collection::Email, document_id) + .get_archive(account_id, Collection::Email, item.document_id) .await .caused_by(trc::location!())? { @@ -791,7 +791,7 @@ impl EmailIngest for Server { let mut new_data = data.deserialize().caused_by(trc::location!())?; new_data.thread_id = thread_id; batch - .update_document(document_id) + .update_document(item.document_id) .custom( ObjectIndexBuilder::new() .with_current(data) diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 34f7a526..ef453f04 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -189,9 +189,8 @@ impl SieveScriptIngest for Server { } else { let mut mailbox_id = u32::MAX; if let Ok(role) = SpecialUse::parse_value(&role) { - if let Some((mailbox_id_, _)) = mailbox_cache.by_role(&role) - { - mailbox_id = *mailbox_id_; + if let Some(m) = mailbox_cache.by_role(&role) { + mailbox_id = m.document_id; } } @@ -205,8 +204,8 @@ impl SieveScriptIngest for Server { Mailbox::Name(name) => { if !matches!( mailbox_cache.by_path(&name), - Some((document_id, _)) if special_use_ids.is_empty() || - special_use_ids.contains(document_id) + Some(item) if special_use_ids.is_empty() || + special_use_ids.contains(&item.document_id) ) { result = false; break; @@ -214,7 +213,7 @@ impl SieveScriptIngest for Server { } Mailbox::Id(id) => { if !matches!(Id::from_bytes(id.as_bytes()), Some(id) if - mailbox_cache.items.contains_key(&id.document_id()) && + mailbox_cache.has_id(&id.document_id()) && (special_use_ids.is_empty() || special_use_ids.contains(&id.document_id()))) { @@ -310,7 +309,7 @@ impl SieveScriptIngest for Server { mailbox_id.and_then(|m| Id::from_bytes(m.as_bytes())) { let mailbox_id = mailbox_id.document_id(); - if mailbox_cache.items.contains_key(&mailbox_id) { + if mailbox_cache.has_id(&mailbox_id) { target_id = mailbox_id; } } @@ -323,8 +322,8 @@ impl SieveScriptIngest for Server { } else if special_use.eq_ignore_ascii_case("trash") { target_id = TRASH_ID; } else if let Ok(role) = SpecialUse::parse_value(&special_use) { - if let Some((mailbox_id_, _)) = mailbox_cache.by_role(&role) { - target_id = *mailbox_id_; + if let Some(item) = mailbox_cache.by_role(&role) { + target_id = item.document_id; } } } @@ -333,8 +332,8 @@ impl SieveScriptIngest for Server { // Find mailbox by name if target_id == u32::MAX { if !create { - if let Some((document_id, _)) = mailbox_cache.by_path(&folder) { - target_id = *document_id; + if let Some(m) = mailbox_cache.by_path(&folder) { + target_id = m.document_id; } } else if let Some(document_id) = self .mailbox_create_path(account_id, &folder) diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index 157d575e..1bb6e0df 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -7,7 +7,7 @@ use common::storage::index::{ IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject, }; -use jmap_proto::types::{collection::Collection, value::AclGrant}; +use jmap_proto::types::value::AclGrant; use store::SerializeInfallible; use crate::{IDX_CARD_UID, IDX_NAME}; @@ -89,10 +89,6 @@ impl IndexableObject for ContactCard { + self.size, }, IndexValue::LogChild { prefix: None }, - IndexValue::LogParent { - collection: Collection::AddressBook, - ids: self.names.iter().map(|v| v.parent_id).collect::>(), - }, ] .into_iter() } @@ -120,14 +116,6 @@ impl IndexableObject for &ArchivedContactCard { + self.size, }, IndexValue::LogChild { prefix: None }, - IndexValue::LogParent { - collection: Collection::AddressBook, - ids: self - .names - .iter() - .map(|v| v.parent_id.to_native()) - .collect::>(), - }, ] .into_iter() } diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 4a73ca61..4a61c21f 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -20,7 +20,7 @@ use email::{ INBOX_ID, cache::{MailboxCacheAccess, MessageMailboxCache}, }, - message::cache::{MessageCache, MessageCacheAccess}, + message::cache::{MessageCacheAccess, MessageCacheFetch}, }; use imap_proto::protocol::list::Attribute; use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, keyword::Keyword}; @@ -124,13 +124,13 @@ impl SessionData { // Build special uses let mut special_uses = AHashMap::new(); - for (&mailbox_id, mailbox) in &cached_mailboxes.items { + for mailbox in &cached_mailboxes.items { if shared_mailbox_ids .as_ref() - .is_none_or(|ids| ids.contains(mailbox_id)) + .is_none_or(|ids| ids.contains(mailbox.document_id)) && !matches!(mailbox.role, SpecialUse::None) { - special_uses.insert(mailbox.role, mailbox_id); + special_uses.insert(mailbox.role, mailbox.document_id); } } @@ -146,10 +146,10 @@ impl SessionData { }, }; - for (&mailbox_id, mailbox) in &cached_mailboxes.items { + for mailbox in &cached_mailboxes.items { if shared_mailbox_ids .as_ref() - .is_some_and(|ids| !ids.contains(mailbox_id)) + .is_some_and(|ids| !ids.contains(mailbox.document_id)) { continue; } @@ -173,17 +173,17 @@ impl SessionData { .find(|f| f.name == mailbox_name || f.aliases.iter().any(|a| a == mailbox_name)) .and_then(|f| special_uses.get(&f.special_use)) .copied() - .unwrap_or(mailbox_id); + .unwrap_or(mailbox.document_id); account .mailbox_names .insert(mailbox_name, effective_mailbox_id); account.mailbox_state.insert( - mailbox_id, + mailbox.document_id, Mailbox { has_children: cached_mailboxes .items - .values() - .any(|child| child.parent_id == mailbox_id), + .iter() + .any(|child| child.parent_id == mailbox.document_id), is_subscribed: mailbox.subscribers.contains(&access_token.primary_id()), special_use: match mailbox.role { SpecialUse::Trash => Some(Attribute::Trash), @@ -194,18 +194,18 @@ impl SessionData { SpecialUse::Important => Some(Attribute::Important), _ => None, }, - total_messages: cached_messages.in_mailbox(mailbox_id).count() as u64, + total_messages: cached_messages.in_mailbox(mailbox.document_id).count() as u64, total_unseen: cached_messages - .in_mailbox_without_keyword(mailbox_id, &Keyword::Seen) + .in_mailbox_without_keyword(mailbox.document_id, &Keyword::Seen) .count() as u64, total_deleted: cached_messages - .in_mailbox_with_keyword(mailbox_id, &Keyword::Deleted) + .in_mailbox_with_keyword(mailbox.document_id, &Keyword::Deleted) .count() as u64, uid_validity: mailbox.uid_validity as u64, uid_next: self .get_uid_next(&MailboxId { account_id, - mailbox_id, + mailbox_id: mailbox.document_id, }) .await .caused_by(trc::location!())? as u64, diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 368c4a00..27eaf26d 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -6,7 +6,7 @@ use ahash::AHashMap; use common::listener::SessionStream; -use email::message::cache::MessageCache; +use email::message::cache::MessageCacheFetch; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; use std::collections::BTreeMap; @@ -39,10 +39,10 @@ impl SessionData { let uid_map = cached_messages .items .iter() - .filter_map(|(document_id, item)| { + .filter_map(|item| { item.mailboxes.iter().find_map(|m| { if m.mailbox_id == mailbox.mailbox_id { - Some((m.uid, *document_id)) + Some((m.uid, item.document_id)) } else { None } diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 34ac5d5a..6ccba07b 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -9,7 +9,7 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashMap; use directory::Permission; use email::message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheFetch, MessageCacheAccess}, delete::EmailDeletion, metadata::MessageData, }; @@ -121,7 +121,7 @@ impl SessionData { .await .caused_by(trc::location!())? .in_mailbox_with_keyword(mailbox.id.mailbox_id, &Keyword::Deleted) - .map(|(id, _)| id), + .map(|m| m.document_id), ); // Filter by sequence diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 7cb94453..401a48df 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -14,7 +14,7 @@ use ahash::AHashMap; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::message::{ - cache::MessageCache, + cache::{MessageCacheAccess, MessageCacheFetch}, metadata::{ ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, @@ -35,11 +35,7 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{ - acl::Acl, - collection::Collection, - id::Id, - keyword::{ArchivedKeyword, Keyword}, - property::Property, + acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, }; use store::{ query::log::{Change, Query}, @@ -171,7 +167,7 @@ impl SessionData { for change in changelog.changes { match change { - Change::Insert(id) | Change::Update(id) | Change::ChildUpdate(id) => { + Change::Insert(id) | Change::Update(id) => { let id = (id & u32::MAX as u64) as u32; if let Some(uid) = ids.get(&id) { changed_ids.insert(id, *uid); @@ -318,7 +314,7 @@ impl SessionData { ) .await .imap_ctx(&arguments.tag, trc::location!())?, - message_cache.items.get(&id), + message_cache.by_id(&id), ) { (email, data) } else { @@ -369,8 +365,7 @@ impl SessionData { // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); - let set_seen_flag = - set_seen_flags && !data.keywords.iter().any(|k| k == &ArchivedKeyword::Seen); + let set_seen_flag = set_seen_flags && !message_cache.has_keyword(data, &Keyword::Seen); for attribute in &arguments.attributes { match attribute { @@ -380,10 +375,8 @@ impl SessionData { }); } Attribute::Flags => { - let mut flags = data - .keywords - .iter() - .cloned() + let mut flags = message_cache + .expand_keywords(data) .map(Flag::from) .collect::>(); if set_seen_flag { @@ -517,10 +510,8 @@ impl SessionData { // Add flags to the response if the message was unseen if set_seen_flag && !arguments.attributes.contains(&Attribute::Flags) { - let mut flags = data - .keywords - .iter() - .cloned() + let mut flags = message_cache + .expand_keywords(data) .map(Flag::from) .collect::>(); flags.push(Flag::Seen); diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 9624044a..62703e69 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -8,7 +8,7 @@ use std::{sync::Arc, time::Instant}; use common::listener::SessionStream; use directory::Permission; -use email::message::cache::{MessageCache, MessageCacheAccess}; +use email::message::cache::{MessageCacheAccess, MessageCacheFetch}; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -266,8 +266,11 @@ impl SessionData { .get_cached_messages(mailbox.id.account_id) .await .caused_by(trc::location!())?; - let message_ids = - RoaringBitmap::from_iter(cache.in_mailbox(mailbox.id.mailbox_id).map(|(id, _)| id)); + let message_ids = RoaringBitmap::from_iter( + cache + .in_mailbox(mailbox.id.mailbox_id) + .map(|m| m.document_id), + ); filters.push(query::Filter::is_in_set(message_ids.clone())); @@ -452,7 +455,9 @@ impl SessionData { } search::Filter::Answered => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Answered).map(|(id, _)| id), + cache + .with_keyword(&Keyword::Answered) + .map(|m| m.document_id), ))); } search::Filter::Before(date) => { @@ -463,24 +468,24 @@ impl SessionData { } search::Filter::Deleted => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Deleted).map(|(id, _)| id), + cache.with_keyword(&Keyword::Deleted).map(|m| m.document_id), ))); } search::Filter::Draft => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Draft).map(|(id, _)| id), + cache.with_keyword(&Keyword::Draft).map(|m| m.document_id), ))); } search::Filter::Flagged => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Flagged).map(|(id, _)| id), + cache.with_keyword(&Keyword::Flagged).map(|m| m.document_id), ))); } search::Filter::Keyword(keyword) => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( cache .with_keyword(&Keyword::from(keyword)) - .map(|(id, _)| id), + .map(|m| m.document_id), ))); } search::Filter::Larger(size) => { @@ -500,7 +505,7 @@ impl SessionData { } search::Filter::Seen => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Seen).map(|(id, _)| id), + cache.with_keyword(&Keyword::Seen).map(|m| m.document_id), ))); } search::Filter::SentBefore(date) => { @@ -537,48 +542,44 @@ impl SessionData { filters.push(query::Filter::lt(Property::Size, size.serialize())); } search::Filter::Unanswered => { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Answered).map(|(id, _)| id), - ))); - filters.push(query::Filter::End); - } - search::Filter::Undeleted => { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Deleted).map(|(id, _)| id), - ))); - filters.push(query::Filter::End); - } - search::Filter::Undraft => { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Draft).map(|(id, _)| id), - ))); - filters.push(query::Filter::End); - } - search::Filter::Unflagged => { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Flagged).map(|(id, _)| id), - ))); - filters.push(query::Filter::End); - } - search::Filter::Unkeyword(keyword) => { - filters.push(query::Filter::Not); filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( cache - .with_keyword(&Keyword::from(keyword)) - .map(|(id, _)| id), + .without_keyword(&Keyword::Answered) + .map(|m| m.document_id), + ))); + } + search::Filter::Undeleted => { + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Deleted) + .map(|m| m.document_id), + ))); + } + search::Filter::Undraft => { + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Draft) + .map(|m| m.document_id), + ))); + } + search::Filter::Unflagged => { + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Flagged) + .map(|m| m.document_id), + ))); + } + search::Filter::Unkeyword(keyword) => { + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::from(keyword)) + .map(|m| m.document_id), ))); - filters.push(query::Filter::End); } search::Filter::Unseen => { - filters.push(query::Filter::Not); filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Seen).map(|(id, _)| id), + cache.without_keyword(&Keyword::Seen).map(|m| m.document_id), ))); - filters.push(query::Filter::End); } search::Filter::And => { filters.push(query::Filter::And); @@ -658,7 +659,7 @@ impl SessionData { search::Filter::ThreadId(id) => { if let Some(id) = Id::from_bytes(id.as_bytes()) { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.in_thread(id.document_id()).map(|(id, _)| id), + cache.in_thread(id.document_id()).map(|m| m.document_id), ))); } else { return Err(trc::ImapEvent::Error diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index 9eb131d4..6cefb75b 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -14,7 +14,7 @@ use crate::{ use common::listener::SessionStream; use compact_str::CompactString; use directory::Permission; -use email::message::cache::{MessageCache, MessageCacheAccess}; +use email::message::cache::{MessageCacheAccess, MessageCacheFetch}; use imap_proto::{ Command, ResponseCode, StatusResponse, parser::PushUnique, @@ -232,7 +232,7 @@ impl SessionData { &RoaringBitmap::from_iter( cache .in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted) - .map(|x| x.0), + .map(|x| x.document_id), ), ) .await @@ -241,7 +241,7 @@ impl SessionData { .calculate_mailbox_size( mailbox.account_id, &RoaringBitmap::from_iter( - cache.in_mailbox(mailbox.mailbox_id).map(|x| x.0), + cache.in_mailbox(mailbox.mailbox_id).map(|x| x.document_id), ), ) .await diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 657047cc..58561f54 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -124,10 +124,7 @@ impl SessionData { // Add all IDs that changed in this mailbox for change in changelog.changes { - let (Change::Insert(id) - | Change::Update(id) - | Change::ChildUpdate(id) - | Change::Delete(id)) = change; + let (Change::Insert(id) | Change::Update(id) | Change::Delete(id)) = change; let id = (id & u32::MAX as u64) as u32; if let Some(imap_id) = ids.remove(&id) { if is_uid { @@ -348,7 +345,7 @@ impl SessionData { // Log mailbox changes if !changed_mailboxes.is_empty() { for parent_id in changed_mailboxes { - batch.log_child_update(Collection::Mailbox, parent_id); + batch.log_parent_update(Collection::Mailbox.as_child_update(), parent_id); } } diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index a86f6212..f4ddf723 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -13,7 +13,7 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::message::cache::MessageCache; +use email::message::cache::MessageCacheFetch; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -89,9 +89,9 @@ impl SessionData { // Group messages by thread let mut threads: AHashMap> = AHashMap::new(); let state = mailbox.state.lock(); - for (document_id, item) in &cache.items { - if result_set.results.contains(*document_id) { - if let Some((imap_id, _)) = state.map_result_id(*document_id, is_uid) { + for item in &cache.items { + if result_set.results.contains(item.document_id) { + if let Some((imap_id, _)) = state.map_result_id(item.document_id, is_uid) { threads.entry(item.thread_id).or_default().push(imap_id); } } diff --git a/crates/jmap-proto/src/method/copy.rs b/crates/jmap-proto/src/method/copy.rs index 6e09162e..e18ab388 100644 --- a/crates/jmap-proto/src/method/copy.rs +++ b/crates/jmap-proto/src/method/copy.rs @@ -14,7 +14,7 @@ use crate::{ types::{ blob::BlobId, id::Id, - state::{State, StateChange}, + state::State, value::{Object, SetValue, Value}, }, }; @@ -52,9 +52,6 @@ pub struct CopyResponse { #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_created: VecMap, - - #[serde(skip)] - pub state_change: Option, } #[derive(Debug, Clone)] diff --git a/crates/jmap-proto/src/method/import.rs b/crates/jmap-proto/src/method/import.rs index d340f686..55212256 100644 --- a/crates/jmap-proto/src/method/import.rs +++ b/crates/jmap-proto/src/method/import.rs @@ -20,7 +20,7 @@ use crate::{ id::Id, keyword::Keyword, property::Property, - state::{State, StateChange}, + state::State, value::{Object, SetValueMap, Value}, }, }; @@ -59,9 +59,6 @@ pub struct ImportEmailResponse { #[serde(rename = "notCreated")] #[serde(skip_serializing_if = "VecMap::is_empty")] pub not_created: VecMap, - - #[serde(skip)] - pub state_change: Option, } impl JsonObjectParser for ImportEmailRequest { diff --git a/crates/jmap-proto/src/types/collection.rs b/crates/jmap-proto/src/types/collection.rs index 2ad07a62..29c1ba17 100644 --- a/crates/jmap-proto/src/types/collection.rs +++ b/crates/jmap-proto/src/types/collection.rs @@ -26,12 +26,11 @@ pub enum Collection { Principal = 7, Calendar = 8, CalendarEvent = 9, - CalendarEventNotification = 10, - AddressBook = 11, - ContactCard = 12, - FileNode = 13, + AddressBook = 10, + ContactCard = 11, + FileNode = 12, #[default] - None = 14, + None = 13, } impl Collection { @@ -79,10 +78,9 @@ impl From for Collection { 7 => Collection::Principal, 8 => Collection::Calendar, 9 => Collection::CalendarEvent, - 10 => Collection::CalendarEventNotification, - 11 => Collection::AddressBook, - 12 => Collection::ContactCard, - 13 => Collection::FileNode, + 10 => Collection::AddressBook, + 11 => Collection::ContactCard, + 12 => Collection::FileNode, _ => Collection::None, } } @@ -101,10 +99,9 @@ impl From for Collection { 7 => Collection::Principal, 8 => Collection::Calendar, 9 => Collection::CalendarEvent, - 10 => Collection::CalendarEventNotification, - 11 => Collection::AddressBook, - 12 => Collection::ContactCard, - 13 => Collection::FileNode, + 10 => Collection::AddressBook, + 11 => Collection::ContactCard, + 12 => Collection::FileNode, _ => Collection::None, } } @@ -158,13 +155,16 @@ impl Collection { Collection::Principal => "principal", Collection::Calendar => "calendar", Collection::CalendarEvent => "calendarEvent", - Collection::CalendarEventNotification => "calendarEventNotification", Collection::AddressBook => "addressBook", Collection::ContactCard => "contactCard", Collection::FileNode => "fileNode", Collection::None => "", } } + + pub fn as_child_update(&self) -> u8 { + u8::MAX - u8::from(*self) + } } impl FromStr for Collection { @@ -182,7 +182,6 @@ impl FromStr for Collection { "principal" => Collection::Principal, "calendar" => Collection::Calendar, "calendarEvent" => Collection::CalendarEvent, - "calendarEventNotification" => Collection::CalendarEventNotification, "addressBook" => Collection::AddressBook, "contactCard" => Collection::ContactCard, "fileNode" => Collection::FileNode, diff --git a/crates/jmap-proto/src/types/keyword.rs b/crates/jmap-proto/src/types/keyword.rs index 693a651d..68abb875 100644 --- a/crates/jmap-proto/src/types/keyword.rs +++ b/crates/jmap-proto/src/types/keyword.rs @@ -255,6 +255,24 @@ impl Keyword { Keyword::Other(string) => Err(string), } } + + pub fn try_from_id(id: usize) -> Result { + match id { + SEEN => Ok(Keyword::Seen), + DRAFT => Ok(Keyword::Draft), + FLAGGED => Ok(Keyword::Flagged), + ANSWERED => Ok(Keyword::Answered), + RECENT => Ok(Keyword::Recent), + IMPORTANT => Ok(Keyword::Important), + PHISHING => Ok(Keyword::Phishing), + JUNK => Ok(Keyword::Junk), + NOTJUNK => Ok(Keyword::NotJunk), + DELETED => Ok(Keyword::Deleted), + FORWARDED => Ok(Keyword::Forwarded), + MDN_SENT => Ok(Keyword::MdnSent), + _ => Err(id), + } + } } impl ArchivedKeyword { diff --git a/crates/jmap-proto/src/types/type_state.rs b/crates/jmap-proto/src/types/type_state.rs index 04bc498f..98473132 100644 --- a/crates/jmap-proto/src/types/type_state.rs +++ b/crates/jmap-proto/src/types/type_state.rs @@ -176,9 +176,10 @@ impl TryFrom for DataType { type Error = (); fn try_from(value: ShortId) -> Result { + const MAILBOX_CHANGE: u8 = u8::MAX - 1; match value.0 { 0 => Ok(DataType::Email), - 1 => Ok(DataType::Mailbox), + 1 | MAILBOX_CHANGE => Ok(DataType::Mailbox), 2 => Ok(DataType::Thread), 3 => Ok(DataType::Identity), 4 => Ok(DataType::EmailSubmission), @@ -186,10 +187,9 @@ impl TryFrom for DataType { 6 => Ok(DataType::PushSubscription), 8 => Ok(DataType::Calendar), 9 => Ok(DataType::CalendarEvent), - 10 => Ok(DataType::CalendarEventNotification), - 11 => Ok(DataType::AddressBook), - 12 => Ok(DataType::ContactCard), - 13 => Ok(DataType::FileNode), + 10 => Ok(DataType::AddressBook), + 11 => Ok(DataType::ContactCard), + 12 => Ok(DataType::FileNode), _ => Err(()), } } diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 5fa6332e..9f0055c7 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -109,17 +109,6 @@ impl RequestHandler for Server { ResponseMethod::ImportEmail(import_response) => { // Add created ids import_response.update_created_ids(&mut response); - - // Publish state changes - if let Some(state_change) = import_response.state_change.take() { - self.broadcast_state_change(state_change).await; - } - } - ResponseMethod::Copy(copy_response) => { - // Publish state changes - if let Some(state_change) = copy_response.state_change.take() { - self.broadcast_state_change(state_change).await; - } } ResponseMethod::UploadBlob(upload_response) => { // Add created blobIds diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 1c34f82f..9189f833 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -9,7 +9,7 @@ use std::ops::Range; use common::{Server, auth::AccessToken}; use email::{ mailbox::cache::MessageMailboxCache, - message::cache::{MessageCache, MessageCacheAccess}, + message::cache::{MessageCacheFetch, MessageCacheAccess}, }; use jmap_proto::types::{acl::Acl, blob::BlobId, collection::Collection}; use std::future::Future; diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index ac44ef08..541a4a42 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -10,7 +10,7 @@ use jmap_proto::{ types::{collection::Collection, property::Property, state::State}, }; use std::future::Future; -use store::query::log::{Change, Query}; +use store::query::log::{Change, Changes, Query}; pub trait ChangesLookup: Sync + Send { fn changes( @@ -80,10 +80,8 @@ impl ChangesLookup for Server { let (items_sent, mut changelog) = match &request.since_state { State::Initial => { - let changelog = self - .store() - .changes(account_id, collection, Query::All) - .await?; + let changelog = + changes(self, account_id, collection, Query::All, &mut response).await?; if changelog.changes.is_empty() && changelog.from_change_id == 0 { return Ok(response); } @@ -92,29 +90,35 @@ impl ChangesLookup for Server { } State::Exact(change_id) => ( 0, - self.store() - .changes(account_id, collection, Query::Since(*change_id)) - .await?, + changes( + self, + account_id, + collection, + Query::Since(*change_id), + &mut response, + ) + .await?, ), State::Intermediate(intermediate_state) => { - let mut changelog = self - .store() - .changes( - account_id, - collection, - Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), - ) - .await?; + let mut changelog = changes( + self, + account_id, + collection, + Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), + &mut response, + ) + .await?; if intermediate_state.items_sent >= changelog.changes.len() { ( 0, - self.store() - .changes( - account_id, - collection, - Query::Since(intermediate_state.to_id), - ) - .await?, + changes( + self, + account_id, + collection, + Query::Since(intermediate_state.to_id), + &mut response, + ) + .await?, ) } else { changelog.changes.drain( @@ -133,19 +137,13 @@ impl ChangesLookup for Server { response.has_more_changes = true; }; - let mut items_changed = false; - let total_changes = changelog.changes.len(); if total_changes > 0 { for change in changelog.changes { match change { Change::Insert(item) => response.created.push(item.into()), - Change::Update(item) => { - items_changed = true; - response.updated.push(item.into()) - } + Change::Update(item) => response.updated.push(item.into()), Change::Delete(item) => response.destroyed.push(item.into()), - Change::ChildUpdate(item) => response.updated.push(item.into()), }; } } @@ -159,16 +157,53 @@ impl ChangesLookup for Server { State::new_exact(changelog.to_change_id) }; - if !response.updated.is_empty() && !items_changed && collection == Collection::Mailbox { - response.updated_properties = vec![ - Property::TotalEmails, - Property::UnreadEmails, - Property::TotalThreads, - Property::UnreadThreads, - ] - .into() - } - Ok(response) } } + +async fn changes( + server: &Server, + account_id: u32, + collection: Collection, + query: Query, + response: &mut ChangesResponse, +) -> trc::Result { + let mut main_changes = server + .store() + .changes(account_id, collection, query) + .await?; + if matches!(collection, Collection::Mailbox) { + let child_changes = server + .store() + .changes(account_id, collection.as_child_update(), query) + .await?; + + if !child_changes.changes.is_empty() { + if child_changes.from_change_id < main_changes.from_change_id { + main_changes.from_change_id = child_changes.from_change_id; + } + if child_changes.to_change_id > main_changes.to_change_id { + main_changes.to_change_id = child_changes.to_change_id; + } + let mut has_child_changes = false; + for change in child_changes.changes { + let id = change.id(); + if !main_changes.changes.iter().any(|c| c.id() == id) { + main_changes.changes.push(change); + has_child_changes = true; + } + } + + if has_child_changes { + response.updated_properties = vec![ + Property::TotalEmails, + Property::UnreadEmails, + Property::TotalThreads, + Property::UnreadThreads, + ] + .into(); + } + } + } + Ok(main_changes) +} diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index f8e38ec2..8be5ec1d 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -9,7 +9,7 @@ use common::{Server, auth::AccessToken}; use email::{ mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}, message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheAccess, MessageCacheFetch}, copy::EmailCopy, }, }; @@ -30,8 +30,6 @@ use jmap_proto::{ acl::Acl, collection::Collection, property::Property, - state::{State, StateChange}, - type_state::DataType, value::{MaybePatchValue, Value}, }, }; @@ -77,7 +75,6 @@ impl JmapEmailCopy for Server { old_state, created: VecMap::with_capacity(request.create.len()), not_created: VecMap::new(), - state_change: None, }; let from_cached_messages = self @@ -208,7 +205,7 @@ impl JmapEmailCopy for Server { // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { - if !cached_mailboxes.items.contains_key(mailbox_id) { + if !cached_mailboxes.has_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -257,13 +254,6 @@ impl JmapEmailCopy for Server { // Update state if !response.created.is_empty() { response.new_state = self.get_state(account_id, Collection::Email).await?; - if let State::Exact(change_id) = &response.new_state { - response.state_change = StateChange::new(account_id) - .with_change(DataType::Email, *change_id) - .with_change(DataType::Mailbox, *change_id) - .with_change(DataType::Thread, *change_id) - .into() - } } // Destroy ids diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 243fb641..5321723e 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -9,10 +9,9 @@ use common::{Server, auth::AccessToken}; use email::{ mailbox::cache::MessageMailboxCache, message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheAccess, MessageCacheFetch}, metadata::{ - ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, MessageData, - MessageMetadata, + ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, MessageMetadata, }, }, }; @@ -126,7 +125,7 @@ impl EmailGet for Server { .items .iter() .take(self.core.jmap.get_max_objects) - .map(|(document_id, item)| Id::from_parts(item.thread_id, *document_id)) + .map(|item| Id::from_parts(item.thread_id, item.document_id)) .collect() }; let mut response = GetResponse { @@ -178,19 +177,13 @@ impl EmailGet for Server { .caused_by(trc::location!())?; // Obtain message data - let data_ = match self - .get_archive(account_id, Collection::Email, id.document_id()) - .await? - { + let data = match cached_messages.by_id(&id.document_id()) { Some(data) => data, None => { response.not_found.push(id.into()); continue; } }; - let data = data_ - .unarchive::() - .caused_by(trc::location!())?; // Retrieve raw message if needed let blob_hash = BlobHash::from(&metadata.blob_hash); @@ -243,17 +236,14 @@ impl EmailGet for Server { let mut obj = Object::with_capacity(data.mailboxes.len()); for id in data.mailboxes.iter() { debug_assert!(id.uid != 0); - obj.append( - Property::_T(Id::from(u32::from(id.mailbox_id)).to_string()), - true, - ); + obj.append(Property::_T(Id::from(id.mailbox_id).to_string()), true); } email.append(property.clone(), Value::Object(obj)); } Property::Keywords => { - let mut obj = Object::with_capacity(data.keywords.len()); - for keyword in data.keywords.iter() { + let mut obj = Object::with_capacity(2); + for keyword in cached_messages.expand_keywords(data) { obj.append(Property::_T(keyword.to_string()), true); } email.append(property.clone(), Value::Object(obj)); diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 4cddd89a..36537a23 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -13,14 +13,7 @@ use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, - types::{ - acl::Acl, - collection::Collection, - id::Id, - property::Property, - state::{State, StateChange}, - type_state::DataType, - }, + types::{acl::Acl, collection::Collection, id::Id, property::Property, state::State}, }; use mail_parser::MessageParser; use utils::map::vec_map::VecMap; @@ -69,7 +62,6 @@ impl EmailImport for Server { old_state: old_state.into(), created: VecMap::with_capacity(request.emails.len()), not_created: VecMap::new(), - state_change: None, }; let can_train_spam = self.email_bayes_can_train(access_token); @@ -91,7 +83,7 @@ impl EmailImport for Server { continue; } for mailbox_id in &mailbox_ids { - if !cached_mailboxes.items.contains_key(mailbox_id) { + if !cached_mailboxes.has_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -174,13 +166,6 @@ impl EmailImport for Server { // Update state if !response.created.is_empty() { response.new_state = self.get_state(account_id, Collection::Email).await?; - if let State::Exact(change_id) = &response.new_state { - response.state_change = StateChange::new(account_id) - .with_change(DataType::Email, *change_id) - .with_change(DataType::Mailbox, *change_id) - .with_change(DataType::Thread, *change_id) - .into() - } } Ok(response) diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index be694650..4511ad24 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,10 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{MessageItemCache, MessageStoreCache, Server, auth::AccessToken}; +use common::{MessageStoreCache, Server, auth::AccessToken}; use email::{ mailbox::cache::MessageMailboxCache, - message::cache::{MessageCache, MessageCacheAccess}, + message::cache::{MessageCacheAccess, MessageCacheFetch}, }; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, @@ -191,7 +191,7 @@ impl EmailQuery for Server { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( cached_messages .in_mailbox(mailbox.document_id()) - .map(|(id, _)| *id), + .map(|item| item.document_id), ))) } Filter::InMailboxOtherThan(mailboxes) => { @@ -201,7 +201,7 @@ impl EmailQuery for Server { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( cached_messages .in_mailbox(mailbox.document_id()) - .map(|(id, _)| *id), + .map(|item| item.document_id), ))); } filters.push(query::Filter::End); @@ -244,13 +244,17 @@ impl EmailQuery for Server { } Filter::HasKeyword(keyword) => { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cached_messages.with_keyword(&keyword).map(|(id, _)| *id), + cached_messages + .with_keyword(&keyword) + .map(|item| item.document_id), ))); } Filter::NotKeyword(keyword) => { filters.push(query::Filter::Not); filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cached_messages.with_keyword(&keyword).map(|(id, _)| *id), + cached_messages + .with_keyword(&keyword) + .map(|item| item.document_id), ))); filters.push(query::Filter::End); } @@ -282,7 +286,7 @@ impl EmailQuery for Server { filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( cached_messages .in_thread(id.document_id()) - .map(|(id, _)| *id), + .map(|item| item.document_id), ))) } Filter::And | Filter::Or | Filter::Not | Filter::Close => { @@ -344,7 +348,7 @@ impl EmailQuery for Server { RoaringBitmap::from_iter( cached_messages .with_keyword(&comparator.keyword.unwrap_or(Keyword::Seen)) - .map(|(id, _)| *id), + .map(|item| item.document_id), ), comparator.is_ascending, ), @@ -387,7 +391,7 @@ impl EmailQuery for Server { &cache .items .iter() - .map(|(id, item)| (*id, item.thread_id)) + .map(|item| (item.document_id, item.thread_id)) .collect(), ) .with_prefix_unique(request.arguments.collapse_threads.unwrap_or(false)), @@ -400,12 +404,9 @@ impl EmailQuery for Server { } } -fn thread_keywords( - cache: &MessageStoreCache, - keyword: Keyword, - match_all: bool, -) -> RoaringBitmap { - let keyword_doc_ids = RoaringBitmap::from_iter(cache.with_keyword(&keyword).map(|(id, _)| *id)); +fn thread_keywords(cache: &MessageStoreCache, keyword: Keyword, match_all: bool) -> RoaringBitmap { + let keyword_doc_ids = + RoaringBitmap::from_iter(cache.with_keyword(&keyword).map(|item| item.document_id)); if keyword_doc_ids.is_empty() { return keyword_doc_ids; } @@ -414,14 +415,15 @@ fn thread_keywords( let mut thread_map: AHashMap = AHashMap::new(); - for (&document_id, item) in &cache.items { + for item in &cache.items { thread_map .entry(item.thread_id) .or_default() - .insert(document_id); + .insert(item.document_id); } - for (&keyword_doc_id, item) in &cache.items { + for item in &cache.items { + let keyword_doc_id = item.document_id; if !keyword_doc_ids.contains(keyword_doc_id) || matched_ids.contains(keyword_doc_id) || not_matched_ids.contains(keyword_doc_id) diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 0ec56192..0d2c8356 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -13,7 +13,7 @@ use email::{ cache::{MailboxCacheAccess, MessageMailboxCache}, }, message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheAccess, MessageCacheFetch}, delete::EmailDeletion, ingest::{EmailIngest, IngestEmail, IngestSource}, metadata::MessageData, @@ -662,7 +662,7 @@ impl EmailSet for Server { // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { - if !cached_mailboxes.items.contains_key(mailbox_id) { + if !cached_mailboxes.has_id(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -878,7 +878,7 @@ impl EmailSet for Server { // Make sure all new mailboxIds are valid for mailbox_id in new_data.added_mailboxes(data.inner) { - if cached_mailboxes.items.contains_key(&mailbox_id.mailbox_id) { + if cached_mailboxes.has_id(&mailbox_id.mailbox_id) { // Verify permissions on shared accounts if !matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(mailbox_id.mailbox_id)) { @@ -958,7 +958,7 @@ impl EmailSet for Server { if !batch.is_empty() { // Log mailbox changes for parent_id in changed_mailboxes { - batch.log_child_update(Collection::Mailbox, parent_id); + batch.log_parent_update(Collection::Mailbox.as_child_update(), parent_id); } match self.commit_batch(batch).await { diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 8c0cc996..4d4673df 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -8,7 +8,7 @@ use common::{Server, auth::AccessToken}; use email::{ mailbox::cache::MessageMailboxCache, message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheFetch, MessageCacheAccess}, metadata::{ ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, DecodedPartContent, MessageMetadata, diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index c5486c6a..eaeef7af 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -7,7 +7,7 @@ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; use email::{ mailbox::cache::{MailboxCacheAccess, MessageMailboxCache}, - message::cache::{MessageCache, MessageCacheAccess}, + message::cache::{MessageCacheAccess, MessageCacheFetch}, }; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, @@ -63,7 +63,7 @@ impl MailboxGet for Server { ids } else { mailbox_cache - .items + .index .keys() .filter(|id| shared_ids.as_ref().is_none_or(|ids| ids.contains(**id))) .copied() @@ -82,7 +82,7 @@ impl MailboxGet for Server { // Obtain the mailbox object let document_id = id.document_id(); let cached_mailbox = if let Some(mailbox) = - mailbox_cache.items.get(&document_id).filter(|_| { + mailbox_cache.by_id(&document_id).filter(|_| { shared_ids .as_ref() .is_none_or(|ids| ids.contains(document_id)) @@ -125,14 +125,14 @@ impl MailboxGet for Server { Property::TotalThreads => Value::UnsignedInt( message_cache .in_mailbox(document_id) - .map(|(_, m)| m.thread_id) + .map(|m| m.thread_id) .collect::>() .len() as u64, ), Property::UnreadThreads => Value::UnsignedInt( message_cache .in_mailbox_without_keyword(document_id, &Keyword::Seen) - .map(|(_, m)| m.thread_id) + .map(|m| m.thread_id) .collect::>() .len() as u64, ), diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 1588263a..987277a6 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -50,8 +50,8 @@ impl MailboxQuery for Server { mailboxes .items .iter() - .filter(|(_, mailbox)| mailbox.parent_id == parent_id) - .map(|(id, _)| id) + .filter(|mailbox| mailbox.parent_id == parent_id) + .map(|m| m.document_id) .collect::(), )); } @@ -68,8 +68,8 @@ impl MailboxQuery for Server { mailboxes .items .iter() - .filter(|(_, mailbox)| mailbox.name.to_lowercase().contains(&name)) - .map(|(id, _)| id) + .filter(|mailbox| mailbox.name.to_lowercase().contains(&name)) + .map(|m| m.document_id) .collect::(), )); } @@ -79,10 +79,8 @@ impl MailboxQuery for Server { mailboxes .items .iter() - .filter(|(_, mailbox)| { - mailbox.role.as_str().is_some_and(|r| r == role) - }) - .map(|(id, _)| id) + .filter(|mailbox| mailbox.role.as_str().is_some_and(|r| r == role)) + .map(|m| m.document_id) .collect::(), )); } else { @@ -91,8 +89,8 @@ impl MailboxQuery for Server { mailboxes .items .iter() - .filter(|(_, mailbox)| matches!(mailbox.role, SpecialUse::None)) - .map(|(id, _)| id) + .filter(|mailbox| matches!(mailbox.role, SpecialUse::None)) + .map(|m| m.document_id) .collect::(), )); filters.push(query::Filter::End); @@ -106,8 +104,8 @@ impl MailboxQuery for Server { mailboxes .items .iter() - .filter(|(_, mailbox)| !matches!(mailbox.role, SpecialUse::None)) - .map(|(id, _)| id) + .filter(|mailbox| !matches!(mailbox.role, SpecialUse::None)) + .map(|m| m.document_id) .collect::(), )); if !has_role { @@ -122,10 +120,10 @@ impl MailboxQuery for Server { mailboxes .items .iter() - .filter(|(_, mailbox)| { + .filter(|mailbox| { mailbox.subscribers.contains(&access_token.primary_id) }) - .map(|(id, _)| id) + .map(|m| m.document_id) .collect::(), )); if !is_subscribed { @@ -159,7 +157,7 @@ impl MailboxQuery for Server { for document_id in &result_set.results { let mut check_id = document_id; for _ in 0..self.core.jmap.mailbox_max_depth { - if let Some(mailbox) = mailboxes.items.get(&check_id) { + if let Some(mailbox) = mailboxes.by_id(&check_id) { if let Some(parent_id) = mailbox.parent_id() { if result_set.results.contains(parent_id) { check_id = parent_id; @@ -194,7 +192,7 @@ impl MailboxQuery for Server { let sorted_list = mailboxes .items .iter() - .map(|(id, mailbox)| (mailbox.path.as_str(), *id)) + .map(|mailbox| (mailbox.path.as_str(), mailbox.document_id)) .collect::>(); comparators.push(query::Comparator::sorted_list( sorted_list.into_values().collect(), @@ -213,7 +211,7 @@ impl MailboxQuery for Server { let sorted_list = mailboxes .items .iter() - .map(|(id, mailbox)| (mailbox.name.as_str(), *id)) + .map(|mailbox| (mailbox.name.as_str(), mailbox.document_id)) .collect::>(); query::Comparator::sorted_list( @@ -225,7 +223,7 @@ impl MailboxQuery for Server { let sorted_list = mailboxes .items .iter() - .map(|(id, mailbox)| (mailbox.sort_order, *id)) + .map(|mailbox| (mailbox.sort_order, mailbox.document_id)) .collect::>(); query::Comparator::sorted_list( @@ -237,10 +235,10 @@ impl MailboxQuery for Server { let sorted_list = mailboxes .items .iter() - .map(|(id, mailbox)| { + .map(|mailbox| { ( mailbox.parent_id().map(|id| id + 1).unwrap_or_default(), - *id, + mailbox.document_id, ) }) .collect::>(); diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 6bef7496..b14027e4 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -83,7 +83,7 @@ impl MailboxSet for Server { .prepare_set_response(&request, Collection::Mailbox) .await?, mailbox_ids: RoaringBitmap::from_iter( - self.get_cached_mailboxes(account_id).await?.items.keys(), + self.get_cached_mailboxes(account_id).await?.index.keys(), ), will_destroy: request.unwrap_destroy(), }; @@ -469,7 +469,7 @@ impl MailboxSet for Server { if update .as_ref() .is_none_or(|(_, m)| m.inner.name != changes.name) - && cached_mailboxes.items.iter().any(|(_, m)| { + && cached_mailboxes.items.iter().any(|m| { m.name.to_lowercase() == lower_name && m.parent_id().map_or(0, |id| id + 1) == changes.parent_id }) diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index d32edd97..73629a31 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -5,7 +5,7 @@ */ use common::Server; -use email::message::cache::MessageCache; +use email::message::cache::MessageCacheFetch; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{collection::Collection, id::Id, property::Property, value::Object}, @@ -34,7 +34,7 @@ impl ThreadGet for Server { ) -> trc::Result { let account_id = request.account_id.document_id(); let mut thread_map: AHashMap = AHashMap::with_capacity(32); - for (document_id, item) in &self + for item in &self .get_cached_messages(account_id) .await .caused_by(trc::location!())? @@ -43,7 +43,7 @@ impl ThreadGet for Server { thread_map .entry(item.thread_id) .or_default() - .insert(*document_id); + .insert(item.document_id); } let ids = if let Some(ids) = request.unwrap_ids(self.core.jmap.get_max_objects)? { diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 894d1120..908bf6b5 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -12,7 +12,7 @@ use email::{ INBOX_ID, cache::{MailboxCacheAccess, MessageMailboxCache}, }, - message::cache::MessageCache, + message::cache::MessageCacheFetch, }; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ @@ -59,7 +59,7 @@ impl Session { .caused_by(trc::location!())?; let uid_validity = mailbox_cache .by_role(&SpecialUse::Inbox) - .map(|x| x.1.uid_validity) + .map(|x| x.uid_validity) .unwrap_or_default(); // Obtain message sizes @@ -88,7 +88,7 @@ impl Session { .no_values(), |key, _| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if mailbox_cache.items.contains_key(&document_id) { + if mailbox_cache.has_id(&document_id) { message_sizes.insert( document_id, key.deserialize_be_u32(key.len() - (U32_LEN * 2))?, @@ -105,11 +105,12 @@ impl Session { let message_map = message_cache .items .iter() - .filter_map(|(document_id, m)| { - m.mailboxes + .filter_map(|message| { + message + .mailboxes .iter() .find(|m| m.mailbox_id == INBOX_ID) - .map(|m| (m.uid, *document_id)) + .map(|m| (m.uid, message.document_id)) }) .collect::>(); diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index 995265f6..9a966e7f 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -13,7 +13,6 @@ use crate::{IterateParams, LogKey, Store, U64_LEN, write::key::DeserializeBigEnd pub enum Change { Insert(u64), Update(u64), - ChildUpdate(u64), Delete(u64), } @@ -24,7 +23,7 @@ pub struct Changes { pub to_change_id: u64, } -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub enum Query { All, Since(u64), @@ -144,7 +143,6 @@ impl Changes { let mut bytes_it = bytes.iter(); let total_inserts: usize = bytes_it.next_leb128()?; let total_updates: usize = bytes_it.next_leb128()?; - let total_child_updates: usize = bytes_it.next_leb128()?; let total_deletes: usize = bytes_it.next_leb128()?; if total_inserts > 0 { @@ -153,10 +151,9 @@ impl Changes { } } - if total_updates > 0 || total_child_updates > 0 { - 'update_outer: for change_pos in 0..(total_updates + total_child_updates) { + if total_updates > 0 { + 'update_outer: for _ in 0..total_updates { let id = bytes_it.next_leb128()?; - let mut is_child_update = change_pos >= total_updates; for (idx, change) in self.changes.iter().enumerate() { match change { @@ -165,12 +162,6 @@ impl Changes { continue 'update_outer; } Change::Update(update_id) if *update_id == id => { - // Move update to the front - is_child_update = false; - self.changes.remove(idx); - break; - } - Change::ChildUpdate(update_id) if *update_id == id => { // Move update to the front self.changes.remove(idx); break; @@ -179,11 +170,7 @@ impl Changes { } } - self.changes.push(if !is_child_update { - Change::Update(id) - } else { - Change::ChildUpdate(id) - }); + self.changes.push(Change::Update(id)); } } @@ -197,9 +184,7 @@ impl Changes { self.changes.remove(idx); continue 'delete_outer; } - Change::Update(update_id) | Change::ChildUpdate(update_id) - if *update_id == id => - { + Change::Update(update_id) if *update_id == id => { self.changes.remove(idx); break 'delete_inner; } @@ -220,7 +205,6 @@ impl Change { match self { Change::Insert(id) => *id, Change::Update(id) => *id, - Change::ChildUpdate(id) => *id, Change::Delete(id) => *id, } } @@ -229,7 +213,6 @@ impl Change { match self { Change::Insert(id) => id, Change::Update(id) => id, - Change::ChildUpdate(id) => id, Change::Delete(id) => id, } } diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 277ed788..3ece7921 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -301,13 +301,13 @@ impl BatchBuilder { self } - pub fn log_child_update(&mut self, collection: impl Into, parent_id: u32) -> &mut Self { + pub fn log_parent_update(&mut self, collection: impl Into, parent_id: u32) -> &mut Self { let collection = collection.into(); if let Some(account_id) = self.current_account_id { self.changes .get_mut_or_insert(account_id) - .log_child_update(collection, None, parent_id); + .log_update(collection, None, parent_id); } if self.current_change_id.is_none() { self.generate_change_id(); @@ -325,7 +325,11 @@ impl BatchBuilder { for (collection, set) in changelog.serialize() { let cc = self.changed_collections.get_mut_or_insert(account_id); cc.0 = change_id; - cc.1.insert(ShortId(collection)); + if collection < 64 { + cc.1.insert(ShortId(collection)); + } else { + cc.1.insert(ShortId(u8::MAX - collection)); + } self.ops.push(Operation::Log { change_id, diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index 50f59f4d..463a82a3 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -19,7 +19,6 @@ pub struct Changes { pub inserts: AHashSet, pub updates: AHashSet, pub deletes: AHashSet, - pub child_updates: AHashSet, } impl ChangeLogBuilder { @@ -49,18 +48,6 @@ impl ChangeLogBuilder { changes.updates.remove(&id); changes.deletes.insert(id); } - - pub fn log_child_update( - &mut self, - collection: impl Into, - prefix: Option, - document_id: u32, - ) { - self.changes - .get_mut_or_insert(collection.into()) - .child_updates - .insert(build_id(prefix, document_id)); - } } #[inline(always)] @@ -95,17 +82,6 @@ impl Changes { } } - pub fn child_update(id: T) -> Self - where - T: IntoIterator, - I: Into, - { - Changes { - child_updates: id.into_iter().map(Into::into).collect(), - ..Default::default() - } - } - pub fn delete(id: T) -> Self where T: IntoIterator, @@ -121,25 +97,15 @@ impl Changes { impl SerializeInfallible for Changes { fn serialize(&self) -> Vec { let mut buf = Vec::with_capacity( - 1 + (self.inserts.len() - + self.updates.len() - + self.child_updates.len() - + self.deletes.len() - + 4) + 1 + (self.inserts.len() + self.updates.len() + self.deletes.len() + 4) * std::mem::size_of::(), ); buf.push_leb128(self.inserts.len()); buf.push_leb128(self.updates.len()); - buf.push_leb128(self.child_updates.len()); buf.push_leb128(self.deletes.len()); - for list in [ - &self.inserts, - &self.updates, - &self.child_updates, - &self.deletes, - ] { + for list in [&self.inserts, &self.updates, &self.deletes] { for id in list { buf.push_leb128(*id); } diff --git a/crates/utils/src/map/bitmap.rs b/crates/utils/src/map/bitmap.rs index 29a038c5..db43a091 100644 --- a/crates/utils/src/map/bitmap.rs +++ b/crates/utils/src/map/bitmap.rs @@ -22,6 +22,7 @@ use std::ops::Deref; Hash, )] #[rkyv(compare(PartialEq), derive(Debug))] +#[repr(transparent)] pub struct Bitmap { pub bitmap: u64, #[serde(skip)] diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 29610d2a..3419ec00 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -8,7 +8,7 @@ use std::time::Duration; use email::{ mailbox::{INBOX_ID, JUNK_ID}, - message::cache::{MessageCache, MessageCacheAccess}, + message::cache::{MessageCacheAccess, MessageCacheFetch}, }; use jmap_proto::types::{collection::Collection, id::Id}; diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index aa9c8f71..5a75c5af 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -148,7 +148,7 @@ pub async fn test(params: &mut JMAPTest) { batch.update_document(id as u32).log_delete(None); } LogAction::UpdateChild(id) => { - batch.log_child_update(Collection::Email, id as u32); + batch.log_parent_update(Collection::Email, id as u32); } LogAction::Move(old_id, new_id) => { batch diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index e8108f18..d9e6e611 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -11,7 +11,7 @@ use crate::{ store::{deflate_test_resource, query::FIELDS}, }; -use ::email::{mailbox::Mailbox, message::cache::MessageCache}; +use ::email::{mailbox::Mailbox, message::cache::MessageCacheFetch}; use ahash::AHashSet; use common::{config::jmap::settings::SpecialUse, storage::index::ObjectIndexBuilder}; use jmap_client::{ @@ -19,12 +19,12 @@ use jmap_client::{ core::query::{Comparator, Filter}, email, }; -use jmap_proto::types::{collection::Collection, id::Id, property::Property}; +use jmap_proto::types::{collection::Collection, id::Id}; use mail_parser::{DateTime, HeaderName}; use store::{ ahash::AHashMap, - write::{BatchBuilder, ValueClass, now}, + write::{BatchBuilder, now}, }; use super::JMAPTest; @@ -73,24 +73,6 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { println!("Inserting JMAP Mail query test messages..."); create(client).await; - // Remove mailboxes - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox); - for mailbox_id in 1545..3010 { - batch - .delete_document(mailbox_id) - .clear(ValueClass::Property(Property::EmailIds.into())); - } - server - .core - .storage - .data - .write(batch.build_all()) - .await - .unwrap(); - assert_eq!( params .server @@ -98,7 +80,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .await .unwrap() .items - .values() + .iter() .map(|m| m.thread_id) .collect::>() .len(), @@ -797,6 +779,27 @@ pub async fn create(client: &mut Client) { total_messages += 1; + let mut keywords = Vec::new(); + for keyword in [ + values_str["medium"].to_string(), + values_str["artistRole"].to_string(), + values_str["accession_number"][0..1].to_string(), + format!( + "N{}", + &values_str["accession_number"][values_str["accession_number"].len() - 1..] + ), + ] { + if keyword == "attributed to" + || keyword == "T" + || keyword == "N0" + || keyword == "N" + || keyword == "artist" + || keyword == "Bronze" + { + keywords.push(keyword); + } + } + client .email_import( format!( @@ -821,15 +824,7 @@ pub async fn create(client: &mut Client) { Id::new(values_int["year"] as u64).to_string(), Id::new((values_int["acquisitionYear"] + 1000) as u64).to_string(), ], - [ - values_str["medium"].to_string(), - values_str["artistRole"].to_string(), - values_str["accession_number"][0..1].to_string(), - format!( - "N{}", - &values_str["accession_number"][values_str["accession_number"].len() - 1..] - ), - ] + keywords .into(), Some(values_int["year"] as i64), ) diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index de4a04d2..7f1e7569 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -373,8 +373,8 @@ pub async fn jmap_tests() { ) .await; - webhooks::test(&mut params).await; - //email_query::test(&mut params, delete).await; + /*webhooks::test(&mut params).await; + email_query::test(&mut params, delete).await; email_get::test(&mut params).await; email_set::test(&mut params).await; email_parse::test(&mut params).await; @@ -382,8 +382,8 @@ pub async fn jmap_tests() { email_changes::test(&mut params).await; email_query_changes::test(&mut params).await; email_copy::test(&mut params).await; - thread_get::test(&mut params).await; - //thread_merge::test(&mut params).await; + thread_get::test(&mut params).await;*/ + thread_merge::test(&mut params).await; mailbox::test(&mut params).await; delivery::test(&mut params).await; auth_acl::test(&mut params).await; diff --git a/tests/src/jmap/purge.rs b/tests/src/jmap/purge.rs index 6e952b57..452525c2 100644 --- a/tests/src/jmap/purge.rs +++ b/tests/src/jmap/purge.rs @@ -10,7 +10,7 @@ use directory::{QueryBy, backend::internal::manage::ManageDirectory}; use email::{ mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheAccess, MessageCacheFetch}, delete::EmailDeletion, }, }; diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index a03e7198..52264fb1 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -10,7 +10,7 @@ use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; use email::message::{ - cache::{MessageCache, MessageCacheAccess}, + cache::{MessageCacheAccess, MessageCacheFetch}, metadata::MessageData, }; use futures::future::join_all; @@ -223,7 +223,7 @@ async fn email_tests(server: Server, client: Arc) { .await .unwrap() .in_mailbox(mailbox_id) - .map(|(id, _)| id), + .map(|m| m.document_id), ); let mut email_ids_check = email_ids_in_mailbox.clone(); email_ids_check &= &email_ids;