/* * Copyright (c) 2023 Stalwart Labs Ltd. * * This file is part of the Stalwart Mail Server. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * in the LICENSE file at the top-level directory of this distribution. * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . * * You can be released from the requirements of the AGPLv3 license by * purchasing a commercial license. Please contact licensing@stalw.art * for more details. */ use std::{ collections::HashSet, hash::Hash, slice::Iter, time::{Duration, SystemTime}, }; use nlp::tokenizers::word::WordTokenizer; use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; use crate::{ backend::MAX_TOKEN_LENGTH, BlobClass, BlobHash, Deserialize, Serialize, BLOB_HASH_LEN, }; use self::assert::AssertValue; pub mod assert; pub mod assign_id; pub mod batch; pub mod bitmap; pub mod blob; pub mod hash; pub mod key; pub mod log; pub mod purge; #[cfg(not(feature = "test_mode"))] pub(crate) const ID_ASSIGNMENT_EXPIRY: u64 = 60 * 60; // seconds #[cfg(not(feature = "test_mode"))] pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 10; #[cfg(not(feature = "test_mode"))] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(10); #[cfg(feature = "test_mode")] pub(crate) const MAX_COMMIT_ATTEMPTS: u32 = 1000; #[cfg(feature = "test_mode")] pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(3600); pub const F_VALUE: u32 = 1 << 0; pub const F_INDEX: u32 = 1 << 1; pub const F_BITMAP: u32 = 1 << 2; pub const F_CLEAR: u32 = 1 << 3; #[derive(Debug)] pub struct Batch { pub ops: Vec, } #[derive(Debug)] pub struct BatchBuilder { pub ops: Vec, } #[derive(Debug, PartialEq, Eq, Hash)] pub enum Operation { AccountId { account_id: u32, }, Collection { collection: u8, }, DocumentId { document_id: u32, }, AssertValue { class: ValueClass, assert_value: AssertValue, }, Value { class: ValueClass, op: ValueOp, }, Index { field: u8, key: Vec, set: bool, }, Bitmap { class: BitmapClass, set: bool, }, Blob { hash: BlobHash, op: BlobOp, set: bool, }, Log { change_id: u64, collection: u8, set: Vec, }, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum BitmapClass { DocumentIds, Tag { field: u8, value: TagValue }, Text { field: u8, token: BitmapHash }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct BitmapHash { pub hash: [u8; 8], pub len: u8, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum TagValue { Id(u32), Text(Vec), Static(u8), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum ValueClass { Property(u8), Acl(u32), Subspace { key: Vec, id: u8 }, Key { key: Vec }, TermIndex, ReservedId, } #[derive(Debug, PartialEq, Eq, Hash, Default)] pub enum ValueOp { Set(Vec), Add(i64), #[default] Clear, } #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] pub enum BlobOp { Reserve { until: u64, size: usize }, Commit, Link, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct AnyKey> { pub subspace: u8, pub key: T, } impl From for TagValue { fn from(value: u32) -> Self { TagValue::Id(value) } } impl From> for TagValue { fn from(value: Vec) -> Self { TagValue::Text(value) } } impl From for TagValue { fn from(value: String) -> Self { TagValue::Text(value.into_bytes()) } } impl From for TagValue { fn from(value: u8) -> Self { TagValue::Static(value) } } impl From<()> for TagValue { fn from(_: ()) -> Self { TagValue::Text(vec![]) } } impl Serialize for u32 { fn serialize(self) -> Vec { self.to_be_bytes().to_vec() } } impl Serialize for u64 { fn serialize(self) -> Vec { self.to_be_bytes().to_vec() } } impl Serialize for u16 { fn serialize(self) -> Vec { self.to_be_bytes().to_vec() } } impl Serialize for f64 { fn serialize(self) -> Vec { self.to_be_bytes().to_vec() } } impl Serialize for &str { fn serialize(self) -> Vec { self.as_bytes().to_vec() } } impl Serialize for &String { fn serialize(self) -> Vec { self.as_bytes().to_vec() } } impl Serialize for String { fn serialize(self) -> Vec { self.into_bytes() } } impl Serialize for Vec { fn serialize(self) -> Vec { self } } impl Deserialize for String { fn deserialize(bytes: &[u8]) -> crate::Result { Ok(String::from_utf8_lossy(bytes).into_owned()) } } impl Deserialize for u64 { fn deserialize(bytes: &[u8]) -> crate::Result { Ok(u64::from_be_bytes(bytes.try_into().map_err(|_| { crate::Error::InternalError("Failed to deserialize u64".to_string()) })?)) } } impl Deserialize for u32 { fn deserialize(bytes: &[u8]) -> crate::Result { Ok(u32::from_be_bytes(bytes.try_into().map_err(|_| { crate::Error::InternalError("Failed to deserialize u64".to_string()) })?)) } } pub trait SerializeInto { fn serialize_into(&self, buf: &mut Vec); } pub trait DeserializeFrom: Sized { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option; } impl Serialize for Vec { fn serialize(self) -> Vec { let mut bytes = Vec::with_capacity(self.len() * 4); bytes.push_leb128(self.len()); for item in self { item.serialize_into(&mut bytes); } bytes } } impl SerializeInto for String { fn serialize_into(&self, buf: &mut Vec) { buf.push_leb128(self.len()); if !self.is_empty() { buf.extend_from_slice(self.as_bytes()); } } } impl SerializeInto for Vec { fn serialize_into(&self, buf: &mut Vec) { buf.push_leb128(self.len()); if !self.is_empty() { buf.extend_from_slice(self.as_slice()); } } } impl SerializeInto for u32 { fn serialize_into(&self, buf: &mut Vec) { buf.push_leb128(*self); } } impl SerializeInto for u64 { fn serialize_into(&self, buf: &mut Vec) { buf.push_leb128(*self); } } impl DeserializeFrom for u32 { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { bytes.next_leb128() } } impl DeserializeFrom for u64 { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { bytes.next_leb128() } } impl DeserializeFrom for String { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { >::deserialize_from(bytes).and_then(|s| String::from_utf8(s).ok()) } } impl DeserializeFrom for Vec { fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { let len: usize = bytes.next_leb128()?; let mut buf = Vec::with_capacity(len); for _ in 0..len { buf.push(*bytes.next()?); } buf.into() } } impl Deserialize for Vec { fn deserialize(bytes: &[u8]) -> crate::Result { let mut bytes = bytes.iter(); let len: usize = bytes .next_leb128() .ok_or_else(|| crate::Error::InternalError("Failed to deserialize Vec".to_string()))?; let mut list = Vec::with_capacity(len); for _ in 0..len { list.push(T::deserialize_from(&mut bytes).ok_or_else(|| { crate::Error::InternalError("Failed to deserialize Vec".to_string()) })?); } Ok(list) } } trait HasFlag { fn has_flag(&self, flag: u32) -> bool; } impl HasFlag for u32 { #[inline(always)] fn has_flag(&self, flag: u32) -> bool { self & flag == flag } } pub trait ToBitmaps { fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool); } pub trait TokenizeText { fn tokenize_into(&self, tokens: &mut HashSet); fn to_tokens(&self) -> HashSet; } impl ToBitmaps for &str { fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { let mut tokens = HashSet::new(); self.tokenize_into(&mut tokens); for token in tokens { ops.push(Operation::Bitmap { class: BitmapClass::Text { field, token: BitmapHash::new(token), }, set, }); } } } impl TokenizeText for &str { fn tokenize_into(&self, tokens: &mut HashSet) { for token in WordTokenizer::new(self, MAX_TOKEN_LENGTH) { tokens.insert(token.word.into_owned()); } } fn to_tokens(&self) -> HashSet { let mut tokens = HashSet::new(); self.tokenize_into(&mut tokens); tokens } } impl ToBitmaps for String { fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { self.as_str().to_bitmaps(ops, field, set) } } impl ToBitmaps for u32 { fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { ops.push(Operation::Bitmap { class: BitmapClass::Tag { field, value: TagValue::Id(*self), }, set, }); } } impl ToBitmaps for u64 { fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { ops.push(Operation::Bitmap { class: BitmapClass::Tag { field, value: TagValue::Id(*self as u32), }, set, }); } } impl ToBitmaps for f64 { fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { unreachable!() } } impl ToBitmaps for Vec { fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { for item in self { item.to_bitmaps(ops, field, set); } } } impl Serialize for () { fn serialize(self) -> Vec { Vec::with_capacity(0) } } impl ToBitmaps for () { fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { unreachable!() } } pub trait IntoOperations { fn build(self, batch: &mut BatchBuilder); } impl Operation { pub fn acl(grant_account_id: u32, set: Option>) -> Self { Operation::Value { class: ValueClass::Acl(grant_account_id), op: set.map(ValueOp::Set).unwrap_or(ValueOp::Clear), } } } #[inline(always)] pub fn now() -> u64 { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()) } impl AsRef for ValueClass { fn as_ref(&self) -> &ValueClass { self } } impl AsRef for BitmapClass { fn as_ref(&self) -> &BitmapClass { self } } impl BitmapClass { pub fn tag_id(property: impl Into, id: u32) -> Self { BitmapClass::Tag { field: property.into(), value: TagValue::Id(id), } } } impl BlobHash { pub fn new_max() -> Self { BlobHash([u8::MAX; BLOB_HASH_LEN]) } pub fn try_from_hash_slice(value: &[u8]) -> Result { value.try_into().map(BlobHash) } pub fn as_slice(&self) -> &[u8] { self.0.as_ref() } } impl From<&[u8]> for BlobHash { fn from(value: &[u8]) -> Self { BlobHash(blake3::hash(value).into()) } } impl From> for BlobHash { fn from(value: Vec) -> Self { value.as_slice().into() } } impl From<&Vec> for BlobHash { fn from(value: &Vec) -> Self { value.as_slice().into() } } impl AsRef for BlobHash { fn as_ref(&self) -> &BlobHash { self } } impl AsRef<[u8]> for BlobHash { fn as_ref(&self) -> &[u8] { self.0.as_ref() } } impl AsMut<[u8]> for BlobHash { fn as_mut(&mut self) -> &mut [u8] { self.0.as_mut() } } impl AsRef for BlobClass { fn as_ref(&self) -> &BlobClass { self } } impl From for Vec { fn from(value: BlobHash) -> Self { value.0.to_vec() } } impl BlobClass { pub fn account_id(&self) -> u32 { match self { BlobClass::Reserved { account_id } | BlobClass::Linked { account_id, .. } => { *account_id } } } }