From 6ba892f7d620ead08c4f4a6a9ba26f7b644418e1 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Fri, 7 Jul 2023 09:27:22 +0200 Subject: [PATCH] FoundationDB changes. --- Cargo.lock | 4 +- crates/imap/Cargo.toml | 2 +- crates/jmap/Cargo.toml | 2 +- crates/jmap/src/auth/mod.rs | 2 +- crates/main/Cargo.toml | 3 +- crates/store/Cargo.toml | 2 +- crates/store/src/backend/foundationdb/read.rs | 3 +- .../store/src/backend/foundationdb/write.rs | 95 ++++++++++--------- tests/Cargo.toml | 3 +- tests/src/jmap/push_subscription.rs | 5 +- tests/src/store/assign_id.rs | 8 +- tests/src/store/mod.rs | 1 + 12 files changed, 71 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 078cc9d7..033e4772 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1881,7 +1881,7 @@ dependencies = [ [[package]] name = "imap" -version = "0.1.0" +version = "0.3.0" dependencies = [ "ahash 0.8.3", "dashmap", @@ -2056,7 +2056,7 @@ dependencies = [ [[package]] name = "jmap" -version = "0.1.0" +version = "0.3.0" dependencies = [ "aes-gcm", "aes-gcm-siv", diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index b5d55157..2878926f 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap" -version = "0.1.0" +version = "0.3.0" edition = "2021" resolver = "2" diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 4368fb05..ef97a882 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jmap" -version = "0.1.0" +version = "0.3.0" edition = "2021" resolver = "2" diff --git a/crates/jmap/src/auth/mod.rs b/crates/jmap/src/auth/mod.rs index ee5f8de0..41344a11 100644 --- a/crates/jmap/src/auth/mod.rs +++ b/crates/jmap/src/auth/mod.rs @@ -46,7 +46,7 @@ pub mod authenticate; pub mod oauth; pub mod rate_limit; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct AccessToken { pub primary_id: u32, pub member_of: Vec, diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index b1849eb6..48506a1e 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -31,7 +31,8 @@ tracing = "0.1" jemallocator = "0.5.0" [features] -default = ["sqlite"] +#default = ["sqlite"] +default = ["foundationdb"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 0739849a..329d6b7a 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -8,7 +8,7 @@ resolver = "2" utils = { path = "../utils" } maybe-async = { path = "../maybe-async" } rocksdb = { version = "0.20.1", optional = true } -foundationdb = { version = "0.8.0", optional = true } +foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"], optional = true } rusqlite = { version = "0.29.0", features = ["bundled"], optional = true } rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls"] } tokio = { version = "1.23", features = ["sync", "fs", "io-util"] } diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index 223974ba..7049929f 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -438,7 +438,8 @@ impl Store { if (key.len() == 4 && value.len() == 8 && u32::deserialize(key).is_ok() - && u64::deserialize(value).is_ok()) || &key[0..4] == u32::MAX.to_be_bytes() { + && u64::deserialize(value).is_ok()) + || &key[0..4] == u32::MAX.to_be_bytes() { continue; } diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 9502b81b..8afd9601 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -34,7 +34,7 @@ use rand::Rng; use crate::{ write::{ key::{DeserializeBigEndian, KeySerializer}, - now, Batch, Operation, + now, Batch, Operation, ValueClass, }, AclKey, BitmapKey, Deserialize, IndexKey, LogKey, Serialize, Store, ValueKey, SUBSPACE_QUOTAS, SUBSPACE_VALUES, @@ -94,15 +94,30 @@ impl Store { } => { document_id = *document_id_; } - Operation::Value { family, field, set } => { - let key = ValueKey { - account_id, - collection, - document_id, - family: *family, - field: *field, - } - .serialize(); + Operation::Value { class, set } => { + let key = match class { + ValueClass::Property { field, family } => ValueKey { + account_id, + collection, + document_id, + family: *family, + field: *field, + } + .serialize(), + ValueClass::Acl { grant_account_id } => AclKey { + grant_account_id: *grant_account_id, + to_account_id: account_id, + to_collection: collection, + to_document_id: document_id, + } + .serialize(), + ValueClass::Custom { bytes } => { + let mut key = Vec::with_capacity(1 + bytes.len()); + key.push(SUBSPACE_VALUES); + key.extend_from_slice(bytes); + key + } + }; if let Some(value) = set { trx.set(&key, value); } else { @@ -151,23 +166,6 @@ impl Store { .set(document_id); } } - Operation::Acl { - grant_account_id, - set, - } => { - let key = AclKey { - grant_account_id: *grant_account_id, - to_account_id: account_id, - to_collection: collection, - to_document_id: document_id, - } - .serialize(); - if let Some(value) = set { - trx.set(&key, value); - } else { - trx.clear(&key); - } - } Operation::Log { collection, change_id, @@ -182,24 +180,38 @@ impl Store { trx.set(&key, set); } Operation::AssertValue { - field, - family, + class, assert_value, } => { - let key = ValueKey { - account_id, - collection, - document_id, - family: *family, - field: *field, - } - .serialize(); + let key = match class { + ValueClass::Property { field, family } => ValueKey { + account_id, + collection, + document_id, + family: *family, + field: *field, + } + .serialize(), + ValueClass::Acl { grant_account_id } => AclKey { + grant_account_id: *grant_account_id, + to_account_id: account_id, + to_collection: collection, + to_document_id: document_id, + } + .serialize(), + ValueClass::Custom { bytes } => { + let mut key = Vec::with_capacity(1 + bytes.len()); + key.push(SUBSPACE_VALUES); + key.extend_from_slice(bytes); + key + } + }; let matches = if let Ok(bytes) = trx.get(&key, false).await { if let Some(bytes) = bytes { assert_value.matches(bytes.as_ref()) } else { - assert_value.is_none(); + assert_value.is_none() } } else { false @@ -315,7 +327,6 @@ impl Store { let collection = collection.into(); loop { - //let mut assign_source = 0; // First try to reuse an expired assigned id let begin = IndexKey { account_id, @@ -375,10 +386,8 @@ impl Store { // Obtain a random id from the expired ids if expired_ids.len() > 1 { document_id = expired_ids[rand::thread_rng().gen_range(0..expired_ids.len())]; - //assign_source = 1; } else { document_id = expired_ids[0]; - //assign_source = 2; } } else { // Find the next available id @@ -420,8 +429,6 @@ impl Store { for document_id_ in 0..BITS_PER_BLOCK { if !reserved_ids.contains(&document_id_) { document_id = document_id_; - //assign_source = 4; - break; } } @@ -441,8 +448,6 @@ impl Store { match trx.commit().await { Ok(_) => { - //println!("assigned id: {document_id} {assign_source}"); - return Ok(document_id); } Err(err) => { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index a7c8bd47..dce7d3b2 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -5,7 +5,8 @@ edition = "2021" resolver = "2" [features] -default = ["sqlite"] +#default = ["sqlite"] +default = ["foundationdb"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation"] diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 89bc4b81..84f9de30 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -38,6 +38,7 @@ use jmap::{ http::{fetch_body, ToHttpResponse}, HtmlResponse, StateChangeResponse, }, + auth::AccessToken, JMAP, }; use jmap_client::{client::Client, mailbox::Role, push_subscription::Keys}; @@ -307,7 +308,9 @@ impl utils::listener::SessionManager for SessionManager { .map_or(false, |encoding| { encoding.to_str().unwrap() == "aes128gcm" }); - let body = fetch_body(&mut req, 1024 * 1024).await.unwrap(); + let body = fetch_body(&mut req, 1024 * 1024, &AccessToken::default()) + .await + .unwrap(); let message = serde_json::from_slice::(&if is_encrypted { ece::decrypt( &push.keypair, diff --git a/tests/src/store/assign_id.rs b/tests/src/store/assign_id.rs index c1b13b3c..eed81298 100644 --- a/tests/src/store/assign_id.rs +++ b/tests/src/store/assign_id.rs @@ -30,16 +30,16 @@ use store::{write::BatchBuilder, Store}; pub async fn test(db: Arc) { println!("Running Store ID assignment tests..."); - /*store::backend::foundationdb::write::ID_ASSIGNMENT_EXPIRY - .store(2, std::sync::atomic::Ordering::Relaxed);*/ + store::backend::foundationdb::write::ID_ASSIGNMENT_EXPIRY + .store(2, std::sync::atomic::Ordering::Relaxed); test_1(db.clone()).await; test_2(db.clone()).await; test_3(db.clone()).await; test_4(db).await; - /*store::backend::foundationdb::write::ID_ASSIGNMENT_EXPIRY - .store(60 * 60, std::sync::atomic::Ordering::Relaxed);*/ + store::backend::foundationdb::write::ID_ASSIGNMENT_EXPIRY + .store(60 * 60, std::sync::atomic::Ordering::Relaxed); } async fn test_1(db: Arc) { diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index c97ebdba..6fbcd4d4 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -21,6 +21,7 @@ * for more details. */ +#[cfg(feature = "foundationdb")] pub mod assign_id; pub mod blob; pub mod query;