Scale-out storage: IMAP CONDSTORE raises the mark, and the two-node check (ST-7, test 11)
A FETCH with CHANGEDSINCE presents its mod-sequence to the read scope, so a replica must have that change before it answers, as a JMAP sinceState already did. replica_cluster_tests covers test 11: with the replica's replay paused, a write on one node is read back through a second store with its own marks, sharing through Redis. Composite stores nest store futures deeply enough to pass rustc's default query depth once both postgres and redis are compiled in, so the server crates raise their recursion limit.
This commit is contained in:
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use crate::auth::{AccessTokenInner, EmailAddress};
|
use crate::auth::{AccessTokenInner, EmailAddress};
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
|
|||||||
@@ -3,6 +3,10 @@
|
|||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
pub mod calendar;
|
pub mod calendar;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use crate::backend::oidc::OpenIdDirectory;
|
use crate::backend::oidc::OpenIdDirectory;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
pub mod cache;
|
pub mod cache;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use calcard::common::timezone::Tz;
|
use calcard::common::timezone::Tz;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
pub mod api;
|
pub mod api;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
|
|||||||
@@ -175,6 +175,8 @@ impl<T: SessionStream> SessionData<T> {
|
|||||||
|
|
||||||
// Convert state to modseq
|
// Convert state to modseq
|
||||||
if let Some(changed_since) = arguments.changed_since {
|
if let Some(changed_since) = arguments.changed_since {
|
||||||
|
// inbuxa: ST-7: a replica must have the state the client knows
|
||||||
|
store::backend::scaleout::replica::present_change(self.account_id, changed_since);
|
||||||
// Obtain changes since the modseq.
|
// Obtain changes since the modseq.
|
||||||
let changelog = self
|
let changelog = self
|
||||||
.server
|
.server
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use jmap_proto::object::JmapObjectId;
|
use jmap_proto::object::JmapObjectId;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
pub mod core;
|
pub mod core;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use std::{net::IpAddr, sync::Arc};
|
use std::{net::IpAddr, sync::Arc};
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use broadcast::publisher::spawn_broadcast_publisher;
|
use broadcast::publisher::spawn_broadcast_publisher;
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
use common::{
|
use common::{
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||||
|
// futures deeply enough to pass rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#![warn(clippy::large_futures)]
|
#![warn(clippy::large_futures)]
|
||||||
|
|
||||||
pub mod analysis;
|
pub mod analysis;
|
||||||
|
|||||||
@@ -141,12 +141,26 @@ pub fn is_replica_subspace(subspace: u8) -> bool {
|
|||||||
/// the client presented (ST-7, step 4), and the replica it settled on.
|
/// the client presented (ST-7, step 4), and the replica it settled on.
|
||||||
pub struct ReadScope {
|
pub struct ReadScope {
|
||||||
accounts: Vec<(u32, u64)>,
|
accounts: Vec<(u32, u64)>,
|
||||||
|
/// Change ids the client presented during the scope (ST-7, step 4).
|
||||||
|
presented: Mutex<Vec<(u32, u64)>>,
|
||||||
choice: tokio::sync::OnceCell<Option<usize>>,
|
choice: tokio::sync::OnceCell<Option<usize>>,
|
||||||
/// Set by any write made inside the scope: from then on it reads from
|
/// Set by any write made inside the scope: from then on it reads from
|
||||||
/// the primary (ST-6: a read in a request that writes).
|
/// the primary (ST-6: a read in a request that writes).
|
||||||
wrote: std::sync::atomic::AtomicBool,
|
wrote: std::sync::atomic::AtomicBool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A state the client presented raises the mark a replica must have
|
||||||
|
/// reached before it may answer this scope (ST-7, step 4).
|
||||||
|
pub fn present_change(account_id: u32, change_id: u64) {
|
||||||
|
let _ = READ_SCOPE.try_with(|scope| {
|
||||||
|
scope
|
||||||
|
.presented
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push((account_id, change_id))
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/// A write happened in the current task: a read scope, if any, stops using
|
/// A write happened in the current task: a read scope, if any, stops using
|
||||||
/// replicas.
|
/// replicas.
|
||||||
pub fn note_scope_write() {
|
pub fn note_scope_write() {
|
||||||
@@ -167,6 +181,7 @@ pub async fn replica_read<F: Future>(
|
|||||||
.scope(
|
.scope(
|
||||||
Arc::new(ReadScope {
|
Arc::new(ReadScope {
|
||||||
accounts: accounts.into_iter().collect(),
|
accounts: accounts.into_iter().collect(),
|
||||||
|
presented: Mutex::new(Vec::new()),
|
||||||
choice: tokio::sync::OnceCell::new(),
|
choice: tokio::sync::OnceCell::new(),
|
||||||
wrote: std::sync::atomic::AtomicBool::new(false),
|
wrote: std::sync::atomic::AtomicBool::new(false),
|
||||||
}),
|
}),
|
||||||
@@ -310,8 +325,15 @@ impl ReplicatedStore {
|
|||||||
if !replica.usable() {
|
if !replica.usable() {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
for (account_id, presented) in &scope.accounts {
|
let presented = scope.presented.lock().unwrap().clone();
|
||||||
let mark = self.mark(*account_id).await.max(*presented);
|
for (account_id, from_request) in &scope.accounts {
|
||||||
|
let highest = presented
|
||||||
|
.iter()
|
||||||
|
.filter(|(id, _)| id == account_id)
|
||||||
|
.map(|(_, change_id)| *change_id)
|
||||||
|
.max()
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mark = self.mark(*account_id).await.max(*from_request).max(highest);
|
||||||
if mark == 0 {
|
if mark == 0 {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,10 @@
|
|||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
// inbuxa: composite stores nest store futures deeply enough to pass
|
||||||
|
// rustc's default query depth
|
||||||
|
#![recursion_limit = "512"]
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use ::store::registry::bootstrap::Bootstrap;
|
use ::store::registry::bootstrap::Bootstrap;
|
||||||
#[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
|
#[cfg(not(any(target_env = "msvc", target_os = "freebsd")))]
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ pub mod registry;
|
|||||||
pub mod replica; // inbuxa: read replicas
|
pub mod replica; // inbuxa: read replicas
|
||||||
#[cfg(feature = "mysql")]
|
#[cfg(feature = "mysql")]
|
||||||
pub mod replica_mysql; // inbuxa: read replicas on MySQL
|
pub mod replica_mysql; // inbuxa: read replicas on MySQL
|
||||||
|
#[cfg(all(feature = "postgres", feature = "redis"))]
|
||||||
|
pub mod replica_cluster; // inbuxa: read replicas across nodes
|
||||||
pub mod scaleout; // inbuxa: scale-out storage
|
pub mod scaleout; // inbuxa: scale-out storage
|
||||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
||||||
pub mod sql_timeout;
|
pub mod sql_timeout;
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
//! Read replicas across two nodes, from
|
||||||
|
//! `docs/spec/features/scale-out-storage.md` (test 11): a write on one node
|
||||||
|
//! is seen by a read on the other straight away, because the high-water
|
||||||
|
//! mark is shared (ST-7, step 2). Built with `postgres` and `redis`.
|
||||||
|
|
||||||
|
use crate::utils::{
|
||||||
|
containers::{PG_REPLICA_CONTAINER, ensure_redis, psql},
|
||||||
|
server::TestServerBuilder,
|
||||||
|
};
|
||||||
|
use registry::schema::structs::{Coordinator, InMemoryStore, RedisStore};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
use store::{
|
||||||
|
SerializeInfallible, Store, U64_LEN, ValueKey,
|
||||||
|
backend::scaleout::replica::{ReplicaState, ReplicatedStore, replica_read},
|
||||||
|
write::{AnyClass, BatchBuilder, ValueClass},
|
||||||
|
};
|
||||||
|
use types::collection::{Collection, SyncCollection};
|
||||||
|
|
||||||
|
fn replicated(store: &Store) -> std::sync::Arc<ReplicatedStore> {
|
||||||
|
match store {
|
||||||
|
Store::Replicated(store) => store.clone(),
|
||||||
|
other => panic!("the data store isn't replicated: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_up(store: &ReplicatedStore, what: &str) {
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(90);
|
||||||
|
while store.replicas[0].state() != ReplicaState::Up {
|
||||||
|
assert!(
|
||||||
|
Instant::now() < deadline,
|
||||||
|
"{what}: the replica is {:?}",
|
||||||
|
store.replicas[0].state()
|
||||||
|
);
|
||||||
|
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn account_key(account_id: u32) -> ValueClass {
|
||||||
|
let mut key = b"Rt".to_vec();
|
||||||
|
key.extend_from_slice(&account_id.to_be_bytes());
|
||||||
|
ValueClass::Any(AnyClass {
|
||||||
|
subspace: store::SUBSPACE_PROPERTY,
|
||||||
|
key,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test 11. `cargo test -p tests --features postgres,redis
|
||||||
|
/// replica_cluster_tests -- --ignored`, with
|
||||||
|
/// `STORE=PostgreSqlReplicated`.
|
||||||
|
#[ignore]
|
||||||
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
|
pub async fn replica_cluster_tests() {
|
||||||
|
assert_eq!(
|
||||||
|
std::env::var("STORE").as_deref(),
|
||||||
|
Ok("PostgreSqlReplicated"),
|
||||||
|
"run with STORE=PostgreSqlReplicated"
|
||||||
|
);
|
||||||
|
ensure_redis().await;
|
||||||
|
let redis = || RedisStore {
|
||||||
|
url: "redis://127.0.0.1".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Node A is a server; node B is a second replicated store with its own
|
||||||
|
// marks, as another node would have. Both share marks through Redis.
|
||||||
|
let node_a = TestServerBuilder::new("replica_cluster_a")
|
||||||
|
.await
|
||||||
|
.with_default_listeners()
|
||||||
|
.await
|
||||||
|
.with_object(Coordinator::Redis(redis()))
|
||||||
|
.await
|
||||||
|
.with_object(InMemoryStore::Redis(redis()))
|
||||||
|
.await
|
||||||
|
.build()
|
||||||
|
.await;
|
||||||
|
let store_a = replicated(node_a.server.store());
|
||||||
|
let shared = store::backend::redis::RedisStore::open_single(redis())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
store_a.share_marks(shared.clone());
|
||||||
|
let node_b = crate::utils::storage::build_data_store(
|
||||||
|
"PostgreSqlReplicated",
|
||||||
|
node_a.temp_dir.path.to_str().unwrap(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let node_b = Store::build(node_b).await.unwrap();
|
||||||
|
let store_b = replicated(&node_b);
|
||||||
|
store_b.share_marks(shared);
|
||||||
|
wait_up(&store_a, "node A").await;
|
||||||
|
wait_up(&store_b, "node B").await;
|
||||||
|
|
||||||
|
// Everything written so far reaches the replica, then replay stops
|
||||||
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
|
psql(PG_REPLICA_CONTAINER, "SELECT pg_wal_replay_pause()");
|
||||||
|
|
||||||
|
// Node A writes for the account, which assigns a change id
|
||||||
|
let account_id = 1234u32;
|
||||||
|
let value = store::rand::random::<u64>();
|
||||||
|
let mut batch = BatchBuilder::new();
|
||||||
|
batch
|
||||||
|
.with_account_id(account_id)
|
||||||
|
.with_collection(Collection::Email)
|
||||||
|
.log_container_insert(SyncCollection::Email)
|
||||||
|
.set(account_key(account_id), value.serialize());
|
||||||
|
node_a
|
||||||
|
.server
|
||||||
|
.store()
|
||||||
|
.write(batch.build_all())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Node B reads it straight away, in a scope that may use the replica
|
||||||
|
let read = replica_read([(account_id, 0)], async {
|
||||||
|
node_b
|
||||||
|
.get_value::<u64>(ValueKey::from(account_key(account_id)))
|
||||||
|
.await
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
read,
|
||||||
|
Some(value),
|
||||||
|
"test 11: node B must see node A's write (the replica is paused)"
|
||||||
|
);
|
||||||
|
let _ = U64_LEN;
|
||||||
|
|
||||||
|
psql(PG_REPLICA_CONTAINER, "SELECT pg_wal_replay_resume()");
|
||||||
|
node_a.temp_dir.delete();
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user