diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index 226d43fe..599be1c8 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -96,6 +96,16 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { return Ok(()); } + // inbuxa: #436: one node at a time, so a node starting beside another on + // a fresh store never takes half-written defaults as the whole set + let store = bp.data_store.clone(); + let lease = super::defaults_lease::DefaultsLease::take(&store).await?; + let result = insert_normal_defaults(bp).await; + lease.release(&store).await; + result +} + +async fn insert_normal_defaults(bp: &mut Bootstrap) -> trc::Result<()> { let is_new_install = is_new_install(bp).await?; // inbuxa: a store brought over from Stalwart doesn't serve upstream's web diff --git a/crates/common/src/manager/defaults_lease.rs b/crates/common/src/manager/defaults_lease.rs new file mode 100644 index 00000000..c163dcb7 --- /dev/null +++ b/crates/common/src/manager/defaults_lease.rs @@ -0,0 +1,174 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: one node at a time sets up the defaults (#436). +//! +//! Every normal start checks what the store is missing (roles, listeners, +//! throttles, spam rules...) and adds it, each as "none of these yet, so +//! insert them". Two nodes starting together on a fresh store both did +//! that at once: one could find some of the default listeners already +//! written by the other, take the store as set up, and come up without the +//! rest (on the lab cluster, every listener but `http`, the last one). The +//! same race could store the default roles twice. +//! +//! So the defaults run under a short lease in the store, taken with a +//! compare-and-swap like the store layout lease. A node that finds it held +//! waits until it's released, or runs out because its holder stopped, and +//! then checks the store itself, which by then has everything. + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; +use store::{ + Deserialize, SUBSPACE_INBUXA, Store, ValueKey, + rand::{RngExt, rng}, + write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now}, +}; +use trc::AddContext; + +/// Under `B`, the first boot defaults (store-layout SL-16). +pub const LEASE_KEY: &[u8] = b"Bl"; +/// How long a lease lasts unless renewed. +const LEASE_SECONDS: u64 = 60; +const RENEWAL: Duration = Duration::from_secs(15); +const POLL: Duration = Duration::from_millis(250); + +struct Raw(Vec); + +impl Deserialize for Raw { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(Raw(bytes.to_vec())) + } +} + +fn lease_class() -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: LEASE_KEY.to_vec(), + }) +} + +/// The holder's token, then when the lease runs out. +fn lease_bytes(token: u64, expires: u64) -> Vec { + let mut bytes = Vec::with_capacity(16); + bytes.extend_from_slice(&token.to_be_bytes()); + bytes.extend_from_slice(&expires.to_be_bytes()); + bytes +} + +/// When a stored lease runs out; 0 (long gone) for one that can't be read. +fn expires_of(bytes: &[u8]) -> u64 { + bytes + .get(8..16) + .and_then(|b| b.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0) +} + +fn hash(bytes: &[u8]) -> AssertValue { + AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes)) +} + +fn is_assert_failure(err: &trc::Error) -> bool { + matches!( + err.as_ref(), + trc::EventType::Store(trc::StoreEvent::AssertValueFailed) + ) +} + +/// Replaces `current` (None: no lease) with `new`; false if another node +/// wrote the lease in between. +async fn swap(store: &Store, current: Option<&[u8]>, new: Vec) -> trc::Result { + let mut batch = BatchBuilder::new(); + batch.assert_value(lease_class(), current.map_or(AssertValue::None, hash)); + batch.set(lease_class(), new); + match store.write(batch.build_all()).await { + Ok(_) => Ok(true), + Err(err) if is_assert_failure(&err) => Ok(false), + Err(err) => Err(err.caused_by(trc::location!())), + } +} + +pub struct DefaultsLease { + bytes: Arc>>, + renewal: tokio::task::JoinHandle<()>, +} + +impl DefaultsLease { + /// Waits for the lease and takes it. + pub async fn take(store: &Store) -> trc::Result { + let token: u64 = rng().random(); + loop { + let current = store + .get_value::(ValueKey::from(lease_class())) + .await + .caused_by(trc::location!())? + .map(|Raw(bytes)| bytes); + let free = current + .as_deref() + .is_none_or(|bytes| expires_of(bytes) <= now()); + if free { + let new = lease_bytes(token, now() + LEASE_SECONDS); + if swap(store, current.as_deref(), new.clone()).await? { + return Ok(Self::held(store, token, new)); + } + } + tokio::time::sleep(POLL).await; + } + } + + fn held(store: &Store, token: u64, bytes: Vec) -> Self { + let bytes = Arc::new(Mutex::new(bytes)); + let renewal = { + let (store, bytes) = (store.clone(), bytes.clone()); + tokio::spawn(async move { + loop { + tokio::time::sleep(RENEWAL).await; + let current = bytes.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let new = lease_bytes(token, now() + LEASE_SECONDS); + match swap(&store, Some(¤t), new.clone()).await { + Ok(true) => *bytes.lock().unwrap_or_else(|e| e.into_inner()) = new, + // Taken over after it ran out: nothing left to renew + Ok(false) => return, + Err(err) => { + trc::error!(err.details("Failed to renew the defaults lease")); + } + } + } + }) + }; + DefaultsLease { bytes, renewal } + } + + pub async fn release(self, store: &Store) { + self.renewal.abort(); + let current = self.bytes.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let mut batch = BatchBuilder::new(); + batch.assert_value(lease_class(), hash(¤t)); + batch.clear(lease_class()); + if let Err(err) = store.write(batch.build_all()).await + && !is_assert_failure(&err) + { + // It runs out on its own + trc::error!(err.details("Failed to release the defaults lease")); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_lease_says_when_it_runs_out() { + let bytes = lease_bytes(7, 1_234); + assert_eq!(bytes.len(), 16); + assert_eq!(expires_of(&bytes), 1_234); + // A value that can't be read counts as run out + assert_eq!(expires_of(b"short"), 0); + } +} diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index e7ad827a..fdc9ad3d 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -23,6 +23,7 @@ pub mod check; // inbuxa: K-C5, --check pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles pub mod console; pub mod defaults; +pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults pub mod first_party; pub mod granted_permissions; // inbuxa: permissions added after roles were stored pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment diff --git a/tests/src/store/defaults_lease.rs b/tests/src/store/defaults_lease.rs new file mode 100644 index 00000000..d3e2624a --- /dev/null +++ b/tests/src/store/defaults_lease.rs @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! #436: the defaults lease lets one node at a time set up the defaults. +//! Nodes are played by tasks sharing one store. + +use common::manager::defaults_lease::{DefaultsLease, LEASE_KEY}; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; +use store::{ + SUBSPACE_INBUXA, Store, + backend::sqlite::SqliteStore, + write::{AnyClass, BatchBuilder, ValueClass, now}, +}; + +fn store() -> Store { + Store::SQLite(SqliteStore::open_memory().unwrap().into()) +} + +/// A lease another node left, running out at `expires`. +async fn foreign_lease(store: &Store, expires: u64) { + let mut value = 42u64.to_be_bytes().to_vec(); + value.extend_from_slice(&expires.to_be_bytes()); + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: LEASE_KEY.to_vec(), + }), + value, + ); + store.write(batch.build_all()).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn one_node_at_a_time() { + let store = store(); + let inside = Arc::new(AtomicBool::new(false)); + let turns = Arc::new(AtomicUsize::new(0)); + + let nodes = (0..4) + .map(|_| { + let (store, inside, turns) = (store.clone(), inside.clone(), turns.clone()); + tokio::spawn(async move { + for _ in 0..3 { + let lease = DefaultsLease::take(&store).await.unwrap(); + assert!( + !inside.swap(true, Ordering::SeqCst), + "two nodes held the lease at once" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + inside.store(false, Ordering::SeqCst); + turns.fetch_add(1, Ordering::SeqCst); + lease.release(&store).await; + } + }) + }) + .collect::>(); + for node in nodes { + tokio::time::timeout(Duration::from_secs(30), node) + .await + .expect("a node never got the lease") + .unwrap(); + } + assert_eq!(turns.load(Ordering::SeqCst), 12); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_live_lease_blocks_and_one_run_out_does_not() { + let store = store(); + + // Another node is setting up the defaults: wait + foreign_lease(&store, now() + 60).await; + assert!( + tokio::time::timeout(Duration::from_secs(1), DefaultsLease::take(&store)) + .await + .is_err(), + "took a lease another node holds" + ); + + // That node stopped and its lease ran out: take it at once + foreign_lease(&store, now() - 1).await; + let started = Instant::now(); + let lease = DefaultsLease::take(&store).await.unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + + // Released, the next node gets it at once + lease.release(&store).await; + let started = Instant::now(); + DefaultsLease::take(&store) + .await + .unwrap() + .release(&store) + .await; + assert!(started.elapsed() < Duration::from_secs(2)); +} diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index b65f810f..4477a7a4 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -7,6 +7,7 @@ */ pub mod blob; +pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults pub mod import_export; pub mod lookup; pub mod ops;