From 3e6300f802a198351c65ee55e3cff146f65c112e Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 09:29:21 -0700 Subject: [PATCH] Defaults: one node at a time on a fresh store (#436) Every normal start adds what the store is missing, each as "none of these yet, so insert them". Two nodes starting together on a freshly bootstrapped store both did that at once. The second could find some of the seven default listeners already written by the first, take the set as complete and come up without the rest: in the Kubernetes lab, every listener but http, the last one, so the node answered nothing on 8080. The default roles and the Authentication record could be stored twice the same way. The normal-mode defaults now run under a short lease in the store (`_` key "Bl", under the first boot defaults' letter), taken with a compare-and-swap like the store layout lease: 60 seconds, renewed every 15 while held, released when done. A node that finds it held waits until it's released, or runs out because its holder stopped, then checks the store itself, which by then has everything. Bootstrap and recovery mode are unchanged. Test: tests/src/store/defaults_lease.rs: four tasks sharing one store never hold the lease together; a live lease another node holds blocks; one that ran out, or was released, is taken at once. --- crates/common/src/manager/defaults.rs | 10 ++ crates/common/src/manager/defaults_lease.rs | 174 ++++++++++++++++++++ crates/common/src/manager/mod.rs | 1 + tests/src/store/defaults_lease.rs | 104 ++++++++++++ tests/src/store/mod.rs | 1 + 5 files changed, 290 insertions(+) create mode 100644 crates/common/src/manager/defaults_lease.rs create mode 100644 tests/src/store/defaults_lease.rs diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index 226d43fe..599be1c8 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -96,6 +96,16 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { return Ok(()); } + // inbuxa: #436: one node at a time, so a node starting beside another on + // a fresh store never takes half-written defaults as the whole set + let store = bp.data_store.clone(); + let lease = super::defaults_lease::DefaultsLease::take(&store).await?; + let result = insert_normal_defaults(bp).await; + lease.release(&store).await; + result +} + +async fn insert_normal_defaults(bp: &mut Bootstrap) -> trc::Result<()> { let is_new_install = is_new_install(bp).await?; // inbuxa: a store brought over from Stalwart doesn't serve upstream's web diff --git a/crates/common/src/manager/defaults_lease.rs b/crates/common/src/manager/defaults_lease.rs new file mode 100644 index 00000000..c163dcb7 --- /dev/null +++ b/crates/common/src/manager/defaults_lease.rs @@ -0,0 +1,174 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: one node at a time sets up the defaults (#436). +//! +//! Every normal start checks what the store is missing (roles, listeners, +//! throttles, spam rules...) and adds it, each as "none of these yet, so +//! insert them". Two nodes starting together on a fresh store both did +//! that at once: one could find some of the default listeners already +//! written by the other, take the store as set up, and come up without the +//! rest (on the lab cluster, every listener but `http`, the last one). The +//! same race could store the default roles twice. +//! +//! So the defaults run under a short lease in the store, taken with a +//! compare-and-swap like the store layout lease. A node that finds it held +//! waits until it's released, or runs out because its holder stopped, and +//! then checks the store itself, which by then has everything. + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; +use store::{ + Deserialize, SUBSPACE_INBUXA, Store, ValueKey, + rand::{RngExt, rng}, + write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now}, +}; +use trc::AddContext; + +/// Under `B`, the first boot defaults (store-layout SL-16). +pub const LEASE_KEY: &[u8] = b"Bl"; +/// How long a lease lasts unless renewed. +const LEASE_SECONDS: u64 = 60; +const RENEWAL: Duration = Duration::from_secs(15); +const POLL: Duration = Duration::from_millis(250); + +struct Raw(Vec); + +impl Deserialize for Raw { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(Raw(bytes.to_vec())) + } +} + +fn lease_class() -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: LEASE_KEY.to_vec(), + }) +} + +/// The holder's token, then when the lease runs out. +fn lease_bytes(token: u64, expires: u64) -> Vec { + let mut bytes = Vec::with_capacity(16); + bytes.extend_from_slice(&token.to_be_bytes()); + bytes.extend_from_slice(&expires.to_be_bytes()); + bytes +} + +/// When a stored lease runs out; 0 (long gone) for one that can't be read. +fn expires_of(bytes: &[u8]) -> u64 { + bytes + .get(8..16) + .and_then(|b| b.try_into().ok()) + .map(u64::from_be_bytes) + .unwrap_or(0) +} + +fn hash(bytes: &[u8]) -> AssertValue { + AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes)) +} + +fn is_assert_failure(err: &trc::Error) -> bool { + matches!( + err.as_ref(), + trc::EventType::Store(trc::StoreEvent::AssertValueFailed) + ) +} + +/// Replaces `current` (None: no lease) with `new`; false if another node +/// wrote the lease in between. +async fn swap(store: &Store, current: Option<&[u8]>, new: Vec) -> trc::Result { + let mut batch = BatchBuilder::new(); + batch.assert_value(lease_class(), current.map_or(AssertValue::None, hash)); + batch.set(lease_class(), new); + match store.write(batch.build_all()).await { + Ok(_) => Ok(true), + Err(err) if is_assert_failure(&err) => Ok(false), + Err(err) => Err(err.caused_by(trc::location!())), + } +} + +pub struct DefaultsLease { + bytes: Arc>>, + renewal: tokio::task::JoinHandle<()>, +} + +impl DefaultsLease { + /// Waits for the lease and takes it. + pub async fn take(store: &Store) -> trc::Result { + let token: u64 = rng().random(); + loop { + let current = store + .get_value::(ValueKey::from(lease_class())) + .await + .caused_by(trc::location!())? + .map(|Raw(bytes)| bytes); + let free = current + .as_deref() + .is_none_or(|bytes| expires_of(bytes) <= now()); + if free { + let new = lease_bytes(token, now() + LEASE_SECONDS); + if swap(store, current.as_deref(), new.clone()).await? { + return Ok(Self::held(store, token, new)); + } + } + tokio::time::sleep(POLL).await; + } + } + + fn held(store: &Store, token: u64, bytes: Vec) -> Self { + let bytes = Arc::new(Mutex::new(bytes)); + let renewal = { + let (store, bytes) = (store.clone(), bytes.clone()); + tokio::spawn(async move { + loop { + tokio::time::sleep(RENEWAL).await; + let current = bytes.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let new = lease_bytes(token, now() + LEASE_SECONDS); + match swap(&store, Some(¤t), new.clone()).await { + Ok(true) => *bytes.lock().unwrap_or_else(|e| e.into_inner()) = new, + // Taken over after it ran out: nothing left to renew + Ok(false) => return, + Err(err) => { + trc::error!(err.details("Failed to renew the defaults lease")); + } + } + } + }) + }; + DefaultsLease { bytes, renewal } + } + + pub async fn release(self, store: &Store) { + self.renewal.abort(); + let current = self.bytes.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let mut batch = BatchBuilder::new(); + batch.assert_value(lease_class(), hash(¤t)); + batch.clear(lease_class()); + if let Err(err) = store.write(batch.build_all()).await + && !is_assert_failure(&err) + { + // It runs out on its own + trc::error!(err.details("Failed to release the defaults lease")); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_lease_says_when_it_runs_out() { + let bytes = lease_bytes(7, 1_234); + assert_eq!(bytes.len(), 16); + assert_eq!(expires_of(&bytes), 1_234); + // A value that can't be read counts as run out + assert_eq!(expires_of(b"short"), 0); + } +} diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index e7ad827a..fdc9ad3d 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -23,6 +23,7 @@ pub mod check; // inbuxa: K-C5, --check pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles pub mod console; pub mod defaults; +pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults pub mod first_party; pub mod granted_permissions; // inbuxa: permissions added after roles were stored pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment diff --git a/tests/src/store/defaults_lease.rs b/tests/src/store/defaults_lease.rs new file mode 100644 index 00000000..d3e2624a --- /dev/null +++ b/tests/src/store/defaults_lease.rs @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! #436: the defaults lease lets one node at a time set up the defaults. +//! Nodes are played by tasks sharing one store. + +use common::manager::defaults_lease::{DefaultsLease, LEASE_KEY}; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; +use store::{ + SUBSPACE_INBUXA, Store, + backend::sqlite::SqliteStore, + write::{AnyClass, BatchBuilder, ValueClass, now}, +}; + +fn store() -> Store { + Store::SQLite(SqliteStore::open_memory().unwrap().into()) +} + +/// A lease another node left, running out at `expires`. +async fn foreign_lease(store: &Store, expires: u64) { + let mut value = 42u64.to_be_bytes().to_vec(); + value.extend_from_slice(&expires.to_be_bytes()); + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: LEASE_KEY.to_vec(), + }), + value, + ); + store.write(batch.build_all()).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn one_node_at_a_time() { + let store = store(); + let inside = Arc::new(AtomicBool::new(false)); + let turns = Arc::new(AtomicUsize::new(0)); + + let nodes = (0..4) + .map(|_| { + let (store, inside, turns) = (store.clone(), inside.clone(), turns.clone()); + tokio::spawn(async move { + for _ in 0..3 { + let lease = DefaultsLease::take(&store).await.unwrap(); + assert!( + !inside.swap(true, Ordering::SeqCst), + "two nodes held the lease at once" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + inside.store(false, Ordering::SeqCst); + turns.fetch_add(1, Ordering::SeqCst); + lease.release(&store).await; + } + }) + }) + .collect::>(); + for node in nodes { + tokio::time::timeout(Duration::from_secs(30), node) + .await + .expect("a node never got the lease") + .unwrap(); + } + assert_eq!(turns.load(Ordering::SeqCst), 12); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_live_lease_blocks_and_one_run_out_does_not() { + let store = store(); + + // Another node is setting up the defaults: wait + foreign_lease(&store, now() + 60).await; + assert!( + tokio::time::timeout(Duration::from_secs(1), DefaultsLease::take(&store)) + .await + .is_err(), + "took a lease another node holds" + ); + + // That node stopped and its lease ran out: take it at once + foreign_lease(&store, now() - 1).await; + let started = Instant::now(); + let lease = DefaultsLease::take(&store).await.unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + + // Released, the next node gets it at once + lease.release(&store).await; + let started = Instant::now(); + DefaultsLease::take(&store) + .await + .unwrap() + .release(&store) + .await; + assert!(started.elapsed() < Duration::from_secs(2)); +} diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index b65f810f..4477a7a4 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -7,6 +7,7 @@ */ pub mod blob; +pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults pub mod import_export; pub mod lookup; pub mod ops; -- 2.54.0