Defaults: one node at a time on a fresh store #482

Merged
jcoffey-dev merged 1 commits from fix/436-defaults-lease into main 2026-11-19 17:08:05 +00:00
5 changed files with 290 additions and 0 deletions

No files matched your search

+10
View File
@@ -96,6 +96,16 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
return Ok(());
}
// inbuxa: #436: one node at a time, so a node starting beside another on
// a fresh store never takes half-written defaults as the whole set
let store = bp.data_store.clone();
let lease = super::defaults_lease::DefaultsLease::take(&store).await?;
let result = insert_normal_defaults(bp).await;
lease.release(&store).await;
result
}
async fn insert_normal_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
let is_new_install = is_new_install(bp).await?;
// inbuxa: a store brought over from Stalwart doesn't serve upstream's web
+174
View File
@@ -0,0 +1,174 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: one node at a time sets up the defaults (#436).
//!
//! Every normal start checks what the store is missing (roles, listeners,
//! throttles, spam rules...) and adds it, each as "none of these yet, so
//! insert them". Two nodes starting together on a fresh store both did
//! that at once: one could find some of the default listeners already
//! written by the other, take the store as set up, and come up without the
//! rest (on the lab cluster, every listener but `http`, the last one). The
//! same race could store the default roles twice.
//!
//! So the defaults run under a short lease in the store, taken with a
//! compare-and-swap like the store layout lease. A node that finds it held
//! waits until it's released, or runs out because its holder stopped, and
//! then checks the store itself, which by then has everything.
use std::{
sync::{Arc, Mutex},
time::Duration,
};
use store::{
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
rand::{RngExt, rng},
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now},
};
use trc::AddContext;
/// Under `B`, the first boot defaults (store-layout SL-16).
pub const LEASE_KEY: &[u8] = b"Bl";
/// How long a lease lasts unless renewed.
const LEASE_SECONDS: u64 = 60;
const RENEWAL: Duration = Duration::from_secs(15);
const POLL: Duration = Duration::from_millis(250);
struct Raw(Vec<u8>);
impl Deserialize for Raw {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
Ok(Raw(bytes.to_vec()))
}
}
fn lease_class() -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: LEASE_KEY.to_vec(),
})
}
/// The holder's token, then when the lease runs out.
fn lease_bytes(token: u64, expires: u64) -> Vec<u8> {
let mut bytes = Vec::with_capacity(16);
bytes.extend_from_slice(&token.to_be_bytes());
bytes.extend_from_slice(&expires.to_be_bytes());
bytes
}
/// When a stored lease runs out; 0 (long gone) for one that can't be read.
fn expires_of(bytes: &[u8]) -> u64 {
bytes
.get(8..16)
.and_then(|b| b.try_into().ok())
.map(u64::from_be_bytes)
.unwrap_or(0)
}
fn hash(bytes: &[u8]) -> AssertValue {
AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes))
}
fn is_assert_failure(err: &trc::Error) -> bool {
matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
)
}
/// Replaces `current` (None: no lease) with `new`; false if another node
/// wrote the lease in between.
async fn swap(store: &Store, current: Option<&[u8]>, new: Vec<u8>) -> trc::Result<bool> {
let mut batch = BatchBuilder::new();
batch.assert_value(lease_class(), current.map_or(AssertValue::None, hash));
batch.set(lease_class(), new);
match store.write(batch.build_all()).await {
Ok(_) => Ok(true),
Err(err) if is_assert_failure(&err) => Ok(false),
Err(err) => Err(err.caused_by(trc::location!())),
}
}
pub struct DefaultsLease {
bytes: Arc<Mutex<Vec<u8>>>,
renewal: tokio::task::JoinHandle<()>,
}
impl DefaultsLease {
/// Waits for the lease and takes it.
pub async fn take(store: &Store) -> trc::Result<Self> {
let token: u64 = rng().random();
loop {
let current = store
.get_value::<Raw>(ValueKey::from(lease_class()))
.await
.caused_by(trc::location!())?
.map(|Raw(bytes)| bytes);
let free = current
.as_deref()
.is_none_or(|bytes| expires_of(bytes) <= now());
if free {
let new = lease_bytes(token, now() + LEASE_SECONDS);
if swap(store, current.as_deref(), new.clone()).await? {
return Ok(Self::held(store, token, new));
}
}
tokio::time::sleep(POLL).await;
}
}
fn held(store: &Store, token: u64, bytes: Vec<u8>) -> Self {
let bytes = Arc::new(Mutex::new(bytes));
let renewal = {
let (store, bytes) = (store.clone(), bytes.clone());
tokio::spawn(async move {
loop {
tokio::time::sleep(RENEWAL).await;
let current = bytes.lock().unwrap_or_else(|e| e.into_inner()).clone();
let new = lease_bytes(token, now() + LEASE_SECONDS);
match swap(&store, Some(&current), new.clone()).await {
Ok(true) => *bytes.lock().unwrap_or_else(|e| e.into_inner()) = new,
// Taken over after it ran out: nothing left to renew
Ok(false) => return,
Err(err) => {
trc::error!(err.details("Failed to renew the defaults lease"));
}
}
}
})
};
DefaultsLease { bytes, renewal }
}
pub async fn release(self, store: &Store) {
self.renewal.abort();
let current = self.bytes.lock().unwrap_or_else(|e| e.into_inner()).clone();
let mut batch = BatchBuilder::new();
batch.assert_value(lease_class(), hash(&current));
batch.clear(lease_class());
if let Err(err) = store.write(batch.build_all()).await
&& !is_assert_failure(&err)
{
// It runs out on its own
trc::error!(err.details("Failed to release the defaults lease"));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_lease_says_when_it_runs_out() {
let bytes = lease_bytes(7, 1_234);
assert_eq!(bytes.len(), 16);
assert_eq!(expires_of(&bytes), 1_234);
// A value that can't be read counts as run out
assert_eq!(expires_of(b"short"), 0);
}
}
+1
View File
@@ -23,6 +23,7 @@ pub mod check; // inbuxa: K-C5, --check
pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles
pub mod console;
pub mod defaults;
pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults
pub mod first_party;
pub mod granted_permissions; // inbuxa: permissions added after roles were stored
pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment
+104
View File
@@ -0,0 +1,104 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! #436: the defaults lease lets one node at a time set up the defaults.
//! Nodes are played by tasks sharing one store.
use common::manager::defaults_lease::{DefaultsLease, LEASE_KEY};
use std::{
sync::{
Arc,
atomic::{AtomicBool, AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use store::{
SUBSPACE_INBUXA, Store,
backend::sqlite::SqliteStore,
write::{AnyClass, BatchBuilder, ValueClass, now},
};
fn store() -> Store {
Store::SQLite(SqliteStore::open_memory().unwrap().into())
}
/// A lease another node left, running out at `expires`.
async fn foreign_lease(store: &Store, expires: u64) {
let mut value = 42u64.to_be_bytes().to_vec();
value.extend_from_slice(&expires.to_be_bytes());
let mut batch = BatchBuilder::new();
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: LEASE_KEY.to_vec(),
}),
value,
);
store.write(batch.build_all()).await.unwrap();
}
#[tokio::test(flavor = "multi_thread")]
async fn one_node_at_a_time() {
let store = store();
let inside = Arc::new(AtomicBool::new(false));
let turns = Arc::new(AtomicUsize::new(0));
let nodes = (0..4)
.map(|_| {
let (store, inside, turns) = (store.clone(), inside.clone(), turns.clone());
tokio::spawn(async move {
for _ in 0..3 {
let lease = DefaultsLease::take(&store).await.unwrap();
assert!(
!inside.swap(true, Ordering::SeqCst),
"two nodes held the lease at once"
);
tokio::time::sleep(Duration::from_millis(50)).await;
inside.store(false, Ordering::SeqCst);
turns.fetch_add(1, Ordering::SeqCst);
lease.release(&store).await;
}
})
})
.collect::<Vec<_>>();
for node in nodes {
tokio::time::timeout(Duration::from_secs(30), node)
.await
.expect("a node never got the lease")
.unwrap();
}
assert_eq!(turns.load(Ordering::SeqCst), 12);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_live_lease_blocks_and_one_run_out_does_not() {
let store = store();
// Another node is setting up the defaults: wait
foreign_lease(&store, now() + 60).await;
assert!(
tokio::time::timeout(Duration::from_secs(1), DefaultsLease::take(&store))
.await
.is_err(),
"took a lease another node holds"
);
// That node stopped and its lease ran out: take it at once
foreign_lease(&store, now() - 1).await;
let started = Instant::now();
let lease = DefaultsLease::take(&store).await.unwrap();
assert!(started.elapsed() < Duration::from_secs(2));
// Released, the next node gets it at once
lease.release(&store).await;
let started = Instant::now();
DefaultsLease::take(&store)
.await
.unwrap()
.release(&store)
.await;
assert!(started.elapsed() < Duration::from_secs(2));
}
+1
View File
@@ -7,6 +7,7 @@
*/
pub mod blob;
pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults
pub mod import_export;
pub mod lookup;
pub mod ops;