Defaults: one node at a time on a fresh store #482
No files matched your search
@@ -96,6 +96,16 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// inbuxa: #436: one node at a time, so a node starting beside another on
|
||||
// a fresh store never takes half-written defaults as the whole set
|
||||
let store = bp.data_store.clone();
|
||||
let lease = super::defaults_lease::DefaultsLease::take(&store).await?;
|
||||
let result = insert_normal_defaults(bp).await;
|
||||
lease.release(&store).await;
|
||||
result
|
||||
}
|
||||
|
||||
async fn insert_normal_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
|
||||
let is_new_install = is_new_install(bp).await?;
|
||||
|
||||
// inbuxa: a store brought over from Stalwart doesn't serve upstream's web
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: one node at a time sets up the defaults (#436).
|
||||
//!
|
||||
//! Every normal start checks what the store is missing (roles, listeners,
|
||||
//! throttles, spam rules...) and adds it, each as "none of these yet, so
|
||||
//! insert them". Two nodes starting together on a fresh store both did
|
||||
//! that at once: one could find some of the default listeners already
|
||||
//! written by the other, take the store as set up, and come up without the
|
||||
//! rest (on the lab cluster, every listener but `http`, the last one). The
|
||||
//! same race could store the default roles twice.
|
||||
//!
|
||||
//! So the defaults run under a short lease in the store, taken with a
|
||||
//! compare-and-swap like the store layout lease. A node that finds it held
|
||||
//! waits until it's released, or runs out because its holder stopped, and
|
||||
//! then checks the store itself, which by then has everything.
|
||||
|
||||
use std::{
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
use store::{
|
||||
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
|
||||
rand::{RngExt, rng},
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now},
|
||||
};
|
||||
use trc::AddContext;
|
||||
|
||||
/// Under `B`, the first boot defaults (store-layout SL-16).
|
||||
pub const LEASE_KEY: &[u8] = b"Bl";
|
||||
/// How long a lease lasts unless renewed.
|
||||
const LEASE_SECONDS: u64 = 60;
|
||||
const RENEWAL: Duration = Duration::from_secs(15);
|
||||
const POLL: Duration = Duration::from_millis(250);
|
||||
|
||||
struct Raw(Vec<u8>);
|
||||
|
||||
impl Deserialize for Raw {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
Ok(Raw(bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
|
||||
fn lease_class() -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: LEASE_KEY.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The holder's token, then when the lease runs out.
|
||||
fn lease_bytes(token: u64, expires: u64) -> Vec<u8> {
|
||||
let mut bytes = Vec::with_capacity(16);
|
||||
bytes.extend_from_slice(&token.to_be_bytes());
|
||||
bytes.extend_from_slice(&expires.to_be_bytes());
|
||||
bytes
|
||||
}
|
||||
|
||||
/// When a stored lease runs out; 0 (long gone) for one that can't be read.
|
||||
fn expires_of(bytes: &[u8]) -> u64 {
|
||||
bytes
|
||||
.get(8..16)
|
||||
.and_then(|b| b.try_into().ok())
|
||||
.map(u64::from_be_bytes)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn hash(bytes: &[u8]) -> AssertValue {
|
||||
AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes))
|
||||
}
|
||||
|
||||
fn is_assert_failure(err: &trc::Error) -> bool {
|
||||
matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
)
|
||||
}
|
||||
|
||||
/// Replaces `current` (None: no lease) with `new`; false if another node
|
||||
/// wrote the lease in between.
|
||||
async fn swap(store: &Store, current: Option<&[u8]>, new: Vec<u8>) -> trc::Result<bool> {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(lease_class(), current.map_or(AssertValue::None, hash));
|
||||
batch.set(lease_class(), new);
|
||||
match store.write(batch.build_all()).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) if is_assert_failure(&err) => Ok(false),
|
||||
Err(err) => Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DefaultsLease {
|
||||
bytes: Arc<Mutex<Vec<u8>>>,
|
||||
renewal: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl DefaultsLease {
|
||||
/// Waits for the lease and takes it.
|
||||
pub async fn take(store: &Store) -> trc::Result<Self> {
|
||||
let token: u64 = rng().random();
|
||||
loop {
|
||||
let current = store
|
||||
.get_value::<Raw>(ValueKey::from(lease_class()))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Raw(bytes)| bytes);
|
||||
let free = current
|
||||
.as_deref()
|
||||
.is_none_or(|bytes| expires_of(bytes) <= now());
|
||||
if free {
|
||||
let new = lease_bytes(token, now() + LEASE_SECONDS);
|
||||
if swap(store, current.as_deref(), new.clone()).await? {
|
||||
return Ok(Self::held(store, token, new));
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(POLL).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn held(store: &Store, token: u64, bytes: Vec<u8>) -> Self {
|
||||
let bytes = Arc::new(Mutex::new(bytes));
|
||||
let renewal = {
|
||||
let (store, bytes) = (store.clone(), bytes.clone());
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(RENEWAL).await;
|
||||
let current = bytes.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
||||
let new = lease_bytes(token, now() + LEASE_SECONDS);
|
||||
match swap(&store, Some(¤t), new.clone()).await {
|
||||
Ok(true) => *bytes.lock().unwrap_or_else(|e| e.into_inner()) = new,
|
||||
// Taken over after it ran out: nothing left to renew
|
||||
Ok(false) => return,
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to renew the defaults lease"));
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
DefaultsLease { bytes, renewal }
|
||||
}
|
||||
|
||||
pub async fn release(self, store: &Store) {
|
||||
self.renewal.abort();
|
||||
let current = self.bytes.lock().unwrap_or_else(|e| e.into_inner()).clone();
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(lease_class(), hash(¤t));
|
||||
batch.clear(lease_class());
|
||||
if let Err(err) = store.write(batch.build_all()).await
|
||||
&& !is_assert_failure(&err)
|
||||
{
|
||||
// It runs out on its own
|
||||
trc::error!(err.details("Failed to release the defaults lease"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_lease_says_when_it_runs_out() {
|
||||
let bytes = lease_bytes(7, 1_234);
|
||||
assert_eq!(bytes.len(), 16);
|
||||
assert_eq!(expires_of(&bytes), 1_234);
|
||||
// A value that can't be read counts as run out
|
||||
assert_eq!(expires_of(b"short"), 0);
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ pub mod check; // inbuxa: K-C5, --check
|
||||
pub mod compliance_roles; // inbuxa: personal-data catalog, the compliance roles
|
||||
pub mod console;
|
||||
pub mod defaults;
|
||||
pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults
|
||||
pub mod first_party;
|
||||
pub mod granted_permissions; // inbuxa: permissions added after roles were stored
|
||||
pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! #436: the defaults lease lets one node at a time set up the defaults.
|
||||
//! Nodes are played by tasks sharing one store.
|
||||
|
||||
use common::manager::defaults_lease::{DefaultsLease, LEASE_KEY};
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use store::{
|
||||
SUBSPACE_INBUXA, Store,
|
||||
backend::sqlite::SqliteStore,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, now},
|
||||
};
|
||||
|
||||
fn store() -> Store {
|
||||
Store::SQLite(SqliteStore::open_memory().unwrap().into())
|
||||
}
|
||||
|
||||
/// A lease another node left, running out at `expires`.
|
||||
async fn foreign_lease(store: &Store, expires: u64) {
|
||||
let mut value = 42u64.to_be_bytes().to_vec();
|
||||
value.extend_from_slice(&expires.to_be_bytes());
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: LEASE_KEY.to_vec(),
|
||||
}),
|
||||
value,
|
||||
);
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn one_node_at_a_time() {
|
||||
let store = store();
|
||||
let inside = Arc::new(AtomicBool::new(false));
|
||||
let turns = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let nodes = (0..4)
|
||||
.map(|_| {
|
||||
let (store, inside, turns) = (store.clone(), inside.clone(), turns.clone());
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..3 {
|
||||
let lease = DefaultsLease::take(&store).await.unwrap();
|
||||
assert!(
|
||||
!inside.swap(true, Ordering::SeqCst),
|
||||
"two nodes held the lease at once"
|
||||
);
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
inside.store(false, Ordering::SeqCst);
|
||||
turns.fetch_add(1, Ordering::SeqCst);
|
||||
lease.release(&store).await;
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for node in nodes {
|
||||
tokio::time::timeout(Duration::from_secs(30), node)
|
||||
.await
|
||||
.expect("a node never got the lease")
|
||||
.unwrap();
|
||||
}
|
||||
assert_eq!(turns.load(Ordering::SeqCst), 12);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn a_live_lease_blocks_and_one_run_out_does_not() {
|
||||
let store = store();
|
||||
|
||||
// Another node is setting up the defaults: wait
|
||||
foreign_lease(&store, now() + 60).await;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(1), DefaultsLease::take(&store))
|
||||
.await
|
||||
.is_err(),
|
||||
"took a lease another node holds"
|
||||
);
|
||||
|
||||
// That node stopped and its lease ran out: take it at once
|
||||
foreign_lease(&store, now() - 1).await;
|
||||
let started = Instant::now();
|
||||
let lease = DefaultsLease::take(&store).await.unwrap();
|
||||
assert!(started.elapsed() < Duration::from_secs(2));
|
||||
|
||||
// Released, the next node gets it at once
|
||||
lease.release(&store).await;
|
||||
let started = Instant::now();
|
||||
DefaultsLease::take(&store)
|
||||
.await
|
||||
.unwrap()
|
||||
.release(&store)
|
||||
.await;
|
||||
assert!(started.elapsed() < Duration::from_secs(2));
|
||||
}
|
||||
@@ -7,6 +7,7 @@
|
||||
*/
|
||||
|
||||
pub mod blob;
|
||||
pub mod defaults_lease; // inbuxa: #436, one node at a time sets up the defaults
|
||||
pub mod import_export;
|
||||
pub mod lookup;
|
||||
pub mod ops;
|
||||
|
||||
Reference in new issue
Block a user