diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 6e0e9a9e..bba1b128 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -95,6 +95,7 @@ impl Data { queue_status: true.into(), settings_reload: Default::default(), store_health: Default::default(), + singleton_leases: Default::default(), applications, logos: Default::default(), smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"), @@ -242,6 +243,7 @@ impl Default for Data { queue_status: true.into(), settings_reload: Default::default(), store_health: Default::default(), + singleton_leases: Default::default(), applications: WebApplications::new(), logos: Default::default(), smtp_connectors: TlsConnectors::try_new().unwrap(), diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index f85dc029..02cab4bb 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -208,6 +208,9 @@ pub struct Data { // inbuxa: the relays outbound mail goes through, and this node's links // to them (relay spec, RL-20, RL-24) pub relay_senders: inbuxa_features::relay::outbound::Senders, + + // inbuxa: #437, the singleton role leases this node holds + pub singleton_leases: manager::singleton_lease::SingletonLeases, } #[derive(Clone)] diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index fdc9ad3d..02a83983 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -28,6 +28,7 @@ pub mod first_party; pub mod granted_permissions; // inbuxa: permissions added after roles were stored pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment pub mod restore; +pub mod singleton_lease; // inbuxa: #437, one node at a time does the singleton roles' work pub mod spam_rules; // inbuxa: rules bundled with the server #[cfg(target_os = "linux")] pub mod systemd; diff --git a/crates/common/src/manager/singleton_lease.rs b/crates/common/src/manager/singleton_lease.rs new file mode 100644 index 00000000..07795467 --- /dev/null +++ b/crates/common/src/manager/singleton_lease.rs @@ -0,0 +1,376 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: one node at a time does the cluster's singleton work (#437). +//! +//! Every node with the `taskScheduler` role used to queue its own copy of +//! each scheduled maintenance task, and every node with `metricsCalculate` +//! evaluated alerts and sent their emails, so each of those roles had to be +//! on exactly one node. Losing that node paused the work until it came back. +//! +//! Now each role has a lease in the store, taken with a compare-and-swap +//! like the store layout lease. Every node with the role tries for it each +//! minute; the one holding it renews it and does the work, the others wait. +//! A holder that stops renewing loses it when it runs out, and another node +//! with the role takes it over. +//! +//! A node without a cluster coordinator is on its own and needs no lease: +//! the role alone decides, as before. + +use crate::{BuildServer, Inner, Server}; +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; +use store::{ + Deserialize, SUBSPACE_INBUXA, Store, ValueKey, + rand::{RngExt, rng}, + write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now}, +}; +use trc::{AddContext, ClusterEvent}; + +/// How long a lease lasts unless renewed. +pub const LEASE_SECONDS: u64 = 180; +/// How often each node with the role tries for its lease. +pub const ROUND: Duration = Duration::from_secs(60); +/// A holder stops acting this long before its lease runs out, so it has +/// stopped by the time another node can take the lease over. +const MARGIN: u64 = 15; + +/// The roles only one node of a cluster acts on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SingletonRole { + /// Queues the scheduled maintenance tasks (`taskScheduler`). + TaskScheduler, + /// Calculates the cluster-wide gauges and evaluates alerts + /// (`metricsCalculate`). + Metrics, +} + +impl SingletonRole { + pub const ALL: [SingletonRole; 2] = [SingletonRole::TaskScheduler, SingletonRole::Metrics]; + + /// Under `X` (cluster-wide singletons) in `_`. + pub fn key(self) -> &'static [u8] { + match self { + SingletonRole::TaskScheduler => b"Xs", + SingletonRole::Metrics => b"Xm", + } + } + + /// The task role's name, as cluster roles list it. + pub fn name(self) -> &'static str { + match self { + SingletonRole::TaskScheduler => "taskScheduler", + SingletonRole::Metrics => "metricsCalculate", + } + } + + fn index(self) -> usize { + match self { + SingletonRole::TaskScheduler => 0, + SingletonRole::Metrics => 1, + } + } + + /// Whether this node has the role. + pub fn assigned(self, server: &Server) -> bool { + let roles = &server.core.network.roles; + match self { + SingletonRole::TaskScheduler => roles.task_scheduler, + SingletonRole::Metrics => roles.metrics_calculate, + } + } +} + +/// Which leases this node holds, and until when. +pub struct SingletonLeases { + /// Says which leases are this process's. + token: u64, + held_until: [AtomicU64; 2], +} + +impl Default for SingletonLeases { + fn default() -> Self { + SingletonLeases { + token: rng().random(), + held_until: Default::default(), + } + } +} + +impl SingletonLeases { + /// Whether this node holds `role`'s lease with time to spare. + pub fn holds(&self, role: SingletonRole) -> bool { + now() + MARGIN < self.held_until[role.index()].load(Ordering::Relaxed) + } + + fn set(&self, role: SingletonRole, until: u64) { + self.held_until[role.index()].store(until, Ordering::Relaxed); + } +} + +impl Server { + /// Whether this node does `role`'s work now: it has the role, and, in a + /// cluster, holds the role's lease. + pub fn runs_singleton(&self, role: SingletonRole) -> bool { + role.assigned(self) + && (!self.core.storage.has_coordinator() + || self.inner.data.singleton_leases.holds(role)) + } +} + +struct Raw(Vec); + +impl Deserialize for Raw { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(Raw(bytes.to_vec())) + } +} + +fn lease_class(role: SingletonRole) -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: role.key().to_vec(), + }) +} + +/// A lease: the holder's token, when it runs out, then the holder's node id +/// (for the log only; the token says whose it is). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Lease { + pub token: u64, + pub expires: u64, + pub node_id: u16, +} + +impl Lease { + pub fn to_bytes(self) -> Vec { + let mut bytes = Vec::with_capacity(18); + bytes.extend_from_slice(&self.token.to_be_bytes()); + bytes.extend_from_slice(&self.expires.to_be_bytes()); + bytes.extend_from_slice(&self.node_id.to_be_bytes()); + bytes + } + + /// None for a value that can't be read, which counts as no lease. + pub fn from_bytes(bytes: &[u8]) -> Option { + Some(Lease { + token: u64::from_be_bytes(bytes.get(0..8)?.try_into().ok()?), + expires: u64::from_be_bytes(bytes.get(8..16)?.try_into().ok()?), + node_id: u16::from_be_bytes(bytes.get(16..18)?.try_into().ok()?), + }) + } +} + +fn is_assert_failure(err: &trc::Error) -> bool { + matches!( + err.as_ref(), + trc::EventType::Store(trc::StoreEvent::AssertValueFailed) + ) +} + +async fn current(store: &Store, role: SingletonRole) -> trc::Result>> { + Ok(store + .get_value::(ValueKey::from(lease_class(role))) + .await + .caused_by(trc::location!())? + .map(|Raw(bytes)| bytes)) +} + +/// Who holds `role`'s lease, if it hasn't run out. +pub async fn holder(store: &Store, role: SingletonRole) -> trc::Result> { + Ok(current(store, role) + .await? + .and_then(|bytes| Lease::from_bytes(&bytes)) + .filter(|lease| lease.expires > now())) +} + +/// One round for `role`: takes the lease if nobody holds it or it ran out, +/// or renews it if it's `token`'s. Returns when it now runs out, or None +/// when another holder has it. +pub async fn try_hold( + store: &Store, + role: SingletonRole, + token: u64, + node_id: u16, +) -> trc::Result> { + let current = current(store, role).await?; + if let Some(lease) = current.as_deref().and_then(Lease::from_bytes) + && lease.token != token + && lease.expires > now() + { + return Ok(None); + } + + let expires = now() + LEASE_SECONDS; + let mut batch = BatchBuilder::new(); + batch.assert_value( + lease_class(role), + current.as_deref().map_or(AssertValue::None, |bytes| { + AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes)) + }), + ); + batch.set( + lease_class(role), + Lease { + token, + expires, + node_id, + } + .to_bytes(), + ); + match store.write(batch.build_all()).await { + Ok(_) => Ok(Some(expires)), + // Another node took or renewed it in between + Err(err) if is_assert_failure(&err) => Ok(None), + Err(err) => Err(err.caused_by(trc::location!())), + } +} + +/// Gives `role`'s lease up if it's `token`'s, so another node can take it +/// at its next round instead of waiting for it to run out. +pub async fn release(store: &Store, role: SingletonRole, token: u64) -> trc::Result<()> { + let Some(bytes) = current(store, role).await? else { + return Ok(()); + }; + if Lease::from_bytes(&bytes).is_none_or(|lease| lease.token != token) { + return Ok(()); + } + let mut batch = BatchBuilder::new(); + batch.assert_value( + lease_class(role), + AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(&bytes)), + ); + batch.clear(lease_class(role)); + match store.write(batch.build_all()).await { + Ok(_) => Ok(()), + Err(err) if is_assert_failure(&err) => Ok(()), + Err(err) => Err(err.caused_by(trc::location!())), + } +} + +/// Keeps this node's singleton leases: each round, every role the node has +/// is tried for, and a role it no longer has is given up. +pub fn spawn_singleton_leases(inner: Arc) { + tokio::spawn(async move { + loop { + let server = inner.build_server(); + if server.core.storage.has_coordinator() { + for role in SingletonRole::ALL { + round(&server, role).await; + } + } + tokio::time::sleep(ROUND).await; + } + }); +} + +/// Gives up every lease this node holds, as it stops, so another node with +/// the role, or this one once it's back, takes over at its next round +/// instead of when the lease runs out. +pub async fn release_singleton_leases(server: &Server) { + let leases = &server.inner.data.singleton_leases; + for role in SingletonRole::ALL { + if leases.held_until[role.index()].swap(0, Ordering::Relaxed) == 0 { + continue; + } + match release(server.store(), role, leases.token).await { + Ok(()) => lost(role, "This node is stopping"), + Err(err) => { + // It runs out on its own + trc::error!(err.details("Failed to give up a singleton role lease")); + } + } + } +} + +async fn round(server: &Server, role: SingletonRole) { + let leases = &server.inner.data.singleton_leases; + let token = leases.token; + let store = server.store(); + let was_holding = leases.holds(role); + + if !role.assigned(server) { + if was_holding { + leases.set(role, 0); + lost(role, "The role was taken off this node"); + if let Err(err) = release(store, role, token).await { + trc::error!(err.details("Failed to give up a singleton role lease")); + } + } + return; + } + + match try_hold(store, role, token, server.registry().node_id()).await { + Ok(Some(expires)) => { + if !was_holding { + if role == SingletonRole::Metrics { + // Carry on from the last holder's firing alerts, so an + // alert that's still firing isn't sent again + crate::telemetry::alerts::load_firing(store).await; + } + trc::event!( + Cluster(ClusterEvent::SingletonLeaseTaken), + Type = role.name(), + Id = server.registry().node_id(), + Expires = trc::Value::Timestamp(expires), + ); + } + leases.set(role, expires); + } + Ok(None) => { + leases.set(role, 0); + if was_holding { + lost(role, "Another node holds the lease"); + } + } + // Keeps what it had: the lease runs out on its own if this goes on + Err(err) => { + trc::error!(err.details("Failed to renew a singleton role lease")); + } + } +} + +fn lost(role: SingletonRole, why: &'static str) { + trc::event!( + Cluster(ClusterEvent::SingletonLeaseLost), + Type = role.name(), + Reason = why, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_lease_reads_back() { + let lease = Lease { + token: 7, + expires: 1_234, + node_id: 3, + }; + assert_eq!(Lease::from_bytes(&lease.to_bytes()), Some(lease)); + // A value that can't be read counts as no lease + assert_eq!(Lease::from_bytes(b"short"), None); + } + + #[test] + fn a_holder_stops_before_its_lease_runs_out() { + let leases = SingletonLeases::default(); + assert!(!leases.holds(SingletonRole::TaskScheduler)); + leases.set(SingletonRole::TaskScheduler, now() + LEASE_SECONDS); + assert!(leases.holds(SingletonRole::TaskScheduler)); + assert!(!leases.holds(SingletonRole::Metrics)); + // Inside the margin it no longer acts, though the lease hasn't run out + leases.set(SingletonRole::TaskScheduler, now() + MARGIN); + assert!(!leases.holds(SingletonRole::TaskScheduler)); + } +} diff --git a/crates/common/src/telemetry/alerts.rs b/crates/common/src/telemetry/alerts.rs index 83e5036f..63a481ee 100644 --- a/crates/common/src/telemetry/alerts.rs +++ b/crates/common/src/telemetry/alerts.rs @@ -25,7 +25,11 @@ use registry::{ types::{id::ObjectId, list::List}, }; use std::sync::Mutex; -use store::registry::{RegistryQuery, bootstrap::Bootstrap}; +use store::{ + Deserialize, SUBSPACE_INBUXA, Store, ValueKey, + registry::{RegistryQuery, bootstrap::Bootstrap}, + write::{AnyClass, BatchBuilder, ValueClass}, +}; use trc::{Collector, MetricType, TelemetryEvent}; use types::id::Id; @@ -37,10 +41,71 @@ pub struct AlertMessage { pub body: Vec, } -/// The alerts whose condition held at the last evaluation (MON-26). In -/// memory, so a restart while a condition holds fires once more. +/// The alerts whose condition held at the last evaluation (MON-26). Kept +/// in the store as well (#437), so the node that takes over alerts from +/// another carries on from it instead of firing them all again. static FIRING: Mutex>> = Mutex::new(None); +/// Under `X` (cluster-wide singletons) in `_`, beside the singleton role +/// leases: the ids +/// of the alerts firing at the last evaluation. +pub const FIRING_KEY: &[u8] = b"Xa"; + +fn firing_class() -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: FIRING_KEY.to_vec(), + }) +} + +struct FiringIds(AHashSet); + +impl Deserialize for FiringIds { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(FiringIds( + bytes + .as_chunks::<8>() + .0 + .iter() + .map(|id| u64::from_be_bytes(*id)) + .collect(), + )) + } +} + +fn firing_bytes(ids: &AHashSet) -> Vec { + let mut ids = ids.iter().copied().collect::>(); + ids.sort_unstable(); + ids.into_iter().flat_map(u64::to_be_bytes).collect() +} + +/// Takes up the alerts the store says are firing, as the node that takes +/// over evaluating them (#437). With none stored, the next evaluation is +/// a first one (MON-26). +pub async fn load_firing(store: &Store) { + match store + .get_value::(ValueKey::from(firing_class())) + .await + { + Ok(stored) => *FIRING.lock().unwrap() = stored.map(|FiringIds(ids)| ids), + Err(err) => { + trc::error!(err.details("Failed to read the firing alerts")); + } + } +} + +/// Stores the firing alerts when they changed since the last evaluation. +async fn store_firing(store: &Store, previous: Option<&AHashSet>, holding: &AHashSet) { + if previous == Some(holding) { + return; + } + let mut batch = BatchBuilder::new(); + batch.set(firing_class(), firing_bytes(holding)); + if let Err(err) = store.write(batch.build_all()).await { + trc::error!(err.details("Failed to store the firing alerts")); + } +} + fn is_ident_char(c: char) -> bool { c.is_ascii_alphanumeric() || c == '_' } @@ -237,7 +302,8 @@ impl Server { messages.push(build_email(email)); } } - *FIRING.lock().unwrap() = Some(holding); + let previous = FIRING.lock().unwrap().replace(holding.clone()); + store_firing(self.store(), previous.as_ref(), &holding).await; Ok(messages) } } @@ -246,6 +312,20 @@ impl Server { mod tests { use super::*; + #[test] + fn firing_alerts_read_back() { + let ids = AHashSet::from_iter([3u64, 1 << 40, 7]); + let FiringIds(read) = FiringIds::deserialize(&firing_bytes(&ids)).unwrap(); + assert_eq!(read, ids); + // Stored in order, so an unchanged set writes the same bytes + assert_eq!( + firing_bytes(&ids), + firing_bytes(&AHashSet::from_iter([7u64, 3, 1 << 40])) + ); + let FiringIds(none) = FiringIds::deserialize(&[]).unwrap(); + assert!(none.is_empty()); + } + #[test] fn rewrites_underscore_names() { assert_eq!(rewrite("domain_count > 1"), "metric('domain.count') > 1"); diff --git a/crates/common/src/telemetry/metrics/store.rs b/crates/common/src/telemetry/metrics/store.rs index dd734745..618fc797 100644 --- a/crates/common/src/telemetry/metrics/store.rs +++ b/crates/common/src/telemetry/metrics/store.rs @@ -197,7 +197,9 @@ impl Server { if store.is_none() { return; } - let samples = sample(self.core.network.roles.metrics_calculate); + // inbuxa: #437: the cluster-wide gauges from the node that sets them + let samples = + sample(self.runs_singleton(crate::manager::singleton_lease::SingletonRole::Metrics)); let count = samples.len(); let started = std::time::Instant::now(); match store.write_metrics(samples, now()).await { diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index 66f37fde..4482bfeb 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -162,6 +162,9 @@ async fn main() -> std::io::Result<()> { // inbuxa: hand back the task locks this node holds, so other nodes can // run those tasks now rather than when the locks expire services::task_manager::lock::release_task_locks(&inner.build_server()).await; + // inbuxa: #437: and its singleton role leases, so another node (or this + // one, restarted) takes those roles over at once + common::manager::singleton_lease::release_singleton_leases(&inner.build_server()).await; // Shutdown collector Collector::shutdown(); diff --git a/crates/migration/src/fork_subspaces.rs b/crates/migration/src/fork_subspaces.rs index e0bce5bf..e4526753 100644 --- a/crates/migration/src/fork_subspaces.rs +++ b/crates/migration/src/fork_subspaces.rs @@ -113,6 +113,7 @@ pub const OWNERS: &[(u8, Owner)] = &[ b'W', Owner::Stays("blob store changes and the sharing policy"), ), + (b'X', Owner::Stays("cluster singleton role leases")), ]; /// Step 2 -> 3. Returns how many rows moved. diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 445da7d9..fe9378db 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -110,6 +110,8 @@ impl SpawnServices for IpcReceivers { inbuxa_scheduled_reports::spawn_scheduled_reports(inner.clone()); // Spawn task scheduler + // inbuxa: #437, before the scheduler, which asks it who acts + common::manager::singleton_lease::spawn_singleton_leases(inner.clone()); spawn_task_scheduler(inner); } } diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index e1cf8125..682ed4af 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -14,8 +14,9 @@ use std::{ }; use common::{ - BuildServer, Inner, LONG_1D_SLUMBER, + BuildServer, Inner, LONG_1D_SLUMBER, Server, config::{mailstore::spamfilter, telemetry::OtelMetrics}, + manager::singleton_lease::SingletonRole, }; use registry::{ schema::{ @@ -70,6 +71,37 @@ async fn metrics_collection_delay(server: &common::Server) -> Duration { .time_to_next() } +/// The cluster-wide gauges, on the node that does the metrics work: the +/// queue every time, accounts and domains when `others` (once a day). +async fn calculate_cluster_metrics(server: &Server, others: bool) { + // inbuxa: MON-7: the queue gauge from the queue itself, so it's right + // after a restart + match server.total_queued_messages().await { + Ok((total, oldest)) => { + Collector::update_gauge(MetricType::QueueCount, total); + Collector::update_gauge(MetricType::QueueOldestMessage, oldest.unwrap_or_default()); + } + Err(err) => { + trc::error!(err.details("Failed to count queued messages")); + } + } + + if others { + match server.total_accounts().await { + Ok(total) => Collector::update_gauge(MetricType::UserCount, total as u64), + Err(err) => { + trc::error!(err.details("Failed to obtain account count")); + } + } + match server.total_domains().await { + Ok(total) => Collector::update_gauge(MetricType::DomainCount, total as u64), + Err(err) => { + trc::error!(err.details("Failed to obtain domain count")); + } + } + } +} + #[derive(Default)] struct Queue { heap: BinaryHeap, @@ -174,13 +206,27 @@ pub fn spawn_task_scheduler(inner: Arc) { } let mut next_metric_update = Instant::now(); + // inbuxa: #437: whether this node calculated the cluster-wide gauges + // since it last took on the metrics role, so a node that just took + // over doesn't evaluate alerts against gauges it never set + let mut calculated = false; loop { tokio::time::sleep(queue.wake_up_time()).await; let server = inner.build_server(); let roles = &server.core.network.roles; - let mut batch = (roles.task_scheduler).then(BatchBuilder::new); + // inbuxa: #437: in a cluster, only the node holding the role's + // lease queues the scheduled tasks and does the metrics work + let mut batch = server + .runs_singleton(SingletonRole::TaskScheduler) + .then(BatchBuilder::new); + let calculates = server.runs_singleton(SingletonRole::Metrics); + if !calculates { + calculated = false; + // Counted again at once if this node takes the role on + next_metric_update = Instant::now(); + } while let Some(event) = queue.pop() { match event.event { @@ -299,61 +345,18 @@ pub fn spawn_task_scheduler(inner: Arc) { Event::CalculateMetrics, ); - let update_other_metrics = if Instant::now() >= next_metric_update { + let update_other_metrics = + calculates && Instant::now() >= next_metric_update; + if update_other_metrics { next_metric_update = Instant::now() + Duration::from_secs(86400); - true - } else { - false - }; + } + calculated |= calculates; let server = server.clone(); tokio::spawn(async move { let elapsed = Instant::now(); - if server.core.network.roles.metrics_calculate { - // inbuxa: MON-7: the queue gauge from the queue itself, - // so it's right after a restart - match server.total_queued_messages().await { - Ok((total, oldest)) => { - Collector::update_gauge(MetricType::QueueCount, total); - Collector::update_gauge( - MetricType::QueueOldestMessage, - oldest.unwrap_or_default(), - ); - } - Err(err) => { - trc::error!(err.details("Failed to count queued messages")); - } - } - - if update_other_metrics { - match server.total_accounts().await { - Ok(total) => { - Collector::update_gauge( - MetricType::UserCount, - total as u64, - ); - } - Err(err) => { - trc::error!( - err.details("Failed to obtain account count") - ); - } - } - - match server.total_domains().await { - Ok(total) => { - Collector::update_gauge( - MetricType::DomainCount, - total as u64, - ); - } - Err(err) => { - trc::error!( - err.details("Failed to obtain domain count") - ); - } - } - } + if calculates { + calculate_cluster_metrics(&server, update_other_metrics).await; } match tokio::task::spawn_blocking(memory_stats::memory_stats).await { @@ -387,9 +390,19 @@ pub fn spawn_task_scheduler(inner: Arc) { Instant::now() + Duration::from_secs(60), Event::EvaluateAlerts, ); - if server.core.network.roles.metrics_calculate { + if calculates { + // #437: a node that just took over sets the gauges + // first, so no alert reads one it never set + let first = !calculated; + if first { + calculated = true; + next_metric_update = Instant::now() + Duration::from_secs(86400); + } let server = server.clone(); tokio::spawn(async move { + if first { + calculate_cluster_metrics(&server, true).await; + } match server.process_alerts().await { Ok(messages) => { use smtp::reporting::send::MtaReportSend; @@ -434,7 +447,7 @@ pub fn spawn_task_scheduler(inner: Arc) { Instant::now() + DOMAIN_TASK_CHECK_INTERVAL, Event::CheckDomainTasks, ); - if roles.task_scheduler { + if batch.is_some() { let server = server.clone(); tokio::spawn(async move { if let Err(err) = diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index e678d72f..695d9f0e 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -16,12 +16,13 @@ // 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is // store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is // spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228); -// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371 +// 664 is auth.passkey-counter (passkeys spec, PK-18); 665 and 666 are +// cluster.singleton-lease-taken and -lost (#437). Metrics 369 to 371 // are gauges for Kubernetes alerts (K-C10): queue.oldest-message, // cluster.lease-renewed, tls.earliest-expiry. // TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one // more than the highest id; the test below holds it to the variant list -pub const TOTAL_EVENT_COUNT: usize = 665; +pub const TOTAL_EVENT_COUNT: usize = 667; pub const TOTAL_METRIC_COUNT: usize = 372; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -170,6 +171,9 @@ pub enum ClusterEvent { CoordinatorConnected = 644, CoordinatorDisconnected = 645, CoordinatorError = 646, + // inbuxa: #437, the singleton role leases + SingletonLeaseTaken = 665, + SingletonLeaseLost = 666, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index 51d410c5..3aa8ded5 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -95,6 +95,8 @@ impl EventType { b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid), b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed), b"cluster.node-id-reassigned" => EventType::Cluster(ClusterEvent::NodeIdReassigned), + b"cluster.singleton-lease-taken" => EventType::Cluster(ClusterEvent::SingletonLeaseTaken), + b"cluster.singleton-lease-lost" => EventType::Cluster(ClusterEvent::SingletonLeaseLost), // inbuxa: coordinator connection b"cluster.coordinator-connected" => EventType::Cluster(ClusterEvent::CoordinatorConnected), b"cluster.coordinator-disconnected" => EventType::Cluster(ClusterEvent::CoordinatorDisconnected), @@ -784,6 +786,10 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid", EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed", EventType::Cluster(ClusterEvent::NodeIdReassigned) => "cluster.node-id-reassigned", + EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => { + "cluster.singleton-lease-taken" + } + EventType::Cluster(ClusterEvent::SingletonLeaseLost) => "cluster.singleton-lease-lost", // inbuxa: coordinator connection EventType::Cluster(ClusterEvent::CoordinatorConnected) => { "cluster.coordinator-connected" @@ -1598,6 +1604,8 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageInvalid) => 49, EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275, EventType::Cluster(ClusterEvent::NodeIdReassigned) => 649, + EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => 665, + EventType::Cluster(ClusterEvent::SingletonLeaseLost) => 666, // inbuxa: coordinator connection EventType::Cluster(ClusterEvent::CoordinatorConnected) => 644, EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => 645, @@ -2278,6 +2286,8 @@ impl EventType { 49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), 275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)), 649 => Some(EventType::Cluster(ClusterEvent::NodeIdReassigned)), + 665 => Some(EventType::Cluster(ClusterEvent::SingletonLeaseTaken)), + 666 => Some(EventType::Cluster(ClusterEvent::SingletonLeaseLost)), // inbuxa: coordinator connection 644 => Some(EventType::Cluster(ClusterEvent::CoordinatorConnected)), 645 => Some(EventType::Cluster(ClusterEvent::CoordinatorDisconnected)), @@ -3256,6 +3266,8 @@ impl EventType { EventType::Cluster(ClusterEvent::CoordinatorConnected) => Level::Info, EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => Level::Warn, EventType::Cluster(ClusterEvent::CoordinatorError) => Level::Warn, + EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => Level::Info, + EventType::Cluster(ClusterEvent::SingletonLeaseLost) => Level::Info, EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn, EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn, EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn, @@ -3407,6 +3419,8 @@ impl EventType { EventType::Cluster(ClusterEvent::CoordinatorConnected) => "Coordinator connected", EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => "Coordinator unavailable", EventType::Cluster(ClusterEvent::CoordinatorError) => "Coordinator error", + EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => "Singleton role lease taken", + EventType::Cluster(ClusterEvent::SingletonLeaseLost) => "Singleton role lease lost", EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful", EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed", EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE", @@ -4529,6 +4543,8 @@ impl EventType { EventType::Cluster(ClusterEvent::CoordinatorConnected), EventType::Cluster(ClusterEvent::CoordinatorDisconnected), EventType::Cluster(ClusterEvent::CoordinatorError), + EventType::Cluster(ClusterEvent::SingletonLeaseTaken), + EventType::Cluster(ClusterEvent::SingletonLeaseLost), EventType::Dane(DaneEvent::AuthenticationSuccess), EventType::Dane(DaneEvent::AuthenticationFailure), EventType::Dane(DaneEvent::NoCertificatesFound), diff --git a/docs/spec/features/monitoring.md b/docs/spec/features/monitoring.md index ea625b30..bc6f8ef1 100644 --- a/docs/spec/features/monitoring.md +++ b/docs/spec/features/monitoring.md @@ -336,9 +336,11 @@ unchanged. ### Alerts - **MON-25.** **Evaluation.** Each enabled alert's condition is evaluated - every 60 seconds (**Decision**) on every node that holds the - `metricsCalculate` task role, against that node's current values. A - one-node install always holds it. The condition language is the server's + every 60 seconds (**Decision**) on the node that does the + `metricsCalculate` task role's work, against that node's current values. A + one-node install always holds it. In a cluster several nodes can have the + role; the one holding its lease in the store does the work (#437, see + "Singleton roles" below). The condition language is the server's expression language, and a metric is read two ways, both accepted: - `metric('queue.count')`, which the AGPL expression code already supports; - the name with dots and hyphens as underscores (`queue_count`), as @@ -350,9 +352,22 @@ unchanged. true, including the first evaluation after start. While it stays true it doesn't fire again. Once false, it can fire again. **Decision**: upstream's docs say email goes out "each time the condition becomes true". State is in - memory, so a restart while the condition holds fires once more. A condition + memory, and in a cluster in the store too (`Xa` in `_`), so a node that + takes the role over from another carries on from it; a node without a + coordinator that restarts while the condition holds fires once more. A condition that fails to evaluate (a metric name that doesn't exist, a type error) is rejected when the alert is saved, and never fires. +- **Singleton roles (#437).** `taskScheduler` (queuing the scheduled + maintenance tasks, and the daily check of each domain's DKIM and ACME + tasks) and `metricsCalculate` (the cluster-wide gauges, alert evaluation + and their emails) each have a lease in the store (`Xs`, `Xm` in `_`). + Every node with the role tries for it each minute; the holder renews it + and does the work. A lease lasts 3 minutes, and its holder stops acting + 15 seconds before it runs out, so when the holder stops another node with + the role takes over within about 4 minutes. A node that takes over sets + the gauges before its first alert evaluation. A node without a cluster + coordinator needs no lease: the role alone decides. Taking and losing a + lease log `cluster.singleton-lease-taken` and `cluster.singleton-lease-lost`. - **MON-27.** **Placeholders.** In `eventMessage`, `subject` and `body`, `%{metric.name}%` (the dotted name) is replaced by the value used in that evaluation. Whole numbers print without decimals ("3", not "3.0"); others diff --git a/docs/spec/features/store-layout.md b/docs/spec/features/store-layout.md index 3ce7e88d..0af18131 100644 --- a/docs/spec/features/store-layout.md +++ b/docs/spec/features/store-layout.md @@ -156,6 +156,8 @@ can't place, before it writes anything. The table is `OWNERS` in | `K` | account locks | stays | | `L` | audit log (`e`, `t`, `o`, `h`, `f`) | moves to `ib_audit` (0x80); its settings (`s`) stay | | `M` | masked email | stays | +| `N` | passkeys | stays | +| `O` | listener status | stays | | `P` | permissions and protocol policies | stays | | `Q` | acceptance records | stays | | `R` | mail flow rules | stays | @@ -164,6 +166,7 @@ can't place, before it writes anything. The table is `OWNERS` in | `U` | undelete notes (`n`, `x`, `b`, `r`, `c`, `k`, `a`, `g`) | moves to `ib_undelete` (0x83) | | `V` | store checks, the layout mark (`Vs`), lease (`Vm`) and step progress (`Vt`) | stays | | `W` | blob store changes, the sharing policy | stays | +| `X` | singleton role leases (`Xs`, `Xm`) and the alerts firing (`Xa`), #437 | stays | A key moves without its first byte, so `Le` + node + seq becomes `e` + node + seq in `ib_audit`. A feature's settings stay in `_`, where a configuration diff --git a/resources/schema/schema.json.gz b/resources/schema/schema.json.gz index 1f4fa662..8182dddc 100644 Binary files a/resources/schema/schema.json.gz and b/resources/schema/schema.json.gz differ diff --git a/resources/schema/schema.json.sha256 b/resources/schema/schema.json.sha256 index f7118806..0cf7fc29 100644 --- a/resources/schema/schema.json.sha256 +++ b/resources/schema/schema.json.sha256 @@ -1 +1 @@ -3CkGF0FiqDTQxYPZ0VjCxnlGmiuzyjDDbmIv1BVmDmg \ No newline at end of file +gBQXaaMdj6J-6RZEk6DUlOzZD58mbuzZ2Uo8M2QksY8 \ No newline at end of file diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 4477a7a4..40aa79fd 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -24,6 +24,7 @@ pub mod replica_mysql; // inbuxa: read replicas on MySQL pub mod scaleout; // inbuxa: scale-out storage #[cfg(feature = "postgres")] pub mod search_gin; // inbuxa: GIN indexes without a pending list +pub mod singleton_lease; // inbuxa: #437, one node at a time does the singleton roles' work #[cfg(any(feature = "postgres", feature = "mysql"))] pub mod sql_timeout; pub mod task_locks; // inbuxa: task locks across nodes diff --git a/tests/src/store/singleton_lease.rs b/tests/src/store/singleton_lease.rs new file mode 100644 index 00000000..95bd7130 --- /dev/null +++ b/tests/src/store/singleton_lease.rs @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! #437: a singleton role's lease has one holder at a time, renews for its +//! holder, and passes to another node once it's given up or runs out. +//! Nodes are played by tokens sharing one store. + +use common::manager::singleton_lease::{ + LEASE_SECONDS, Lease, SingletonRole, holder, release, try_hold, +}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +use store::{ + SUBSPACE_INBUXA, Store, + backend::sqlite::SqliteStore, + write::{AnyClass, BatchBuilder, ValueClass, now}, +}; + +const SCHEDULER: SingletonRole = SingletonRole::TaskScheduler; + +fn store() -> Store { + Store::SQLite(SqliteStore::open_memory().unwrap().into()) +} + +/// A lease node `node_id` (token `token`) left, running out at `expires`. +async fn write_lease(store: &Store, token: u64, node_id: u16, expires: u64) { + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: SCHEDULER.key().to_vec(), + }), + Lease { + token, + expires, + node_id, + } + .to_bytes(), + ); + store.write(batch.build_all()).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn one_holder_among_many() { + let store = store(); + let won = Arc::new(AtomicUsize::new(0)); + + // Eight nodes try at once: exactly one gets it + let nodes = (1..=8u64) + .map(|token| { + let (store, won) = (store.clone(), won.clone()); + tokio::spawn(async move { + if try_hold(&store, SCHEDULER, token, token as u16) + .await + .unwrap() + .is_some() + { + won.fetch_add(1, Ordering::SeqCst); + } + }) + }) + .collect::>(); + for node in nodes { + node.await.unwrap(); + } + assert_eq!(won.load(Ordering::SeqCst), 1, "not exactly one holder"); + + // The holder renews it; the others still can't have it + let lease = holder(&store, SCHEDULER).await.unwrap().unwrap(); + let renewed = try_hold(&store, SCHEDULER, lease.token, lease.node_id) + .await + .unwrap() + .expect("the holder couldn't renew its lease"); + assert!(renewed >= lease.expires); + for token in (1..=8u64).filter(|token| *token != lease.token) { + assert_eq!( + try_hold(&store, SCHEDULER, token, token as u16) + .await + .unwrap(), + None, + "node {token} took a lease another node holds" + ); + } + + // The roles have leases of their own + assert!( + try_hold(&store, SingletonRole::Metrics, 99, 99) + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_lease_passes_on_when_it_runs_out_or_is_given_up() { + let store = store(); + + // Node 1 holds it: node 2 waits + write_lease(&store, 1, 1, now() + LEASE_SECONDS).await; + assert_eq!(try_hold(&store, SCHEDULER, 2, 2).await.unwrap(), None); + + // Node 1 stopped and its lease ran out: node 2 takes it over + write_lease(&store, 1, 1, now() - 1).await; + assert!(holder(&store, SCHEDULER).await.unwrap().is_none()); + let expires = try_hold(&store, SCHEDULER, 2, 2) + .await + .unwrap() + .expect("node 2 didn't take over a lease that ran out"); + assert!(expires >= now() + LEASE_SECONDS - 1); + assert_eq!(holder(&store, SCHEDULER).await.unwrap().unwrap().node_id, 2); + + // Node 1 comes back: it's node 2's now + assert_eq!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap(), None); + + // Giving up someone else's lease does nothing + release(&store, SCHEDULER, 1).await.unwrap(); + assert_eq!(holder(&store, SCHEDULER).await.unwrap().unwrap().token, 2); + + // Node 2 gives it up (the role was taken off it): node 1 takes it at once + release(&store, SCHEDULER, 2).await.unwrap(); + assert!(holder(&store, SCHEDULER).await.unwrap().is_none()); + assert!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap().is_some()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_lease_that_cant_be_read_counts_as_none() { + let store = store(); + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: SCHEDULER.key().to_vec(), + }), + b"short".to_vec(), + ); + store.write(batch.build_all()).await.unwrap(); + assert!(holder(&store, SCHEDULER).await.unwrap().is_none()); + assert!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap().is_some()); +}