Cluster: one node at a time does the singleton roles' work (#437) #524

Open
jcoffey-dev wants to merge 1 commits from feat/singleton-role-lease into main
pull from: feat/singleton-role-lease
18 changed files with 732 additions and 66 deletions

No files matched your search

+2
View File
@@ -95,6 +95,7 @@ impl Data {
queue_status: true.into(),
settings_reload: Default::default(),
store_health: Default::default(),
singleton_leases: Default::default(),
applications,
logos: Default::default(),
smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"),
@@ -242,6 +243,7 @@ impl Default for Data {
queue_status: true.into(),
settings_reload: Default::default(),
store_health: Default::default(),
singleton_leases: Default::default(),
applications: WebApplications::new(),
logos: Default::default(),
smtp_connectors: TlsConnectors::try_new().unwrap(),
+3
View File
@@ -208,6 +208,9 @@ pub struct Data {
// inbuxa: the relays outbound mail goes through, and this node's links
// to them (relay spec, RL-20, RL-24)
pub relay_senders: inbuxa_features::relay::outbound::Senders,
// inbuxa: #437, the singleton role leases this node holds
pub singleton_leases: manager::singleton_lease::SingletonLeases,
}
#[derive(Clone)]
+1
View File
@@ -28,6 +28,7 @@ pub mod first_party;
pub mod granted_permissions; // inbuxa: permissions added after roles were stored
pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment
pub mod restore;
pub mod singleton_lease; // inbuxa: #437, one node at a time does the singleton roles' work
pub mod spam_rules; // inbuxa: rules bundled with the server
#[cfg(target_os = "linux")]
pub mod systemd;
@@ -0,0 +1,376 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! inbuxa: one node at a time does the cluster's singleton work (#437).
//!
//! Every node with the `taskScheduler` role used to queue its own copy of
//! each scheduled maintenance task, and every node with `metricsCalculate`
//! evaluated alerts and sent their emails, so each of those roles had to be
//! on exactly one node. Losing that node paused the work until it came back.
//!
//! Now each role has a lease in the store, taken with a compare-and-swap
//! like the store layout lease. Every node with the role tries for it each
//! minute; the one holding it renews it and does the work, the others wait.
//! A holder that stops renewing loses it when it runs out, and another node
//! with the role takes it over.
//!
//! A node without a cluster coordinator is on its own and needs no lease:
//! the role alone decides, as before.
use crate::{BuildServer, Inner, Server};
use std::{
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
time::Duration,
};
use store::{
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
rand::{RngExt, rng},
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now},
};
use trc::{AddContext, ClusterEvent};
/// How long a lease lasts unless renewed.
pub const LEASE_SECONDS: u64 = 180;
/// How often each node with the role tries for its lease.
pub const ROUND: Duration = Duration::from_secs(60);
/// A holder stops acting this long before its lease runs out, so it has
/// stopped by the time another node can take the lease over.
const MARGIN: u64 = 15;
/// The roles only one node of a cluster acts on.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SingletonRole {
/// Queues the scheduled maintenance tasks (`taskScheduler`).
TaskScheduler,
/// Calculates the cluster-wide gauges and evaluates alerts
/// (`metricsCalculate`).
Metrics,
}
impl SingletonRole {
pub const ALL: [SingletonRole; 2] = [SingletonRole::TaskScheduler, SingletonRole::Metrics];
/// Under `X` (cluster-wide singletons) in `_`.
pub fn key(self) -> &'static [u8] {
match self {
SingletonRole::TaskScheduler => b"Xs",
SingletonRole::Metrics => b"Xm",
}
}
/// The task role's name, as cluster roles list it.
pub fn name(self) -> &'static str {
match self {
SingletonRole::TaskScheduler => "taskScheduler",
SingletonRole::Metrics => "metricsCalculate",
}
}
fn index(self) -> usize {
match self {
SingletonRole::TaskScheduler => 0,
SingletonRole::Metrics => 1,
}
}
/// Whether this node has the role.
pub fn assigned(self, server: &Server) -> bool {
let roles = &server.core.network.roles;
match self {
SingletonRole::TaskScheduler => roles.task_scheduler,
SingletonRole::Metrics => roles.metrics_calculate,
}
}
}
/// Which leases this node holds, and until when.
pub struct SingletonLeases {
/// Says which leases are this process's.
token: u64,
held_until: [AtomicU64; 2],
}
impl Default for SingletonLeases {
fn default() -> Self {
SingletonLeases {
token: rng().random(),
held_until: Default::default(),
}
}
}
impl SingletonLeases {
/// Whether this node holds `role`'s lease with time to spare.
pub fn holds(&self, role: SingletonRole) -> bool {
now() + MARGIN < self.held_until[role.index()].load(Ordering::Relaxed)
}
fn set(&self, role: SingletonRole, until: u64) {
self.held_until[role.index()].store(until, Ordering::Relaxed);
}
}
impl Server {
/// Whether this node does `role`'s work now: it has the role, and, in a
/// cluster, holds the role's lease.
pub fn runs_singleton(&self, role: SingletonRole) -> bool {
role.assigned(self)
&& (!self.core.storage.has_coordinator()
|| self.inner.data.singleton_leases.holds(role))
}
}
struct Raw(Vec<u8>);
impl Deserialize for Raw {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
Ok(Raw(bytes.to_vec()))
}
}
fn lease_class(role: SingletonRole) -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: role.key().to_vec(),
})
}
/// A lease: the holder's token, when it runs out, then the holder's node id
/// (for the log only; the token says whose it is).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Lease {
pub token: u64,
pub expires: u64,
pub node_id: u16,
}
impl Lease {
pub fn to_bytes(self) -> Vec<u8> {
let mut bytes = Vec::with_capacity(18);
bytes.extend_from_slice(&self.token.to_be_bytes());
bytes.extend_from_slice(&self.expires.to_be_bytes());
bytes.extend_from_slice(&self.node_id.to_be_bytes());
bytes
}
/// None for a value that can't be read, which counts as no lease.
pub fn from_bytes(bytes: &[u8]) -> Option<Self> {
Some(Lease {
token: u64::from_be_bytes(bytes.get(0..8)?.try_into().ok()?),
expires: u64::from_be_bytes(bytes.get(8..16)?.try_into().ok()?),
node_id: u16::from_be_bytes(bytes.get(16..18)?.try_into().ok()?),
})
}
}
fn is_assert_failure(err: &trc::Error) -> bool {
matches!(
err.as_ref(),
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
)
}
async fn current(store: &Store, role: SingletonRole) -> trc::Result<Option<Vec<u8>>> {
Ok(store
.get_value::<Raw>(ValueKey::from(lease_class(role)))
.await
.caused_by(trc::location!())?
.map(|Raw(bytes)| bytes))
}
/// Who holds `role`'s lease, if it hasn't run out.
pub async fn holder(store: &Store, role: SingletonRole) -> trc::Result<Option<Lease>> {
Ok(current(store, role)
.await?
.and_then(|bytes| Lease::from_bytes(&bytes))
.filter(|lease| lease.expires > now()))
}
/// One round for `role`: takes the lease if nobody holds it or it ran out,
/// or renews it if it's `token`'s. Returns when it now runs out, or None
/// when another holder has it.
pub async fn try_hold(
store: &Store,
role: SingletonRole,
token: u64,
node_id: u16,
) -> trc::Result<Option<u64>> {
let current = current(store, role).await?;
if let Some(lease) = current.as_deref().and_then(Lease::from_bytes)
&& lease.token != token
&& lease.expires > now()
{
return Ok(None);
}
let expires = now() + LEASE_SECONDS;
let mut batch = BatchBuilder::new();
batch.assert_value(
lease_class(role),
current.as_deref().map_or(AssertValue::None, |bytes| {
AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes))
}),
);
batch.set(
lease_class(role),
Lease {
token,
expires,
node_id,
}
.to_bytes(),
);
match store.write(batch.build_all()).await {
Ok(_) => Ok(Some(expires)),
// Another node took or renewed it in between
Err(err) if is_assert_failure(&err) => Ok(None),
Err(err) => Err(err.caused_by(trc::location!())),
}
}
/// Gives `role`'s lease up if it's `token`'s, so another node can take it
/// at its next round instead of waiting for it to run out.
pub async fn release(store: &Store, role: SingletonRole, token: u64) -> trc::Result<()> {
let Some(bytes) = current(store, role).await? else {
return Ok(());
};
if Lease::from_bytes(&bytes).is_none_or(|lease| lease.token != token) {
return Ok(());
}
let mut batch = BatchBuilder::new();
batch.assert_value(
lease_class(role),
AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(&bytes)),
);
batch.clear(lease_class(role));
match store.write(batch.build_all()).await {
Ok(_) => Ok(()),
Err(err) if is_assert_failure(&err) => Ok(()),
Err(err) => Err(err.caused_by(trc::location!())),
}
}
/// Keeps this node's singleton leases: each round, every role the node has
/// is tried for, and a role it no longer has is given up.
pub fn spawn_singleton_leases(inner: Arc<Inner>) {
tokio::spawn(async move {
loop {
let server = inner.build_server();
if server.core.storage.has_coordinator() {
for role in SingletonRole::ALL {
round(&server, role).await;
}
}
tokio::time::sleep(ROUND).await;
}
});
}
/// Gives up every lease this node holds, as it stops, so another node with
/// the role, or this one once it's back, takes over at its next round
/// instead of when the lease runs out.
pub async fn release_singleton_leases(server: &Server) {
let leases = &server.inner.data.singleton_leases;
for role in SingletonRole::ALL {
if leases.held_until[role.index()].swap(0, Ordering::Relaxed) == 0 {
continue;
}
match release(server.store(), role, leases.token).await {
Ok(()) => lost(role, "This node is stopping"),
Err(err) => {
// It runs out on its own
trc::error!(err.details("Failed to give up a singleton role lease"));
}
}
}
}
async fn round(server: &Server, role: SingletonRole) {
let leases = &server.inner.data.singleton_leases;
let token = leases.token;
let store = server.store();
let was_holding = leases.holds(role);
if !role.assigned(server) {
if was_holding {
leases.set(role, 0);
lost(role, "The role was taken off this node");
if let Err(err) = release(store, role, token).await {
trc::error!(err.details("Failed to give up a singleton role lease"));
}
}
return;
}
match try_hold(store, role, token, server.registry().node_id()).await {
Ok(Some(expires)) => {
if !was_holding {
if role == SingletonRole::Metrics {
// Carry on from the last holder's firing alerts, so an
// alert that's still firing isn't sent again
crate::telemetry::alerts::load_firing(store).await;
}
trc::event!(
Cluster(ClusterEvent::SingletonLeaseTaken),
Type = role.name(),
Id = server.registry().node_id(),
Expires = trc::Value::Timestamp(expires),
);
}
leases.set(role, expires);
}
Ok(None) => {
leases.set(role, 0);
if was_holding {
lost(role, "Another node holds the lease");
}
}
// Keeps what it had: the lease runs out on its own if this goes on
Err(err) => {
trc::error!(err.details("Failed to renew a singleton role lease"));
}
}
}
fn lost(role: SingletonRole, why: &'static str) {
trc::event!(
Cluster(ClusterEvent::SingletonLeaseLost),
Type = role.name(),
Reason = why,
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_lease_reads_back() {
let lease = Lease {
token: 7,
expires: 1_234,
node_id: 3,
};
assert_eq!(Lease::from_bytes(&lease.to_bytes()), Some(lease));
// A value that can't be read counts as no lease
assert_eq!(Lease::from_bytes(b"short"), None);
}
#[test]
fn a_holder_stops_before_its_lease_runs_out() {
let leases = SingletonLeases::default();
assert!(!leases.holds(SingletonRole::TaskScheduler));
leases.set(SingletonRole::TaskScheduler, now() + LEASE_SECONDS);
assert!(leases.holds(SingletonRole::TaskScheduler));
assert!(!leases.holds(SingletonRole::Metrics));
// Inside the margin it no longer acts, though the lease hasn't run out
leases.set(SingletonRole::TaskScheduler, now() + MARGIN);
assert!(!leases.holds(SingletonRole::TaskScheduler));
}
}
+84 -4
View File
@@ -25,7 +25,11 @@ use registry::{
types::{id::ObjectId, list::List},
};
use std::sync::Mutex;
use store::registry::{RegistryQuery, bootstrap::Bootstrap};
use store::{
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
registry::{RegistryQuery, bootstrap::Bootstrap},
write::{AnyClass, BatchBuilder, ValueClass},
};
use trc::{Collector, MetricType, TelemetryEvent};
use types::id::Id;
@@ -37,10 +41,71 @@ pub struct AlertMessage {
pub body: Vec<u8>,
}
/// The alerts whose condition held at the last evaluation (MON-26). In
/// memory, so a restart while a condition holds fires once more.
/// The alerts whose condition held at the last evaluation (MON-26). Kept
/// in the store as well (#437), so the node that takes over alerts from
/// another carries on from it instead of firing them all again.
static FIRING: Mutex<Option<AHashSet<u64>>> = Mutex::new(None);
/// Under `X` (cluster-wide singletons) in `_`, beside the singleton role
/// leases: the ids
/// of the alerts firing at the last evaluation.
pub const FIRING_KEY: &[u8] = b"Xa";
fn firing_class() -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: FIRING_KEY.to_vec(),
})
}
struct FiringIds(AHashSet<u64>);
impl Deserialize for FiringIds {
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
Ok(FiringIds(
bytes
.as_chunks::<8>()
.0
.iter()
.map(|id| u64::from_be_bytes(*id))
.collect(),
))
}
}
fn firing_bytes(ids: &AHashSet<u64>) -> Vec<u8> {
let mut ids = ids.iter().copied().collect::<Vec<_>>();
ids.sort_unstable();
ids.into_iter().flat_map(u64::to_be_bytes).collect()
}
/// Takes up the alerts the store says are firing, as the node that takes
/// over evaluating them (#437). With none stored, the next evaluation is
/// a first one (MON-26).
pub async fn load_firing(store: &Store) {
match store
.get_value::<FiringIds>(ValueKey::from(firing_class()))
.await
{
Ok(stored) => *FIRING.lock().unwrap() = stored.map(|FiringIds(ids)| ids),
Err(err) => {
trc::error!(err.details("Failed to read the firing alerts"));
}
}
}
/// Stores the firing alerts when they changed since the last evaluation.
async fn store_firing(store: &Store, previous: Option<&AHashSet<u64>>, holding: &AHashSet<u64>) {
if previous == Some(holding) {
return;
}
let mut batch = BatchBuilder::new();
batch.set(firing_class(), firing_bytes(holding));
if let Err(err) = store.write(batch.build_all()).await {
trc::error!(err.details("Failed to store the firing alerts"));
}
}
fn is_ident_char(c: char) -> bool {
c.is_ascii_alphanumeric() || c == '_'
}
@@ -237,7 +302,8 @@ impl Server {
messages.push(build_email(email));
}
}
*FIRING.lock().unwrap() = Some(holding);
let previous = FIRING.lock().unwrap().replace(holding.clone());
store_firing(self.store(), previous.as_ref(), &holding).await;
Ok(messages)
}
}
@@ -246,6 +312,20 @@ impl Server {
mod tests {
use super::*;
#[test]
fn firing_alerts_read_back() {
let ids = AHashSet::from_iter([3u64, 1 << 40, 7]);
let FiringIds(read) = FiringIds::deserialize(&firing_bytes(&ids)).unwrap();
assert_eq!(read, ids);
// Stored in order, so an unchanged set writes the same bytes
assert_eq!(
firing_bytes(&ids),
firing_bytes(&AHashSet::from_iter([7u64, 3, 1 << 40]))
);
let FiringIds(none) = FiringIds::deserialize(&[]).unwrap();
assert!(none.is_empty());
}
#[test]
fn rewrites_underscore_names() {
assert_eq!(rewrite("domain_count > 1"), "metric('domain.count') > 1");
+3 -1
View File
@@ -197,7 +197,9 @@ impl Server {
if store.is_none() {
return;
}
let samples = sample(self.core.network.roles.metrics_calculate);
// inbuxa: #437: the cluster-wide gauges from the node that sets them
let samples =
sample(self.runs_singleton(crate::manager::singleton_lease::SingletonRole::Metrics));
let count = samples.len();
let started = std::time::Instant::now();
match store.write_metrics(samples, now()).await {
+3
View File
@@ -162,6 +162,9 @@ async fn main() -> std::io::Result<()> {
// inbuxa: hand back the task locks this node holds, so other nodes can
// run those tasks now rather than when the locks expire
services::task_manager::lock::release_task_locks(&inner.build_server()).await;
// inbuxa: #437: and its singleton role leases, so another node (or this
// one, restarted) takes those roles over at once
common::manager::singleton_lease::release_singleton_leases(&inner.build_server()).await;
// Shutdown collector
Collector::shutdown();
+1
View File
@@ -113,6 +113,7 @@ pub const OWNERS: &[(u8, Owner)] = &[
b'W',
Owner::Stays("blob store changes and the sharing policy"),
),
(b'X', Owner::Stays("cluster singleton role leases")),
];
/// Step 2 -> 3. Returns how many rows moved.
+2
View File
@@ -110,6 +110,8 @@ impl SpawnServices for IpcReceivers {
inbuxa_scheduled_reports::spawn_scheduled_reports(inner.clone());
// Spawn task scheduler
// inbuxa: #437, before the scheduler, which asks it who acts
common::manager::singleton_lease::spawn_singleton_leases(inner.clone());
spawn_task_scheduler(inner);
}
}
+67 -54
View File
@@ -14,8 +14,9 @@ use std::{
};
use common::{
BuildServer, Inner, LONG_1D_SLUMBER,
BuildServer, Inner, LONG_1D_SLUMBER, Server,
config::{mailstore::spamfilter, telemetry::OtelMetrics},
manager::singleton_lease::SingletonRole,
};
use registry::{
schema::{
@@ -70,6 +71,37 @@ async fn metrics_collection_delay(server: &common::Server) -> Duration {
.time_to_next()
}
/// The cluster-wide gauges, on the node that does the metrics work: the
/// queue every time, accounts and domains when `others` (once a day).
async fn calculate_cluster_metrics(server: &Server, others: bool) {
// inbuxa: MON-7: the queue gauge from the queue itself, so it's right
// after a restart
match server.total_queued_messages().await {
Ok((total, oldest)) => {
Collector::update_gauge(MetricType::QueueCount, total);
Collector::update_gauge(MetricType::QueueOldestMessage, oldest.unwrap_or_default());
}
Err(err) => {
trc::error!(err.details("Failed to count queued messages"));
}
}
if others {
match server.total_accounts().await {
Ok(total) => Collector::update_gauge(MetricType::UserCount, total as u64),
Err(err) => {
trc::error!(err.details("Failed to obtain account count"));
}
}
match server.total_domains().await {
Ok(total) => Collector::update_gauge(MetricType::DomainCount, total as u64),
Err(err) => {
trc::error!(err.details("Failed to obtain domain count"));
}
}
}
}
#[derive(Default)]
struct Queue {
heap: BinaryHeap<Action>,
@@ -174,13 +206,27 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
}
let mut next_metric_update = Instant::now();
// inbuxa: #437: whether this node calculated the cluster-wide gauges
// since it last took on the metrics role, so a node that just took
// over doesn't evaluate alerts against gauges it never set
let mut calculated = false;
loop {
tokio::time::sleep(queue.wake_up_time()).await;
let server = inner.build_server();
let roles = &server.core.network.roles;
let mut batch = (roles.task_scheduler).then(BatchBuilder::new);
// inbuxa: #437: in a cluster, only the node holding the role's
// lease queues the scheduled tasks and does the metrics work
let mut batch = server
.runs_singleton(SingletonRole::TaskScheduler)
.then(BatchBuilder::new);
let calculates = server.runs_singleton(SingletonRole::Metrics);
if !calculates {
calculated = false;
// Counted again at once if this node takes the role on
next_metric_update = Instant::now();
}
while let Some(event) = queue.pop() {
match event.event {
@@ -299,61 +345,18 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
Event::CalculateMetrics,
);
let update_other_metrics = if Instant::now() >= next_metric_update {
let update_other_metrics =
calculates && Instant::now() >= next_metric_update;
if update_other_metrics {
next_metric_update = Instant::now() + Duration::from_secs(86400);
true
} else {
false
};
}
calculated |= calculates;
let server = server.clone();
tokio::spawn(async move {
let elapsed = Instant::now();
if server.core.network.roles.metrics_calculate {
// inbuxa: MON-7: the queue gauge from the queue itself,
// so it's right after a restart
match server.total_queued_messages().await {
Ok((total, oldest)) => {
Collector::update_gauge(MetricType::QueueCount, total);
Collector::update_gauge(
MetricType::QueueOldestMessage,
oldest.unwrap_or_default(),
);
}
Err(err) => {
trc::error!(err.details("Failed to count queued messages"));
}
}
if update_other_metrics {
match server.total_accounts().await {
Ok(total) => {
Collector::update_gauge(
MetricType::UserCount,
total as u64,
);
}
Err(err) => {
trc::error!(
err.details("Failed to obtain account count")
);
}
}
match server.total_domains().await {
Ok(total) => {
Collector::update_gauge(
MetricType::DomainCount,
total as u64,
);
}
Err(err) => {
trc::error!(
err.details("Failed to obtain domain count")
);
}
}
}
if calculates {
calculate_cluster_metrics(&server, update_other_metrics).await;
}
match tokio::task::spawn_blocking(memory_stats::memory_stats).await {
@@ -387,9 +390,19 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
Instant::now() + Duration::from_secs(60),
Event::EvaluateAlerts,
);
if server.core.network.roles.metrics_calculate {
if calculates {
// #437: a node that just took over sets the gauges
// first, so no alert reads one it never set
let first = !calculated;
if first {
calculated = true;
next_metric_update = Instant::now() + Duration::from_secs(86400);
}
let server = server.clone();
tokio::spawn(async move {
if first {
calculate_cluster_metrics(&server, true).await;
}
match server.process_alerts().await {
Ok(messages) => {
use smtp::reporting::send::MtaReportSend;
@@ -434,7 +447,7 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
Instant::now() + DOMAIN_TASK_CHECK_INTERVAL,
Event::CheckDomainTasks,
);
if roles.task_scheduler {
if batch.is_some() {
let server = server.clone();
tokio::spawn(async move {
if let Err(err) =
+6 -2
View File
@@ -16,12 +16,13 @@
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228);
// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371
// 664 is auth.passkey-counter (passkeys spec, PK-18); 665 and 666 are
// cluster.singleton-lease-taken and -lost (#437). Metrics 369 to 371
// are gauges for Kubernetes alerts (K-C10): queue.oldest-message,
// cluster.lease-renewed, tls.earliest-expiry.
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
// more than the highest id; the test below holds it to the variant list
pub const TOTAL_EVENT_COUNT: usize = 665;
pub const TOTAL_EVENT_COUNT: usize = 667;
pub const TOTAL_METRIC_COUNT: usize = 372;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -170,6 +171,9 @@ pub enum ClusterEvent {
CoordinatorConnected = 644,
CoordinatorDisconnected = 645,
CoordinatorError = 646,
// inbuxa: #437, the singleton role leases
SingletonLeaseTaken = 665,
SingletonLeaseLost = 666,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+16
View File
@@ -95,6 +95,8 @@ impl EventType {
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
b"cluster.node-id-reassigned" => EventType::Cluster(ClusterEvent::NodeIdReassigned),
b"cluster.singleton-lease-taken" => EventType::Cluster(ClusterEvent::SingletonLeaseTaken),
b"cluster.singleton-lease-lost" => EventType::Cluster(ClusterEvent::SingletonLeaseLost),
// inbuxa: coordinator connection
b"cluster.coordinator-connected" => EventType::Cluster(ClusterEvent::CoordinatorConnected),
b"cluster.coordinator-disconnected" => EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
@@ -784,6 +786,10 @@ impl EventType {
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
EventType::Cluster(ClusterEvent::NodeIdReassigned) => "cluster.node-id-reassigned",
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => {
"cluster.singleton-lease-taken"
}
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => "cluster.singleton-lease-lost",
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected) => {
"cluster.coordinator-connected"
@@ -1598,6 +1604,8 @@ impl EventType {
EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
EventType::Cluster(ClusterEvent::NodeIdReassigned) => 649,
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => 665,
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => 666,
// inbuxa: coordinator connection
EventType::Cluster(ClusterEvent::CoordinatorConnected) => 644,
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => 645,
@@ -2278,6 +2286,8 @@ impl EventType {
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
649 => Some(EventType::Cluster(ClusterEvent::NodeIdReassigned)),
665 => Some(EventType::Cluster(ClusterEvent::SingletonLeaseTaken)),
666 => Some(EventType::Cluster(ClusterEvent::SingletonLeaseLost)),
// inbuxa: coordinator connection
644 => Some(EventType::Cluster(ClusterEvent::CoordinatorConnected)),
645 => Some(EventType::Cluster(ClusterEvent::CoordinatorDisconnected)),
@@ -3256,6 +3266,8 @@ impl EventType {
EventType::Cluster(ClusterEvent::CoordinatorConnected) => Level::Info,
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => Level::Warn,
EventType::Cluster(ClusterEvent::CoordinatorError) => Level::Warn,
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => Level::Info,
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => Level::Info,
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
@@ -3407,6 +3419,8 @@ impl EventType {
EventType::Cluster(ClusterEvent::CoordinatorConnected) => "Coordinator connected",
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => "Coordinator unavailable",
EventType::Cluster(ClusterEvent::CoordinatorError) => "Coordinator error",
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => "Singleton role lease taken",
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => "Singleton role lease lost",
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
@@ -4529,6 +4543,8 @@ impl EventType {
EventType::Cluster(ClusterEvent::CoordinatorConnected),
EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
EventType::Cluster(ClusterEvent::CoordinatorError),
EventType::Cluster(ClusterEvent::SingletonLeaseTaken),
EventType::Cluster(ClusterEvent::SingletonLeaseLost),
EventType::Dane(DaneEvent::AuthenticationSuccess),
EventType::Dane(DaneEvent::AuthenticationFailure),
EventType::Dane(DaneEvent::NoCertificatesFound),
+19 -4
View File
@@ -336,9 +336,11 @@ unchanged.
### Alerts
- **MON-25.** **Evaluation.** Each enabled alert's condition is evaluated
every 60 seconds (**Decision**) on every node that holds the
`metricsCalculate` task role, against that node's current values. A
one-node install always holds it. The condition language is the server's
every 60 seconds (**Decision**) on the node that does the
`metricsCalculate` task role's work, against that node's current values. A
one-node install always holds it. In a cluster several nodes can have the
role; the one holding its lease in the store does the work (#437, see
"Singleton roles" below). The condition language is the server's
expression language, and a metric is read two ways, both accepted:
- `metric('queue.count')`, which the AGPL expression code already supports;
- the name with dots and hyphens as underscores (`queue_count`), as
@@ -350,9 +352,22 @@ unchanged.
true, including the first evaluation after start. While it stays true it
doesn't fire again. Once false, it can fire again. **Decision**: upstream's
docs say email goes out "each time the condition becomes true". State is in
memory, so a restart while the condition holds fires once more. A condition
memory, and in a cluster in the store too (`Xa` in `_`), so a node that
takes the role over from another carries on from it; a node without a
coordinator that restarts while the condition holds fires once more. A condition
that fails to evaluate (a metric name that doesn't exist, a type error) is
rejected when the alert is saved, and never fires.
- **Singleton roles (#437).** `taskScheduler` (queuing the scheduled
maintenance tasks, and the daily check of each domain's DKIM and ACME
tasks) and `metricsCalculate` (the cluster-wide gauges, alert evaluation
and their emails) each have a lease in the store (`Xs`, `Xm` in `_`).
Every node with the role tries for it each minute; the holder renews it
and does the work. A lease lasts 3 minutes, and its holder stops acting
15 seconds before it runs out, so when the holder stops another node with
the role takes over within about 4 minutes. A node that takes over sets
the gauges before its first alert evaluation. A node without a cluster
coordinator needs no lease: the role alone decides. Taking and losing a
lease log `cluster.singleton-lease-taken` and `cluster.singleton-lease-lost`.
- **MON-27.** **Placeholders.** In `eventMessage`, `subject` and `body`,
`%{metric.name}%` (the dotted name) is replaced by the value used in that
evaluation. Whole numbers print without decimals ("3", not "3.0"); others
+3
View File
@@ -156,6 +156,8 @@ can't place, before it writes anything. The table is `OWNERS` in
| `K` | account locks | stays |
| `L` | audit log (`e`, `t`, `o`, `h`, `f`) | moves to `ib_audit` (0x80); its settings (`s`) stay |
| `M` | masked email | stays |
| `N` | passkeys | stays |
| `O` | listener status | stays |
| `P` | permissions and protocol policies | stays |
| `Q` | acceptance records | stays |
| `R` | mail flow rules | stays |
@@ -164,6 +166,7 @@ can't place, before it writes anything. The table is `OWNERS` in
| `U` | undelete notes (`n`, `x`, `b`, `r`, `c`, `k`, `a`, `g`) | moves to `ib_undelete` (0x83) |
| `V` | store checks, the layout mark (`Vs`), lease (`Vm`) and step progress (`Vt`) | stays |
| `W` | blob store changes, the sharing policy | stays |
| `X` | singleton role leases (`Xs`, `Xm`) and the alerts firing (`Xa`), #437 | stays |
A key moves without its first byte, so `Le` + node + seq becomes `e` + node
+ seq in `ib_audit`. A feature's settings stay in `_`, where a configuration
Binary file not shown.
+1 -1
View File
@@ -1 +1 @@
3CkGF0FiqDTQxYPZ0VjCxnlGmiuzyjDDbmIv1BVmDmg
gBQXaaMdj6J-6RZEk6DUlOzZD58mbuzZ2Uo8M2QksY8
+1
View File
@@ -24,6 +24,7 @@ pub mod replica_mysql; // inbuxa: read replicas on MySQL
pub mod scaleout; // inbuxa: scale-out storage
#[cfg(feature = "postgres")]
pub mod search_gin; // inbuxa: GIN indexes without a pending list
pub mod singleton_lease; // inbuxa: #437, one node at a time does the singleton roles' work
#[cfg(any(feature = "postgres", feature = "mysql"))]
pub mod sql_timeout;
pub mod task_locks; // inbuxa: task locks across nodes
+144
View File
@@ -0,0 +1,144 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! #437: a singleton role's lease has one holder at a time, renews for its
//! holder, and passes to another node once it's given up or runs out.
//! Nodes are played by tokens sharing one store.
use common::manager::singleton_lease::{
LEASE_SECONDS, Lease, SingletonRole, holder, release, try_hold,
};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use store::{
SUBSPACE_INBUXA, Store,
backend::sqlite::SqliteStore,
write::{AnyClass, BatchBuilder, ValueClass, now},
};
const SCHEDULER: SingletonRole = SingletonRole::TaskScheduler;
fn store() -> Store {
Store::SQLite(SqliteStore::open_memory().unwrap().into())
}
/// A lease node `node_id` (token `token`) left, running out at `expires`.
async fn write_lease(store: &Store, token: u64, node_id: u16, expires: u64) {
let mut batch = BatchBuilder::new();
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: SCHEDULER.key().to_vec(),
}),
Lease {
token,
expires,
node_id,
}
.to_bytes(),
);
store.write(batch.build_all()).await.unwrap();
}
#[tokio::test(flavor = "multi_thread")]
async fn one_holder_among_many() {
let store = store();
let won = Arc::new(AtomicUsize::new(0));
// Eight nodes try at once: exactly one gets it
let nodes = (1..=8u64)
.map(|token| {
let (store, won) = (store.clone(), won.clone());
tokio::spawn(async move {
if try_hold(&store, SCHEDULER, token, token as u16)
.await
.unwrap()
.is_some()
{
won.fetch_add(1, Ordering::SeqCst);
}
})
})
.collect::<Vec<_>>();
for node in nodes {
node.await.unwrap();
}
assert_eq!(won.load(Ordering::SeqCst), 1, "not exactly one holder");
// The holder renews it; the others still can't have it
let lease = holder(&store, SCHEDULER).await.unwrap().unwrap();
let renewed = try_hold(&store, SCHEDULER, lease.token, lease.node_id)
.await
.unwrap()
.expect("the holder couldn't renew its lease");
assert!(renewed >= lease.expires);
for token in (1..=8u64).filter(|token| *token != lease.token) {
assert_eq!(
try_hold(&store, SCHEDULER, token, token as u16)
.await
.unwrap(),
None,
"node {token} took a lease another node holds"
);
}
// The roles have leases of their own
assert!(
try_hold(&store, SingletonRole::Metrics, 99, 99)
.await
.unwrap()
.is_some()
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_lease_passes_on_when_it_runs_out_or_is_given_up() {
let store = store();
// Node 1 holds it: node 2 waits
write_lease(&store, 1, 1, now() + LEASE_SECONDS).await;
assert_eq!(try_hold(&store, SCHEDULER, 2, 2).await.unwrap(), None);
// Node 1 stopped and its lease ran out: node 2 takes it over
write_lease(&store, 1, 1, now() - 1).await;
assert!(holder(&store, SCHEDULER).await.unwrap().is_none());
let expires = try_hold(&store, SCHEDULER, 2, 2)
.await
.unwrap()
.expect("node 2 didn't take over a lease that ran out");
assert!(expires >= now() + LEASE_SECONDS - 1);
assert_eq!(holder(&store, SCHEDULER).await.unwrap().unwrap().node_id, 2);
// Node 1 comes back: it's node 2's now
assert_eq!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap(), None);
// Giving up someone else's lease does nothing
release(&store, SCHEDULER, 1).await.unwrap();
assert_eq!(holder(&store, SCHEDULER).await.unwrap().unwrap().token, 2);
// Node 2 gives it up (the role was taken off it): node 1 takes it at once
release(&store, SCHEDULER, 2).await.unwrap();
assert!(holder(&store, SCHEDULER).await.unwrap().is_none());
assert!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap().is_some());
}
#[tokio::test(flavor = "multi_thread")]
async fn a_lease_that_cant_be_read_counts_as_none() {
let store = store();
let mut batch = BatchBuilder::new();
batch.set(
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: SCHEDULER.key().to_vec(),
}),
b"short".to_vec(),
);
store.write(batch.build_all()).await.unwrap();
assert!(holder(&store, SCHEDULER).await.unwrap().is_none());
assert!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap().is_some());
}