Author SHA1 Message Date
jcoffey-dev 96b54ede4e Merge pull request 'Release 2026.9.25.1' (#55) from bump/2026.9.25.1 into main
ci / fork-checks (push) Successful in 18s
publish / version (push) Successful in 33s
publish / publish-amd64 (push) Successful in 23m45s
publish / release (push) Successful in 1s
ci / build (push) Successful in 37m8s
publish / publish-arm64 (push) Successful in 35m1s
publish / binaries (push) Successful in 39s
2026-09-25 08:23:55 +00:00
jcoffey-dev 1f9b3174de Release 2026.9.25.1
ci / fork-checks (pull_request) Successful in 17s
ci / build (pull_request) Successful in 7m17s
2026-09-25 01:15:55 -07:00
jcoffey-dev 5e2ddf644f Merge pull request 'Report a node unhealthy after three minutes of silence' (#54) from feature/node-heartbeat into main
ci / fork-checks (push) Successful in 45s
ci / build (push) Canceled after 8m9s
2026-09-25 08:15:44 +00:00
jcoffey-dev 5245abd08d Report a node unhealthy after three minutes of silence
ci / fork-checks (pull_request) Successful in 38s
ci / build (pull_request) Successful in 7m24s
Every node renews its lease once a minute instead of every 30 minutes,
so the lease works as a heartbeat. x:ClusterNode reports a node Stale
once it has gone three minutes without renewing (it used to take an
hour), and Inactive after a day, as before.

Taking over a lease still needs a full hour of silence. A node that is
slow rather than gone never loses its id to another host, so snowflake
ids stay unique.

The admin dashboard's Cluster Health card counts these statuses.
2026-09-25 01:05:15 -07:00
2 changed files with 58 additions and 5 deletions
+57 -4
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]> * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
* *
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/ */
use crate::{ use crate::{
@@ -23,6 +25,14 @@ use utils::snowflake::MAX_NODE_ID;
const STALE_NODE_TIMEOUT: u64 = 60 * 60; // 1 hour const STALE_NODE_TIMEOUT: u64 = 60 * 60; // 1 hour
const DEAD_NODE_TIMEOUT: u64 = 60 * 60 * 24; // 24 hours const DEAD_NODE_TIMEOUT: u64 = 60 * 60 * 24; // 24 hours
// INBUXA: every node renews its lease once a minute, so the lease doubles as
// a heartbeat. A node not heard from in three minutes is reported Stale, which
// is what Cluster Health on the dashboard counts. Taking over a lease still
// needs the full hour of silence, so a node that is slow rather than gone
// never loses its id to another host.
const HEARTBEAT_INTERVAL: u64 = 60; // 1 minute
const UNRESPONSIVE_NODE_TIMEOUT: u64 = 3 * HEARTBEAT_INTERVAL;
const MAX_LEASE_RETRIES: u32 = 5; const MAX_LEASE_RETRIES: u32 = 5;
struct NodeSlot { struct NodeSlot {
@@ -96,7 +106,7 @@ impl RegistryStore {
} }
pub fn refresh_node_id_interval(&self) -> Duration { pub fn refresh_node_id_interval(&self) -> Duration {
Duration::from_secs(STALE_NODE_TIMEOUT / 2) Duration::from_secs(HEARTBEAT_INTERVAL)
} }
pub async fn cluster_node_list(&self) -> trc::Result<Vec<ClusterNode>> { pub async fn cluster_node_list(&self) -> trc::Result<Vec<ClusterNode>> {
@@ -289,6 +299,10 @@ impl NodeSlot {
self.elapsed > DEAD_NODE_TIMEOUT self.elapsed > DEAD_NODE_TIMEOUT
} }
fn is_responsive(&self) -> bool {
self.elapsed <= UNRESPONSIVE_NODE_TIMEOUT
}
fn is_assignable(&self) -> bool { fn is_assignable(&self) -> bool {
self.node_id <= MAX_NODE_ID self.node_id <= MAX_NODE_ID
} }
@@ -296,10 +310,10 @@ impl NodeSlot {
fn status(&self) -> ClusterNodeStatus { fn status(&self) -> ClusterNodeStatus {
if self.is_dead() { if self.is_dead() {
ClusterNodeStatus::Inactive ClusterNodeStatus::Inactive
} else if self.is_stale() { } else if self.is_responsive() {
ClusterNodeStatus::Stale
} else {
ClusterNodeStatus::Active ClusterNodeStatus::Active
} else {
ClusterNodeStatus::Stale
} }
} }
} }
@@ -314,3 +328,42 @@ impl From<NodeSlot> for ClusterNode {
} }
} }
} }
#[cfg(test)]
mod tests {
use super::*;
fn slot(elapsed: u64) -> NodeSlot {
NodeSlot {
node_id: 1,
hostname: "mx2.example.org".into(),
last_renewal: 0,
elapsed,
hash: 0,
}
}
#[test]
fn status_follows_the_heartbeat() {
assert_eq!(slot(0).status(), ClusterNodeStatus::Active);
assert_eq!(slot(UNRESPONSIVE_NODE_TIMEOUT).status(), ClusterNodeStatus::Active);
assert_eq!(slot(UNRESPONSIVE_NODE_TIMEOUT + 1).status(), ClusterNodeStatus::Stale);
assert_eq!(slot(DEAD_NODE_TIMEOUT).status(), ClusterNodeStatus::Stale);
assert_eq!(slot(DEAD_NODE_TIMEOUT + 1).status(), ClusterNodeStatus::Inactive);
}
#[test]
fn a_silent_node_keeps_its_id_for_an_hour() {
// Reported Stale after three minutes, but not free to take over.
let quiet = slot(UNRESPONSIVE_NODE_TIMEOUT + 1);
assert_eq!(quiet.status(), ClusterNodeStatus::Stale);
assert!(!quiet.is_stale());
assert!(slot(STALE_NODE_TIMEOUT + 1).is_stale());
}
#[test]
fn several_renewals_fit_before_a_node_looks_unresponsive() {
assert!(UNRESPONSIVE_NODE_TIMEOUT >= 3 * HEARTBEAT_INTERVAL);
assert!(HEARTBEAT_INTERVAL * 2 < STALE_NODE_TIMEOUT);
}
}
+1 -1
View File
@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
#[macro_export] #[macro_export]
macro_rules! brand_version { macro_rules! brand_version {
() => { () => {
"2026.9.25" "2026.9.25.1"
}; };
} }