From 281f3ce771eaf79d08789ea46bd024c9be6f7c0a Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 13:43:29 -0700 Subject: [PATCH] Cluster: drop the old node lease after a server is renamed Renaming a server left its old node behind on the dashboard as "not responding" for a day. The lease is keyed by hostname, and the installer sets the container's hostname to the mail host, so a rename restarted the server under a new lease while the old one sat there until it went inactive 24 hours later. It can't be deleted from the console. Each node now keeps the name it last leased under in node-hostname, next to its local config (/etc/inbuxa, a volume in the installer's compose file). When the name has changed, the per-minute lease renewal releases the old name's lease once it has missed two renewals, before the dashboard would call it stale. Only a lease last renewed before this node started counts, so a host still running under that name (two nodes sharing a config directory, say) keeps its lease and still shows up when it stops. A read-only config directory just means the 24-hour expiry, as before. Leases left by a rename before this release still expire on their own. Test: unit tests for the release rule, and two store-backed tests (sqlite, test_mode): a rename drops the old lease after the grace period, and a host renewing the old name after start is left alone. Fixes #458 --- crates/store/src/build/cluster.rs | 241 +++++++++++++++++++++++++++-- crates/store/src/build/registry.rs | 1 + crates/store/src/lib.rs | 3 + crates/store/src/registry/local.rs | 1 + 4 files changed, 237 insertions(+), 9 deletions(-) diff --git a/crates/store/src/build/cluster.rs b/crates/store/src/build/cluster.rs index 65d366ce..cd5064d2 100644 --- a/crates/store/src/build/cluster.rs +++ b/crates/store/src/build/cluster.rs @@ -19,7 +19,11 @@ use registry::{ schema::{enums::ClusterNodeStatus, structs::ClusterNode}, types::datetime::UTCDateTime, }; -use std::{sync::atomic::Ordering, time::Duration}; +use std::{ + path::{Path, PathBuf}, + sync::atomic::Ordering, + time::Duration, +}; use trc::{AddContext, ClusterEvent}; use utils::snowflake::{MAX_NODE_ID, SnowflakeIdGenerator}; @@ -34,6 +38,9 @@ const DEAD_NODE_TIMEOUT: u64 = 60 * 60 * 24; // 24 hours const HEARTBEAT_INTERVAL: u64 = 60; // 1 minute const UNRESPONSIVE_NODE_TIMEOUT: u64 = 3 * HEARTBEAT_INTERVAL; const MAX_LEASE_RETRIES: u32 = 5; +// INBUXA: #458, see `release_renamed_lease`. +const RENAMED_LEASE_GRACE: u64 = 2 * HEARTBEAT_INTERVAL; +const HOSTNAME_MARKER: &str = "node-hostname"; struct NodeSlot { node_id: u16, @@ -52,6 +59,7 @@ impl RegistryStoreInner { pub(super) async fn acquire_node_id(&mut self) -> Result<(), String> { let (node_id, slots) = NodeSlot::acquire(&self.store, &self.env_hostname).await?; self.node_id.store(node_id, Ordering::Relaxed); + self.remember_hostname().await; if let Err(err) = NodeSlot::release( &self.store, @@ -66,6 +74,38 @@ impl RegistryStoreInner { Ok(()) } + + // INBUXA: #458. The lease is keyed by hostname, so renaming a server + // (the installer sets the container's hostname to the mail host) left the + // old name's lease behind, reported as a node not responding until it went + // inactive a day later. Each node keeps the name it last leased under in a + // file next to its local config. When that name changes, the old name's + // lease is this node's own and is released once it stops renewing; see + // `release_renamed_lease`. The file is best effort: a read-only config + // directory only means a rename falls back to the 24-hour expiry. + async fn remember_hostname(&mut self) { + let path = hostname_marker(&self.local_path); + if let Ok(previous) = tokio::fs::read_to_string(&path).await { + let previous = previous.trim(); + if !previous.is_empty() && previous != self.env_hostname { + self.renamed_from = Some((previous.to_string(), now())); + } + } + if self.renamed_from.is_some() || !path.exists() { + let _ = tokio::fs::write(&path, &self.env_hostname).await; + } + } +} + +// INBUXA: #458, the file a node records its leased hostname in. +// It sits beside the local config file, or inside the directory when the +// path is one (as in the test harness). +fn hostname_marker(local_path: &Path) -> PathBuf { + if local_path.is_dir() { + local_path.join(HOSTNAME_MARKER) + } else { + local_path.with_file_name(HOSTNAME_MARKER) + } } impl RegistryStore { @@ -85,12 +125,10 @@ impl RegistryStore { pub async fn refresh_node_id_lease(&self) -> trc::Result<()> { let node_id = self.node_id(); - let assert = match NodeSlot::list(&self.0.store, now()) + let slots = NodeSlot::list(&self.0.store, now()) .await - .caused_by(trc::location!())? - .into_iter() - .find(|slot| slot.node_id == node_id) - { + .caused_by(trc::location!())?; + let assert = match slots.iter().find(|slot| slot.node_id == node_id) { Some(slot) if slot.is_owned_by(&self.0.env_hostname) => AssertValue::Hash(slot.hash), Some(slot) => { // INBUXA: another host took this id after the hour of silence, @@ -112,7 +150,7 @@ impl RegistryStore { trc::event!( Cluster(ClusterEvent::NodeIdReassigned), Id = new_node_id, - Hostname = slot.hostname, + Hostname = slot.hostname.clone(), Details = format!("Node id {node_id} lease is held by another host"), ); @@ -134,8 +172,30 @@ impl RegistryStore { .store .write(batch.build_all()) .await - .caused_by(trc::location!()) - .map(|_| ()) + .caused_by(trc::location!())?; + + self.release_renamed_lease(&slots).await + } + + // INBUXA: #458. Drops the lease this node held under its previous + // hostname once that lease has missed two renewals, before the dashboard + // would report it as a node not responding. Only a lease last renewed + // before this node started counts: the old name stops renewing when the + // server is restarted under the new one. A host still running under that + // name (two nodes sharing a config directory, say) renews after that, so + // its lease is left to the usual expiry and still shows up when it stops. + async fn release_renamed_lease(&self, slots: &[NodeSlot]) -> trc::Result<()> { + let Some((previous, started)) = self.0.renamed_from.as_ref() else { + return Ok(()); + }; + NodeSlot::release( + &self.0.store, + slots + .iter() + .filter(|slot| slot.is_renamed_from(previous, &self.0.env_hostname, *started)), + ) + .await + .caused_by(trc::location!()) } pub async fn purge_dead_nodes(&self) -> trc::Result<()> { @@ -315,6 +375,15 @@ impl NodeSlot { self.hostname == hostname } + // INBUXA: #458, a lease left under this node's previous hostname: last + // renewed before this node started, and two renewals missed since. + fn is_renamed_from(&self, previous: &str, current: &str, started: u64) -> bool { + previous != current + && self.is_owned_by(previous) + && self.last_renewal <= started + && self.elapsed > RENAMED_LEASE_GRACE + } + fn is_stale(&self) -> bool { self.elapsed > STALE_NODE_TIMEOUT } @@ -385,6 +454,40 @@ mod tests { ); } + fn renewed(hostname: &str, last_renewal: u64, now: u64) -> NodeSlot { + NodeSlot { + node_id: 1, + hostname: hostname.into(), + last_renewal, + elapsed: now - last_renewal, + hash: 0, + } + } + + #[test] + fn a_renamed_node_drops_its_old_lease() { + // mail.old.example renewed at 1000, the server came back as + // mail.new.example at 1010. + let (old, new, started) = ("mail.old.example", "mail.new.example", 1010); + // Not before two renewals are missed... + assert!(!renewed(old, 1000, 1000 + RENAMED_LEASE_GRACE).is_renamed_from(old, new, started)); + // ...and then well before the dashboard would call it stale. + let left = renewed(old, 1000, 1001 + RENAMED_LEASE_GRACE); + assert!(left.is_renamed_from(old, new, started)); + assert_eq!(left.status(), ClusterNodeStatus::Active); + // Only the previous name's lease, and never this node's own. + assert!(!renewed("mx2.example", 1000, 2000).is_renamed_from(old, new, started)); + assert!(!renewed(new, 1000, 2000).is_renamed_from(new, new, started)); + } + + #[test] + fn a_live_host_under_the_old_name_keeps_its_lease() { + // Renewed after this node started, so it isn't a lease left behind + // by a rename: it goes through the usual Stale/Inactive path. + let (old, new, started) = ("mx1.example", "mx2.example", 1010); + assert!(!renewed(old, 1020, 5000).is_renamed_from(old, new, started)); + } + #[test] fn a_silent_node_keeps_its_id_for_an_hour() { // Reported Stale after three minutes, but not free to take over. @@ -433,3 +536,123 @@ mod tests { assert!(HEARTBEAT_INTERVAL * 2 < STALE_NODE_TIMEOUT); } } + +// INBUXA: #458, a rename against a real store: the same config directory, +// the server restarted under a new hostname. +#[cfg(all(test, feature = "sqlite", feature = "test_mode"))] +mod rename_tests { + use super::*; + use crate::{RegistryStore, backend::sqlite::SqliteStore}; + + struct Dir(PathBuf); + + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn node(dir: &Path, store: &Store, hostname: &str) -> RegistryStoreInner { + let mut inner = RegistryStoreInner::new(dir.join("config.json")); + inner.store = store.clone(); + inner.env_hostname = hostname.to_string(); + inner + } + + // Moves a lease's last renewal back, as if its host had gone quiet. + async fn age(store: &Store, hostname: &str, last_renewal: u64) { + let slot = NodeSlot::list(store, now()) + .await + .unwrap() + .into_iter() + .find(|slot| slot.hostname == hostname) + .unwrap(); + let mut batch = BatchBuilder::new(); + batch.set( + ValueClass::NodeId(slot.node_id), + KeySerializer::new(hostname.len() + U64_LEN) + .write(last_renewal) + .write(hostname) + .finalize(), + ); + store.write(batch.build_all()).await.unwrap(); + } + + async fn hostnames(store: &Store) -> Vec { + let mut names = NodeSlot::list(store, now()) + .await + .unwrap() + .into_iter() + .map(|slot| slot.hostname) + .collect::>(); + names.sort(); + names + } + + fn temp_dir(name: &str) -> Dir { + let dir = std::env::temp_dir().join(format!("inbuxa-458-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + Dir(dir) + } + + #[tokio::test] + async fn a_renamed_server_drops_its_old_lease() { + let dir = temp_dir("rename"); + let store = Store::SQLite(SqliteStore::open_memory().unwrap().into()); + + let mut old = node(&dir.0, &store, "mail.old.example"); + old.acquire_node_id().await.unwrap(); + assert_eq!( + std::fs::read_to_string(dir.0.join(HOSTNAME_MARKER)).unwrap(), + "mail.old.example" + ); + + // Shut down a little while ago, then started again as mail.new.example. + age(&store, "mail.old.example", now() - 10).await; + let mut new = node(&dir.0, &store, "mail.new.example"); + new.acquire_node_id().await.unwrap(); + assert_eq!( + std::fs::read_to_string(dir.0.join(HOSTNAME_MARKER)).unwrap(), + "mail.new.example" + ); + let new = RegistryStore(new.into()); + + // Inside the grace period both leases are there. + new.refresh_node_id_lease().await.unwrap(); + assert_eq!( + hostnames(&store).await, + ["mail.new.example", "mail.old.example"] + ); + + // Two missed renewals later, the old one is gone. + age(&store, "mail.old.example", now() - RENAMED_LEASE_GRACE - 10).await; + new.refresh_node_id_lease().await.unwrap(); + assert_eq!(hostnames(&store).await, ["mail.new.example"]); + } + + #[tokio::test] + async fn a_host_still_renewing_the_old_name_is_left_alone() { + let dir = temp_dir("shared"); + let store = Store::SQLite(SqliteStore::open_memory().unwrap().into()); + + let mut one = node(&dir.0, &store, "mx1.example"); + one.acquire_node_id().await.unwrap(); + // mx2 shares the config directory, so it reads mx1's name as its own + // previous one... + let mut two = node(&dir.0, &store, "mx2.example"); + two.acquire_node_id().await.unwrap(); + let (_, started) = two.renamed_from.clone().unwrap(); + let two = RegistryStore(two.into()); + + // ...but mx1 renewed after mx2 started, then went quiet: a node down, + // which the dashboard has to keep showing. + age(&store, "mx1.example", started + 1).await; + // Pretend the clock moved on so the grace period has passed. + let slots = NodeSlot::list(&store, started + 1 + RENAMED_LEASE_GRACE + 60) + .await + .unwrap(); + two.release_renamed_lease(&slots).await.unwrap(); + assert_eq!(hostnames(&store).await, ["mx1.example", "mx2.example"]); + } +} diff --git a/crates/store/src/build/registry.rs b/crates/store/src/build/registry.rs index 7fab5cdd..699a9aa3 100644 --- a/crates/store/src/build/registry.rs +++ b/crates/store/src/build/registry.rs @@ -174,6 +174,7 @@ impl RegistryStore { env_cluster_role: cluster_role, env_push_shard_id: push_shard_id, env_hostname: hostname, + renamed_from: None, env_public_url: None, id_generator: utils::snowflake::SnowflakeIdGenerator::new(), write_hook: Default::default(), diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 7771b45f..440c0595 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -265,6 +265,9 @@ pub struct RegistryStoreInner { pub(crate) env_cluster_role: Option, pub(crate) env_push_shard_id: u32, pub(crate) env_hostname: String, + // inbuxa: #458, the name this node leased under before a rename, if any, + // and when this node started (unix seconds) + pub(crate) renamed_from: Option<(String, u64)>, pub(crate) env_public_url: Option, pub(crate) id_generator: SnowflakeIdGenerator, // inbuxa: AU-1.10, shared by every clone of this registry diff --git a/crates/store/src/registry/local.rs b/crates/store/src/registry/local.rs index 03b0d2d3..23450b2c 100644 --- a/crates/store/src/registry/local.rs +++ b/crates/store/src/registry/local.rs @@ -67,6 +67,7 @@ impl RegistryStoreInner { }) }), env_hostname, + renamed_from: None, write_hook: Default::default(), } } -- 2.54.0