Cluster: one node at a time does the singleton roles' work (#437) #524
No files matched your search
@@ -95,6 +95,7 @@ impl Data {
|
||||
queue_status: true.into(),
|
||||
settings_reload: Default::default(),
|
||||
store_health: Default::default(),
|
||||
singleton_leases: Default::default(),
|
||||
applications,
|
||||
logos: Default::default(),
|
||||
smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"),
|
||||
@@ -242,6 +243,7 @@ impl Default for Data {
|
||||
queue_status: true.into(),
|
||||
settings_reload: Default::default(),
|
||||
store_health: Default::default(),
|
||||
singleton_leases: Default::default(),
|
||||
applications: WebApplications::new(),
|
||||
logos: Default::default(),
|
||||
smtp_connectors: TlsConnectors::try_new().unwrap(),
|
||||
|
||||
@@ -208,6 +208,9 @@ pub struct Data {
|
||||
// inbuxa: the relays outbound mail goes through, and this node's links
|
||||
// to them (relay spec, RL-20, RL-24)
|
||||
pub relay_senders: inbuxa_features::relay::outbound::Senders,
|
||||
|
||||
// inbuxa: #437, the singleton role leases this node holds
|
||||
pub singleton_leases: manager::singleton_lease::SingletonLeases,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
||||
@@ -28,6 +28,7 @@ pub mod first_party;
|
||||
pub mod granted_permissions; // inbuxa: permissions added after roles were stored
|
||||
pub mod listener_ports; // inbuxa: K-S1, default listener ports from the environment
|
||||
pub mod restore;
|
||||
pub mod singleton_lease; // inbuxa: #437, one node at a time does the singleton roles' work
|
||||
pub mod spam_rules; // inbuxa: rules bundled with the server
|
||||
#[cfg(target_os = "linux")]
|
||||
pub mod systemd;
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! inbuxa: one node at a time does the cluster's singleton work (#437).
|
||||
//!
|
||||
//! Every node with the `taskScheduler` role used to queue its own copy of
|
||||
//! each scheduled maintenance task, and every node with `metricsCalculate`
|
||||
//! evaluated alerts and sent their emails, so each of those roles had to be
|
||||
//! on exactly one node. Losing that node paused the work until it came back.
|
||||
//!
|
||||
//! Now each role has a lease in the store, taken with a compare-and-swap
|
||||
//! like the store layout lease. Every node with the role tries for it each
|
||||
//! minute; the one holding it renews it and does the work, the others wait.
|
||||
//! A holder that stops renewing loses it when it runs out, and another node
|
||||
//! with the role takes it over.
|
||||
//!
|
||||
//! A node without a cluster coordinator is on its own and needs no lease:
|
||||
//! the role alone decides, as before.
|
||||
|
||||
use crate::{BuildServer, Inner, Server};
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use store::{
|
||||
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
|
||||
rand::{RngExt, rng},
|
||||
write::{AnyClass, BatchBuilder, ValueClass, assert::AssertValue, now},
|
||||
};
|
||||
use trc::{AddContext, ClusterEvent};
|
||||
|
||||
/// How long a lease lasts unless renewed.
|
||||
pub const LEASE_SECONDS: u64 = 180;
|
||||
/// How often each node with the role tries for its lease.
|
||||
pub const ROUND: Duration = Duration::from_secs(60);
|
||||
/// A holder stops acting this long before its lease runs out, so it has
|
||||
/// stopped by the time another node can take the lease over.
|
||||
const MARGIN: u64 = 15;
|
||||
|
||||
/// The roles only one node of a cluster acts on.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SingletonRole {
|
||||
/// Queues the scheduled maintenance tasks (`taskScheduler`).
|
||||
TaskScheduler,
|
||||
/// Calculates the cluster-wide gauges and evaluates alerts
|
||||
/// (`metricsCalculate`).
|
||||
Metrics,
|
||||
}
|
||||
|
||||
impl SingletonRole {
|
||||
pub const ALL: [SingletonRole; 2] = [SingletonRole::TaskScheduler, SingletonRole::Metrics];
|
||||
|
||||
/// Under `X` (cluster-wide singletons) in `_`.
|
||||
pub fn key(self) -> &'static [u8] {
|
||||
match self {
|
||||
SingletonRole::TaskScheduler => b"Xs",
|
||||
SingletonRole::Metrics => b"Xm",
|
||||
}
|
||||
}
|
||||
|
||||
/// The task role's name, as cluster roles list it.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
SingletonRole::TaskScheduler => "taskScheduler",
|
||||
SingletonRole::Metrics => "metricsCalculate",
|
||||
}
|
||||
}
|
||||
|
||||
fn index(self) -> usize {
|
||||
match self {
|
||||
SingletonRole::TaskScheduler => 0,
|
||||
SingletonRole::Metrics => 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this node has the role.
|
||||
pub fn assigned(self, server: &Server) -> bool {
|
||||
let roles = &server.core.network.roles;
|
||||
match self {
|
||||
SingletonRole::TaskScheduler => roles.task_scheduler,
|
||||
SingletonRole::Metrics => roles.metrics_calculate,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which leases this node holds, and until when.
|
||||
pub struct SingletonLeases {
|
||||
/// Says which leases are this process's.
|
||||
token: u64,
|
||||
held_until: [AtomicU64; 2],
|
||||
}
|
||||
|
||||
impl Default for SingletonLeases {
|
||||
fn default() -> Self {
|
||||
SingletonLeases {
|
||||
token: rng().random(),
|
||||
held_until: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SingletonLeases {
|
||||
/// Whether this node holds `role`'s lease with time to spare.
|
||||
pub fn holds(&self, role: SingletonRole) -> bool {
|
||||
now() + MARGIN < self.held_until[role.index()].load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn set(&self, role: SingletonRole, until: u64) {
|
||||
self.held_until[role.index()].store(until, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl Server {
|
||||
/// Whether this node does `role`'s work now: it has the role, and, in a
|
||||
/// cluster, holds the role's lease.
|
||||
pub fn runs_singleton(&self, role: SingletonRole) -> bool {
|
||||
role.assigned(self)
|
||||
&& (!self.core.storage.has_coordinator()
|
||||
|| self.inner.data.singleton_leases.holds(role))
|
||||
}
|
||||
}
|
||||
|
||||
struct Raw(Vec<u8>);
|
||||
|
||||
impl Deserialize for Raw {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
Ok(Raw(bytes.to_vec()))
|
||||
}
|
||||
}
|
||||
|
||||
fn lease_class(role: SingletonRole) -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: role.key().to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
/// A lease: the holder's token, when it runs out, then the holder's node id
|
||||
/// (for the log only; the token says whose it is).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Lease {
|
||||
pub token: u64,
|
||||
pub expires: u64,
|
||||
pub node_id: u16,
|
||||
}
|
||||
|
||||
impl Lease {
|
||||
pub fn to_bytes(self) -> Vec<u8> {
|
||||
let mut bytes = Vec::with_capacity(18);
|
||||
bytes.extend_from_slice(&self.token.to_be_bytes());
|
||||
bytes.extend_from_slice(&self.expires.to_be_bytes());
|
||||
bytes.extend_from_slice(&self.node_id.to_be_bytes());
|
||||
bytes
|
||||
}
|
||||
|
||||
/// None for a value that can't be read, which counts as no lease.
|
||||
pub fn from_bytes(bytes: &[u8]) -> Option<Self> {
|
||||
Some(Lease {
|
||||
token: u64::from_be_bytes(bytes.get(0..8)?.try_into().ok()?),
|
||||
expires: u64::from_be_bytes(bytes.get(8..16)?.try_into().ok()?),
|
||||
node_id: u16::from_be_bytes(bytes.get(16..18)?.try_into().ok()?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn is_assert_failure(err: &trc::Error) -> bool {
|
||||
matches!(
|
||||
err.as_ref(),
|
||||
trc::EventType::Store(trc::StoreEvent::AssertValueFailed)
|
||||
)
|
||||
}
|
||||
|
||||
async fn current(store: &Store, role: SingletonRole) -> trc::Result<Option<Vec<u8>>> {
|
||||
Ok(store
|
||||
.get_value::<Raw>(ValueKey::from(lease_class(role)))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.map(|Raw(bytes)| bytes))
|
||||
}
|
||||
|
||||
/// Who holds `role`'s lease, if it hasn't run out.
|
||||
pub async fn holder(store: &Store, role: SingletonRole) -> trc::Result<Option<Lease>> {
|
||||
Ok(current(store, role)
|
||||
.await?
|
||||
.and_then(|bytes| Lease::from_bytes(&bytes))
|
||||
.filter(|lease| lease.expires > now()))
|
||||
}
|
||||
|
||||
/// One round for `role`: takes the lease if nobody holds it or it ran out,
|
||||
/// or renews it if it's `token`'s. Returns when it now runs out, or None
|
||||
/// when another holder has it.
|
||||
pub async fn try_hold(
|
||||
store: &Store,
|
||||
role: SingletonRole,
|
||||
token: u64,
|
||||
node_id: u16,
|
||||
) -> trc::Result<Option<u64>> {
|
||||
let current = current(store, role).await?;
|
||||
if let Some(lease) = current.as_deref().and_then(Lease::from_bytes)
|
||||
&& lease.token != token
|
||||
&& lease.expires > now()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let expires = now() + LEASE_SECONDS;
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(
|
||||
lease_class(role),
|
||||
current.as_deref().map_or(AssertValue::None, |bytes| {
|
||||
AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(bytes))
|
||||
}),
|
||||
);
|
||||
batch.set(
|
||||
lease_class(role),
|
||||
Lease {
|
||||
token,
|
||||
expires,
|
||||
node_id,
|
||||
}
|
||||
.to_bytes(),
|
||||
);
|
||||
match store.write(batch.build_all()).await {
|
||||
Ok(_) => Ok(Some(expires)),
|
||||
// Another node took or renewed it in between
|
||||
Err(err) if is_assert_failure(&err) => Ok(None),
|
||||
Err(err) => Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Gives `role`'s lease up if it's `token`'s, so another node can take it
|
||||
/// at its next round instead of waiting for it to run out.
|
||||
pub async fn release(store: &Store, role: SingletonRole, token: u64) -> trc::Result<()> {
|
||||
let Some(bytes) = current(store, role).await? else {
|
||||
return Ok(());
|
||||
};
|
||||
if Lease::from_bytes(&bytes).is_none_or(|lease| lease.token != token) {
|
||||
return Ok(());
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.assert_value(
|
||||
lease_class(role),
|
||||
AssertValue::Hash(store::xxhash_rust::xxh3::xxh3_64(&bytes)),
|
||||
);
|
||||
batch.clear(lease_class(role));
|
||||
match store.write(batch.build_all()).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if is_assert_failure(&err) => Ok(()),
|
||||
Err(err) => Err(err.caused_by(trc::location!())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps this node's singleton leases: each round, every role the node has
|
||||
/// is tried for, and a role it no longer has is given up.
|
||||
pub fn spawn_singleton_leases(inner: Arc<Inner>) {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let server = inner.build_server();
|
||||
if server.core.storage.has_coordinator() {
|
||||
for role in SingletonRole::ALL {
|
||||
round(&server, role).await;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(ROUND).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Gives up every lease this node holds, as it stops, so another node with
|
||||
/// the role, or this one once it's back, takes over at its next round
|
||||
/// instead of when the lease runs out.
|
||||
pub async fn release_singleton_leases(server: &Server) {
|
||||
let leases = &server.inner.data.singleton_leases;
|
||||
for role in SingletonRole::ALL {
|
||||
if leases.held_until[role.index()].swap(0, Ordering::Relaxed) == 0 {
|
||||
continue;
|
||||
}
|
||||
match release(server.store(), role, leases.token).await {
|
||||
Ok(()) => lost(role, "This node is stopping"),
|
||||
Err(err) => {
|
||||
// It runs out on its own
|
||||
trc::error!(err.details("Failed to give up a singleton role lease"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn round(server: &Server, role: SingletonRole) {
|
||||
let leases = &server.inner.data.singleton_leases;
|
||||
let token = leases.token;
|
||||
let store = server.store();
|
||||
let was_holding = leases.holds(role);
|
||||
|
||||
if !role.assigned(server) {
|
||||
if was_holding {
|
||||
leases.set(role, 0);
|
||||
lost(role, "The role was taken off this node");
|
||||
if let Err(err) = release(store, role, token).await {
|
||||
trc::error!(err.details("Failed to give up a singleton role lease"));
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
match try_hold(store, role, token, server.registry().node_id()).await {
|
||||
Ok(Some(expires)) => {
|
||||
if !was_holding {
|
||||
if role == SingletonRole::Metrics {
|
||||
// Carry on from the last holder's firing alerts, so an
|
||||
// alert that's still firing isn't sent again
|
||||
crate::telemetry::alerts::load_firing(store).await;
|
||||
}
|
||||
trc::event!(
|
||||
Cluster(ClusterEvent::SingletonLeaseTaken),
|
||||
Type = role.name(),
|
||||
Id = server.registry().node_id(),
|
||||
Expires = trc::Value::Timestamp(expires),
|
||||
);
|
||||
}
|
||||
leases.set(role, expires);
|
||||
}
|
||||
Ok(None) => {
|
||||
leases.set(role, 0);
|
||||
if was_holding {
|
||||
lost(role, "Another node holds the lease");
|
||||
}
|
||||
}
|
||||
// Keeps what it had: the lease runs out on its own if this goes on
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to renew a singleton role lease"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn lost(role: SingletonRole, why: &'static str) {
|
||||
trc::event!(
|
||||
Cluster(ClusterEvent::SingletonLeaseLost),
|
||||
Type = role.name(),
|
||||
Reason = why,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_lease_reads_back() {
|
||||
let lease = Lease {
|
||||
token: 7,
|
||||
expires: 1_234,
|
||||
node_id: 3,
|
||||
};
|
||||
assert_eq!(Lease::from_bytes(&lease.to_bytes()), Some(lease));
|
||||
// A value that can't be read counts as no lease
|
||||
assert_eq!(Lease::from_bytes(b"short"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_holder_stops_before_its_lease_runs_out() {
|
||||
let leases = SingletonLeases::default();
|
||||
assert!(!leases.holds(SingletonRole::TaskScheduler));
|
||||
leases.set(SingletonRole::TaskScheduler, now() + LEASE_SECONDS);
|
||||
assert!(leases.holds(SingletonRole::TaskScheduler));
|
||||
assert!(!leases.holds(SingletonRole::Metrics));
|
||||
// Inside the margin it no longer acts, though the lease hasn't run out
|
||||
leases.set(SingletonRole::TaskScheduler, now() + MARGIN);
|
||||
assert!(!leases.holds(SingletonRole::TaskScheduler));
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,11 @@ use registry::{
|
||||
types::{id::ObjectId, list::List},
|
||||
};
|
||||
use std::sync::Mutex;
|
||||
use store::registry::{RegistryQuery, bootstrap::Bootstrap};
|
||||
use store::{
|
||||
Deserialize, SUBSPACE_INBUXA, Store, ValueKey,
|
||||
registry::{RegistryQuery, bootstrap::Bootstrap},
|
||||
write::{AnyClass, BatchBuilder, ValueClass},
|
||||
};
|
||||
use trc::{Collector, MetricType, TelemetryEvent};
|
||||
use types::id::Id;
|
||||
|
||||
@@ -37,10 +41,71 @@ pub struct AlertMessage {
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
/// The alerts whose condition held at the last evaluation (MON-26). In
|
||||
/// memory, so a restart while a condition holds fires once more.
|
||||
/// The alerts whose condition held at the last evaluation (MON-26). Kept
|
||||
/// in the store as well (#437), so the node that takes over alerts from
|
||||
/// another carries on from it instead of firing them all again.
|
||||
static FIRING: Mutex<Option<AHashSet<u64>>> = Mutex::new(None);
|
||||
|
||||
/// Under `X` (cluster-wide singletons) in `_`, beside the singleton role
|
||||
/// leases: the ids
|
||||
/// of the alerts firing at the last evaluation.
|
||||
pub const FIRING_KEY: &[u8] = b"Xa";
|
||||
|
||||
fn firing_class() -> ValueClass {
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: FIRING_KEY.to_vec(),
|
||||
})
|
||||
}
|
||||
|
||||
struct FiringIds(AHashSet<u64>);
|
||||
|
||||
impl Deserialize for FiringIds {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
Ok(FiringIds(
|
||||
bytes
|
||||
.as_chunks::<8>()
|
||||
.0
|
||||
.iter()
|
||||
.map(|id| u64::from_be_bytes(*id))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn firing_bytes(ids: &AHashSet<u64>) -> Vec<u8> {
|
||||
let mut ids = ids.iter().copied().collect::<Vec<_>>();
|
||||
ids.sort_unstable();
|
||||
ids.into_iter().flat_map(u64::to_be_bytes).collect()
|
||||
}
|
||||
|
||||
/// Takes up the alerts the store says are firing, as the node that takes
|
||||
/// over evaluating them (#437). With none stored, the next evaluation is
|
||||
/// a first one (MON-26).
|
||||
pub async fn load_firing(store: &Store) {
|
||||
match store
|
||||
.get_value::<FiringIds>(ValueKey::from(firing_class()))
|
||||
.await
|
||||
{
|
||||
Ok(stored) => *FIRING.lock().unwrap() = stored.map(|FiringIds(ids)| ids),
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to read the firing alerts"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores the firing alerts when they changed since the last evaluation.
|
||||
async fn store_firing(store: &Store, previous: Option<&AHashSet<u64>>, holding: &AHashSet<u64>) {
|
||||
if previous == Some(holding) {
|
||||
return;
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(firing_class(), firing_bytes(holding));
|
||||
if let Err(err) = store.write(batch.build_all()).await {
|
||||
trc::error!(err.details("Failed to store the firing alerts"));
|
||||
}
|
||||
}
|
||||
|
||||
fn is_ident_char(c: char) -> bool {
|
||||
c.is_ascii_alphanumeric() || c == '_'
|
||||
}
|
||||
@@ -237,7 +302,8 @@ impl Server {
|
||||
messages.push(build_email(email));
|
||||
}
|
||||
}
|
||||
*FIRING.lock().unwrap() = Some(holding);
|
||||
let previous = FIRING.lock().unwrap().replace(holding.clone());
|
||||
store_firing(self.store(), previous.as_ref(), &holding).await;
|
||||
Ok(messages)
|
||||
}
|
||||
}
|
||||
@@ -246,6 +312,20 @@ impl Server {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn firing_alerts_read_back() {
|
||||
let ids = AHashSet::from_iter([3u64, 1 << 40, 7]);
|
||||
let FiringIds(read) = FiringIds::deserialize(&firing_bytes(&ids)).unwrap();
|
||||
assert_eq!(read, ids);
|
||||
// Stored in order, so an unchanged set writes the same bytes
|
||||
assert_eq!(
|
||||
firing_bytes(&ids),
|
||||
firing_bytes(&AHashSet::from_iter([7u64, 3, 1 << 40]))
|
||||
);
|
||||
let FiringIds(none) = FiringIds::deserialize(&[]).unwrap();
|
||||
assert!(none.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_underscore_names() {
|
||||
assert_eq!(rewrite("domain_count > 1"), "metric('domain.count') > 1");
|
||||
|
||||
@@ -197,7 +197,9 @@ impl Server {
|
||||
if store.is_none() {
|
||||
return;
|
||||
}
|
||||
let samples = sample(self.core.network.roles.metrics_calculate);
|
||||
// inbuxa: #437: the cluster-wide gauges from the node that sets them
|
||||
let samples =
|
||||
sample(self.runs_singleton(crate::manager::singleton_lease::SingletonRole::Metrics));
|
||||
let count = samples.len();
|
||||
let started = std::time::Instant::now();
|
||||
match store.write_metrics(samples, now()).await {
|
||||
|
||||
@@ -162,6 +162,9 @@ async fn main() -> std::io::Result<()> {
|
||||
// inbuxa: hand back the task locks this node holds, so other nodes can
|
||||
// run those tasks now rather than when the locks expire
|
||||
services::task_manager::lock::release_task_locks(&inner.build_server()).await;
|
||||
// inbuxa: #437: and its singleton role leases, so another node (or this
|
||||
// one, restarted) takes those roles over at once
|
||||
common::manager::singleton_lease::release_singleton_leases(&inner.build_server()).await;
|
||||
|
||||
// Shutdown collector
|
||||
Collector::shutdown();
|
||||
|
||||
@@ -113,6 +113,7 @@ pub const OWNERS: &[(u8, Owner)] = &[
|
||||
b'W',
|
||||
Owner::Stays("blob store changes and the sharing policy"),
|
||||
),
|
||||
(b'X', Owner::Stays("cluster singleton role leases")),
|
||||
];
|
||||
|
||||
/// Step 2 -> 3. Returns how many rows moved.
|
||||
|
||||
@@ -110,6 +110,8 @@ impl SpawnServices for IpcReceivers {
|
||||
inbuxa_scheduled_reports::spawn_scheduled_reports(inner.clone());
|
||||
|
||||
// Spawn task scheduler
|
||||
// inbuxa: #437, before the scheduler, which asks it who acts
|
||||
common::manager::singleton_lease::spawn_singleton_leases(inner.clone());
|
||||
spawn_task_scheduler(inner);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,8 +14,9 @@ use std::{
|
||||
};
|
||||
|
||||
use common::{
|
||||
BuildServer, Inner, LONG_1D_SLUMBER,
|
||||
BuildServer, Inner, LONG_1D_SLUMBER, Server,
|
||||
config::{mailstore::spamfilter, telemetry::OtelMetrics},
|
||||
manager::singleton_lease::SingletonRole,
|
||||
};
|
||||
use registry::{
|
||||
schema::{
|
||||
@@ -70,6 +71,37 @@ async fn metrics_collection_delay(server: &common::Server) -> Duration {
|
||||
.time_to_next()
|
||||
}
|
||||
|
||||
/// The cluster-wide gauges, on the node that does the metrics work: the
|
||||
/// queue every time, accounts and domains when `others` (once a day).
|
||||
async fn calculate_cluster_metrics(server: &Server, others: bool) {
|
||||
// inbuxa: MON-7: the queue gauge from the queue itself, so it's right
|
||||
// after a restart
|
||||
match server.total_queued_messages().await {
|
||||
Ok((total, oldest)) => {
|
||||
Collector::update_gauge(MetricType::QueueCount, total);
|
||||
Collector::update_gauge(MetricType::QueueOldestMessage, oldest.unwrap_or_default());
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to count queued messages"));
|
||||
}
|
||||
}
|
||||
|
||||
if others {
|
||||
match server.total_accounts().await {
|
||||
Ok(total) => Collector::update_gauge(MetricType::UserCount, total as u64),
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to obtain account count"));
|
||||
}
|
||||
}
|
||||
match server.total_domains().await {
|
||||
Ok(total) => Collector::update_gauge(MetricType::DomainCount, total as u64),
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to obtain domain count"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Queue {
|
||||
heap: BinaryHeap<Action>,
|
||||
@@ -174,13 +206,27 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
}
|
||||
|
||||
let mut next_metric_update = Instant::now();
|
||||
// inbuxa: #437: whether this node calculated the cluster-wide gauges
|
||||
// since it last took on the metrics role, so a node that just took
|
||||
// over doesn't evaluate alerts against gauges it never set
|
||||
let mut calculated = false;
|
||||
|
||||
loop {
|
||||
tokio::time::sleep(queue.wake_up_time()).await;
|
||||
|
||||
let server = inner.build_server();
|
||||
let roles = &server.core.network.roles;
|
||||
let mut batch = (roles.task_scheduler).then(BatchBuilder::new);
|
||||
// inbuxa: #437: in a cluster, only the node holding the role's
|
||||
// lease queues the scheduled tasks and does the metrics work
|
||||
let mut batch = server
|
||||
.runs_singleton(SingletonRole::TaskScheduler)
|
||||
.then(BatchBuilder::new);
|
||||
let calculates = server.runs_singleton(SingletonRole::Metrics);
|
||||
if !calculates {
|
||||
calculated = false;
|
||||
// Counted again at once if this node takes the role on
|
||||
next_metric_update = Instant::now();
|
||||
}
|
||||
|
||||
while let Some(event) = queue.pop() {
|
||||
match event.event {
|
||||
@@ -299,61 +345,18 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
Event::CalculateMetrics,
|
||||
);
|
||||
|
||||
let update_other_metrics = if Instant::now() >= next_metric_update {
|
||||
let update_other_metrics =
|
||||
calculates && Instant::now() >= next_metric_update;
|
||||
if update_other_metrics {
|
||||
next_metric_update = Instant::now() + Duration::from_secs(86400);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
}
|
||||
calculated |= calculates;
|
||||
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
let elapsed = Instant::now();
|
||||
if server.core.network.roles.metrics_calculate {
|
||||
// inbuxa: MON-7: the queue gauge from the queue itself,
|
||||
// so it's right after a restart
|
||||
match server.total_queued_messages().await {
|
||||
Ok((total, oldest)) => {
|
||||
Collector::update_gauge(MetricType::QueueCount, total);
|
||||
Collector::update_gauge(
|
||||
MetricType::QueueOldestMessage,
|
||||
oldest.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to count queued messages"));
|
||||
}
|
||||
}
|
||||
|
||||
if update_other_metrics {
|
||||
match server.total_accounts().await {
|
||||
Ok(total) => {
|
||||
Collector::update_gauge(
|
||||
MetricType::UserCount,
|
||||
total as u64,
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.details("Failed to obtain account count")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
match server.total_domains().await {
|
||||
Ok(total) => {
|
||||
Collector::update_gauge(
|
||||
MetricType::DomainCount,
|
||||
total as u64,
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(
|
||||
err.details("Failed to obtain domain count")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if calculates {
|
||||
calculate_cluster_metrics(&server, update_other_metrics).await;
|
||||
}
|
||||
|
||||
match tokio::task::spawn_blocking(memory_stats::memory_stats).await {
|
||||
@@ -387,9 +390,19 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
Instant::now() + Duration::from_secs(60),
|
||||
Event::EvaluateAlerts,
|
||||
);
|
||||
if server.core.network.roles.metrics_calculate {
|
||||
if calculates {
|
||||
// #437: a node that just took over sets the gauges
|
||||
// first, so no alert reads one it never set
|
||||
let first = !calculated;
|
||||
if first {
|
||||
calculated = true;
|
||||
next_metric_update = Instant::now() + Duration::from_secs(86400);
|
||||
}
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
if first {
|
||||
calculate_cluster_metrics(&server, true).await;
|
||||
}
|
||||
match server.process_alerts().await {
|
||||
Ok(messages) => {
|
||||
use smtp::reporting::send::MtaReportSend;
|
||||
@@ -434,7 +447,7 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
Instant::now() + DOMAIN_TASK_CHECK_INTERVAL,
|
||||
Event::CheckDomainTasks,
|
||||
);
|
||||
if roles.task_scheduler {
|
||||
if batch.is_some() {
|
||||
let server = server.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) =
|
||||
|
||||
@@ -16,12 +16,13 @@
|
||||
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
|
||||
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
|
||||
// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228);
|
||||
// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371
|
||||
// 664 is auth.passkey-counter (passkeys spec, PK-18); 665 and 666 are
|
||||
// cluster.singleton-lease-taken and -lost (#437). Metrics 369 to 371
|
||||
// are gauges for Kubernetes alerts (K-C10): queue.oldest-message,
|
||||
// cluster.lease-renewed, tls.earliest-expiry.
|
||||
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
|
||||
// more than the highest id; the test below holds it to the variant list
|
||||
pub const TOTAL_EVENT_COUNT: usize = 665;
|
||||
pub const TOTAL_EVENT_COUNT: usize = 667;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 372;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -170,6 +171,9 @@ pub enum ClusterEvent {
|
||||
CoordinatorConnected = 644,
|
||||
CoordinatorDisconnected = 645,
|
||||
CoordinatorError = 646,
|
||||
// inbuxa: #437, the singleton role leases
|
||||
SingletonLeaseTaken = 665,
|
||||
SingletonLeaseLost = 666,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
|
||||
@@ -95,6 +95,8 @@ impl EventType {
|
||||
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
|
||||
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
||||
b"cluster.node-id-reassigned" => EventType::Cluster(ClusterEvent::NodeIdReassigned),
|
||||
b"cluster.singleton-lease-taken" => EventType::Cluster(ClusterEvent::SingletonLeaseTaken),
|
||||
b"cluster.singleton-lease-lost" => EventType::Cluster(ClusterEvent::SingletonLeaseLost),
|
||||
// inbuxa: coordinator connection
|
||||
b"cluster.coordinator-connected" => EventType::Cluster(ClusterEvent::CoordinatorConnected),
|
||||
b"cluster.coordinator-disconnected" => EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
|
||||
@@ -784,6 +786,10 @@ impl EventType {
|
||||
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
|
||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
|
||||
EventType::Cluster(ClusterEvent::NodeIdReassigned) => "cluster.node-id-reassigned",
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => {
|
||||
"cluster.singleton-lease-taken"
|
||||
}
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => "cluster.singleton-lease-lost",
|
||||
// inbuxa: coordinator connection
|
||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => {
|
||||
"cluster.coordinator-connected"
|
||||
@@ -1598,6 +1604,8 @@ impl EventType {
|
||||
EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
|
||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
|
||||
EventType::Cluster(ClusterEvent::NodeIdReassigned) => 649,
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => 665,
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => 666,
|
||||
// inbuxa: coordinator connection
|
||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => 644,
|
||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => 645,
|
||||
@@ -2278,6 +2286,8 @@ impl EventType {
|
||||
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
|
||||
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
|
||||
649 => Some(EventType::Cluster(ClusterEvent::NodeIdReassigned)),
|
||||
665 => Some(EventType::Cluster(ClusterEvent::SingletonLeaseTaken)),
|
||||
666 => Some(EventType::Cluster(ClusterEvent::SingletonLeaseLost)),
|
||||
// inbuxa: coordinator connection
|
||||
644 => Some(EventType::Cluster(ClusterEvent::CoordinatorConnected)),
|
||||
645 => Some(EventType::Cluster(ClusterEvent::CoordinatorDisconnected)),
|
||||
@@ -3256,6 +3266,8 @@ impl EventType {
|
||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => Level::Info,
|
||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => Level::Warn,
|
||||
EventType::Cluster(ClusterEvent::CoordinatorError) => Level::Warn,
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => Level::Info,
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => Level::Info,
|
||||
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
|
||||
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
|
||||
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
|
||||
@@ -3407,6 +3419,8 @@ impl EventType {
|
||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => "Coordinator connected",
|
||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => "Coordinator unavailable",
|
||||
EventType::Cluster(ClusterEvent::CoordinatorError) => "Coordinator error",
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseTaken) => "Singleton role lease taken",
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseLost) => "Singleton role lease lost",
|
||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
|
||||
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
|
||||
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
|
||||
@@ -4529,6 +4543,8 @@ impl EventType {
|
||||
EventType::Cluster(ClusterEvent::CoordinatorConnected),
|
||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
|
||||
EventType::Cluster(ClusterEvent::CoordinatorError),
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseTaken),
|
||||
EventType::Cluster(ClusterEvent::SingletonLeaseLost),
|
||||
EventType::Dane(DaneEvent::AuthenticationSuccess),
|
||||
EventType::Dane(DaneEvent::AuthenticationFailure),
|
||||
EventType::Dane(DaneEvent::NoCertificatesFound),
|
||||
|
||||
@@ -336,9 +336,11 @@ unchanged.
|
||||
### Alerts
|
||||
|
||||
- **MON-25.** **Evaluation.** Each enabled alert's condition is evaluated
|
||||
every 60 seconds (**Decision**) on every node that holds the
|
||||
`metricsCalculate` task role, against that node's current values. A
|
||||
one-node install always holds it. The condition language is the server's
|
||||
every 60 seconds (**Decision**) on the node that does the
|
||||
`metricsCalculate` task role's work, against that node's current values. A
|
||||
one-node install always holds it. In a cluster several nodes can have the
|
||||
role; the one holding its lease in the store does the work (#437, see
|
||||
"Singleton roles" below). The condition language is the server's
|
||||
expression language, and a metric is read two ways, both accepted:
|
||||
- `metric('queue.count')`, which the AGPL expression code already supports;
|
||||
- the name with dots and hyphens as underscores (`queue_count`), as
|
||||
@@ -350,9 +352,22 @@ unchanged.
|
||||
true, including the first evaluation after start. While it stays true it
|
||||
doesn't fire again. Once false, it can fire again. **Decision**: upstream's
|
||||
docs say email goes out "each time the condition becomes true". State is in
|
||||
memory, so a restart while the condition holds fires once more. A condition
|
||||
memory, and in a cluster in the store too (`Xa` in `_`), so a node that
|
||||
takes the role over from another carries on from it; a node without a
|
||||
coordinator that restarts while the condition holds fires once more. A condition
|
||||
that fails to evaluate (a metric name that doesn't exist, a type error) is
|
||||
rejected when the alert is saved, and never fires.
|
||||
- **Singleton roles (#437).** `taskScheduler` (queuing the scheduled
|
||||
maintenance tasks, and the daily check of each domain's DKIM and ACME
|
||||
tasks) and `metricsCalculate` (the cluster-wide gauges, alert evaluation
|
||||
and their emails) each have a lease in the store (`Xs`, `Xm` in `_`).
|
||||
Every node with the role tries for it each minute; the holder renews it
|
||||
and does the work. A lease lasts 3 minutes, and its holder stops acting
|
||||
15 seconds before it runs out, so when the holder stops another node with
|
||||
the role takes over within about 4 minutes. A node that takes over sets
|
||||
the gauges before its first alert evaluation. A node without a cluster
|
||||
coordinator needs no lease: the role alone decides. Taking and losing a
|
||||
lease log `cluster.singleton-lease-taken` and `cluster.singleton-lease-lost`.
|
||||
- **MON-27.** **Placeholders.** In `eventMessage`, `subject` and `body`,
|
||||
`%{metric.name}%` (the dotted name) is replaced by the value used in that
|
||||
evaluation. Whole numbers print without decimals ("3", not "3.0"); others
|
||||
|
||||
@@ -156,6 +156,8 @@ can't place, before it writes anything. The table is `OWNERS` in
|
||||
| `K` | account locks | stays |
|
||||
| `L` | audit log (`e`, `t`, `o`, `h`, `f`) | moves to `ib_audit` (0x80); its settings (`s`) stay |
|
||||
| `M` | masked email | stays |
|
||||
| `N` | passkeys | stays |
|
||||
| `O` | listener status | stays |
|
||||
| `P` | permissions and protocol policies | stays |
|
||||
| `Q` | acceptance records | stays |
|
||||
| `R` | mail flow rules | stays |
|
||||
@@ -164,6 +166,7 @@ can't place, before it writes anything. The table is `OWNERS` in
|
||||
| `U` | undelete notes (`n`, `x`, `b`, `r`, `c`, `k`, `a`, `g`) | moves to `ib_undelete` (0x83) |
|
||||
| `V` | store checks, the layout mark (`Vs`), lease (`Vm`) and step progress (`Vt`) | stays |
|
||||
| `W` | blob store changes, the sharing policy | stays |
|
||||
| `X` | singleton role leases (`Xs`, `Xm`) and the alerts firing (`Xa`), #437 | stays |
|
||||
|
||||
A key moves without its first byte, so `Le` + node + seq becomes `e` + node
|
||||
+ seq in `ib_audit`. A feature's settings stay in `_`, where a configuration
|
||||
|
||||
Binary file not shown.
@@ -1 +1 @@
|
||||
3CkGF0FiqDTQxYPZ0VjCxnlGmiuzyjDDbmIv1BVmDmg
|
||||
gBQXaaMdj6J-6RZEk6DUlOzZD58mbuzZ2Uo8M2QksY8
|
||||
@@ -24,6 +24,7 @@ pub mod replica_mysql; // inbuxa: read replicas on MySQL
|
||||
pub mod scaleout; // inbuxa: scale-out storage
|
||||
#[cfg(feature = "postgres")]
|
||||
pub mod search_gin; // inbuxa: GIN indexes without a pending list
|
||||
pub mod singleton_lease; // inbuxa: #437, one node at a time does the singleton roles' work
|
||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
||||
pub mod sql_timeout;
|
||||
pub mod task_locks; // inbuxa: task locks across nodes
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! #437: a singleton role's lease has one holder at a time, renews for its
|
||||
//! holder, and passes to another node once it's given up or runs out.
|
||||
//! Nodes are played by tokens sharing one store.
|
||||
|
||||
use common::manager::singleton_lease::{
|
||||
LEASE_SECONDS, Lease, SingletonRole, holder, release, try_hold,
|
||||
};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use store::{
|
||||
SUBSPACE_INBUXA, Store,
|
||||
backend::sqlite::SqliteStore,
|
||||
write::{AnyClass, BatchBuilder, ValueClass, now},
|
||||
};
|
||||
|
||||
const SCHEDULER: SingletonRole = SingletonRole::TaskScheduler;
|
||||
|
||||
fn store() -> Store {
|
||||
Store::SQLite(SqliteStore::open_memory().unwrap().into())
|
||||
}
|
||||
|
||||
/// A lease node `node_id` (token `token`) left, running out at `expires`.
|
||||
async fn write_lease(store: &Store, token: u64, node_id: u16, expires: u64) {
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: SCHEDULER.key().to_vec(),
|
||||
}),
|
||||
Lease {
|
||||
token,
|
||||
expires,
|
||||
node_id,
|
||||
}
|
||||
.to_bytes(),
|
||||
);
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn one_holder_among_many() {
|
||||
let store = store();
|
||||
let won = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// Eight nodes try at once: exactly one gets it
|
||||
let nodes = (1..=8u64)
|
||||
.map(|token| {
|
||||
let (store, won) = (store.clone(), won.clone());
|
||||
tokio::spawn(async move {
|
||||
if try_hold(&store, SCHEDULER, token, token as u16)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
{
|
||||
won.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for node in nodes {
|
||||
node.await.unwrap();
|
||||
}
|
||||
assert_eq!(won.load(Ordering::SeqCst), 1, "not exactly one holder");
|
||||
|
||||
// The holder renews it; the others still can't have it
|
||||
let lease = holder(&store, SCHEDULER).await.unwrap().unwrap();
|
||||
let renewed = try_hold(&store, SCHEDULER, lease.token, lease.node_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("the holder couldn't renew its lease");
|
||||
assert!(renewed >= lease.expires);
|
||||
for token in (1..=8u64).filter(|token| *token != lease.token) {
|
||||
assert_eq!(
|
||||
try_hold(&store, SCHEDULER, token, token as u16)
|
||||
.await
|
||||
.unwrap(),
|
||||
None,
|
||||
"node {token} took a lease another node holds"
|
||||
);
|
||||
}
|
||||
|
||||
// The roles have leases of their own
|
||||
assert!(
|
||||
try_hold(&store, SingletonRole::Metrics, 99, 99)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn a_lease_passes_on_when_it_runs_out_or_is_given_up() {
|
||||
let store = store();
|
||||
|
||||
// Node 1 holds it: node 2 waits
|
||||
write_lease(&store, 1, 1, now() + LEASE_SECONDS).await;
|
||||
assert_eq!(try_hold(&store, SCHEDULER, 2, 2).await.unwrap(), None);
|
||||
|
||||
// Node 1 stopped and its lease ran out: node 2 takes it over
|
||||
write_lease(&store, 1, 1, now() - 1).await;
|
||||
assert!(holder(&store, SCHEDULER).await.unwrap().is_none());
|
||||
let expires = try_hold(&store, SCHEDULER, 2, 2)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("node 2 didn't take over a lease that ran out");
|
||||
assert!(expires >= now() + LEASE_SECONDS - 1);
|
||||
assert_eq!(holder(&store, SCHEDULER).await.unwrap().unwrap().node_id, 2);
|
||||
|
||||
// Node 1 comes back: it's node 2's now
|
||||
assert_eq!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap(), None);
|
||||
|
||||
// Giving up someone else's lease does nothing
|
||||
release(&store, SCHEDULER, 1).await.unwrap();
|
||||
assert_eq!(holder(&store, SCHEDULER).await.unwrap().unwrap().token, 2);
|
||||
|
||||
// Node 2 gives it up (the role was taken off it): node 1 takes it at once
|
||||
release(&store, SCHEDULER, 2).await.unwrap();
|
||||
assert!(holder(&store, SCHEDULER).await.unwrap().is_none());
|
||||
assert!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn a_lease_that_cant_be_read_counts_as_none() {
|
||||
let store = store();
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch.set(
|
||||
ValueClass::Any(AnyClass {
|
||||
subspace: SUBSPACE_INBUXA,
|
||||
key: SCHEDULER.key().to_vec(),
|
||||
}),
|
||||
b"short".to_vec(),
|
||||
);
|
||||
store.write(batch.build_all()).await.unwrap();
|
||||
assert!(holder(&store, SCHEDULER).await.unwrap().is_none());
|
||||
assert!(try_hold(&store, SCHEDULER, 1, 1).await.unwrap().is_some());
|
||||
}
|
||||
Reference in new issue
Block a user