From 9cffec3a6bdc6f114c297c6ede4eac35ea225179 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 08:32:12 -0700 Subject: [PATCH 1/6] Metrics for Kubernetes: scraping and OTLP from the environment, three gauges A pod's metrics settings can come from its deployment instead of the store, so a Helm chart can turn them on for every pod: - INBUXA_METRICS_PROMETHEUS_USERNAME and INBUXA_METRICS_PROMETHEUS_SECRET_FILE open /metrics/prometheus with that username and the password in the file. Both or neither; the endpoint is never opened without a password this way. - When the store has no OpenTelemetry metrics set, OTEL_EXPORTER_OTLP_ENDPOINT (or _METRICS_ENDPOINT) turns the exporter on from the standard OTEL_ variables: protocol, headers, timeout, OTEL_METRIC_EXPORT_INTERVAL and OTEL_RESOURCE_ATTRIBUTES. Three gauges for alerts, each a Unix time: - queue.oldest-message: when the oldest queued message arrived, read from the first queue id (a snowflake) when the queue is counted, without reading any message. 0 when the queue is empty. - cluster.lease-renewed: when this node last renewed its cluster lease. A pod that runs on but can't reach the store stops moving it. (A count of Stale nodes wouldn't do on Kubernetes: a pod that rolls away never gives its lease back, so it reads Stale for a day after every rollout.) - tls.earliest-expiry: when the first name served runs out of certificate. --- crates/common/src/config/server/tls.rs | 12 ++ crates/common/src/config/telemetry.rs | 127 +++++++++++++++++- crates/common/src/network/mta.rs | 20 ++- crates/services/src/task_manager/scheduler.rs | 6 +- crates/store/src/build/cluster.rs | 9 ++ crates/trc/src/event/enums.rs | 9 +- crates/trc/src/event/enums_impl.rs | 27 ++++ crates/trc/src/ipc/metrics.rs | 21 ++- 8 files changed, 220 insertions(+), 11 deletions(-) diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index 58a26046..6b04bb21 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -288,6 +288,18 @@ pub(crate) async fn parse_certificates( } } + // inbuxa: K-C10: when the first name served runs out, for an alert that + // warns before a renewal that isn't happening matters + trc::Collector::update_gauge( + trc::MetricType::TlsEarliestExpiry, + certs_expirations + .iter() + .filter(|(name, _)| name.as_str() != "*") + .map(|(_, expires)| (*expires).max(0) as u64) + .min() + .unwrap_or_default(), + ); + // No default certificate set: pick one now, once per load, so handshakes // without SNI or for an unknown name get a predictable certificate. if !certificates.contains_key("*") && certificates.len() > 1 { diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 5fd8351c..84b210a4 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -613,8 +613,19 @@ impl Metrics { .with_version(types::brand_version!()) .build(); + // inbuxa: K-C10: scraping and OTLP set by the deployment (Kubernetes), + // over what the store says. A pod reads the scrape password from a + // mounted file, and the OTLP exporter from the standard OTEL_ variables. + let env_prometheus = prometheus_from_env(bp); + let env_otel = if matches!(metrics.open_telemetry, structs::MetricsOtel::Disabled) { + otel_from_env(bp, &resource, &instrumentation) + } else { + None + }; + Metrics { prometheus: match metrics.prometheus { + _ if env_prometheus.is_some() => env_prometheus, MetricsPrometheus::Enabled(prom) => { let secret = prom .auth_secret @@ -705,7 +716,7 @@ impl Metrics { } } } - structs::MetricsOtel::Disabled => None, + structs::MetricsOtel::Disabled => env_otel, }, log_path: bp .list_infallible::() @@ -833,6 +844,120 @@ fn apply_metrics( } } +/// Prometheus scraping from INBUXA_METRICS_PROMETHEUS_USERNAME and +/// INBUXA_METRICS_PROMETHEUS_SECRET_FILE. Both or neither: the endpoint is +/// never opened without a password this way. +fn prometheus_from_env(bp: &mut Bootstrap) -> Option { + let username = types::branding::env_var("METRICS_PROMETHEUS_USERNAME").ok(); + let secret_file = types::branding::env_var("METRICS_PROMETHEUS_SECRET_FILE").ok(); + let (username, secret_file) = match (username, secret_file) { + (None, None) => return None, + (Some(username), Some(secret_file)) if !username.is_empty() && !secret_file.is_empty() => { + (username, secret_file) + } + _ => { + bp.build_error( + ObjectType::Metrics.singleton(), + "Set both INBUXA_METRICS_PROMETHEUS_USERNAME and \ + INBUXA_METRICS_PROMETHEUS_SECRET_FILE, or neither. The setting in the store \ + applies.", + ); + return None; + } + }; + match std::fs::read_to_string(&secret_file) { + Ok(secret) if !secret.trim_end_matches(['\r', '\n']).is_empty() => { + let secret = secret.trim_end_matches(['\r', '\n']); + Some(PrometheusMetrics { + auth: Some(STANDARD.encode(format!("{username}:{secret}"))), + }) + } + Ok(_) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!("{secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE) is empty"), + ); + None + } + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!( + "Unable to read {secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE): {err}" + ), + ); + None + } + } +} + +/// OTLP metrics from the standard variables, when the store has none set: +/// OTEL_EXPORTER_OTLP_METRICS_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT turns +/// it on; the exporter reads those, the _HEADERS and _TIMEOUT variables +/// itself. OTEL_EXPORTER_OTLP_(METRICS_)PROTOCOL picks grpc or http/protobuf +/// (the default), and OTEL_METRIC_EXPORT_INTERVAL the interval in +/// milliseconds (60000). OTEL_RESOURCE_ATTRIBUTES is added to the resource. +fn otel_from_env( + bp: &mut Bootstrap, + resource: &Resource, + instrumentation: &InstrumentationScope, +) -> Option> { + let var = |name: &str| std::env::var(name).ok().filter(|value| !value.is_empty()); + var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT").or_else(|| var("OTEL_EXPORTER_OTLP_ENDPOINT"))?; + let protocol = var("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL") + .or_else(|| var("OTEL_EXPORTER_OTLP_PROTOCOL")) + .unwrap_or_else(|| "http/protobuf".to_string()); + let interval = match var("OTEL_METRIC_EXPORT_INTERVAL").map(|value| value.parse::()) { + None => Duration::from_secs(60), + Some(Ok(millis)) if millis >= 1000 => Duration::from_millis(millis), + Some(_) => { + bp.build_warning( + ObjectType::Metrics.singleton(), + "OTEL_METRIC_EXPORT_INTERVAL is not a number of milliseconds of at least 1000; \ + exporting every 60 seconds.", + ); + Duration::from_secs(60) + } + }; + let exporter = match protocol.as_str() { + "grpc" => MetricExporter::builder() + .with_temporality(Temporality::Delta) + .with_tonic() + .build(), + "http/protobuf" => MetricExporter::builder() + .with_temporality(Temporality::Delta) + .with_http() + .build(), + _ => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!( + "OTEL_EXPORTER_OTLP_PROTOCOL is {protocol:?}; expected grpc or http/protobuf. \ + No OTLP metrics." + ), + ); + return None; + } + }; + match exporter { + Ok(exporter) => Some(Arc::new(OtelMetrics { + exporter, + interval, + resource: resource.clone(), + instrumentation: instrumentation.clone(), + })), + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!( + "Failed to build the OpenTelemetry metrics exporter from OTEL_ variables: {err}" + ), + ); + None + } + } +} + impl std::fmt::Debug for OtelMetrics { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OtelMetrics") diff --git a/crates/common/src/network/mta.rs b/crates/common/src/network/mta.rs index d0fa9630..f9d2cb71 100644 --- a/crates/common/src/network/mta.rs +++ b/crates/common/src/network/mta.rs @@ -35,10 +35,10 @@ use std::{ }; use store::{ Deserialize, IterateParams, ValueKey, - write::{AlignedBytes, Archive, QueueClass, ValueClass}, + write::{AlignedBytes, Archive, QueueClass, ValueClass, key::DeserializeBigEndian}, }; use trc::{AddContext, SpamEvent}; -use utils::DomainPart; +use utils::{DomainPart, snowflake::SnowflakeIdGenerator}; impl Server { pub async fn rcpt_resolve( @@ -505,16 +505,26 @@ impl Server { Ok(()) } - pub async fn total_queued_messages(&self) -> trc::Result { + // inbuxa: K-C10: also when the oldest message arrived. Queue ids are + // snowflakes, so the first key is the oldest message and its id holds + // the time, without reading any message. + pub async fn total_queued_messages(&self) -> trc::Result<(u64, Option)> { let mut total = 0; + let mut oldest = None; self.store() .iterate( IterateParams::new( ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), ) + .ascending() .no_values(), - |_, _| { + |key, _| { + if total == 0 { + oldest = Some(SnowflakeIdGenerator::to_timestamp( + key.deserialize_be_u64(0)?, + )); + } total += 1; Ok(true) @@ -522,7 +532,7 @@ impl Server { ) .await .caused_by(trc::location!()) - .map(|_| total) + .map(|_| (total, oldest)) } } diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index 16a025b0..36d0477b 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -308,8 +308,12 @@ pub fn spawn_task_scheduler(inner: Arc) { // inbuxa: MON-7: the queue gauge from the queue itself, // so it's right after a restart match server.total_queued_messages().await { - Ok(total) => { + Ok((total, oldest)) => { Collector::update_gauge(MetricType::QueueCount, total); + Collector::update_gauge( + MetricType::QueueOldestMessage, + oldest.unwrap_or_default(), + ); } Err(err) => { trc::error!(err.details("Failed to count queued messages")); diff --git a/crates/store/src/build/cluster.rs b/crates/store/src/build/cluster.rs index cd5064d2..d531d2cc 100644 --- a/crates/store/src/build/cluster.rs +++ b/crates/store/src/build/cluster.rs @@ -58,6 +58,7 @@ struct NodeClaim { impl RegistryStoreInner { pub(super) async fn acquire_node_id(&mut self) -> Result<(), String> { let (node_id, slots) = NodeSlot::acquire(&self.store, &self.env_hostname).await?; + lease_renewed(); self.node_id.store(node_id, Ordering::Relaxed); self.remember_hostname().await; @@ -97,6 +98,12 @@ impl RegistryStoreInner { } } +// inbuxa: K-C10: the gauge an alert watches. A pod that keeps running but +// can't renew its lease (the store unreachable, say) stops moving it. +fn lease_renewed() { + trc::Collector::update_gauge(trc::MetricType::ClusterLeaseRenewed, now()); +} + // INBUXA: #458, the file a node records its leased hostname in. // It sits beside the local config file, or inside the directory when the // path is one (as in the test harness). @@ -146,6 +153,7 @@ impl RegistryStore { })?; self.0.node_id.store(new_node_id, Ordering::Relaxed); SnowflakeIdGenerator::set_node_id(new_node_id as u64); + lease_renewed(); trc::event!( Cluster(ClusterEvent::NodeIdReassigned), @@ -173,6 +181,7 @@ impl RegistryStore { .write(batch.build_all()) .await .caused_by(trc::location!())?; + lease_renewed(); self.release_renamed_lease(&slots).await } diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index b12b7745..e678d72f 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -16,11 +16,13 @@ // 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is // store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is // spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228); -// 664 is auth.passkey-counter (passkeys spec, PK-18). +// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371 +// are gauges for Kubernetes alerts (K-C10): queue.oldest-message, +// cluster.lease-renewed, tls.earliest-expiry. // TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one // more than the highest id; the test below holds it to the variant list pub const TOTAL_EVENT_COUNT: usize = 665; -pub const TOTAL_METRIC_COUNT: usize = 369; +pub const TOTAL_METRIC_COUNT: usize = 372; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EventType { @@ -1080,6 +1082,9 @@ pub enum MetricType { DkimMailFromDomainMismatch = 353, DkimNextDomainMismatch = 354, DkimCustodyBreak = 368, + QueueOldestMessage = 369, + ClusterLeaseRenewed = 370, + TlsEarliestExpiry = 371, DkimPublicKeyFetch = 355, DkimPublicKeyMissing = 356, DkimPublicKeyMultiple = 357, diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index 6a6a2665..51d410c5 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -5499,6 +5499,9 @@ impl MetricType { b"telemetry.journal-error" => MetricType::TelemetryJournalError, b"tls.handshake-error" => MetricType::TlsHandshakeError, b"user.count" => MetricType::UserCount, + b"queue.oldest-message" => MetricType::QueueOldestMessage, + b"cluster.lease-renewed" => MetricType::ClusterLeaseRenewed, + b"tls.earliest-expiry" => MetricType::TlsEarliestExpiry, } .copied() } @@ -5886,6 +5889,9 @@ impl MetricType { MetricType::TelemetryJournalError => "telemetry.journal-error", MetricType::TlsHandshakeError => "tls.handshake-error", MetricType::UserCount => "user.count", + MetricType::QueueOldestMessage => "queue.oldest-message", + MetricType::ClusterLeaseRenewed => "cluster.lease-renewed", + MetricType::TlsEarliestExpiry => "tls.earliest-expiry", } } @@ -6260,6 +6266,9 @@ impl MetricType { MetricType::TelemetryJournalError => 335, MetricType::TlsHandshakeError => 336, MetricType::UserCount => 25, + MetricType::QueueOldestMessage => 369, + MetricType::ClusterLeaseRenewed => 370, + MetricType::TlsEarliestExpiry => 371, } } @@ -6634,6 +6643,9 @@ impl MetricType { 335 => Some(MetricType::TelemetryJournalError), 336 => Some(MetricType::TlsHandshakeError), 25 => Some(MetricType::UserCount), + 369 => Some(MetricType::QueueOldestMessage), + 370 => Some(MetricType::ClusterLeaseRenewed), + 371 => Some(MetricType::TlsEarliestExpiry), _ => None, } } @@ -7369,6 +7381,15 @@ impl MetricType { MetricType::TelemetryJournalError => "Journal collector error", MetricType::TlsHandshakeError => "TLS handshake error", MetricType::UserCount => "Total number of users", + MetricType::QueueOldestMessage => { + "When the oldest message in the queue arrived, as a Unix time (0: the queue is empty)" + } + MetricType::ClusterLeaseRenewed => { + "When this node last renewed its cluster lease, as a Unix time" + } + MetricType::TlsEarliestExpiry => { + "When the first certificate in use expires, as a Unix time (0: none)" + } } } @@ -7743,6 +7764,9 @@ impl MetricType { | MetricType::StoreBlobReadTime | MetricType::StoreBlobWriteTime => "milliseconds", MetricType::UserCount => "users", + MetricType::QueueOldestMessage + | MetricType::ClusterLeaseRenewed + | MetricType::TlsEarliestExpiry => "seconds", } } @@ -8117,6 +8141,9 @@ impl MetricType { MetricType::TelemetryJournalError, MetricType::TlsHandshakeError, MetricType::UserCount, + MetricType::QueueOldestMessage, + MetricType::ClusterLeaseRenewed, + MetricType::TlsEarliestExpiry, ] } } diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index bf83ef5c..ecfdcfc0 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -51,6 +51,10 @@ static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory); static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount); static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount); static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount); +// inbuxa: K-C10: gauges for alerts, set where the value is known +static QUEUE_OLDEST_MESSAGE: AtomicGauge = AtomicGauge::new(MetricType::QueueOldestMessage); +static CLUSTER_LEASE_RENEWED: AtomicGauge = AtomicGauge::new(MetricType::ClusterLeaseRenewed); +static TLS_EARLIEST_EXPIRY: AtomicGauge = AtomicGauge::new(MetricType::TlsEarliestExpiry); const CONN_SMTP_IN: usize = 0; const CONN_SMTP_OUT: usize = 1; @@ -229,8 +233,15 @@ impl Collector { // inbuxa: MON-6: one edition, every gauge pub fn collect_gauges() -> impl Iterator { - static GAUGES: &[&AtomicGauge] = - &[&SERVER_MEMORY, &QUEUE_COUNT, &USER_COUNT, &DOMAIN_COUNT]; + static GAUGES: &[&AtomicGauge] = &[ + &SERVER_MEMORY, + &QUEUE_COUNT, + &USER_COUNT, + &DOMAIN_COUNT, + &QUEUE_OLDEST_MESSAGE, + &CLUSTER_LEASE_RENEWED, + &TLS_EARLIEST_EXPIRY, + ]; GAUGES .iter() @@ -306,6 +317,9 @@ impl Collector { MetricType::SieveRequestTime => CONNECTION_METRICS[CONN_SIEVE].elapsed.average(), MetricType::UserCount => USER_COUNT.get() as f64, MetricType::DomainCount => DOMAIN_COUNT.get() as f64, + MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.get() as f64, + MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.get() as f64, + MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.get() as f64, _ => EVENT_COUNTERS.get(metric_type.event_id()) as f64, } } @@ -316,6 +330,9 @@ impl Collector { MetricType::QueueCount => QUEUE_COUNT.set(value), MetricType::UserCount => USER_COUNT.set(value), MetricType::DomainCount => DOMAIN_COUNT.set(value), + MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.set(value), + MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.set(value), + MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.set(value), _ => {} } } -- 2.54.0 From 4472616ae84e8773a65d8a77274bce857df02ea2 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 08:37:42 -0700 Subject: [PATCH 2/6] OTLP headers from a file: INBUXA_OTLP_HEADERS_FILE A collector's token belongs in a mounted file, not the environment, as for the other secrets a pod reads. key=value pairs, by commas or lines. --- crates/common/src/config/telemetry.rs | 103 +++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 84b210a4..8fdc48f4 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -12,7 +12,7 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use hyper::HeaderMap; use opentelemetry::{InstrumentationScope, KeyValue}; use opentelemetry_otlp::{ - LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, + LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig, }; use opentelemetry_sdk::{Resource, metrics::Temporality}; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; @@ -897,6 +897,9 @@ fn prometheus_from_env(bp: &mut Bootstrap) -> Option { /// itself. OTEL_EXPORTER_OTLP_(METRICS_)PROTOCOL picks grpc or http/protobuf /// (the default), and OTEL_METRIC_EXPORT_INTERVAL the interval in /// milliseconds (60000). OTEL_RESOURCE_ATTRIBUTES is added to the resource. +/// Headers that carry a token can come from a file instead of the +/// environment: INBUXA_OTLP_HEADERS_FILE, key=value pairs separated by +/// commas or lines. fn otel_from_env( bp: &mut Bootstrap, resource: &Resource, @@ -919,15 +922,55 @@ fn otel_from_env( Duration::from_secs(60) } }; + let headers = match types::branding::env_var("OTLP_HEADERS_FILE") { + Ok(path) if !path.is_empty() => match otlp_headers_file(&path) { + Ok(headers) => headers, + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!("{path} (INBUXA_OTLP_HEADERS_FILE): {err}. No OTLP metrics."), + ); + return None; + } + }, + _ => HeaderMap::new(), + }; let exporter = match protocol.as_str() { - "grpc" => MetricExporter::builder() - .with_temporality(Temporality::Delta) - .with_tonic() - .build(), - "http/protobuf" => MetricExporter::builder() - .with_temporality(Temporality::Delta) - .with_http() - .build(), + "grpc" => { + let builder = MetricExporter::builder() + .with_temporality(Temporality::Delta) + .with_tonic(); + if headers.is_empty() { + builder.build() + } else { + builder + .with_metadata( + opentelemetry_otlp::tonic_types::metadata::MetadataMap::from_headers( + headers, + ), + ) + .build() + } + } + "http/protobuf" => { + let builder = MetricExporter::builder() + .with_temporality(Temporality::Delta) + .with_http(); + if headers.is_empty() { + builder.build() + } else { + builder + .with_headers( + headers + .iter() + .filter_map(|(k, v)| { + Some((k.to_string(), v.to_str().ok()?.to_string())) + }) + .collect(), + ) + .build() + } + } _ => { bp.build_error( ObjectType::Metrics.singleton(), @@ -958,6 +1001,31 @@ fn otel_from_env( } } +fn otlp_headers_file(path: &str) -> Result { + let contents = std::fs::read_to_string(path).map_err(|err| err.to_string())?; + parse_otlp_headers(&contents) +} + +fn parse_otlp_headers(contents: &str) -> Result { + let mut headers = HeaderMap::new(); + for pair in contents + .split([',', '\n']) + .map(str::trim) + .filter(|pair| !pair.is_empty()) + { + let (name, value) = pair + .split_once('=') + .ok_or_else(|| format!("{pair:?} is not key=value"))?; + let name = + hyper::header::HeaderName::from_bytes(name.trim().to_ascii_lowercase().as_bytes()) + .map_err(|_| format!("{:?} is not a header name", name.trim()))?; + let value = hyper::header::HeaderValue::from_str(value.trim()) + .map_err(|_| format!("the value for {name} is not a header value"))?; + headers.append(name, value); + } + Ok(headers) +} + impl std::fmt::Debug for OtelMetrics { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OtelMetrics") @@ -1025,3 +1093,20 @@ mod tests { assert!(!wants(own, Level::Trace, &[own])); } } + +#[cfg(test)] +mod inbuxa_env_tests { + use super::parse_otlp_headers; + + #[test] + fn otlp_headers_file_pairs() { + let headers = + parse_otlp_headers("Authorization=Bearer abc==\nx-scope = team-a,\n").unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer abc=="); + assert_eq!(headers.get("x-scope").unwrap(), "team-a"); + assert_eq!(headers.len(), 2); + assert!(parse_otlp_headers("").unwrap().is_empty()); + assert!(parse_otlp_headers("no-equals").is_err()); + assert!(parse_otlp_headers("bad name=1").is_err()); + } +} -- 2.54.0 From 5527d7fb393e68b8239e6d791cd7789fdbed6f0c Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 09:01:56 -0700 Subject: [PATCH 3/6] OTLP from the environment: every pod pushes Each process counts only its own traffic, so an exporter that pushes from the metricsPush nodes alone leaves the rest out. One set from the environment names its pod in the resource, so every pod pushes, whatever its role, and the collector adds them up. The store's exporter still follows metricsPush. --- crates/common/src/config/telemetry.rs | 8 ++++++++ crates/services/src/task_manager/scheduler.rs | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 8fdc48f4..843264d7 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -71,6 +71,11 @@ pub struct OtelMetrics { pub instrumentation: InstrumentationScope, pub exporter: MetricExporter, pub interval: Duration, + // inbuxa: K-C10: pushed by every node, whatever its role. Each process + // counts only its own traffic; an exporter set from the environment + // names its pod in the resource (OTEL_RESOURCE_ATTRIBUTES), so every + // pod pushes and the collector adds them up. + pub every_node: bool, } #[derive(Debug)] @@ -680,6 +685,7 @@ impl Metrics { interval: otel.interval.into_inner(), resource, instrumentation, + every_node: false, })), Err(err) => { bp.build_error( @@ -706,6 +712,7 @@ impl Metrics { interval: otel.interval.into_inner(), resource, instrumentation, + every_node: false, })), Err(err) => { bp.build_error( @@ -988,6 +995,7 @@ fn otel_from_env( interval, resource: resource.clone(), instrumentation: instrumentation.clone(), + every_node: true, })), Err(err) => { bp.build_error( diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index 36d0477b..ea0ca5e7 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -272,7 +272,7 @@ pub fn spawn_task_scheduler(inner: Arc) { if let Some(otel) = &server.core.metrics.otel { queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics); - if roles.metrics_push { + if roles.metrics_push || otel.every_node { let otel = otel.clone(); tokio::spawn(async move { -- 2.54.0 From 09ac5d9fef88583de36a90d2a0932f6882cf0f49 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 09:48:40 -0700 Subject: [PATCH 4/6] Liveness answers before the anonymous request limit /healthz/live went through the limit for anonymous requests, which lives in the in-memory store. Where that is the data store, a data store outage made the check wait on it until the kubelet's probe timed out, and every pod was restarted at once, though each process was fine. Found on the Kubernetes lab, cutting one pod off from Postgres: it was killed within a minute. The liveness answer is a fixed 200 and costs nothing to serve, so it no longer waits for the limit; readiness still does. --- crates/http/src/request.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/http/src/request.rs b/crates/http/src/request.rs index dcffe90a..b09e2ed3 100644 --- a/crates/http/src/request.rs +++ b/crates/http/src/request.rs @@ -597,14 +597,22 @@ impl ParseHttp for Server { ); } "healthz" => { + let check = path.next().unwrap_or_default(); + + // inbuxa: K-C10: liveness answers before the limit. The limit + // is kept in the in-memory store, which may be the data store, + // so with that down the check waited on it, the probe timed + // out, and the orchestrator restarted a process that was + // fine, every one at once. A fixed 200 costs nothing to serve. + if check == "live" { + return Ok(JsonProblemResponse(StatusCode::OK).into_http_response()); + } + // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - match path.next().unwrap_or_default() { - "live" => { - return Ok(JsonProblemResponse(StatusCode::OK).into_http_response()); - } + match check { "ready" => { // inbuxa: ready only while the data store answers // (a cached, time-limited read); liveness stays 200 -- 2.54.0 From baf7d7ba93e3d3f064473d2025b28a425fd8dd0e Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 10:22:11 -0700 Subject: [PATCH 5/6] Metrics: a gauge never wraps below zero; counters scraped from zero - The queue count on a node that delivers what another queued went below zero and wrapped: an mta pod reported 18446744073709552000 messages. Gauges now stop at zero. - /metrics/prometheus left out counters still at zero, so one first seen at 3 never read as having risen, and rate() and increase() alerts missed it. Every collected counter is in the scrape now, zeros too. --- .../src/telemetry/metrics/prometheus.rs | 2 +- crates/trc/src/atomics/gauge.rs | 31 +++++++++++++++++-- crates/trc/src/ipc/metrics.rs | 21 +++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/crates/common/src/telemetry/metrics/prometheus.rs b/crates/common/src/telemetry/metrics/prometheus.rs index 8c34eee7..c005a6e0 100644 --- a/crates/common/src/telemetry/metrics/prometheus.rs +++ b/crates/common/src/telemetry/metrics/prometheus.rs @@ -19,7 +19,7 @@ impl Server { let mut metrics = Vec::new(); // Add counters - for counter in Collector::collect_counters() { + for counter in Collector::collect_metric_counters() { let mut metric = MetricFamily::default(); metric.set_name(metric_name(counter.id().as_str())); metric.set_help(counter.id().description().into()); diff --git a/crates/trc/src/atomics/gauge.rs b/crates/trc/src/atomics/gauge.rs index 5a92e222..0f3152cc 100644 --- a/crates/trc/src/atomics/gauge.rs +++ b/crates/trc/src/atomics/gauge.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use std::sync::atomic::{AtomicU64, Ordering}; @@ -31,9 +33,12 @@ impl AtomicGauge { self.value.store(value, Ordering::Relaxed); } + // inbuxa: K-C10: never below zero. A process can take off what another + // added (a message queued on one node, delivered on another), and a + // wrapped u64 read as 18446744073709552000 messages in the queue. #[inline(always)] pub fn decrement(&self) { - self.value.fetch_sub(1, Ordering::Relaxed); + self.subtract(1); } #[inline(always)] @@ -48,10 +53,32 @@ impl AtomicGauge { #[inline(always)] pub fn subtract(&self, value: u64) { - self.value.fetch_sub(value, Ordering::Relaxed); + let _ = self + .value + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_sub(value)) + }); } pub fn id(&self) -> MetricType { self.id } } + +#[cfg(test)] +mod tests { + use super::AtomicGauge; + use crate::MetricType; + + #[test] + fn gauge_stops_at_zero() { + let gauge = AtomicGauge::new(MetricType::QueueCount); + gauge.increment(); + gauge.decrement(); + gauge.decrement(); + assert_eq!(gauge.get(), 0); + gauge.add(3); + gauge.subtract(5); + assert_eq!(gauge.get(), 0); + } +} diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index ecfdcfc0..6924bed4 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -213,6 +213,27 @@ impl Collector { METRIC_INTERESTS.update(interests); } + // inbuxa: K-C10: for a scrape, every counter that is collected, at zero + // too. Prometheus measures an increase from the first sample, so a + // counter that first shows up at 3 never reads as having risen. + pub fn collect_metric_counters() -> impl Iterator { + EVENT_COUNTERS + .inner() + .iter() + .enumerate() + .filter_map(|(event_id, value)| { + let value = value.load(Ordering::Relaxed); + if value > 0 || METRIC_INTERESTS.get(event_id) { + Some(EventCounter { + id: EventType::from_id(event_id as u16)?, + value, + }) + } else { + None + } + }) + } + pub fn collect_counters() -> impl Iterator { EVENT_COUNTERS .inner() -- 2.54.0 From 4b8644710a75b5bb7715adbe11626bbccb49e128 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 10:53:45 -0700 Subject: [PATCH 6/6] Queue gauge in a cluster: only the count from the store Each node moved queue.count by what it queued and delivered itself, so in a cluster the node that counts the queue subtracted deliveries of messages other nodes had queued, and read low until the next count: three messages waiting showed as none. With a coordinator, the gauge changes only when the queue is counted (every five minutes, by the nodes that calculate metrics). A single server keeps its live count. --- crates/common/src/config/storage.rs | 7 ++++++ crates/services/src/task_manager/scheduler.rs | 5 ++++ crates/trc/src/ipc/metrics.rs | 23 ++++++++++++++++--- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/crates/common/src/config/storage.rs b/crates/common/src/config/storage.rs index 47dcbaeb..29ff79b6 100644 --- a/crates/common/src/config/storage.rs +++ b/crates/common/src/config/storage.rs @@ -30,6 +30,13 @@ pub struct Storage { pub directories: IdMap, } +impl Storage { + // inbuxa: K-C10: whether nodes share this store as a cluster + pub fn has_coordinator(&self) -> bool { + !matches!(self.coordinator, Coordinator::None) + } +} + impl Storage { pub async fn parse(bp: &mut Bootstrap) -> Self { let memory = InMemoryStore::build(bp).await.unwrap_or_default(); diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index ea0ca5e7..e1cf8125 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -136,6 +136,11 @@ pub fn spawn_task_scheduler(inner: Arc) { ); } + // inbuxa: K-C10: nodes sharing a queue count it from the store + if server.core.storage.has_coordinator() { + Collector::count_queue_from_store_only(); + } + // OTEL Push Metrics if let Some(otel) = &server.core.metrics.otel { OtelMetrics::enable_errors(); diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index 6924bed4..c3ed0878 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -49,6 +49,12 @@ static DNS_LOOKUP_TIME: AtomicHistogram<12> = static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory); static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount); +// inbuxa: K-C10: in a cluster the queue gauge is only what the node that +// calculates metrics counted in the store. Each node seeing only its own +// queued and delivered messages, live changes there were noise: the core +// subtracted deliveries of messages other nodes had queued. +static QUEUE_COUNT_FROM_STORE: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount); static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount); // inbuxa: K-C10: gauges for alerts, set where the value is known @@ -148,7 +154,9 @@ impl Collector { conn.elapsed.observe(elapsed); } EventType::Delivery(DeliveryEvent::Completed) => { - QUEUE_COUNT.decrement(); + if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) { + QUEUE_COUNT.decrement(); + } MESSAGE_DELIVERY_TIME.observe(elapsed); } EventType::Delivery( @@ -171,11 +179,15 @@ impl Collector { } EventType::Queue(QueueEvent::MessageQueued) => { MESSAGE_INCOMING_SIZE.observe(size); - QUEUE_COUNT.increment(); + if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) { + QUEUE_COUNT.increment(); + } } EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { MESSAGE_SUBMISSION_SIZE.observe(size); - QUEUE_COUNT.increment(); + if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) { + QUEUE_COUNT.increment(); + } } EventType::Queue(QueueEvent::ReportQueued) => { MESSAGE_OUT_REPORT_SIZE.observe(size); @@ -345,6 +357,11 @@ impl Collector { } } + /// In a cluster: the queue gauge changes only when the queue is counted. + pub fn count_queue_from_store_only() { + QUEUE_COUNT_FROM_STORE.store(true, Ordering::Relaxed); + } + pub fn update_gauge(metric_type: MetricType, value: u64) { match metric_type { MetricType::ServerMemory => SERVER_MEMORY.set(value), -- 2.54.0