diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index 58a26046..6b04bb21 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -288,6 +288,18 @@ pub(crate) async fn parse_certificates( } } + // inbuxa: K-C10: when the first name served runs out, for an alert that + // warns before a renewal that isn't happening matters + trc::Collector::update_gauge( + trc::MetricType::TlsEarliestExpiry, + certs_expirations + .iter() + .filter(|(name, _)| name.as_str() != "*") + .map(|(_, expires)| (*expires).max(0) as u64) + .min() + .unwrap_or_default(), + ); + // No default certificate set: pick one now, once per load, so handshakes // without SNI or for an unknown name get a predictable certificate. if !certificates.contains_key("*") && certificates.len() > 1 { diff --git a/crates/common/src/config/storage.rs b/crates/common/src/config/storage.rs index 47dcbaeb..29ff79b6 100644 --- a/crates/common/src/config/storage.rs +++ b/crates/common/src/config/storage.rs @@ -30,6 +30,13 @@ pub struct Storage { pub directories: IdMap, } +impl Storage { + // inbuxa: K-C10: whether nodes share this store as a cluster + pub fn has_coordinator(&self) -> bool { + !matches!(self.coordinator, Coordinator::None) + } +} + impl Storage { pub async fn parse(bp: &mut Bootstrap) -> Self { let memory = InMemoryStore::build(bp).await.unwrap_or_default(); diff --git a/crates/common/src/config/telemetry.rs b/crates/common/src/config/telemetry.rs index 5fd8351c..843264d7 100644 --- a/crates/common/src/config/telemetry.rs +++ b/crates/common/src/config/telemetry.rs @@ -12,7 +12,7 @@ use base64::{Engine, engine::general_purpose::STANDARD}; use hyper::HeaderMap; use opentelemetry::{InstrumentationScope, KeyValue}; use opentelemetry_otlp::{ - LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, + LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig, }; use opentelemetry_sdk::{Resource, metrics::Temporality}; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; @@ -71,6 +71,11 @@ pub struct OtelMetrics { pub instrumentation: InstrumentationScope, pub exporter: MetricExporter, pub interval: Duration, + // inbuxa: K-C10: pushed by every node, whatever its role. Each process + // counts only its own traffic; an exporter set from the environment + // names its pod in the resource (OTEL_RESOURCE_ATTRIBUTES), so every + // pod pushes and the collector adds them up. + pub every_node: bool, } #[derive(Debug)] @@ -613,8 +618,19 @@ impl Metrics { .with_version(types::brand_version!()) .build(); + // inbuxa: K-C10: scraping and OTLP set by the deployment (Kubernetes), + // over what the store says. A pod reads the scrape password from a + // mounted file, and the OTLP exporter from the standard OTEL_ variables. + let env_prometheus = prometheus_from_env(bp); + let env_otel = if matches!(metrics.open_telemetry, structs::MetricsOtel::Disabled) { + otel_from_env(bp, &resource, &instrumentation) + } else { + None + }; + Metrics { prometheus: match metrics.prometheus { + _ if env_prometheus.is_some() => env_prometheus, MetricsPrometheus::Enabled(prom) => { let secret = prom .auth_secret @@ -669,6 +685,7 @@ impl Metrics { interval: otel.interval.into_inner(), resource, instrumentation, + every_node: false, })), Err(err) => { bp.build_error( @@ -695,6 +712,7 @@ impl Metrics { interval: otel.interval.into_inner(), resource, instrumentation, + every_node: false, })), Err(err) => { bp.build_error( @@ -705,7 +723,7 @@ impl Metrics { } } } - structs::MetricsOtel::Disabled => None, + structs::MetricsOtel::Disabled => env_otel, }, log_path: bp .list_infallible::() @@ -833,6 +851,189 @@ fn apply_metrics( } } +/// Prometheus scraping from INBUXA_METRICS_PROMETHEUS_USERNAME and +/// INBUXA_METRICS_PROMETHEUS_SECRET_FILE. Both or neither: the endpoint is +/// never opened without a password this way. +fn prometheus_from_env(bp: &mut Bootstrap) -> Option { + let username = types::branding::env_var("METRICS_PROMETHEUS_USERNAME").ok(); + let secret_file = types::branding::env_var("METRICS_PROMETHEUS_SECRET_FILE").ok(); + let (username, secret_file) = match (username, secret_file) { + (None, None) => return None, + (Some(username), Some(secret_file)) if !username.is_empty() && !secret_file.is_empty() => { + (username, secret_file) + } + _ => { + bp.build_error( + ObjectType::Metrics.singleton(), + "Set both INBUXA_METRICS_PROMETHEUS_USERNAME and \ + INBUXA_METRICS_PROMETHEUS_SECRET_FILE, or neither. The setting in the store \ + applies.", + ); + return None; + } + }; + match std::fs::read_to_string(&secret_file) { + Ok(secret) if !secret.trim_end_matches(['\r', '\n']).is_empty() => { + let secret = secret.trim_end_matches(['\r', '\n']); + Some(PrometheusMetrics { + auth: Some(STANDARD.encode(format!("{username}:{secret}"))), + }) + } + Ok(_) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!("{secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE) is empty"), + ); + None + } + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!( + "Unable to read {secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE): {err}" + ), + ); + None + } + } +} + +/// OTLP metrics from the standard variables, when the store has none set: +/// OTEL_EXPORTER_OTLP_METRICS_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT turns +/// it on; the exporter reads those, the _HEADERS and _TIMEOUT variables +/// itself. OTEL_EXPORTER_OTLP_(METRICS_)PROTOCOL picks grpc or http/protobuf +/// (the default), and OTEL_METRIC_EXPORT_INTERVAL the interval in +/// milliseconds (60000). OTEL_RESOURCE_ATTRIBUTES is added to the resource. +/// Headers that carry a token can come from a file instead of the +/// environment: INBUXA_OTLP_HEADERS_FILE, key=value pairs separated by +/// commas or lines. +fn otel_from_env( + bp: &mut Bootstrap, + resource: &Resource, + instrumentation: &InstrumentationScope, +) -> Option> { + let var = |name: &str| std::env::var(name).ok().filter(|value| !value.is_empty()); + var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT").or_else(|| var("OTEL_EXPORTER_OTLP_ENDPOINT"))?; + let protocol = var("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL") + .or_else(|| var("OTEL_EXPORTER_OTLP_PROTOCOL")) + .unwrap_or_else(|| "http/protobuf".to_string()); + let interval = match var("OTEL_METRIC_EXPORT_INTERVAL").map(|value| value.parse::()) { + None => Duration::from_secs(60), + Some(Ok(millis)) if millis >= 1000 => Duration::from_millis(millis), + Some(_) => { + bp.build_warning( + ObjectType::Metrics.singleton(), + "OTEL_METRIC_EXPORT_INTERVAL is not a number of milliseconds of at least 1000; \ + exporting every 60 seconds.", + ); + Duration::from_secs(60) + } + }; + let headers = match types::branding::env_var("OTLP_HEADERS_FILE") { + Ok(path) if !path.is_empty() => match otlp_headers_file(&path) { + Ok(headers) => headers, + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!("{path} (INBUXA_OTLP_HEADERS_FILE): {err}. No OTLP metrics."), + ); + return None; + } + }, + _ => HeaderMap::new(), + }; + let exporter = match protocol.as_str() { + "grpc" => { + let builder = MetricExporter::builder() + .with_temporality(Temporality::Delta) + .with_tonic(); + if headers.is_empty() { + builder.build() + } else { + builder + .with_metadata( + opentelemetry_otlp::tonic_types::metadata::MetadataMap::from_headers( + headers, + ), + ) + .build() + } + } + "http/protobuf" => { + let builder = MetricExporter::builder() + .with_temporality(Temporality::Delta) + .with_http(); + if headers.is_empty() { + builder.build() + } else { + builder + .with_headers( + headers + .iter() + .filter_map(|(k, v)| { + Some((k.to_string(), v.to_str().ok()?.to_string())) + }) + .collect(), + ) + .build() + } + } + _ => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!( + "OTEL_EXPORTER_OTLP_PROTOCOL is {protocol:?}; expected grpc or http/protobuf. \ + No OTLP metrics." + ), + ); + return None; + } + }; + match exporter { + Ok(exporter) => Some(Arc::new(OtelMetrics { + exporter, + interval, + resource: resource.clone(), + instrumentation: instrumentation.clone(), + every_node: true, + })), + Err(err) => { + bp.build_error( + ObjectType::Metrics.singleton(), + format!( + "Failed to build the OpenTelemetry metrics exporter from OTEL_ variables: {err}" + ), + ); + None + } + } +} + +fn otlp_headers_file(path: &str) -> Result { + let contents = std::fs::read_to_string(path).map_err(|err| err.to_string())?; + parse_otlp_headers(&contents) +} + +fn parse_otlp_headers(contents: &str) -> Result { + let mut headers = HeaderMap::new(); + for pair in contents + .split([',', '\n']) + .map(str::trim) + .filter(|pair| !pair.is_empty()) + { + let (name, value) = pair + .split_once('=') + .ok_or_else(|| format!("{pair:?} is not key=value"))?; + let name = + hyper::header::HeaderName::from_bytes(name.trim().to_ascii_lowercase().as_bytes()) + .map_err(|_| format!("{:?} is not a header name", name.trim()))?; + let value = hyper::header::HeaderValue::from_str(value.trim()) + .map_err(|_| format!("the value for {name} is not a header value"))?; + headers.append(name, value); + } + Ok(headers) +} + impl std::fmt::Debug for OtelMetrics { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OtelMetrics") @@ -900,3 +1101,20 @@ mod tests { assert!(!wants(own, Level::Trace, &[own])); } } + +#[cfg(test)] +mod inbuxa_env_tests { + use super::parse_otlp_headers; + + #[test] + fn otlp_headers_file_pairs() { + let headers = + parse_otlp_headers("Authorization=Bearer abc==\nx-scope = team-a,\n").unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer abc=="); + assert_eq!(headers.get("x-scope").unwrap(), "team-a"); + assert_eq!(headers.len(), 2); + assert!(parse_otlp_headers("").unwrap().is_empty()); + assert!(parse_otlp_headers("no-equals").is_err()); + assert!(parse_otlp_headers("bad name=1").is_err()); + } +} diff --git a/crates/common/src/network/mta.rs b/crates/common/src/network/mta.rs index d0fa9630..f9d2cb71 100644 --- a/crates/common/src/network/mta.rs +++ b/crates/common/src/network/mta.rs @@ -35,10 +35,10 @@ use std::{ }; use store::{ Deserialize, IterateParams, ValueKey, - write::{AlignedBytes, Archive, QueueClass, ValueClass}, + write::{AlignedBytes, Archive, QueueClass, ValueClass, key::DeserializeBigEndian}, }; use trc::{AddContext, SpamEvent}; -use utils::DomainPart; +use utils::{DomainPart, snowflake::SnowflakeIdGenerator}; impl Server { pub async fn rcpt_resolve( @@ -505,16 +505,26 @@ impl Server { Ok(()) } - pub async fn total_queued_messages(&self) -> trc::Result { + // inbuxa: K-C10: also when the oldest message arrived. Queue ids are + // snowflakes, so the first key is the oldest message and its id holds + // the time, without reading any message. + pub async fn total_queued_messages(&self) -> trc::Result<(u64, Option)> { let mut total = 0; + let mut oldest = None; self.store() .iterate( IterateParams::new( ValueKey::from(ValueClass::Queue(QueueClass::Message(0))), ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))), ) + .ascending() .no_values(), - |_, _| { + |key, _| { + if total == 0 { + oldest = Some(SnowflakeIdGenerator::to_timestamp( + key.deserialize_be_u64(0)?, + )); + } total += 1; Ok(true) @@ -522,7 +532,7 @@ impl Server { ) .await .caused_by(trc::location!()) - .map(|_| total) + .map(|_| (total, oldest)) } } diff --git a/crates/common/src/telemetry/metrics/prometheus.rs b/crates/common/src/telemetry/metrics/prometheus.rs index 8c34eee7..c005a6e0 100644 --- a/crates/common/src/telemetry/metrics/prometheus.rs +++ b/crates/common/src/telemetry/metrics/prometheus.rs @@ -19,7 +19,7 @@ impl Server { let mut metrics = Vec::new(); // Add counters - for counter in Collector::collect_counters() { + for counter in Collector::collect_metric_counters() { let mut metric = MetricFamily::default(); metric.set_name(metric_name(counter.id().as_str())); metric.set_help(counter.id().description().into()); diff --git a/crates/http/src/request.rs b/crates/http/src/request.rs index dcffe90a..b09e2ed3 100644 --- a/crates/http/src/request.rs +++ b/crates/http/src/request.rs @@ -597,14 +597,22 @@ impl ParseHttp for Server { ); } "healthz" => { + let check = path.next().unwrap_or_default(); + + // inbuxa: K-C10: liveness answers before the limit. The limit + // is kept in the in-memory store, which may be the data store, + // so with that down the check waited on it, the probe timed + // out, and the orchestrator restarted a process that was + // fine, every one at once. A fixed 200 costs nothing to serve. + if check == "live" { + return Ok(JsonProblemResponse(StatusCode::OK).into_http_response()); + } + // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - match path.next().unwrap_or_default() { - "live" => { - return Ok(JsonProblemResponse(StatusCode::OK).into_http_response()); - } + match check { "ready" => { // inbuxa: ready only while the data store answers // (a cached, time-limited read); liveness stays 200 diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index 16a025b0..e1cf8125 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -136,6 +136,11 @@ pub fn spawn_task_scheduler(inner: Arc) { ); } + // inbuxa: K-C10: nodes sharing a queue count it from the store + if server.core.storage.has_coordinator() { + Collector::count_queue_from_store_only(); + } + // OTEL Push Metrics if let Some(otel) = &server.core.metrics.otel { OtelMetrics::enable_errors(); @@ -272,7 +277,7 @@ pub fn spawn_task_scheduler(inner: Arc) { if let Some(otel) = &server.core.metrics.otel { queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics); - if roles.metrics_push { + if roles.metrics_push || otel.every_node { let otel = otel.clone(); tokio::spawn(async move { @@ -308,8 +313,12 @@ pub fn spawn_task_scheduler(inner: Arc) { // inbuxa: MON-7: the queue gauge from the queue itself, // so it's right after a restart match server.total_queued_messages().await { - Ok(total) => { + Ok((total, oldest)) => { Collector::update_gauge(MetricType::QueueCount, total); + Collector::update_gauge( + MetricType::QueueOldestMessage, + oldest.unwrap_or_default(), + ); } Err(err) => { trc::error!(err.details("Failed to count queued messages")); diff --git a/crates/store/src/build/cluster.rs b/crates/store/src/build/cluster.rs index cd5064d2..d531d2cc 100644 --- a/crates/store/src/build/cluster.rs +++ b/crates/store/src/build/cluster.rs @@ -58,6 +58,7 @@ struct NodeClaim { impl RegistryStoreInner { pub(super) async fn acquire_node_id(&mut self) -> Result<(), String> { let (node_id, slots) = NodeSlot::acquire(&self.store, &self.env_hostname).await?; + lease_renewed(); self.node_id.store(node_id, Ordering::Relaxed); self.remember_hostname().await; @@ -97,6 +98,12 @@ impl RegistryStoreInner { } } +// inbuxa: K-C10: the gauge an alert watches. A pod that keeps running but +// can't renew its lease (the store unreachable, say) stops moving it. +fn lease_renewed() { + trc::Collector::update_gauge(trc::MetricType::ClusterLeaseRenewed, now()); +} + // INBUXA: #458, the file a node records its leased hostname in. // It sits beside the local config file, or inside the directory when the // path is one (as in the test harness). @@ -146,6 +153,7 @@ impl RegistryStore { })?; self.0.node_id.store(new_node_id, Ordering::Relaxed); SnowflakeIdGenerator::set_node_id(new_node_id as u64); + lease_renewed(); trc::event!( Cluster(ClusterEvent::NodeIdReassigned), @@ -173,6 +181,7 @@ impl RegistryStore { .write(batch.build_all()) .await .caused_by(trc::location!())?; + lease_renewed(); self.release_renamed_lease(&slots).await } diff --git a/crates/trc/src/atomics/gauge.rs b/crates/trc/src/atomics/gauge.rs index 5a92e222..0f3152cc 100644 --- a/crates/trc/src/atomics/gauge.rs +++ b/crates/trc/src/atomics/gauge.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use std::sync::atomic::{AtomicU64, Ordering}; @@ -31,9 +33,12 @@ impl AtomicGauge { self.value.store(value, Ordering::Relaxed); } + // inbuxa: K-C10: never below zero. A process can take off what another + // added (a message queued on one node, delivered on another), and a + // wrapped u64 read as 18446744073709552000 messages in the queue. #[inline(always)] pub fn decrement(&self) { - self.value.fetch_sub(1, Ordering::Relaxed); + self.subtract(1); } #[inline(always)] @@ -48,10 +53,32 @@ impl AtomicGauge { #[inline(always)] pub fn subtract(&self, value: u64) { - self.value.fetch_sub(value, Ordering::Relaxed); + let _ = self + .value + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + Some(current.saturating_sub(value)) + }); } pub fn id(&self) -> MetricType { self.id } } + +#[cfg(test)] +mod tests { + use super::AtomicGauge; + use crate::MetricType; + + #[test] + fn gauge_stops_at_zero() { + let gauge = AtomicGauge::new(MetricType::QueueCount); + gauge.increment(); + gauge.decrement(); + gauge.decrement(); + assert_eq!(gauge.get(), 0); + gauge.add(3); + gauge.subtract(5); + assert_eq!(gauge.get(), 0); + } +} diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index b12b7745..e678d72f 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -16,11 +16,13 @@ // 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is // store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is // spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228); -// 664 is auth.passkey-counter (passkeys spec, PK-18). +// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371 +// are gauges for Kubernetes alerts (K-C10): queue.oldest-message, +// cluster.lease-renewed, tls.earliest-expiry. // TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one // more than the highest id; the test below holds it to the variant list pub const TOTAL_EVENT_COUNT: usize = 665; -pub const TOTAL_METRIC_COUNT: usize = 369; +pub const TOTAL_METRIC_COUNT: usize = 372; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EventType { @@ -1080,6 +1082,9 @@ pub enum MetricType { DkimMailFromDomainMismatch = 353, DkimNextDomainMismatch = 354, DkimCustodyBreak = 368, + QueueOldestMessage = 369, + ClusterLeaseRenewed = 370, + TlsEarliestExpiry = 371, DkimPublicKeyFetch = 355, DkimPublicKeyMissing = 356, DkimPublicKeyMultiple = 357, diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index 6a6a2665..51d410c5 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -5499,6 +5499,9 @@ impl MetricType { b"telemetry.journal-error" => MetricType::TelemetryJournalError, b"tls.handshake-error" => MetricType::TlsHandshakeError, b"user.count" => MetricType::UserCount, + b"queue.oldest-message" => MetricType::QueueOldestMessage, + b"cluster.lease-renewed" => MetricType::ClusterLeaseRenewed, + b"tls.earliest-expiry" => MetricType::TlsEarliestExpiry, } .copied() } @@ -5886,6 +5889,9 @@ impl MetricType { MetricType::TelemetryJournalError => "telemetry.journal-error", MetricType::TlsHandshakeError => "tls.handshake-error", MetricType::UserCount => "user.count", + MetricType::QueueOldestMessage => "queue.oldest-message", + MetricType::ClusterLeaseRenewed => "cluster.lease-renewed", + MetricType::TlsEarliestExpiry => "tls.earliest-expiry", } } @@ -6260,6 +6266,9 @@ impl MetricType { MetricType::TelemetryJournalError => 335, MetricType::TlsHandshakeError => 336, MetricType::UserCount => 25, + MetricType::QueueOldestMessage => 369, + MetricType::ClusterLeaseRenewed => 370, + MetricType::TlsEarliestExpiry => 371, } } @@ -6634,6 +6643,9 @@ impl MetricType { 335 => Some(MetricType::TelemetryJournalError), 336 => Some(MetricType::TlsHandshakeError), 25 => Some(MetricType::UserCount), + 369 => Some(MetricType::QueueOldestMessage), + 370 => Some(MetricType::ClusterLeaseRenewed), + 371 => Some(MetricType::TlsEarliestExpiry), _ => None, } } @@ -7369,6 +7381,15 @@ impl MetricType { MetricType::TelemetryJournalError => "Journal collector error", MetricType::TlsHandshakeError => "TLS handshake error", MetricType::UserCount => "Total number of users", + MetricType::QueueOldestMessage => { + "When the oldest message in the queue arrived, as a Unix time (0: the queue is empty)" + } + MetricType::ClusterLeaseRenewed => { + "When this node last renewed its cluster lease, as a Unix time" + } + MetricType::TlsEarliestExpiry => { + "When the first certificate in use expires, as a Unix time (0: none)" + } } } @@ -7743,6 +7764,9 @@ impl MetricType { | MetricType::StoreBlobReadTime | MetricType::StoreBlobWriteTime => "milliseconds", MetricType::UserCount => "users", + MetricType::QueueOldestMessage + | MetricType::ClusterLeaseRenewed + | MetricType::TlsEarliestExpiry => "seconds", } } @@ -8117,6 +8141,9 @@ impl MetricType { MetricType::TelemetryJournalError, MetricType::TlsHandshakeError, MetricType::UserCount, + MetricType::QueueOldestMessage, + MetricType::ClusterLeaseRenewed, + MetricType::TlsEarliestExpiry, ] } } diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index bf83ef5c..c3ed0878 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -49,8 +49,18 @@ static DNS_LOOKUP_TIME: AtomicHistogram<12> = static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory); static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount); +// inbuxa: K-C10: in a cluster the queue gauge is only what the node that +// calculates metrics counted in the store. Each node seeing only its own +// queued and delivered messages, live changes there were noise: the core +// subtracted deliveries of messages other nodes had queued. +static QUEUE_COUNT_FROM_STORE: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount); static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount); +// inbuxa: K-C10: gauges for alerts, set where the value is known +static QUEUE_OLDEST_MESSAGE: AtomicGauge = AtomicGauge::new(MetricType::QueueOldestMessage); +static CLUSTER_LEASE_RENEWED: AtomicGauge = AtomicGauge::new(MetricType::ClusterLeaseRenewed); +static TLS_EARLIEST_EXPIRY: AtomicGauge = AtomicGauge::new(MetricType::TlsEarliestExpiry); const CONN_SMTP_IN: usize = 0; const CONN_SMTP_OUT: usize = 1; @@ -144,7 +154,9 @@ impl Collector { conn.elapsed.observe(elapsed); } EventType::Delivery(DeliveryEvent::Completed) => { - QUEUE_COUNT.decrement(); + if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) { + QUEUE_COUNT.decrement(); + } MESSAGE_DELIVERY_TIME.observe(elapsed); } EventType::Delivery( @@ -167,11 +179,15 @@ impl Collector { } EventType::Queue(QueueEvent::MessageQueued) => { MESSAGE_INCOMING_SIZE.observe(size); - QUEUE_COUNT.increment(); + if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) { + QUEUE_COUNT.increment(); + } } EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { MESSAGE_SUBMISSION_SIZE.observe(size); - QUEUE_COUNT.increment(); + if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) { + QUEUE_COUNT.increment(); + } } EventType::Queue(QueueEvent::ReportQueued) => { MESSAGE_OUT_REPORT_SIZE.observe(size); @@ -209,6 +225,27 @@ impl Collector { METRIC_INTERESTS.update(interests); } + // inbuxa: K-C10: for a scrape, every counter that is collected, at zero + // too. Prometheus measures an increase from the first sample, so a + // counter that first shows up at 3 never reads as having risen. + pub fn collect_metric_counters() -> impl Iterator { + EVENT_COUNTERS + .inner() + .iter() + .enumerate() + .filter_map(|(event_id, value)| { + let value = value.load(Ordering::Relaxed); + if value > 0 || METRIC_INTERESTS.get(event_id) { + Some(EventCounter { + id: EventType::from_id(event_id as u16)?, + value, + }) + } else { + None + } + }) + } + pub fn collect_counters() -> impl Iterator { EVENT_COUNTERS .inner() @@ -229,8 +266,15 @@ impl Collector { // inbuxa: MON-6: one edition, every gauge pub fn collect_gauges() -> impl Iterator { - static GAUGES: &[&AtomicGauge] = - &[&SERVER_MEMORY, &QUEUE_COUNT, &USER_COUNT, &DOMAIN_COUNT]; + static GAUGES: &[&AtomicGauge] = &[ + &SERVER_MEMORY, + &QUEUE_COUNT, + &USER_COUNT, + &DOMAIN_COUNT, + &QUEUE_OLDEST_MESSAGE, + &CLUSTER_LEASE_RENEWED, + &TLS_EARLIEST_EXPIRY, + ]; GAUGES .iter() @@ -306,16 +350,27 @@ impl Collector { MetricType::SieveRequestTime => CONNECTION_METRICS[CONN_SIEVE].elapsed.average(), MetricType::UserCount => USER_COUNT.get() as f64, MetricType::DomainCount => DOMAIN_COUNT.get() as f64, + MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.get() as f64, + MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.get() as f64, + MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.get() as f64, _ => EVENT_COUNTERS.get(metric_type.event_id()) as f64, } } + /// In a cluster: the queue gauge changes only when the queue is counted. + pub fn count_queue_from_store_only() { + QUEUE_COUNT_FROM_STORE.store(true, Ordering::Relaxed); + } + pub fn update_gauge(metric_type: MetricType, value: u64) { match metric_type { MetricType::ServerMemory => SERVER_MEMORY.set(value), MetricType::QueueCount => QUEUE_COUNT.set(value), MetricType::UserCount => USER_COUNT.set(value), MetricType::DomainCount => DOMAIN_COUNT.set(value), + MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.set(value), + MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.set(value), + MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.set(value), _ => {} } }