Metrics for Kubernetes: scraping and OTLP from the environment, alert gauges, four metric fixes #488

Merged
jcoffey-dev merged 6 commits from feat/k8s-metrics into main 2026-11-19 18:35:13 +00:00
12 changed files with 410 additions and 23 deletions

No files matched your search

+12
View File
@@ -288,6 +288,18 @@ pub(crate) async fn parse_certificates(
}
}
// inbuxa: K-C10: when the first name served runs out, for an alert that
// warns before a renewal that isn't happening matters
trc::Collector::update_gauge(
trc::MetricType::TlsEarliestExpiry,
certs_expirations
.iter()
.filter(|(name, _)| name.as_str() != "*")
.map(|(_, expires)| (*expires).max(0) as u64)
.min()
.unwrap_or_default(),
);
// No default certificate set: pick one now, once per load, so handshakes
// without SNI or for an unknown name get a predictable certificate.
if !certificates.contains_key("*") && certificates.len() > 1 {
+7
View File
@@ -30,6 +30,13 @@ pub struct Storage {
pub directories: IdMap<Directory>,
}
impl Storage {
// inbuxa: K-C10: whether nodes share this store as a cluster
pub fn has_coordinator(&self) -> bool {
!matches!(self.coordinator, Coordinator::None)
}
}
impl Storage {
pub async fn parse(bp: &mut Bootstrap) -> Self {
let memory = InMemoryStore::build(bp).await.unwrap_or_default();
+220 -2
View File
@@ -12,7 +12,7 @@ use base64::{Engine, engine::general_purpose::STANDARD};
use hyper::HeaderMap;
use opentelemetry::{InstrumentationScope, KeyValue};
use opentelemetry_otlp::{
LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig,
LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig,
};
use opentelemetry_sdk::{Resource, metrics::Temporality};
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
@@ -71,6 +71,11 @@ pub struct OtelMetrics {
pub instrumentation: InstrumentationScope,
pub exporter: MetricExporter,
pub interval: Duration,
// inbuxa: K-C10: pushed by every node, whatever its role. Each process
// counts only its own traffic; an exporter set from the environment
// names its pod in the resource (OTEL_RESOURCE_ATTRIBUTES), so every
// pod pushes and the collector adds them up.
pub every_node: bool,
}
#[derive(Debug)]
@@ -613,8 +618,19 @@ impl Metrics {
.with_version(types::brand_version!())
.build();
// inbuxa: K-C10: scraping and OTLP set by the deployment (Kubernetes),
// over what the store says. A pod reads the scrape password from a
// mounted file, and the OTLP exporter from the standard OTEL_ variables.
let env_prometheus = prometheus_from_env(bp);
let env_otel = if matches!(metrics.open_telemetry, structs::MetricsOtel::Disabled) {
otel_from_env(bp, &resource, &instrumentation)
} else {
None
};
Metrics {
prometheus: match metrics.prometheus {
_ if env_prometheus.is_some() => env_prometheus,
MetricsPrometheus::Enabled(prom) => {
let secret = prom
.auth_secret
@@ -669,6 +685,7 @@ impl Metrics {
interval: otel.interval.into_inner(),
resource,
instrumentation,
every_node: false,
})),
Err(err) => {
bp.build_error(
@@ -695,6 +712,7 @@ impl Metrics {
interval: otel.interval.into_inner(),
resource,
instrumentation,
every_node: false,
})),
Err(err) => {
bp.build_error(
@@ -705,7 +723,7 @@ impl Metrics {
}
}
}
structs::MetricsOtel::Disabled => None,
structs::MetricsOtel::Disabled => env_otel,
},
log_path: bp
.list_infallible::<Tracer>()
@@ -833,6 +851,189 @@ fn apply_metrics(
}
}
/// Prometheus scraping from INBUXA_METRICS_PROMETHEUS_USERNAME and
/// INBUXA_METRICS_PROMETHEUS_SECRET_FILE. Both or neither: the endpoint is
/// never opened without a password this way.
fn prometheus_from_env(bp: &mut Bootstrap) -> Option<PrometheusMetrics> {
let username = types::branding::env_var("METRICS_PROMETHEUS_USERNAME").ok();
let secret_file = types::branding::env_var("METRICS_PROMETHEUS_SECRET_FILE").ok();
let (username, secret_file) = match (username, secret_file) {
(None, None) => return None,
(Some(username), Some(secret_file)) if !username.is_empty() && !secret_file.is_empty() => {
(username, secret_file)
}
_ => {
bp.build_error(
ObjectType::Metrics.singleton(),
"Set both INBUXA_METRICS_PROMETHEUS_USERNAME and \
INBUXA_METRICS_PROMETHEUS_SECRET_FILE, or neither. The setting in the store \
applies.",
);
return None;
}
};
match std::fs::read_to_string(&secret_file) {
Ok(secret) if !secret.trim_end_matches(['\r', '\n']).is_empty() => {
let secret = secret.trim_end_matches(['\r', '\n']);
Some(PrometheusMetrics {
auth: Some(STANDARD.encode(format!("{username}:{secret}"))),
})
}
Ok(_) => {
bp.build_error(
ObjectType::Metrics.singleton(),
format!("{secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE) is empty"),
);
None
}
Err(err) => {
bp.build_error(
ObjectType::Metrics.singleton(),
format!(
"Unable to read {secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE): {err}"
),
);
None
}
}
}
/// OTLP metrics from the standard variables, when the store has none set:
/// OTEL_EXPORTER_OTLP_METRICS_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT turns
/// it on; the exporter reads those, the _HEADERS and _TIMEOUT variables
/// itself. OTEL_EXPORTER_OTLP_(METRICS_)PROTOCOL picks grpc or http/protobuf
/// (the default), and OTEL_METRIC_EXPORT_INTERVAL the interval in
/// milliseconds (60000). OTEL_RESOURCE_ATTRIBUTES is added to the resource.
/// Headers that carry a token can come from a file instead of the
/// environment: INBUXA_OTLP_HEADERS_FILE, key=value pairs separated by
/// commas or lines.
fn otel_from_env(
bp: &mut Bootstrap,
resource: &Resource,
instrumentation: &InstrumentationScope,
) -> Option<Arc<OtelMetrics>> {
let var = |name: &str| std::env::var(name).ok().filter(|value| !value.is_empty());
var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT").or_else(|| var("OTEL_EXPORTER_OTLP_ENDPOINT"))?;
let protocol = var("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL")
.or_else(|| var("OTEL_EXPORTER_OTLP_PROTOCOL"))
.unwrap_or_else(|| "http/protobuf".to_string());
let interval = match var("OTEL_METRIC_EXPORT_INTERVAL").map(|value| value.parse::<u64>()) {
None => Duration::from_secs(60),
Some(Ok(millis)) if millis >= 1000 => Duration::from_millis(millis),
Some(_) => {
bp.build_warning(
ObjectType::Metrics.singleton(),
"OTEL_METRIC_EXPORT_INTERVAL is not a number of milliseconds of at least 1000; \
exporting every 60 seconds.",
);
Duration::from_secs(60)
}
};
let headers = match types::branding::env_var("OTLP_HEADERS_FILE") {
Ok(path) if !path.is_empty() => match otlp_headers_file(&path) {
Ok(headers) => headers,
Err(err) => {
bp.build_error(
ObjectType::Metrics.singleton(),
format!("{path} (INBUXA_OTLP_HEADERS_FILE): {err}. No OTLP metrics."),
);
return None;
}
},
_ => HeaderMap::new(),
};
let exporter = match protocol.as_str() {
"grpc" => {
let builder = MetricExporter::builder()
.with_temporality(Temporality::Delta)
.with_tonic();
if headers.is_empty() {
builder.build()
} else {
builder
.with_metadata(
opentelemetry_otlp::tonic_types::metadata::MetadataMap::from_headers(
headers,
),
)
.build()
}
}
"http/protobuf" => {
let builder = MetricExporter::builder()
.with_temporality(Temporality::Delta)
.with_http();
if headers.is_empty() {
builder.build()
} else {
builder
.with_headers(
headers
.iter()
.filter_map(|(k, v)| {
Some((k.to_string(), v.to_str().ok()?.to_string()))
})
.collect(),
)
.build()
}
}
_ => {
bp.build_error(
ObjectType::Metrics.singleton(),
format!(
"OTEL_EXPORTER_OTLP_PROTOCOL is {protocol:?}; expected grpc or http/protobuf. \
No OTLP metrics."
),
);
return None;
}
};
match exporter {
Ok(exporter) => Some(Arc::new(OtelMetrics {
exporter,
interval,
resource: resource.clone(),
instrumentation: instrumentation.clone(),
every_node: true,
})),
Err(err) => {
bp.build_error(
ObjectType::Metrics.singleton(),
format!(
"Failed to build the OpenTelemetry metrics exporter from OTEL_ variables: {err}"
),
);
None
}
}
}
fn otlp_headers_file(path: &str) -> Result<HeaderMap, String> {
let contents = std::fs::read_to_string(path).map_err(|err| err.to_string())?;
parse_otlp_headers(&contents)
}
fn parse_otlp_headers(contents: &str) -> Result<HeaderMap, String> {
let mut headers = HeaderMap::new();
for pair in contents
.split([',', '\n'])
.map(str::trim)
.filter(|pair| !pair.is_empty())
{
let (name, value) = pair
.split_once('=')
.ok_or_else(|| format!("{pair:?} is not key=value"))?;
let name =
hyper::header::HeaderName::from_bytes(name.trim().to_ascii_lowercase().as_bytes())
.map_err(|_| format!("{:?} is not a header name", name.trim()))?;
let value = hyper::header::HeaderValue::from_str(value.trim())
.map_err(|_| format!("the value for {name} is not a header value"))?;
headers.append(name, value);
}
Ok(headers)
}
impl std::fmt::Debug for OtelMetrics {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OtelMetrics")
@@ -900,3 +1101,20 @@ mod tests {
assert!(!wants(own, Level::Trace, &[own]));
}
}
#[cfg(test)]
mod inbuxa_env_tests {
use super::parse_otlp_headers;
#[test]
fn otlp_headers_file_pairs() {
let headers =
parse_otlp_headers("Authorization=Bearer abc==\nx-scope = team-a,\n").unwrap();
assert_eq!(headers.get("authorization").unwrap(), "Bearer abc==");
assert_eq!(headers.get("x-scope").unwrap(), "team-a");
assert_eq!(headers.len(), 2);
assert!(parse_otlp_headers("").unwrap().is_empty());
assert!(parse_otlp_headers("no-equals").is_err());
assert!(parse_otlp_headers("bad name=1").is_err());
}
}
+15 -5
View File
@@ -35,10 +35,10 @@ use std::{
};
use store::{
Deserialize, IterateParams, ValueKey,
write::{AlignedBytes, Archive, QueueClass, ValueClass},
write::{AlignedBytes, Archive, QueueClass, ValueClass, key::DeserializeBigEndian},
};
use trc::{AddContext, SpamEvent};
use utils::DomainPart;
use utils::{DomainPart, snowflake::SnowflakeIdGenerator};
impl Server {
pub async fn rcpt_resolve(
@@ -505,16 +505,26 @@ impl Server {
Ok(())
}
pub async fn total_queued_messages(&self) -> trc::Result<u64> {
// inbuxa: K-C10: also when the oldest message arrived. Queue ids are
// snowflakes, so the first key is the oldest message and its id holds
// the time, without reading any message.
pub async fn total_queued_messages(&self) -> trc::Result<(u64, Option<u64>)> {
let mut total = 0;
let mut oldest = None;
self.store()
.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Queue(QueueClass::Message(0))),
ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))),
)
.ascending()
.no_values(),
|_, _| {
|key, _| {
if total == 0 {
oldest = Some(SnowflakeIdGenerator::to_timestamp(
key.deserialize_be_u64(0)?,
));
}
total += 1;
Ok(true)
@@ -522,7 +532,7 @@ impl Server {
)
.await
.caused_by(trc::location!())
.map(|_| total)
.map(|_| (total, oldest))
}
}
@@ -19,7 +19,7 @@ impl Server {
let mut metrics = Vec::new();
// Add counters
for counter in Collector::collect_counters() {
for counter in Collector::collect_metric_counters() {
let mut metric = MetricFamily::default();
metric.set_name(metric_name(counter.id().as_str()));
metric.set_help(counter.id().description().into());
+12 -4
View File
@@ -597,14 +597,22 @@ impl ParseHttp for Server {
);
}
"healthz" => {
let check = path.next().unwrap_or_default();
// inbuxa: K-C10: liveness answers before the limit. The limit
// is kept in the in-memory store, which may be the data store,
// so with that down the check waited on it, the probe timed
// out, and the orchestrator restarted a process that was
// fine, every one at once. A fixed 200 costs nothing to serve.
if check == "live" {
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
}
// Limit anonymous requests
self.is_http_anonymous_request_allowed(session.remote_ip)
.await?;
match path.next().unwrap_or_default() {
"live" => {
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
}
match check {
"ready" => {
// inbuxa: ready only while the data store answers
// (a cached, time-limited read); liveness stays 200
+11 -2
View File
@@ -136,6 +136,11 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
);
}
// inbuxa: K-C10: nodes sharing a queue count it from the store
if server.core.storage.has_coordinator() {
Collector::count_queue_from_store_only();
}
// OTEL Push Metrics
if let Some(otel) = &server.core.metrics.otel {
OtelMetrics::enable_errors();
@@ -272,7 +277,7 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
if let Some(otel) = &server.core.metrics.otel {
queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics);
if roles.metrics_push {
if roles.metrics_push || otel.every_node {
let otel = otel.clone();
tokio::spawn(async move {
@@ -308,8 +313,12 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
// inbuxa: MON-7: the queue gauge from the queue itself,
// so it's right after a restart
match server.total_queued_messages().await {
Ok(total) => {
Ok((total, oldest)) => {
Collector::update_gauge(MetricType::QueueCount, total);
Collector::update_gauge(
MetricType::QueueOldestMessage,
oldest.unwrap_or_default(),
);
}
Err(err) => {
trc::error!(err.details("Failed to count queued messages"));
+9
View File
@@ -58,6 +58,7 @@ struct NodeClaim {
impl RegistryStoreInner {
pub(super) async fn acquire_node_id(&mut self) -> Result<(), String> {
let (node_id, slots) = NodeSlot::acquire(&self.store, &self.env_hostname).await?;
lease_renewed();
self.node_id.store(node_id, Ordering::Relaxed);
self.remember_hostname().await;
@@ -97,6 +98,12 @@ impl RegistryStoreInner {
}
}
// inbuxa: K-C10: the gauge an alert watches. A pod that keeps running but
// can't renew its lease (the store unreachable, say) stops moving it.
fn lease_renewed() {
trc::Collector::update_gauge(trc::MetricType::ClusterLeaseRenewed, now());
}
// INBUXA: #458, the file a node records its leased hostname in.
// It sits beside the local config file, or inside the directory when the
// path is one (as in the test harness).
@@ -146,6 +153,7 @@ impl RegistryStore {
})?;
self.0.node_id.store(new_node_id, Ordering::Relaxed);
SnowflakeIdGenerator::set_node_id(new_node_id as u64);
lease_renewed();
trc::event!(
Cluster(ClusterEvent::NodeIdReassigned),
@@ -173,6 +181,7 @@ impl RegistryStore {
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
lease_renewed();
self.release_renamed_lease(&slots).await
}
+29 -2
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use std::sync::atomic::{AtomicU64, Ordering};
@@ -31,9 +33,12 @@ impl AtomicGauge {
self.value.store(value, Ordering::Relaxed);
}
// inbuxa: K-C10: never below zero. A process can take off what another
// added (a message queued on one node, delivered on another), and a
// wrapped u64 read as 18446744073709552000 messages in the queue.
#[inline(always)]
pub fn decrement(&self) {
self.value.fetch_sub(1, Ordering::Relaxed);
self.subtract(1);
}
#[inline(always)]
@@ -48,10 +53,32 @@ impl AtomicGauge {
#[inline(always)]
pub fn subtract(&self, value: u64) {
self.value.fetch_sub(value, Ordering::Relaxed);
let _ = self
.value
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
Some(current.saturating_sub(value))
});
}
pub fn id(&self) -> MetricType {
self.id
}
}
#[cfg(test)]
mod tests {
use super::AtomicGauge;
use crate::MetricType;
#[test]
fn gauge_stops_at_zero() {
let gauge = AtomicGauge::new(MetricType::QueueCount);
gauge.increment();
gauge.decrement();
gauge.decrement();
assert_eq!(gauge.get(), 0);
gauge.add(3);
gauge.subtract(5);
assert_eq!(gauge.get(), 0);
}
}
+7 -2
View File
@@ -16,11 +16,13 @@
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228);
// 664 is auth.passkey-counter (passkeys spec, PK-18).
// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371
// are gauges for Kubernetes alerts (K-C10): queue.oldest-message,
// cluster.lease-renewed, tls.earliest-expiry.
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
// more than the highest id; the test below holds it to the variant list
pub const TOTAL_EVENT_COUNT: usize = 665;
pub const TOTAL_METRIC_COUNT: usize = 369;
pub const TOTAL_METRIC_COUNT: usize = 372;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum EventType {
@@ -1080,6 +1082,9 @@ pub enum MetricType {
DkimMailFromDomainMismatch = 353,
DkimNextDomainMismatch = 354,
DkimCustodyBreak = 368,
QueueOldestMessage = 369,
ClusterLeaseRenewed = 370,
TlsEarliestExpiry = 371,
DkimPublicKeyFetch = 355,
DkimPublicKeyMissing = 356,
DkimPublicKeyMultiple = 357,
+27
View File
@@ -5499,6 +5499,9 @@ impl MetricType {
b"telemetry.journal-error" => MetricType::TelemetryJournalError,
b"tls.handshake-error" => MetricType::TlsHandshakeError,
b"user.count" => MetricType::UserCount,
b"queue.oldest-message" => MetricType::QueueOldestMessage,
b"cluster.lease-renewed" => MetricType::ClusterLeaseRenewed,
b"tls.earliest-expiry" => MetricType::TlsEarliestExpiry,
}
.copied()
}
@@ -5886,6 +5889,9 @@ impl MetricType {
MetricType::TelemetryJournalError => "telemetry.journal-error",
MetricType::TlsHandshakeError => "tls.handshake-error",
MetricType::UserCount => "user.count",
MetricType::QueueOldestMessage => "queue.oldest-message",
MetricType::ClusterLeaseRenewed => "cluster.lease-renewed",
MetricType::TlsEarliestExpiry => "tls.earliest-expiry",
}
}
@@ -6260,6 +6266,9 @@ impl MetricType {
MetricType::TelemetryJournalError => 335,
MetricType::TlsHandshakeError => 336,
MetricType::UserCount => 25,
MetricType::QueueOldestMessage => 369,
MetricType::ClusterLeaseRenewed => 370,
MetricType::TlsEarliestExpiry => 371,
}
}
@@ -6634,6 +6643,9 @@ impl MetricType {
335 => Some(MetricType::TelemetryJournalError),
336 => Some(MetricType::TlsHandshakeError),
25 => Some(MetricType::UserCount),
369 => Some(MetricType::QueueOldestMessage),
370 => Some(MetricType::ClusterLeaseRenewed),
371 => Some(MetricType::TlsEarliestExpiry),
_ => None,
}
}
@@ -7369,6 +7381,15 @@ impl MetricType {
MetricType::TelemetryJournalError => "Journal collector error",
MetricType::TlsHandshakeError => "TLS handshake error",
MetricType::UserCount => "Total number of users",
MetricType::QueueOldestMessage => {
"When the oldest message in the queue arrived, as a Unix time (0: the queue is empty)"
}
MetricType::ClusterLeaseRenewed => {
"When this node last renewed its cluster lease, as a Unix time"
}
MetricType::TlsEarliestExpiry => {
"When the first certificate in use expires, as a Unix time (0: none)"
}
}
}
@@ -7743,6 +7764,9 @@ impl MetricType {
| MetricType::StoreBlobReadTime
| MetricType::StoreBlobWriteTime => "milliseconds",
MetricType::UserCount => "users",
MetricType::QueueOldestMessage
| MetricType::ClusterLeaseRenewed
| MetricType::TlsEarliestExpiry => "seconds",
}
}
@@ -8117,6 +8141,9 @@ impl MetricType {
MetricType::TelemetryJournalError,
MetricType::TlsHandshakeError,
MetricType::UserCount,
MetricType::QueueOldestMessage,
MetricType::ClusterLeaseRenewed,
MetricType::TlsEarliestExpiry,
]
}
}
+60 -5
View File
@@ -49,8 +49,18 @@ static DNS_LOOKUP_TIME: AtomicHistogram<12> =
static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory);
static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount);
// inbuxa: K-C10: in a cluster the queue gauge is only what the node that
// calculates metrics counted in the store. Each node seeing only its own
// queued and delivered messages, live changes there were noise: the core
// subtracted deliveries of messages other nodes had queued.
static QUEUE_COUNT_FROM_STORE: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount);
static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount);
// inbuxa: K-C10: gauges for alerts, set where the value is known
static QUEUE_OLDEST_MESSAGE: AtomicGauge = AtomicGauge::new(MetricType::QueueOldestMessage);
static CLUSTER_LEASE_RENEWED: AtomicGauge = AtomicGauge::new(MetricType::ClusterLeaseRenewed);
static TLS_EARLIEST_EXPIRY: AtomicGauge = AtomicGauge::new(MetricType::TlsEarliestExpiry);
const CONN_SMTP_IN: usize = 0;
const CONN_SMTP_OUT: usize = 1;
@@ -144,7 +154,9 @@ impl Collector {
conn.elapsed.observe(elapsed);
}
EventType::Delivery(DeliveryEvent::Completed) => {
QUEUE_COUNT.decrement();
if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) {
QUEUE_COUNT.decrement();
}
MESSAGE_DELIVERY_TIME.observe(elapsed);
}
EventType::Delivery(
@@ -167,11 +179,15 @@ impl Collector {
}
EventType::Queue(QueueEvent::MessageQueued) => {
MESSAGE_INCOMING_SIZE.observe(size);
QUEUE_COUNT.increment();
if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) {
QUEUE_COUNT.increment();
}
}
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => {
MESSAGE_SUBMISSION_SIZE.observe(size);
QUEUE_COUNT.increment();
if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) {
QUEUE_COUNT.increment();
}
}
EventType::Queue(QueueEvent::ReportQueued) => {
MESSAGE_OUT_REPORT_SIZE.observe(size);
@@ -209,6 +225,27 @@ impl Collector {
METRIC_INTERESTS.update(interests);
}
// inbuxa: K-C10: for a scrape, every counter that is collected, at zero
// too. Prometheus measures an increase from the first sample, so a
// counter that first shows up at 3 never reads as having risen.
pub fn collect_metric_counters() -> impl Iterator<Item = EventCounter> {
EVENT_COUNTERS
.inner()
.iter()
.enumerate()
.filter_map(|(event_id, value)| {
let value = value.load(Ordering::Relaxed);
if value > 0 || METRIC_INTERESTS.get(event_id) {
Some(EventCounter {
id: EventType::from_id(event_id as u16)?,
value,
})
} else {
None
}
})
}
pub fn collect_counters() -> impl Iterator<Item = EventCounter> {
EVENT_COUNTERS
.inner()
@@ -229,8 +266,15 @@ impl Collector {
// inbuxa: MON-6: one edition, every gauge
pub fn collect_gauges() -> impl Iterator<Item = &'static AtomicGauge> {
static GAUGES: &[&AtomicGauge] =
&[&SERVER_MEMORY, &QUEUE_COUNT, &USER_COUNT, &DOMAIN_COUNT];
static GAUGES: &[&AtomicGauge] = &[
&SERVER_MEMORY,
&QUEUE_COUNT,
&USER_COUNT,
&DOMAIN_COUNT,
&QUEUE_OLDEST_MESSAGE,
&CLUSTER_LEASE_RENEWED,
&TLS_EARLIEST_EXPIRY,
];
GAUGES
.iter()
@@ -306,16 +350,27 @@ impl Collector {
MetricType::SieveRequestTime => CONNECTION_METRICS[CONN_SIEVE].elapsed.average(),
MetricType::UserCount => USER_COUNT.get() as f64,
MetricType::DomainCount => DOMAIN_COUNT.get() as f64,
MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.get() as f64,
MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.get() as f64,
MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.get() as f64,
_ => EVENT_COUNTERS.get(metric_type.event_id()) as f64,
}
}
/// In a cluster: the queue gauge changes only when the queue is counted.
pub fn count_queue_from_store_only() {
QUEUE_COUNT_FROM_STORE.store(true, Ordering::Relaxed);
}
pub fn update_gauge(metric_type: MetricType, value: u64) {
match metric_type {
MetricType::ServerMemory => SERVER_MEMORY.set(value),
MetricType::QueueCount => QUEUE_COUNT.set(value),
MetricType::UserCount => USER_COUNT.set(value),
MetricType::DomainCount => DOMAIN_COUNT.set(value),
MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.set(value),
MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.set(value),
MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.set(value),
_ => {}
}
}