Metrics for Kubernetes: scraping and OTLP from the environment, alert gauges, four metric fixes #488
No files matched your search
@@ -288,6 +288,18 @@ pub(crate) async fn parse_certificates(
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: K-C10: when the first name served runs out, for an alert that
|
||||
// warns before a renewal that isn't happening matters
|
||||
trc::Collector::update_gauge(
|
||||
trc::MetricType::TlsEarliestExpiry,
|
||||
certs_expirations
|
||||
.iter()
|
||||
.filter(|(name, _)| name.as_str() != "*")
|
||||
.map(|(_, expires)| (*expires).max(0) as u64)
|
||||
.min()
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
|
||||
// No default certificate set: pick one now, once per load, so handshakes
|
||||
// without SNI or for an unknown name get a predictable certificate.
|
||||
if !certificates.contains_key("*") && certificates.len() > 1 {
|
||||
|
||||
@@ -30,6 +30,13 @@ pub struct Storage {
|
||||
pub directories: IdMap<Directory>,
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
// inbuxa: K-C10: whether nodes share this store as a cluster
|
||||
pub fn has_coordinator(&self) -> bool {
|
||||
!matches!(self.coordinator, Coordinator::None)
|
||||
}
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
pub async fn parse(bp: &mut Bootstrap) -> Self {
|
||||
let memory = InMemoryStore::build(bp).await.unwrap_or_default();
|
||||
|
||||
@@ -12,7 +12,7 @@ use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use hyper::HeaderMap;
|
||||
use opentelemetry::{InstrumentationScope, KeyValue};
|
||||
use opentelemetry_otlp::{
|
||||
LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig,
|
||||
LogExporter, MetricExporter, SpanExporter, WithExportConfig, WithHttpConfig, WithTonicConfig,
|
||||
};
|
||||
use opentelemetry_sdk::{Resource, metrics::Temporality};
|
||||
use opentelemetry_semantic_conventions::resource::SERVICE_VERSION;
|
||||
@@ -71,6 +71,11 @@ pub struct OtelMetrics {
|
||||
pub instrumentation: InstrumentationScope,
|
||||
pub exporter: MetricExporter,
|
||||
pub interval: Duration,
|
||||
// inbuxa: K-C10: pushed by every node, whatever its role. Each process
|
||||
// counts only its own traffic; an exporter set from the environment
|
||||
// names its pod in the resource (OTEL_RESOURCE_ATTRIBUTES), so every
|
||||
// pod pushes and the collector adds them up.
|
||||
pub every_node: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -613,8 +618,19 @@ impl Metrics {
|
||||
.with_version(types::brand_version!())
|
||||
.build();
|
||||
|
||||
// inbuxa: K-C10: scraping and OTLP set by the deployment (Kubernetes),
|
||||
// over what the store says. A pod reads the scrape password from a
|
||||
// mounted file, and the OTLP exporter from the standard OTEL_ variables.
|
||||
let env_prometheus = prometheus_from_env(bp);
|
||||
let env_otel = if matches!(metrics.open_telemetry, structs::MetricsOtel::Disabled) {
|
||||
otel_from_env(bp, &resource, &instrumentation)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Metrics {
|
||||
prometheus: match metrics.prometheus {
|
||||
_ if env_prometheus.is_some() => env_prometheus,
|
||||
MetricsPrometheus::Enabled(prom) => {
|
||||
let secret = prom
|
||||
.auth_secret
|
||||
@@ -669,6 +685,7 @@ impl Metrics {
|
||||
interval: otel.interval.into_inner(),
|
||||
resource,
|
||||
instrumentation,
|
||||
every_node: false,
|
||||
})),
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
@@ -695,6 +712,7 @@ impl Metrics {
|
||||
interval: otel.interval.into_inner(),
|
||||
resource,
|
||||
instrumentation,
|
||||
every_node: false,
|
||||
})),
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
@@ -705,7 +723,7 @@ impl Metrics {
|
||||
}
|
||||
}
|
||||
}
|
||||
structs::MetricsOtel::Disabled => None,
|
||||
structs::MetricsOtel::Disabled => env_otel,
|
||||
},
|
||||
log_path: bp
|
||||
.list_infallible::<Tracer>()
|
||||
@@ -833,6 +851,189 @@ fn apply_metrics(
|
||||
}
|
||||
}
|
||||
|
||||
/// Prometheus scraping from INBUXA_METRICS_PROMETHEUS_USERNAME and
|
||||
/// INBUXA_METRICS_PROMETHEUS_SECRET_FILE. Both or neither: the endpoint is
|
||||
/// never opened without a password this way.
|
||||
fn prometheus_from_env(bp: &mut Bootstrap) -> Option<PrometheusMetrics> {
|
||||
let username = types::branding::env_var("METRICS_PROMETHEUS_USERNAME").ok();
|
||||
let secret_file = types::branding::env_var("METRICS_PROMETHEUS_SECRET_FILE").ok();
|
||||
let (username, secret_file) = match (username, secret_file) {
|
||||
(None, None) => return None,
|
||||
(Some(username), Some(secret_file)) if !username.is_empty() && !secret_file.is_empty() => {
|
||||
(username, secret_file)
|
||||
}
|
||||
_ => {
|
||||
bp.build_error(
|
||||
ObjectType::Metrics.singleton(),
|
||||
"Set both INBUXA_METRICS_PROMETHEUS_USERNAME and \
|
||||
INBUXA_METRICS_PROMETHEUS_SECRET_FILE, or neither. The setting in the store \
|
||||
applies.",
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match std::fs::read_to_string(&secret_file) {
|
||||
Ok(secret) if !secret.trim_end_matches(['\r', '\n']).is_empty() => {
|
||||
let secret = secret.trim_end_matches(['\r', '\n']);
|
||||
Some(PrometheusMetrics {
|
||||
auth: Some(STANDARD.encode(format!("{username}:{secret}"))),
|
||||
})
|
||||
}
|
||||
Ok(_) => {
|
||||
bp.build_error(
|
||||
ObjectType::Metrics.singleton(),
|
||||
format!("{secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE) is empty"),
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
ObjectType::Metrics.singleton(),
|
||||
format!(
|
||||
"Unable to read {secret_file} (INBUXA_METRICS_PROMETHEUS_SECRET_FILE): {err}"
|
||||
),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// OTLP metrics from the standard variables, when the store has none set:
|
||||
/// OTEL_EXPORTER_OTLP_METRICS_ENDPOINT or OTEL_EXPORTER_OTLP_ENDPOINT turns
|
||||
/// it on; the exporter reads those, the _HEADERS and _TIMEOUT variables
|
||||
/// itself. OTEL_EXPORTER_OTLP_(METRICS_)PROTOCOL picks grpc or http/protobuf
|
||||
/// (the default), and OTEL_METRIC_EXPORT_INTERVAL the interval in
|
||||
/// milliseconds (60000). OTEL_RESOURCE_ATTRIBUTES is added to the resource.
|
||||
/// Headers that carry a token can come from a file instead of the
|
||||
/// environment: INBUXA_OTLP_HEADERS_FILE, key=value pairs separated by
|
||||
/// commas or lines.
|
||||
fn otel_from_env(
|
||||
bp: &mut Bootstrap,
|
||||
resource: &Resource,
|
||||
instrumentation: &InstrumentationScope,
|
||||
) -> Option<Arc<OtelMetrics>> {
|
||||
let var = |name: &str| std::env::var(name).ok().filter(|value| !value.is_empty());
|
||||
var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT").or_else(|| var("OTEL_EXPORTER_OTLP_ENDPOINT"))?;
|
||||
let protocol = var("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL")
|
||||
.or_else(|| var("OTEL_EXPORTER_OTLP_PROTOCOL"))
|
||||
.unwrap_or_else(|| "http/protobuf".to_string());
|
||||
let interval = match var("OTEL_METRIC_EXPORT_INTERVAL").map(|value| value.parse::<u64>()) {
|
||||
None => Duration::from_secs(60),
|
||||
Some(Ok(millis)) if millis >= 1000 => Duration::from_millis(millis),
|
||||
Some(_) => {
|
||||
bp.build_warning(
|
||||
ObjectType::Metrics.singleton(),
|
||||
"OTEL_METRIC_EXPORT_INTERVAL is not a number of milliseconds of at least 1000; \
|
||||
exporting every 60 seconds.",
|
||||
);
|
||||
Duration::from_secs(60)
|
||||
}
|
||||
};
|
||||
let headers = match types::branding::env_var("OTLP_HEADERS_FILE") {
|
||||
Ok(path) if !path.is_empty() => match otlp_headers_file(&path) {
|
||||
Ok(headers) => headers,
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
ObjectType::Metrics.singleton(),
|
||||
format!("{path} (INBUXA_OTLP_HEADERS_FILE): {err}. No OTLP metrics."),
|
||||
);
|
||||
return None;
|
||||
}
|
||||
},
|
||||
_ => HeaderMap::new(),
|
||||
};
|
||||
let exporter = match protocol.as_str() {
|
||||
"grpc" => {
|
||||
let builder = MetricExporter::builder()
|
||||
.with_temporality(Temporality::Delta)
|
||||
.with_tonic();
|
||||
if headers.is_empty() {
|
||||
builder.build()
|
||||
} else {
|
||||
builder
|
||||
.with_metadata(
|
||||
opentelemetry_otlp::tonic_types::metadata::MetadataMap::from_headers(
|
||||
headers,
|
||||
),
|
||||
)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
"http/protobuf" => {
|
||||
let builder = MetricExporter::builder()
|
||||
.with_temporality(Temporality::Delta)
|
||||
.with_http();
|
||||
if headers.is_empty() {
|
||||
builder.build()
|
||||
} else {
|
||||
builder
|
||||
.with_headers(
|
||||
headers
|
||||
.iter()
|
||||
.filter_map(|(k, v)| {
|
||||
Some((k.to_string(), v.to_str().ok()?.to_string()))
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
bp.build_error(
|
||||
ObjectType::Metrics.singleton(),
|
||||
format!(
|
||||
"OTEL_EXPORTER_OTLP_PROTOCOL is {protocol:?}; expected grpc or http/protobuf. \
|
||||
No OTLP metrics."
|
||||
),
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match exporter {
|
||||
Ok(exporter) => Some(Arc::new(OtelMetrics {
|
||||
exporter,
|
||||
interval,
|
||||
resource: resource.clone(),
|
||||
instrumentation: instrumentation.clone(),
|
||||
every_node: true,
|
||||
})),
|
||||
Err(err) => {
|
||||
bp.build_error(
|
||||
ObjectType::Metrics.singleton(),
|
||||
format!(
|
||||
"Failed to build the OpenTelemetry metrics exporter from OTEL_ variables: {err}"
|
||||
),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn otlp_headers_file(path: &str) -> Result<HeaderMap, String> {
|
||||
let contents = std::fs::read_to_string(path).map_err(|err| err.to_string())?;
|
||||
parse_otlp_headers(&contents)
|
||||
}
|
||||
|
||||
fn parse_otlp_headers(contents: &str) -> Result<HeaderMap, String> {
|
||||
let mut headers = HeaderMap::new();
|
||||
for pair in contents
|
||||
.split([',', '\n'])
|
||||
.map(str::trim)
|
||||
.filter(|pair| !pair.is_empty())
|
||||
{
|
||||
let (name, value) = pair
|
||||
.split_once('=')
|
||||
.ok_or_else(|| format!("{pair:?} is not key=value"))?;
|
||||
let name =
|
||||
hyper::header::HeaderName::from_bytes(name.trim().to_ascii_lowercase().as_bytes())
|
||||
.map_err(|_| format!("{:?} is not a header name", name.trim()))?;
|
||||
let value = hyper::header::HeaderValue::from_str(value.trim())
|
||||
.map_err(|_| format!("the value for {name} is not a header value"))?;
|
||||
headers.append(name, value);
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for OtelMetrics {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("OtelMetrics")
|
||||
@@ -900,3 +1101,20 @@ mod tests {
|
||||
assert!(!wants(own, Level::Trace, &[own]));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod inbuxa_env_tests {
|
||||
use super::parse_otlp_headers;
|
||||
|
||||
#[test]
|
||||
fn otlp_headers_file_pairs() {
|
||||
let headers =
|
||||
parse_otlp_headers("Authorization=Bearer abc==\nx-scope = team-a,\n").unwrap();
|
||||
assert_eq!(headers.get("authorization").unwrap(), "Bearer abc==");
|
||||
assert_eq!(headers.get("x-scope").unwrap(), "team-a");
|
||||
assert_eq!(headers.len(), 2);
|
||||
assert!(parse_otlp_headers("").unwrap().is_empty());
|
||||
assert!(parse_otlp_headers("no-equals").is_err());
|
||||
assert!(parse_otlp_headers("bad name=1").is_err());
|
||||
}
|
||||
}
|
||||
@@ -35,10 +35,10 @@ use std::{
|
||||
};
|
||||
use store::{
|
||||
Deserialize, IterateParams, ValueKey,
|
||||
write::{AlignedBytes, Archive, QueueClass, ValueClass},
|
||||
write::{AlignedBytes, Archive, QueueClass, ValueClass, key::DeserializeBigEndian},
|
||||
};
|
||||
use trc::{AddContext, SpamEvent};
|
||||
use utils::DomainPart;
|
||||
use utils::{DomainPart, snowflake::SnowflakeIdGenerator};
|
||||
|
||||
impl Server {
|
||||
pub async fn rcpt_resolve(
|
||||
@@ -505,16 +505,26 @@ impl Server {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn total_queued_messages(&self) -> trc::Result<u64> {
|
||||
// inbuxa: K-C10: also when the oldest message arrived. Queue ids are
|
||||
// snowflakes, so the first key is the oldest message and its id holds
|
||||
// the time, without reading any message.
|
||||
pub async fn total_queued_messages(&self) -> trc::Result<(u64, Option<u64>)> {
|
||||
let mut total = 0;
|
||||
let mut oldest = None;
|
||||
self.store()
|
||||
.iterate(
|
||||
IterateParams::new(
|
||||
ValueKey::from(ValueClass::Queue(QueueClass::Message(0))),
|
||||
ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))),
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|_, _| {
|
||||
|key, _| {
|
||||
if total == 0 {
|
||||
oldest = Some(SnowflakeIdGenerator::to_timestamp(
|
||||
key.deserialize_be_u64(0)?,
|
||||
));
|
||||
}
|
||||
total += 1;
|
||||
|
||||
Ok(true)
|
||||
@@ -522,7 +532,7 @@ impl Server {
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|_| total)
|
||||
.map(|_| (total, oldest))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ impl Server {
|
||||
let mut metrics = Vec::new();
|
||||
|
||||
// Add counters
|
||||
for counter in Collector::collect_counters() {
|
||||
for counter in Collector::collect_metric_counters() {
|
||||
let mut metric = MetricFamily::default();
|
||||
metric.set_name(metric_name(counter.id().as_str()));
|
||||
metric.set_help(counter.id().description().into());
|
||||
|
||||
@@ -597,14 +597,22 @@ impl ParseHttp for Server {
|
||||
);
|
||||
}
|
||||
"healthz" => {
|
||||
let check = path.next().unwrap_or_default();
|
||||
|
||||
// inbuxa: K-C10: liveness answers before the limit. The limit
|
||||
// is kept in the in-memory store, which may be the data store,
|
||||
// so with that down the check waited on it, the probe timed
|
||||
// out, and the orchestrator restarted a process that was
|
||||
// fine, every one at once. A fixed 200 costs nothing to serve.
|
||||
if check == "live" {
|
||||
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
|
||||
}
|
||||
|
||||
// Limit anonymous requests
|
||||
self.is_http_anonymous_request_allowed(session.remote_ip)
|
||||
.await?;
|
||||
|
||||
match path.next().unwrap_or_default() {
|
||||
"live" => {
|
||||
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
|
||||
}
|
||||
match check {
|
||||
"ready" => {
|
||||
// inbuxa: ready only while the data store answers
|
||||
// (a cached, time-limited read); liveness stays 200
|
||||
|
||||
@@ -136,6 +136,11 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
);
|
||||
}
|
||||
|
||||
// inbuxa: K-C10: nodes sharing a queue count it from the store
|
||||
if server.core.storage.has_coordinator() {
|
||||
Collector::count_queue_from_store_only();
|
||||
}
|
||||
|
||||
// OTEL Push Metrics
|
||||
if let Some(otel) = &server.core.metrics.otel {
|
||||
OtelMetrics::enable_errors();
|
||||
@@ -272,7 +277,7 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
if let Some(otel) = &server.core.metrics.otel {
|
||||
queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics);
|
||||
|
||||
if roles.metrics_push {
|
||||
if roles.metrics_push || otel.every_node {
|
||||
let otel = otel.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
@@ -308,8 +313,12 @@ pub fn spawn_task_scheduler(inner: Arc<Inner>) {
|
||||
// inbuxa: MON-7: the queue gauge from the queue itself,
|
||||
// so it's right after a restart
|
||||
match server.total_queued_messages().await {
|
||||
Ok(total) => {
|
||||
Ok((total, oldest)) => {
|
||||
Collector::update_gauge(MetricType::QueueCount, total);
|
||||
Collector::update_gauge(
|
||||
MetricType::QueueOldestMessage,
|
||||
oldest.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err.details("Failed to count queued messages"));
|
||||
|
||||
@@ -58,6 +58,7 @@ struct NodeClaim {
|
||||
impl RegistryStoreInner {
|
||||
pub(super) async fn acquire_node_id(&mut self) -> Result<(), String> {
|
||||
let (node_id, slots) = NodeSlot::acquire(&self.store, &self.env_hostname).await?;
|
||||
lease_renewed();
|
||||
self.node_id.store(node_id, Ordering::Relaxed);
|
||||
self.remember_hostname().await;
|
||||
|
||||
@@ -97,6 +98,12 @@ impl RegistryStoreInner {
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: K-C10: the gauge an alert watches. A pod that keeps running but
|
||||
// can't renew its lease (the store unreachable, say) stops moving it.
|
||||
fn lease_renewed() {
|
||||
trc::Collector::update_gauge(trc::MetricType::ClusterLeaseRenewed, now());
|
||||
}
|
||||
|
||||
// INBUXA: #458, the file a node records its leased hostname in.
|
||||
// It sits beside the local config file, or inside the directory when the
|
||||
// path is one (as in the test harness).
|
||||
@@ -146,6 +153,7 @@ impl RegistryStore {
|
||||
})?;
|
||||
self.0.node_id.store(new_node_id, Ordering::Relaxed);
|
||||
SnowflakeIdGenerator::set_node_id(new_node_id as u64);
|
||||
lease_renewed();
|
||||
|
||||
trc::event!(
|
||||
Cluster(ClusterEvent::NodeIdReassigned),
|
||||
@@ -173,6 +181,7 @@ impl RegistryStore {
|
||||
.write(batch.build_all())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
lease_renewed();
|
||||
|
||||
self.release_renamed_lease(&slots).await
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
@@ -31,9 +33,12 @@ impl AtomicGauge {
|
||||
self.value.store(value, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// inbuxa: K-C10: never below zero. A process can take off what another
|
||||
// added (a message queued on one node, delivered on another), and a
|
||||
// wrapped u64 read as 18446744073709552000 messages in the queue.
|
||||
#[inline(always)]
|
||||
pub fn decrement(&self) {
|
||||
self.value.fetch_sub(1, Ordering::Relaxed);
|
||||
self.subtract(1);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
@@ -48,10 +53,32 @@ impl AtomicGauge {
|
||||
|
||||
#[inline(always)]
|
||||
pub fn subtract(&self, value: u64) {
|
||||
self.value.fetch_sub(value, Ordering::Relaxed);
|
||||
let _ = self
|
||||
.value
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_sub(value))
|
||||
});
|
||||
}
|
||||
|
||||
pub fn id(&self) -> MetricType {
|
||||
self.id
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::AtomicGauge;
|
||||
use crate::MetricType;
|
||||
|
||||
#[test]
|
||||
fn gauge_stops_at_zero() {
|
||||
let gauge = AtomicGauge::new(MetricType::QueueCount);
|
||||
gauge.increment();
|
||||
gauge.decrement();
|
||||
gauge.decrement();
|
||||
assert_eq!(gauge.get(), 0);
|
||||
gauge.add(3);
|
||||
gauge.subtract(5);
|
||||
assert_eq!(gauge.get(), 0);
|
||||
}
|
||||
}
|
||||
@@ -16,11 +16,13 @@
|
||||
// 650 to 659 are inbuxa Relay's (relay spec, RL-23, RL-T23); 660 is
|
||||
// store.blob-read-through; 661 is smtp.directory-no-answer (#364); 662 is
|
||||
// spam.result (#397); 663 is queue.message-subject (inbuxa-admin #228);
|
||||
// 664 is auth.passkey-counter (passkeys spec, PK-18).
|
||||
// 664 is auth.passkey-counter (passkeys spec, PK-18). Metrics 369 to 371
|
||||
// are gauges for Kubernetes alerts (K-C10): queue.oldest-message,
|
||||
// cluster.lease-renewed, tls.earliest-expiry.
|
||||
// TOTAL_EVENT_COUNT sizes the collector's per-event tables, so it is one
|
||||
// more than the highest id; the test below holds it to the variant list
|
||||
pub const TOTAL_EVENT_COUNT: usize = 665;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||
pub const TOTAL_METRIC_COUNT: usize = 372;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum EventType {
|
||||
@@ -1080,6 +1082,9 @@ pub enum MetricType {
|
||||
DkimMailFromDomainMismatch = 353,
|
||||
DkimNextDomainMismatch = 354,
|
||||
DkimCustodyBreak = 368,
|
||||
QueueOldestMessage = 369,
|
||||
ClusterLeaseRenewed = 370,
|
||||
TlsEarliestExpiry = 371,
|
||||
DkimPublicKeyFetch = 355,
|
||||
DkimPublicKeyMissing = 356,
|
||||
DkimPublicKeyMultiple = 357,
|
||||
|
||||
@@ -5499,6 +5499,9 @@ impl MetricType {
|
||||
b"telemetry.journal-error" => MetricType::TelemetryJournalError,
|
||||
b"tls.handshake-error" => MetricType::TlsHandshakeError,
|
||||
b"user.count" => MetricType::UserCount,
|
||||
b"queue.oldest-message" => MetricType::QueueOldestMessage,
|
||||
b"cluster.lease-renewed" => MetricType::ClusterLeaseRenewed,
|
||||
b"tls.earliest-expiry" => MetricType::TlsEarliestExpiry,
|
||||
}
|
||||
.copied()
|
||||
}
|
||||
@@ -5886,6 +5889,9 @@ impl MetricType {
|
||||
MetricType::TelemetryJournalError => "telemetry.journal-error",
|
||||
MetricType::TlsHandshakeError => "tls.handshake-error",
|
||||
MetricType::UserCount => "user.count",
|
||||
MetricType::QueueOldestMessage => "queue.oldest-message",
|
||||
MetricType::ClusterLeaseRenewed => "cluster.lease-renewed",
|
||||
MetricType::TlsEarliestExpiry => "tls.earliest-expiry",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6260,6 +6266,9 @@ impl MetricType {
|
||||
MetricType::TelemetryJournalError => 335,
|
||||
MetricType::TlsHandshakeError => 336,
|
||||
MetricType::UserCount => 25,
|
||||
MetricType::QueueOldestMessage => 369,
|
||||
MetricType::ClusterLeaseRenewed => 370,
|
||||
MetricType::TlsEarliestExpiry => 371,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6634,6 +6643,9 @@ impl MetricType {
|
||||
335 => Some(MetricType::TelemetryJournalError),
|
||||
336 => Some(MetricType::TlsHandshakeError),
|
||||
25 => Some(MetricType::UserCount),
|
||||
369 => Some(MetricType::QueueOldestMessage),
|
||||
370 => Some(MetricType::ClusterLeaseRenewed),
|
||||
371 => Some(MetricType::TlsEarliestExpiry),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -7369,6 +7381,15 @@ impl MetricType {
|
||||
MetricType::TelemetryJournalError => "Journal collector error",
|
||||
MetricType::TlsHandshakeError => "TLS handshake error",
|
||||
MetricType::UserCount => "Total number of users",
|
||||
MetricType::QueueOldestMessage => {
|
||||
"When the oldest message in the queue arrived, as a Unix time (0: the queue is empty)"
|
||||
}
|
||||
MetricType::ClusterLeaseRenewed => {
|
||||
"When this node last renewed its cluster lease, as a Unix time"
|
||||
}
|
||||
MetricType::TlsEarliestExpiry => {
|
||||
"When the first certificate in use expires, as a Unix time (0: none)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7743,6 +7764,9 @@ impl MetricType {
|
||||
| MetricType::StoreBlobReadTime
|
||||
| MetricType::StoreBlobWriteTime => "milliseconds",
|
||||
MetricType::UserCount => "users",
|
||||
MetricType::QueueOldestMessage
|
||||
| MetricType::ClusterLeaseRenewed
|
||||
| MetricType::TlsEarliestExpiry => "seconds",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8117,6 +8141,9 @@ impl MetricType {
|
||||
MetricType::TelemetryJournalError,
|
||||
MetricType::TlsHandshakeError,
|
||||
MetricType::UserCount,
|
||||
MetricType::QueueOldestMessage,
|
||||
MetricType::ClusterLeaseRenewed,
|
||||
MetricType::TlsEarliestExpiry,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,8 +49,18 @@ static DNS_LOOKUP_TIME: AtomicHistogram<12> =
|
||||
|
||||
static SERVER_MEMORY: AtomicGauge = AtomicGauge::new(MetricType::ServerMemory);
|
||||
static QUEUE_COUNT: AtomicGauge = AtomicGauge::new(MetricType::QueueCount);
|
||||
// inbuxa: K-C10: in a cluster the queue gauge is only what the node that
|
||||
// calculates metrics counted in the store. Each node seeing only its own
|
||||
// queued and delivered messages, live changes there were noise: the core
|
||||
// subtracted deliveries of messages other nodes had queued.
|
||||
static QUEUE_COUNT_FROM_STORE: std::sync::atomic::AtomicBool =
|
||||
std::sync::atomic::AtomicBool::new(false);
|
||||
static USER_COUNT: AtomicGauge = AtomicGauge::new(MetricType::UserCount);
|
||||
static DOMAIN_COUNT: AtomicGauge = AtomicGauge::new(MetricType::DomainCount);
|
||||
// inbuxa: K-C10: gauges for alerts, set where the value is known
|
||||
static QUEUE_OLDEST_MESSAGE: AtomicGauge = AtomicGauge::new(MetricType::QueueOldestMessage);
|
||||
static CLUSTER_LEASE_RENEWED: AtomicGauge = AtomicGauge::new(MetricType::ClusterLeaseRenewed);
|
||||
static TLS_EARLIEST_EXPIRY: AtomicGauge = AtomicGauge::new(MetricType::TlsEarliestExpiry);
|
||||
|
||||
const CONN_SMTP_IN: usize = 0;
|
||||
const CONN_SMTP_OUT: usize = 1;
|
||||
@@ -144,7 +154,9 @@ impl Collector {
|
||||
conn.elapsed.observe(elapsed);
|
||||
}
|
||||
EventType::Delivery(DeliveryEvent::Completed) => {
|
||||
QUEUE_COUNT.decrement();
|
||||
if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) {
|
||||
QUEUE_COUNT.decrement();
|
||||
}
|
||||
MESSAGE_DELIVERY_TIME.observe(elapsed);
|
||||
}
|
||||
EventType::Delivery(
|
||||
@@ -167,11 +179,15 @@ impl Collector {
|
||||
}
|
||||
EventType::Queue(QueueEvent::MessageQueued) => {
|
||||
MESSAGE_INCOMING_SIZE.observe(size);
|
||||
QUEUE_COUNT.increment();
|
||||
if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) {
|
||||
QUEUE_COUNT.increment();
|
||||
}
|
||||
}
|
||||
EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => {
|
||||
MESSAGE_SUBMISSION_SIZE.observe(size);
|
||||
QUEUE_COUNT.increment();
|
||||
if !QUEUE_COUNT_FROM_STORE.load(Ordering::Relaxed) {
|
||||
QUEUE_COUNT.increment();
|
||||
}
|
||||
}
|
||||
EventType::Queue(QueueEvent::ReportQueued) => {
|
||||
MESSAGE_OUT_REPORT_SIZE.observe(size);
|
||||
@@ -209,6 +225,27 @@ impl Collector {
|
||||
METRIC_INTERESTS.update(interests);
|
||||
}
|
||||
|
||||
// inbuxa: K-C10: for a scrape, every counter that is collected, at zero
|
||||
// too. Prometheus measures an increase from the first sample, so a
|
||||
// counter that first shows up at 3 never reads as having risen.
|
||||
pub fn collect_metric_counters() -> impl Iterator<Item = EventCounter> {
|
||||
EVENT_COUNTERS
|
||||
.inner()
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(event_id, value)| {
|
||||
let value = value.load(Ordering::Relaxed);
|
||||
if value > 0 || METRIC_INTERESTS.get(event_id) {
|
||||
Some(EventCounter {
|
||||
id: EventType::from_id(event_id as u16)?,
|
||||
value,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn collect_counters() -> impl Iterator<Item = EventCounter> {
|
||||
EVENT_COUNTERS
|
||||
.inner()
|
||||
@@ -229,8 +266,15 @@ impl Collector {
|
||||
|
||||
// inbuxa: MON-6: one edition, every gauge
|
||||
pub fn collect_gauges() -> impl Iterator<Item = &'static AtomicGauge> {
|
||||
static GAUGES: &[&AtomicGauge] =
|
||||
&[&SERVER_MEMORY, &QUEUE_COUNT, &USER_COUNT, &DOMAIN_COUNT];
|
||||
static GAUGES: &[&AtomicGauge] = &[
|
||||
&SERVER_MEMORY,
|
||||
&QUEUE_COUNT,
|
||||
&USER_COUNT,
|
||||
&DOMAIN_COUNT,
|
||||
&QUEUE_OLDEST_MESSAGE,
|
||||
&CLUSTER_LEASE_RENEWED,
|
||||
&TLS_EARLIEST_EXPIRY,
|
||||
];
|
||||
|
||||
GAUGES
|
||||
.iter()
|
||||
@@ -306,16 +350,27 @@ impl Collector {
|
||||
MetricType::SieveRequestTime => CONNECTION_METRICS[CONN_SIEVE].elapsed.average(),
|
||||
MetricType::UserCount => USER_COUNT.get() as f64,
|
||||
MetricType::DomainCount => DOMAIN_COUNT.get() as f64,
|
||||
MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.get() as f64,
|
||||
MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.get() as f64,
|
||||
MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.get() as f64,
|
||||
_ => EVENT_COUNTERS.get(metric_type.event_id()) as f64,
|
||||
}
|
||||
}
|
||||
|
||||
/// In a cluster: the queue gauge changes only when the queue is counted.
|
||||
pub fn count_queue_from_store_only() {
|
||||
QUEUE_COUNT_FROM_STORE.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn update_gauge(metric_type: MetricType, value: u64) {
|
||||
match metric_type {
|
||||
MetricType::ServerMemory => SERVER_MEMORY.set(value),
|
||||
MetricType::QueueCount => QUEUE_COUNT.set(value),
|
||||
MetricType::UserCount => USER_COUNT.set(value),
|
||||
MetricType::DomainCount => DOMAIN_COUNT.set(value),
|
||||
MetricType::QueueOldestMessage => QUEUE_OLDEST_MESSAGE.set(value),
|
||||
MetricType::ClusterLeaseRenewed => CLUSTER_LEASE_RENEWED.set(value),
|
||||
MetricType::TlsEarliestExpiry => TLS_EARLIEST_EXPIRY.set(value),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user