Metrics: scraping, alert rules, OTLP and a Grafana dashboard (K-C10) #27

Merged
jcoffey-dev merged 4 commits from feat/observability into main 2026-11-19 21:06:52 +00:00
10 changed files with 1345 additions and 0 deletions

No files matched your search

+11
View File
@@ -78,6 +78,17 @@ runs, pinned by digest.
status names the part and why. The server container keeps its last log
lines as its termination message too. Needs server 27.1-rc.33: an older
server doesn't know `--check`.
- Metrics (`server.metrics`, off by default). `prometheus`: every server
pod serves `/metrics/prometheus` behind a username and a password the pods
read as a file, and a headless Service lists them all, with a
ServiceMonitor that scrapes each pod and labels its group. `rules`:
PrometheusRules for a pod down, a cluster lease not renewed, a large or
stuck queue, deliveries bouncing, a certificate about to expire, a slow
data store, and (from kube-state-metrics) restarts and flapping readiness.
`otlp`: push to an OpenTelemetry collector, the token from a Secret read as
a file. `grafanaDashboard`: a dashboard for Grafana's sidecar.
`networkPolicy.metricsFrom` lets Prometheus through when `httpFrom` is
narrowed. Needs server 27.1-rc.34.
- `networkPolicy`, on by default: every pod starts from deny all, in and
out, and gets only what it needs. The server takes mail on its listeners
from anywhere and connects anywhere; `networkPolicy.httpFrom` and
+14
View File
@@ -59,6 +59,17 @@ server:
mail:
loadBalancerClass: example.org/lb
annotations: {example.org/lb-pool: mail}
metrics:
prometheus:
enabled: true
serviceMonitor: {enabled: true, interval: 15s, labels: {release: kube-prometheus}}
rules: {enabled: true, labels: {release: kube-prometheus}, certificateDays: 21}
otlp:
enabled: true
endpoint: http://otel-collector.monitoring:4317
protocol: grpc
headersSecret: otel-headers
grafanaDashboard: {enabled: true}
seed:
enabled: true
request:
@@ -103,6 +114,9 @@ networkPolicy:
httpFrom:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: gateway}
metricsFrom:
- namespaceSelector:
matchLabels: {kubernetes.io/metadata.name: monitoring}
serverEgress:
- to:
- namespaceSelector:
+571
View File
@@ -0,0 +1,571 @@
{
"title": "inbuxa",
"uid": "inbuxa-server",
"tags": [
"inbuxa",
"mail"
],
"schemaVersion": 39,
"version": 1,
"editable": true,
"time": {
"from": "now-6h",
"to": "now"
},
"refresh": "1m",
"timezone": "browser",
"templating": {
"list": [
{
"name": "datasource",
"type": "datasource",
"query": "prometheus",
"label": "Data source"
},
{
"name": "namespace",
"type": "query",
"label": "Namespace",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"query": {
"query": "label_values(server_memory, namespace)",
"refId": "ns"
},
"refresh": 2
},
{
"name": "job",
"type": "query",
"label": "Release",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"query": {
"query": "label_values(server_memory{namespace=\"$namespace\"}, job)",
"refId": "job"
},
"refresh": 2
}
]
},
"panels": [
{
"id": 1,
"type": "stat",
"title": "Pods up",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "sum(up{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "up",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {}
},
{
"id": 2,
"type": "stat",
"title": "Queued messages",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "max(queue_count{job=\"$job\",namespace=\"$namespace\",inbuxa_org_group=\"core\"})",
"legendFormat": "queued",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {},
"description": "Counted from the store every five minutes by the core pod"
},
{
"id": 3,
"type": "stat",
"title": "Oldest queued message",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "time() - max(queue_oldest_message{job=\"$job\",namespace=\"$namespace\",inbuxa_org_group=\"core\"} > 0)",
"legendFormat": "age",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 4,
"type": "stat",
"title": "First certificate expires in",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
},
"targets": [
{
"refId": "A",
"expr": "min(tls_earliest_expiry{job=\"$job\",namespace=\"$namespace\"} > 0) - time()",
"legendFormat": "expires",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 5,
"type": "timeseries",
"title": "Deliveries",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 0,
"y": 4,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(delivery_delivered{job=\"$job\",namespace=\"$namespace\"}[5m]))",
"legendFormat": "delivered",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "B",
"expr": "sum(rate(delivery_dsn_temp_fail{job=\"$job\",namespace=\"$namespace\"}[5m]))",
"legendFormat": "delayed (DSN)",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "C",
"expr": "sum(rate(delivery_dsn_perm_fail{job=\"$job\",namespace=\"$namespace\"}[5m]))",
"legendFormat": "bounced (DSN)",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {}
},
{
"id": 6,
"type": "timeseries",
"title": "Messages received",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 12,
"y": 4,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(message_ingest_time_count{job=\"$job\",namespace=\"$namespace\"}[5m]))",
"legendFormat": "stored",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {}
},
{
"id": 7,
"type": "timeseries",
"title": "Open connections",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 0,
"y": 12,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (inbuxa_org_group) (smtp_active_connections{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "smtp {{inbuxa_org_group}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "B",
"expr": "sum by (inbuxa_org_group) (imap_active_connections{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "imap {{inbuxa_org_group}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "C",
"expr": "sum by (inbuxa_org_group) (pop3_active_connections{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "pop3 {{inbuxa_org_group}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "D",
"expr": "sum by (inbuxa_org_group) (http_active_connections{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "http {{inbuxa_org_group}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "E",
"expr": "sum by (inbuxa_org_group) (sieve_active_connections{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "sieve {{inbuxa_org_group}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "F",
"expr": "sum by (inbuxa_org_group) (delivery_active_connections{job=\"$job\",namespace=\"$namespace\"})",
"legendFormat": "delivery {{inbuxa_org_group}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 8,
"type": "timeseries",
"title": "Request time, 99th percentile",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 12,
"y": 12,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum by (le) (rate(smtp_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "smtp",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.99, sum by (le) (rate(imap_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "imap",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "http",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "ms"
},
"overrides": []
},
"options": {}
},
{
"id": 9,
"type": "timeseries",
"title": "Data store, 99th percentile",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_data_read_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "data read",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_data_write_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "data write",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_blob_read_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "blob read",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "D",
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_blob_write_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
"legendFormat": "blob write",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "ms"
},
"overrides": []
},
"options": {}
},
{
"id": 10,
"type": "timeseries",
"title": "Cluster lease, seconds since renewed",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 12,
"y": 20,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "time() - (cluster_lease_renewed{job=\"$job\",namespace=\"$namespace\"} > 0)",
"legendFormat": "{{pod}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {},
"description": "Each pod renews once a minute; over five minutes and the cluster counts it gone"
},
{
"id": 11,
"type": "timeseries",
"title": "Memory",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 0,
"y": 28,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "server_memory{job=\"$job\",namespace=\"$namespace\"}",
"legendFormat": "{{pod}}",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {}
},
{
"id": 12,
"type": "timeseries",
"title": "Failed sign-ins and bans",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"gridPos": {
"x": 12,
"y": 28,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(auth_failed{job=\"$job\",namespace=\"$namespace\"}[5m]))",
"legendFormat": "failed sign-ins",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
},
{
"refId": "B",
"expr": "sum(rate(security_authentication_ban{job=\"$job\",namespace=\"$namespace\"}[5m]))",
"legendFormat": "bans",
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {}
}
],
"annotations": {
"list": []
}
}
+5
View File
@@ -177,3 +177,8 @@ chart.
{{- end }}
{{- join "," $dirs }}
{{- end }}
{{/* The Secret with the metrics scrape username and password. */}}
{{- define "inbuxa.server.metricsSecret" -}}
{{- default (printf "%s-metrics" (include "inbuxa.fullname" .)) .Values.server.metrics.prometheus.existingSecret -}}
{{- end -}}
@@ -9,6 +9,7 @@ all, in and out; the policies after it open only what each one needs:
server its mail listeners and https from anywhere (networkPolicy.serverEgress
anywhere; http from the webmail narrows it)
and from networkPolicy.httpFrom
and .metricsFrom
seed, upgrade, nothing as the server
egress-check
webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80
@@ -74,6 +75,9 @@ spec:
{{- if $np.httpFrom }}
from:
{{- toYaml $np.httpFrom | nindent 8 }}
{{- with $np.metricsFrom }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if $root.Values.webmail.enabled }}
- podSelector:
matchLabels:
@@ -223,6 +223,34 @@ spec:
- name: INBUXA_ADMIN_URL
value: {{ trimSuffix "/" $root.Values.console.publicUrl | quote }}
{{- end }}
{{- with $server.metrics }}
{{- if .prometheus.enabled }}
# /metrics/prometheus, behind this username and password (K-C10)
- name: INBUXA_METRICS_PROMETHEUS_USERNAME
value: {{ .prometheus.username | quote }}
- name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE
value: /etc/inbuxa/metrics/password
{{- end }}
{{- if .otlp.enabled }}
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: {{ .otlp.endpoint | quote }}
- name: OTEL_EXPORTER_OTLP_PROTOCOL
value: {{ .otlp.protocol | quote }}
- name: OTEL_METRIC_EXPORT_INTERVAL
value: {{ mul (int .otlp.intervalSeconds) 1000 | quote }}
- name: OTEL_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OTEL_RESOURCE_ATTRIBUTES
value: {{ printf "service.instance.id=$(OTEL_POD_NAME),k8s.pod.name=$(OTEL_POD_NAME),k8s.namespace.name=%s,k8s.deployment.name=%s-%s,inbuxa.group=%s" $root.Release.Namespace (include "inbuxa.fullname" $root) $group $group | quote }}
{{- if .otlp.headersSecret }}
# A file, not the environment: the headers carry the token
- name: INBUXA_OTLP_HEADERS_FILE
value: /etc/inbuxa/otlp/headers
{{- end }}
{{- end }}
{{- end }}
{{- with $g.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}
@@ -293,6 +321,16 @@ spec:
mountPath: /etc/inbuxa/tls/{{ $name }}
readOnly: true
{{- end }}
{{- if $server.metrics.prometheus.enabled }}
- name: metrics
mountPath: /etc/inbuxa/metrics
readOnly: true
{{- end }}
{{- if and $server.metrics.otlp.enabled $server.metrics.otlp.headersSecret }}
- name: otlp
mountPath: /etc/inbuxa/otlp
readOnly: true
{{- end }}
- name: data
mountPath: /var/lib/inbuxa
- name: tmp
@@ -334,6 +372,24 @@ spec:
- {key: tls.crt, path: tls.crt}
- {key: tls.key, path: tls.key}
{{- end }}
{{- if $server.metrics.prometheus.enabled }}
- name: metrics
secret:
secretName: {{ include "inbuxa.server.metricsSecret" $root }}
defaultMode: 0440
items:
- key: password
path: password
{{- end }}
{{- if and $server.metrics.otlp.enabled $server.metrics.otlp.headersSecret }}
- name: otlp
secret:
secretName: {{ $server.metrics.otlp.headersSecret }}
defaultMode: 0440
items:
- key: headers
path: headers
{{- end }}
# Scratch only: everything lasting is in the data store
- name: data
emptyDir: {}
+236
View File
@@ -0,0 +1,236 @@
{{/*
SPDX-FileCopyrightText: 2026 Coffey Labs LLC
SPDX-License-Identifier: Apache-2.0
Metrics: the scrape password, a headless Service listing every server pod,
a ServiceMonitor, PrometheusRules and a Grafana dashboard.
Each pod counts its own traffic, so Prometheus scrapes every pod, not a
Service that picks one. The queue gauges are read from the core pod, which
always counts the queue itself: another pod's queue count is only what it
queued and delivered since the last count.
*/}}
{{- $root := . }}
{{- $full := include "inbuxa.fullname" . }}
{{- $m := .Values.server.metrics }}
{{- $prom := $m.prometheus }}
{{- if and (or $prom.serviceMonitor.enabled $prom.rules.enabled) (not $prom.enabled) }}
{{- fail "server.metrics.prometheus: turn on enabled for the ServiceMonitor or the rules" }}
{{- end }}
{{- if and $prom.rules.enabled (not $prom.serviceMonitor.enabled) }}
{{- fail "server.metrics.prometheus.rules: they need serviceMonitor on, which labels each pod's series with its group" }}
{{- end }}
{{- if and $m.otlp.enabled (not $m.otlp.endpoint) }}
{{- fail "server.metrics.otlp.endpoint: name the collector, e.g. http://otel-collector.monitoring:4318" }}
{{- end }}
{{- if $prom.enabled }}
{{- $secret := include "inbuxa.server.metricsSecret" . }}
{{- if not $prom.existingSecret }}
{{- /* Random once, then kept, so Prometheus and the pods agree across upgrades */}}
{{- $existing := (lookup "v1" "Secret" .Release.Namespace $secret).data | default dict }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $secret }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "inbuxa.labels" . | nindent 4 }}
app.kubernetes.io/component: server
type: Opaque
data:
username: {{ $prom.username | b64enc }}
password: {{ index $existing "password" | default (randAlphaNum 40 | b64enc) }}
---
{{- end }}
apiVersion: v1
kind: Service
metadata:
name: {{ $full }}-metrics
namespace: {{ .Release.Namespace }}
labels:
{{- include "inbuxa.labels" . | nindent 4 }}
app.kubernetes.io/component: server
inbuxa.org/metrics: "true"
spec:
# Headless: every pod is its own target
clusterIP: None
selector:
{{- include "inbuxa.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: server
ports:
- name: http
port: 8080
targetPort: http
protocol: TCP
{{- if $prom.serviceMonitor.enabled }}
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ $full }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "inbuxa.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with $prom.serviceMonitor.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
selector:
matchLabels:
{{- include "inbuxa.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: server
inbuxa.org/metrics: "true"
namespaceSelector:
matchNames: [{{ .Release.Namespace }}]
# inbuxa_org_group on every series: front, mta or core
podTargetLabels: [inbuxa.org/group]
endpoints:
- port: http
path: /metrics/prometheus
interval: {{ $prom.serviceMonitor.interval }}
basicAuth:
username:
name: {{ $secret }}
key: username
password:
name: {{ $secret }}
key: password
{{- end }}
{{- if $prom.rules.enabled }}
{{- $r := $prom.rules }}
{{- $sel := printf "job=\"%s-metrics\",namespace=\"%s\"" $full .Release.Namespace }}
{{- $core := printf "%s,inbuxa_org_group=\"core\"" $sel }}
{{- $pods := printf "namespace=\"%s\",pod=~\"%s-(front|mta|core)-.*\"" .Release.Namespace $full }}
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: {{ $full }}
namespace: {{ .Release.Namespace }}
labels:
{{- include "inbuxa.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with $r.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
groups:
- name: inbuxa.server
rules:
- alert: InbuxaServerDown
expr: up{ {{- $sel -}} } == 0
for: 5m
labels:
severity: critical
annotations:
summary: {{`"{{ $labels.pod }} doesn't answer the metrics scrape"`}}
description: Prometheus has failed to scrape this server pod for five minutes.
- alert: InbuxaClusterLeaseNotRenewed
expr: time() - (cluster_lease_renewed{ {{- $sel -}} } > 0) > 300
for: 2m
labels:
severity: critical
annotations:
summary: {{`"{{ $labels.pod }} has stopped renewing its cluster lease"`}}
description: >-
The pod runs, but hasn't written its heartbeat to the data store
for over five minutes, so the rest of the cluster sees it as
gone. Its logs say why; usually the data store.
- alert: InbuxaQueueLarge
expr: max(queue_count{ {{- $core -}} }) > {{ int $r.queueMessages }}
for: 30m
labels:
severity: warning
annotations:
summary: {{`"{{ $value }} messages are waiting in the queue"`}}
description: >-
More than {{ int $r.queueMessages }} messages for half an hour.
Delivery may be failing or slow; the console's queue says to
where.
- alert: InbuxaQueueMessageWaiting
expr: time() - max(queue_oldest_message{ {{- $core -}} } > 0) > {{ printf "%.0f" (mulf $r.queueOldestHours 3600) }}
for: 30m
labels:
severity: warning
annotations:
summary: A message has been in the queue for over {{ $r.queueOldestHours }} hours
description: >-
The oldest queued message arrived {{`{{ $value | humanizeDuration }}`}}
ago. The console's queue shows its recipients and the last
error.
- alert: InbuxaDeliveriesFailing
expr: >-
sum(rate(delivery_dsn_perm_fail{ {{- $sel -}} }[30m]))
/ (sum(rate(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) + (sum(rate(delivery_delivered{ {{- $sel -}} }[30m])) or vector(0)))
> {{ $r.deliveryFailureRatio }}
and sum(increase(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) >= {{ int $r.deliveryFailureMinimum }}
for: 15m
labels:
severity: warning
annotations:
summary: {{`"{{ $value | humanizePercentage }} of deliveries are bouncing"`}}
description: >-
Over half an hour, more than {{ printf "%g" (mulf $r.deliveryFailureRatio 100) }}% of
outbound deliveries ended in a bounce. Check whether the mta
pods' address is listed, or a relay refuses them.
- alert: InbuxaCertificateExpiring
expr: min(tls_earliest_expiry{ {{- $sel -}} } > 0) - time() < {{ printf "%.0f" (mulf $r.certificateDays 86400) }}
for: 1h
labels:
severity: warning
annotations:
summary: {{`"A certificate the server uses expires in {{ $value | humanizeDuration }}"`}}
description: >-
Renewal should have replaced it by now. For ACME, the server's
logs and the console's certificates page say why it hasn't; for
a mounted Secret, cert-manager's Certificate status.
- alert: InbuxaDataStoreSlow
expr: >-
histogram_quantile(0.99, sum by (le) (rate(store_data_read_time_bucket{ {{- $sel -}} }[10m]))) > {{ $r.storeLatencyMs }}
or histogram_quantile(0.99, sum by (le) (rate(store_data_write_time_bucket{ {{- $sel -}} }[10m]))) > {{ $r.storeLatencyMs }}
for: 15m
labels:
severity: warning
annotations:
summary: {{`"Data store reads or writes take {{ $value }} ms at the 99th percentile"`}}
description: Everything waits on the data store. Check its load and the network to it.
{{- if $r.kubeStateMetrics }}
- alert: InbuxaServerRestarting
expr: increase(kube_pod_container_status_restarts_total{ {{- $pods -}} ,container="server"}[30m]) > 2
labels:
severity: warning
annotations:
summary: {{`"{{ $labels.pod }} restarted {{ $value }} times in half an hour"`}}
description: The container's last log lines are its termination message (kubectl describe pod).
- alert: InbuxaServerReadinessFlapping
expr: changes(kube_pod_status_ready{ {{- $pods -}} ,condition="true"}[30m]) > 4
labels:
severity: warning
annotations:
summary: {{`"{{ $labels.pod }} keeps going in and out of ready"`}}
description: A pod is ready while its data store answers; flapping readiness usually means the store is struggling.
{{- end }}
{{- end }}
{{- end }}
{{- if $m.grafanaDashboard.enabled }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $full }}-dashboard
namespace: {{ .Release.Namespace }}
labels:
{{- include "inbuxa.labels" . | nindent 4 }}
app.kubernetes.io/component: server
{{- with $m.grafanaDashboard.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with $m.grafanaDashboard.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
data:
inbuxa.json: |-
{{- .Files.Get "files/grafana-dashboard.json" | nindent 4 }}
{{- end }}
+243
View File
@@ -0,0 +1,243 @@
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
suite: metrics
templates:
- server/metrics.yaml
- server/deployment.yaml
- networkpolicy.yaml
release:
name: mail
namespace: inbuxa
values:
- ../ci/defaults-values.yaml
tests:
- it: makes nothing by default
template: server/metrics.yaml
asserts:
- hasDocuments:
count: 0
- it: leaves the pods' metrics settings to the store by default
template: server/deployment.yaml
documentIndex: 0
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content: {name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE, value: /etc/inbuxa/metrics/password}
- notContains:
path: spec.template.spec.volumes
content: {name: metrics}
any: true
- it: makes a password and a headless Service over every server pod
template: server/metrics.yaml
set:
server.metrics.prometheus.enabled: true
asserts:
- hasDocuments:
count: 2
- equal:
path: data.username
value: cHJvbWV0aGV1cw==
documentSelector: {path: kind, value: Secret}
- matchRegex:
path: data.password
pattern: ^[A-Za-z0-9+/=]{40,}$
documentSelector: {path: kind, value: Secret}
- equal:
path: spec.clusterIP
value: None
documentSelector: {path: kind, value: Service}
- notExists:
path: spec.selector["inbuxa.org/group"]
documentSelector: {path: kind, value: Service}
- it: uses an existing Secret instead of making one
template: server/metrics.yaml
set:
server.metrics.prometheus.enabled: true
server.metrics.prometheus.existingSecret: scrape
server.metrics.prometheus.serviceMonitor.enabled: true
asserts:
- hasDocuments:
count: 2
- equal:
path: spec.endpoints[0].basicAuth.password
value: {name: scrape, key: password}
documentSelector: {path: kind, value: ServiceMonitor}
- it: gives every pod the username and the password as a file
template: server/deployment.yaml
set:
server.metrics.prometheus.enabled: true
server.metrics.prometheus.username: scraper
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: INBUXA_METRICS_PROMETHEUS_USERNAME, value: scraper}
- contains:
path: spec.template.spec.containers[0].env
content: {name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE, value: /etc/inbuxa/metrics/password}
- contains:
path: spec.template.spec.containers[0].volumeMounts
content: {name: metrics, mountPath: /etc/inbuxa/metrics, readOnly: true}
- contains:
path: spec.template.spec.volumes
content:
name: metrics
secret:
secretName: mail-inbuxa-metrics
defaultMode: 0440
items: [{key: password, path: password}]
- it: scrapes each pod with basic auth and labels its group
template: server/metrics.yaml
set:
server.metrics.prometheus.enabled: true
server.metrics.prometheus.serviceMonitor.enabled: true
server.metrics.prometheus.serviceMonitor.labels: {release: prom}
documentSelector: {path: kind, value: ServiceMonitor}
asserts:
- equal:
path: metadata.labels.release
value: prom
- equal:
path: spec.podTargetLabels
value: [inbuxa.org/group]
- equal:
path: spec.endpoints[0].path
value: /metrics/prometheus
- equal:
path: spec.endpoints[0].basicAuth.username
value: {name: mail-inbuxa-metrics, key: username}
- equal:
path: spec.selector.matchLabels["inbuxa.org/metrics"]
value: "true"
- it: needs scraping on for a ServiceMonitor
template: server/metrics.yaml
set:
server.metrics.prometheus.serviceMonitor.enabled: true
asserts:
- failedTemplate:
errorPattern: turn on enabled
- it: needs the ServiceMonitor for the rules
template: server/metrics.yaml
set:
server.metrics.prometheus.enabled: true
server.metrics.prometheus.rules.enabled: true
asserts:
- failedTemplate:
errorPattern: they need serviceMonitor on
- it: alerts on this release's pods, with the thresholds given
template: server/metrics.yaml
set:
server.metrics.prometheus.enabled: true
server.metrics.prometheus.serviceMonitor.enabled: true
server.metrics.prometheus.rules.enabled: true
server.metrics.prometheus.rules.queueOldestHours: 12
server.metrics.prometheus.rules.certificateDays: 7
documentSelector: {path: kind, value: PrometheusRule}
asserts:
- lengthEqual:
path: spec.groups[0].rules
count: 9
- equal:
path: spec.groups[0].rules[0].expr
value: up{job="mail-inbuxa-metrics",namespace="inbuxa"} == 0
- equal:
path: spec.groups[0].rules[1].expr
value: time() - (cluster_lease_renewed{job="mail-inbuxa-metrics",namespace="inbuxa"} > 0) > 300
- equal:
path: spec.groups[0].rules[3].expr
value: time() - max(queue_oldest_message{job="mail-inbuxa-metrics",namespace="inbuxa",inbuxa_org_group="core"} > 0) > 43200
- equal:
path: spec.groups[0].rules[5].expr
value: min(tls_earliest_expiry{job="mail-inbuxa-metrics",namespace="inbuxa"} > 0) - time() < 604800
- equal:
path: spec.groups[0].rules[0].annotations.summary
value: "{{ $labels.pod }} doesn't answer the metrics scrape"
- matchRegex:
path: spec.groups[0].rules[7].expr
pattern: pod=~"mail-inbuxa-\(front\|mta\|core\)-\.\*",container="server"
- it: leaves out the kube-state-metrics alerts when asked
template: server/metrics.yaml
set:
server.metrics.prometheus.enabled: true
server.metrics.prometheus.serviceMonitor.enabled: true
server.metrics.prometheus.rules.enabled: true
server.metrics.prometheus.rules.kubeStateMetrics: false
documentSelector: {path: kind, value: PrometheusRule}
asserts:
- lengthEqual:
path: spec.groups[0].rules
count: 7
- it: pushes OTLP from every pod, the headers from a file
template: server/deployment.yaml
documentIndex: 2
set:
server.metrics.otlp.enabled: true
server.metrics.otlp.endpoint: http://otel.monitoring:4317
server.metrics.otlp.protocol: grpc
server.metrics.otlp.intervalSeconds: 30
server.metrics.otlp.headersSecret: otel
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: OTEL_EXPORTER_OTLP_ENDPOINT, value: http://otel.monitoring:4317}
- contains:
path: spec.template.spec.containers[0].env
content: {name: OTEL_EXPORTER_OTLP_PROTOCOL, value: grpc}
- contains:
path: spec.template.spec.containers[0].env
content: {name: OTEL_METRIC_EXPORT_INTERVAL, value: "30000"}
- contains:
path: spec.template.spec.containers[0].env
content:
name: OTEL_RESOURCE_ATTRIBUTES
value: service.instance.id=$(OTEL_POD_NAME),k8s.pod.name=$(OTEL_POD_NAME),k8s.namespace.name=inbuxa,k8s.deployment.name=mail-inbuxa-core,inbuxa.group=core
- contains:
path: spec.template.spec.containers[0].env
content: {name: INBUXA_OTLP_HEADERS_FILE, value: /etc/inbuxa/otlp/headers}
- contains:
path: spec.template.spec.volumes
content:
name: otlp
secret: {secretName: otel, defaultMode: 0440, items: [{key: headers, path: headers}]}
- it: needs a collector for OTLP
template: server/metrics.yaml
set:
server.metrics.otlp.enabled: true
asserts:
- failedTemplate:
errorPattern: name the collector
- it: ships the dashboard for Grafana's sidecar
template: server/metrics.yaml
set:
server.metrics.grafanaDashboard.enabled: true
asserts:
- equal:
path: metadata.labels.grafana_dashboard
value: "1"
- matchRegex:
path: data["inbuxa.json"]
pattern: '"uid": "inbuxa-server"'
- it: lets Prometheus through to the http port when that is narrowed
template: networkpolicy.yaml
set:
networkPolicy.httpFrom:
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: ingress}}
networkPolicy.metricsFrom:
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
documentSelector: {path: metadata.name, value: mail-inbuxa-server}
asserts:
- contains:
path: spec.ingress[1].from
content: {namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}}
+137
View File
@@ -557,6 +557,137 @@
}
}
}
},
"metrics": {
"type": "object",
"additionalProperties": false,
"properties": {
"prometheus": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"username": {
"type": "string",
"pattern": "^[^:\\s]+$"
},
"existingSecret": {
"type": "string"
},
"serviceMonitor": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"interval": {
"type": "string",
"pattern": "^[0-9]+(ms|s|m|h)$"
},
"labels": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
},
"rules": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"labels": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"kubeStateMetrics": {
"type": "boolean"
},
"queueMessages": {
"type": "integer",
"minimum": 1
},
"queueOldestHours": {
"type": "number",
"exclusiveMinimum": 0
},
"deliveryFailureRatio": {
"type": "number",
"exclusiveMinimum": 0,
"maximum": 1
},
"deliveryFailureMinimum": {
"type": "integer",
"minimum": 1
},
"certificateDays": {
"type": "number",
"exclusiveMinimum": 0
},
"storeLatencyMs": {
"type": "number",
"exclusiveMinimum": 0
}
}
}
}
},
"otlp": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"endpoint": {
"type": "string",
"pattern": "^$|^https?://"
},
"protocol": {
"enum": [
"http/protobuf",
"grpc"
]
},
"intervalSeconds": {
"type": "integer",
"minimum": 1
},
"headersSecret": {
"type": "string"
}
}
},
"grafanaDashboard": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"labels": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"annotations": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
}
}
},
@@ -782,6 +913,12 @@
},
"webmailInternet": {
"type": "boolean"
},
"metricsFrom": {
"type": "array",
"items": {
"type": "object"
}
}
}
},
+68
View File
@@ -272,6 +272,69 @@ server:
pop3s: 10995
sieve: 14190
# Metrics, for Prometheus or an OpenTelemetry collector. Needs server
# 27.1-rc.34; an older server ignores these settings.
metrics:
prometheus:
# Every server pod serves /metrics/prometheus on its http port, behind
# basic auth. A headless Service, <release>-metrics, lists them all.
enabled: false
username: prometheus
# A Secret with the keys username (the same as above) and password.
# Empty: the chart makes one, with a random password kept across
# upgrades. The pods read the password as a file.
existingSecret: ""
# A ServiceMonitor for the Prometheus Operator, scraping each pod
serviceMonitor:
enabled: false
interval: 30s
labels: {}
# PrometheusRules: alerts for a pod down, a cluster lease not renewed,
# a large or stuck queue, failing deliveries, a certificate about to
# expire and a slow data store. They read the pods' group from the
# label the ServiceMonitor adds, so they need it on.
rules:
enabled: false
labels: {}
# Alerts on restarts and readiness from kube-state-metrics, when the
# cluster has it
kubeStateMetrics: true
# Messages in the queue
queueMessages: 5000
# Hours the oldest message may wait. A queue keeps retrying a
# message for days, so one that waits longer is worth a look.
queueOldestHours: 24
# Share of deliveries that bounce, over half an hour, once at least
# deliveryFailureMinimum have
deliveryFailureRatio: 0.25
deliveryFailureMinimum: 10
# Days before the first certificate in use expires
certificateDays: 14
# 99th-percentile data store read or write, in milliseconds
storeLatencyMs: 1000
# Push metrics to an OpenTelemetry collector over OTLP. Only when the
# store has no OpenTelemetry metrics set: the console's setting wins.
# Every pod pushes its own, named by service.instance.id and
# k8s.pod.name; the collector adds them up.
otlp:
enabled: false
# The collector, e.g. http://otel-collector.monitoring:4318
endpoint: ""
# http/protobuf or grpc
protocol: http/protobuf
intervalSeconds: 60
# A Secret whose key headers holds headers to send, key=value pairs by
# commas or lines, for a collector that wants a token. The pods read it
# as a file.
headersSecret: ""
# A Grafana dashboard as a ConfigMap, for Grafana's dashboard sidecar
grafanaDashboard:
enabled: false
# The label the sidecar looks for
labels:
grafana_dashboard: "1"
annotations: {}
# The webmail, signing people in through the server's own page (OAuth). Off
# by default: it needs its own address, and the server's publicUrl.
webmail:
@@ -365,6 +428,11 @@ networkPolicy:
# matchLabels: {kubernetes.io/metadata.name: ingress-nginx}
# Empty: from anywhere. The webmail always reaches the server.
httpFrom: []
# Prometheus, when httpFrom is set: it scrapes the server pods' http
# port. As NetworkPolicy peers, e.g.
# - namespaceSelector:
# matchLabels: {kubernetes.io/metadata.name: monitoring}
metricsFrom: []
# Where the server and its Jobs may connect, as NetworkPolicy egress rules,
# besides DNS. Empty: anywhere, which a mail server needs to deliver mail
# unless it hands everything to a relay. Narrow it to your data store,