diff --git a/CHANGELOG.md b/CHANGELOG.md index c873410..f82a2a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,17 @@ runs, pinned by digest. status names the part and why. The server container keeps its last log lines as its termination message too. Needs server 27.1-rc.33: an older server doesn't know `--check`. +- Metrics (`server.metrics`, off by default). `prometheus`: every server + pod serves `/metrics/prometheus` behind a username and a password the pods + read as a file, and a headless Service lists them all, with a + ServiceMonitor that scrapes each pod and labels its group. `rules`: + PrometheusRules for a pod down, a cluster lease not renewed, a large or + stuck queue, deliveries bouncing, a certificate about to expire, a slow + data store, and (from kube-state-metrics) restarts and flapping readiness. + `otlp`: push to an OpenTelemetry collector, the token from a Secret read as + a file. `grafanaDashboard`: a dashboard for Grafana's sidecar. + `networkPolicy.metricsFrom` lets Prometheus through when `httpFrom` is + narrowed. Needs server 27.1-rc.34. - `networkPolicy`, on by default: every pod starts from deny all, in and out, and gets only what it needs. The server takes mail on its listeners from anywhere and connects anywhere; `networkPolicy.httpFrom` and diff --git a/charts/inbuxa/ci/everything-values.yaml b/charts/inbuxa/ci/everything-values.yaml index aad1485..38e8a90 100644 --- a/charts/inbuxa/ci/everything-values.yaml +++ b/charts/inbuxa/ci/everything-values.yaml @@ -59,6 +59,17 @@ server: mail: loadBalancerClass: example.org/lb annotations: {example.org/lb-pool: mail} + metrics: + prometheus: + enabled: true + serviceMonitor: {enabled: true, interval: 15s, labels: {release: kube-prometheus}} + rules: {enabled: true, labels: {release: kube-prometheus}, certificateDays: 21} + otlp: + enabled: true + endpoint: http://otel-collector.monitoring:4317 + protocol: grpc + headersSecret: otel-headers + grafanaDashboard: {enabled: true} seed: enabled: true request: @@ -103,6 +114,9 @@ networkPolicy: httpFrom: - namespaceSelector: matchLabels: {kubernetes.io/metadata.name: gateway} + metricsFrom: + - namespaceSelector: + matchLabels: {kubernetes.io/metadata.name: monitoring} serverEgress: - to: - namespaceSelector: diff --git a/charts/inbuxa/files/grafana-dashboard.json b/charts/inbuxa/files/grafana-dashboard.json new file mode 100644 index 0000000..d706102 --- /dev/null +++ b/charts/inbuxa/files/grafana-dashboard.json @@ -0,0 +1,571 @@ +{ + "title": "inbuxa", + "uid": "inbuxa-server", + "tags": [ + "inbuxa", + "mail" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "time": { + "from": "now-6h", + "to": "now" + }, + "refresh": "1m", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "label": "Data source" + }, + { + "name": "namespace", + "type": "query", + "label": "Namespace", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(server_memory, namespace)", + "refId": "ns" + }, + "refresh": 2 + }, + { + "name": "job", + "type": "query", + "label": "Release", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "query": { + "query": "label_values(server_memory{namespace=\"$namespace\"}, job)", + "refId": "job" + }, + "refresh": 2 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Pods up", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(up{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "up", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + { + "id": 2, + "type": "stat", + "title": "Queued messages", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "max(queue_count{job=\"$job\",namespace=\"$namespace\",inbuxa_org_group=\"core\"})", + "legendFormat": "queued", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "description": "Counted from the store every five minutes by the core pod" + }, + { + "id": 3, + "type": "stat", + "title": "Oldest queued message", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "time() - max(queue_oldest_message{job=\"$job\",namespace=\"$namespace\",inbuxa_org_group=\"core\"} > 0)", + "legendFormat": "age", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 4, + "type": "stat", + "title": "First certificate expires in", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "targets": [ + { + "refId": "A", + "expr": "min(tls_earliest_expiry{job=\"$job\",namespace=\"$namespace\"} > 0) - time()", + "legendFormat": "expires", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 5, + "type": "timeseries", + "title": "Deliveries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(delivery_delivered{job=\"$job\",namespace=\"$namespace\"}[5m]))", + "legendFormat": "delivered", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "sum(rate(delivery_dsn_temp_fail{job=\"$job\",namespace=\"$namespace\"}[5m]))", + "legendFormat": "delayed (DSN)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "C", + "expr": "sum(rate(delivery_dsn_perm_fail{job=\"$job\",namespace=\"$namespace\"}[5m]))", + "legendFormat": "bounced (DSN)", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 6, + "type": "timeseries", + "title": "Messages received", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(message_ingest_time_count{job=\"$job\",namespace=\"$namespace\"}[5m]))", + "legendFormat": "stored", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 7, + "type": "timeseries", + "title": "Open connections", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (inbuxa_org_group) (smtp_active_connections{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "smtp {{inbuxa_org_group}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "sum by (inbuxa_org_group) (imap_active_connections{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "imap {{inbuxa_org_group}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "C", + "expr": "sum by (inbuxa_org_group) (pop3_active_connections{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "pop3 {{inbuxa_org_group}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "D", + "expr": "sum by (inbuxa_org_group) (http_active_connections{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "http {{inbuxa_org_group}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "E", + "expr": "sum by (inbuxa_org_group) (sieve_active_connections{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "sieve {{inbuxa_org_group}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "F", + "expr": "sum by (inbuxa_org_group) (delivery_active_connections{job=\"$job\",namespace=\"$namespace\"})", + "legendFormat": "delivery {{inbuxa_org_group}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 8, + "type": "timeseries", + "title": "Request time, 99th percentile", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.99, sum by (le) (rate(smtp_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "smtp", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "histogram_quantile(0.99, sum by (le) (rate(imap_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "imap", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "http", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 9, + "type": "timeseries", + "title": "Data store, 99th percentile", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.99, sum by (le) (rate(store_data_read_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "data read", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "histogram_quantile(0.99, sum by (le) (rate(store_data_write_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "data write", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(store_blob_read_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "blob read", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "D", + "expr": "histogram_quantile(0.99, sum by (le) (rate(store_blob_write_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))", + "legendFormat": "blob write", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 10, + "type": "timeseries", + "title": "Cluster lease, seconds since renewed", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "time() - (cluster_lease_renewed{job=\"$job\",namespace=\"$namespace\"} > 0)", + "legendFormat": "{{pod}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": {}, + "description": "Each pod renews once a minute; over five minutes and the cluster counts it gone" + }, + { + "id": 11, + "type": "timeseries", + "title": "Memory", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "server_memory{job=\"$job\",namespace=\"$namespace\"}", + "legendFormat": "{{pod}}", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": {} + }, + { + "id": 12, + "type": "timeseries", + "title": "Failed sign-ins and bans", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "x": 12, + "y": 28, + "w": 12, + "h": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(auth_failed{job=\"$job\",namespace=\"$namespace\"}[5m]))", + "legendFormat": "failed sign-ins", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + }, + { + "refId": "B", + "expr": "sum(rate(security_authentication_ban{job=\"$job\",namespace=\"$namespace\"}[5m]))", + "legendFormat": "bans", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": {} + } + ], + "annotations": { + "list": [] + } +} diff --git a/charts/inbuxa/templates/_helpers.tpl b/charts/inbuxa/templates/_helpers.tpl index 146ca49..74b305d 100644 --- a/charts/inbuxa/templates/_helpers.tpl +++ b/charts/inbuxa/templates/_helpers.tpl @@ -177,3 +177,8 @@ chart. {{- end }} {{- join "," $dirs }} {{- end }} + +{{/* The Secret with the metrics scrape username and password. */}} +{{- define "inbuxa.server.metricsSecret" -}} +{{- default (printf "%s-metrics" (include "inbuxa.fullname" .)) .Values.server.metrics.prometheus.existingSecret -}} +{{- end -}} diff --git a/charts/inbuxa/templates/networkpolicy.yaml b/charts/inbuxa/templates/networkpolicy.yaml index 9653027..07d65e7 100644 --- a/charts/inbuxa/templates/networkpolicy.yaml +++ b/charts/inbuxa/templates/networkpolicy.yaml @@ -9,6 +9,7 @@ all, in and out; the policies after it open only what each one needs: server its mail listeners and https from anywhere (networkPolicy.serverEgress anywhere; http from the webmail narrows it) and from networkPolicy.httpFrom + and .metricsFrom seed, upgrade, nothing as the server egress-check webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80 @@ -74,6 +75,9 @@ spec: {{- if $np.httpFrom }} from: {{- toYaml $np.httpFrom | nindent 8 }} + {{- with $np.metricsFrom }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- if $root.Values.webmail.enabled }} - podSelector: matchLabels: diff --git a/charts/inbuxa/templates/server/deployment.yaml b/charts/inbuxa/templates/server/deployment.yaml index c6ad2fa..74ccac6 100644 --- a/charts/inbuxa/templates/server/deployment.yaml +++ b/charts/inbuxa/templates/server/deployment.yaml @@ -223,6 +223,34 @@ spec: - name: INBUXA_ADMIN_URL value: {{ trimSuffix "/" $root.Values.console.publicUrl | quote }} {{- end }} + {{- with $server.metrics }} + {{- if .prometheus.enabled }} + # /metrics/prometheus, behind this username and password (K-C10) + - name: INBUXA_METRICS_PROMETHEUS_USERNAME + value: {{ .prometheus.username | quote }} + - name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE + value: /etc/inbuxa/metrics/password + {{- end }} + {{- if .otlp.enabled }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .otlp.endpoint | quote }} + - name: OTEL_EXPORTER_OTLP_PROTOCOL + value: {{ .otlp.protocol | quote }} + - name: OTEL_METRIC_EXPORT_INTERVAL + value: {{ mul (int .otlp.intervalSeconds) 1000 | quote }} + - name: OTEL_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES + value: {{ printf "service.instance.id=$(OTEL_POD_NAME),k8s.pod.name=$(OTEL_POD_NAME),k8s.namespace.name=%s,k8s.deployment.name=%s-%s,inbuxa.group=%s" $root.Release.Namespace (include "inbuxa.fullname" $root) $group $group | quote }} + {{- if .otlp.headersSecret }} + # A file, not the environment: the headers carry the token + - name: INBUXA_OTLP_HEADERS_FILE + value: /etc/inbuxa/otlp/headers + {{- end }} + {{- end }} + {{- end }} {{- with $g.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} @@ -293,6 +321,16 @@ spec: mountPath: /etc/inbuxa/tls/{{ $name }} readOnly: true {{- end }} + {{- if $server.metrics.prometheus.enabled }} + - name: metrics + mountPath: /etc/inbuxa/metrics + readOnly: true + {{- end }} + {{- if and $server.metrics.otlp.enabled $server.metrics.otlp.headersSecret }} + - name: otlp + mountPath: /etc/inbuxa/otlp + readOnly: true + {{- end }} - name: data mountPath: /var/lib/inbuxa - name: tmp @@ -334,6 +372,24 @@ spec: - {key: tls.crt, path: tls.crt} - {key: tls.key, path: tls.key} {{- end }} + {{- if $server.metrics.prometheus.enabled }} + - name: metrics + secret: + secretName: {{ include "inbuxa.server.metricsSecret" $root }} + defaultMode: 0440 + items: + - key: password + path: password + {{- end }} + {{- if and $server.metrics.otlp.enabled $server.metrics.otlp.headersSecret }} + - name: otlp + secret: + secretName: {{ $server.metrics.otlp.headersSecret }} + defaultMode: 0440 + items: + - key: headers + path: headers + {{- end }} # Scratch only: everything lasting is in the data store - name: data emptyDir: {} diff --git a/charts/inbuxa/templates/server/metrics.yaml b/charts/inbuxa/templates/server/metrics.yaml new file mode 100644 index 0000000..e5d80cd --- /dev/null +++ b/charts/inbuxa/templates/server/metrics.yaml @@ -0,0 +1,236 @@ +{{/* +SPDX-FileCopyrightText: 2026 Coffey Labs LLC +SPDX-License-Identifier: Apache-2.0 + +Metrics: the scrape password, a headless Service listing every server pod, +a ServiceMonitor, PrometheusRules and a Grafana dashboard. + +Each pod counts its own traffic, so Prometheus scrapes every pod, not a +Service that picks one. The queue gauges are read from the core pod, which +always counts the queue itself: another pod's queue count is only what it +queued and delivered since the last count. +*/}} +{{- $root := . }} +{{- $full := include "inbuxa.fullname" . }} +{{- $m := .Values.server.metrics }} +{{- $prom := $m.prometheus }} +{{- if and (or $prom.serviceMonitor.enabled $prom.rules.enabled) (not $prom.enabled) }} +{{- fail "server.metrics.prometheus: turn on enabled for the ServiceMonitor or the rules" }} +{{- end }} +{{- if and $prom.rules.enabled (not $prom.serviceMonitor.enabled) }} +{{- fail "server.metrics.prometheus.rules: they need serviceMonitor on, which labels each pod's series with its group" }} +{{- end }} +{{- if and $m.otlp.enabled (not $m.otlp.endpoint) }} +{{- fail "server.metrics.otlp.endpoint: name the collector, e.g. http://otel-collector.monitoring:4318" }} +{{- end }} +{{- if $prom.enabled }} +{{- $secret := include "inbuxa.server.metricsSecret" . }} +{{- if not $prom.existingSecret }} +{{- /* Random once, then kept, so Prometheus and the pods agree across upgrades */}} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace $secret).data | default dict }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secret }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "inbuxa.labels" . | nindent 4 }} + app.kubernetes.io/component: server +type: Opaque +data: + username: {{ $prom.username | b64enc }} + password: {{ index $existing "password" | default (randAlphaNum 40 | b64enc) }} +--- +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $full }}-metrics + namespace: {{ .Release.Namespace }} + labels: + {{- include "inbuxa.labels" . | nindent 4 }} + app.kubernetes.io/component: server + inbuxa.org/metrics: "true" +spec: + # Headless: every pod is its own target + clusterIP: None + selector: + {{- include "inbuxa.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: server + ports: + - name: http + port: 8080 + targetPort: http + protocol: TCP +{{- if $prom.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ $full }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "inbuxa.labels" . | nindent 4 }} + app.kubernetes.io/component: server + {{- with $prom.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "inbuxa.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + inbuxa.org/metrics: "true" + namespaceSelector: + matchNames: [{{ .Release.Namespace }}] + # inbuxa_org_group on every series: front, mta or core + podTargetLabels: [inbuxa.org/group] + endpoints: + - port: http + path: /metrics/prometheus + interval: {{ $prom.serviceMonitor.interval }} + basicAuth: + username: + name: {{ $secret }} + key: username + password: + name: {{ $secret }} + key: password +{{- end }} +{{- if $prom.rules.enabled }} +{{- $r := $prom.rules }} +{{- $sel := printf "job=\"%s-metrics\",namespace=\"%s\"" $full .Release.Namespace }} +{{- $core := printf "%s,inbuxa_org_group=\"core\"" $sel }} +{{- $pods := printf "namespace=\"%s\",pod=~\"%s-(front|mta|core)-.*\"" .Release.Namespace $full }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ $full }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "inbuxa.labels" . | nindent 4 }} + app.kubernetes.io/component: server + {{- with $r.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: inbuxa.server + rules: + - alert: InbuxaServerDown + expr: up{ {{- $sel -}} } == 0 + for: 5m + labels: + severity: critical + annotations: + summary: {{`"{{ $labels.pod }} doesn't answer the metrics scrape"`}} + description: Prometheus has failed to scrape this server pod for five minutes. + - alert: InbuxaClusterLeaseNotRenewed + expr: time() - (cluster_lease_renewed{ {{- $sel -}} } > 0) > 300 + for: 2m + labels: + severity: critical + annotations: + summary: {{`"{{ $labels.pod }} has stopped renewing its cluster lease"`}} + description: >- + The pod runs, but hasn't written its heartbeat to the data store + for over five minutes, so the rest of the cluster sees it as + gone. Its logs say why; usually the data store. + - alert: InbuxaQueueLarge + expr: max(queue_count{ {{- $core -}} }) > {{ int $r.queueMessages }} + for: 30m + labels: + severity: warning + annotations: + summary: {{`"{{ $value }} messages are waiting in the queue"`}} + description: >- + More than {{ int $r.queueMessages }} messages for half an hour. + Delivery may be failing or slow; the console's queue says to + where. + - alert: InbuxaQueueMessageWaiting + expr: time() - max(queue_oldest_message{ {{- $core -}} } > 0) > {{ printf "%.0f" (mulf $r.queueOldestHours 3600) }} + for: 30m + labels: + severity: warning + annotations: + summary: A message has been in the queue for over {{ $r.queueOldestHours }} hours + description: >- + The oldest queued message arrived {{`{{ $value | humanizeDuration }}`}} + ago. The console's queue shows its recipients and the last + error. + - alert: InbuxaDeliveriesFailing + expr: >- + sum(rate(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) + / (sum(rate(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) + (sum(rate(delivery_delivered{ {{- $sel -}} }[30m])) or vector(0))) + > {{ $r.deliveryFailureRatio }} + and sum(increase(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) >= {{ int $r.deliveryFailureMinimum }} + for: 15m + labels: + severity: warning + annotations: + summary: {{`"{{ $value | humanizePercentage }} of deliveries are bouncing"`}} + description: >- + Over half an hour, more than {{ printf "%g" (mulf $r.deliveryFailureRatio 100) }}% of + outbound deliveries ended in a bounce. Check whether the mta + pods' address is listed, or a relay refuses them. + - alert: InbuxaCertificateExpiring + expr: min(tls_earliest_expiry{ {{- $sel -}} } > 0) - time() < {{ printf "%.0f" (mulf $r.certificateDays 86400) }} + for: 1h + labels: + severity: warning + annotations: + summary: {{`"A certificate the server uses expires in {{ $value | humanizeDuration }}"`}} + description: >- + Renewal should have replaced it by now. For ACME, the server's + logs and the console's certificates page say why it hasn't; for + a mounted Secret, cert-manager's Certificate status. + - alert: InbuxaDataStoreSlow + expr: >- + histogram_quantile(0.99, sum by (le) (rate(store_data_read_time_bucket{ {{- $sel -}} }[10m]))) > {{ $r.storeLatencyMs }} + or histogram_quantile(0.99, sum by (le) (rate(store_data_write_time_bucket{ {{- $sel -}} }[10m]))) > {{ $r.storeLatencyMs }} + for: 15m + labels: + severity: warning + annotations: + summary: {{`"Data store reads or writes take {{ $value }} ms at the 99th percentile"`}} + description: Everything waits on the data store. Check its load and the network to it. + {{- if $r.kubeStateMetrics }} + - alert: InbuxaServerRestarting + expr: increase(kube_pod_container_status_restarts_total{ {{- $pods -}} ,container="server"}[30m]) > 2 + labels: + severity: warning + annotations: + summary: {{`"{{ $labels.pod }} restarted {{ $value }} times in half an hour"`}} + description: The container's last log lines are its termination message (kubectl describe pod). + - alert: InbuxaServerReadinessFlapping + expr: changes(kube_pod_status_ready{ {{- $pods -}} ,condition="true"}[30m]) > 4 + labels: + severity: warning + annotations: + summary: {{`"{{ $labels.pod }} keeps going in and out of ready"`}} + description: A pod is ready while its data store answers; flapping readiness usually means the store is struggling. + {{- end }} +{{- end }} +{{- end }} +{{- if $m.grafanaDashboard.enabled }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $full }}-dashboard + namespace: {{ .Release.Namespace }} + labels: + {{- include "inbuxa.labels" . | nindent 4 }} + app.kubernetes.io/component: server + {{- with $m.grafanaDashboard.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $m.grafanaDashboard.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + inbuxa.json: |- + {{- .Files.Get "files/grafana-dashboard.json" | nindent 4 }} +{{- end }} diff --git a/charts/inbuxa/tests/metrics_test.yaml b/charts/inbuxa/tests/metrics_test.yaml new file mode 100644 index 0000000..d96f1c5 --- /dev/null +++ b/charts/inbuxa/tests/metrics_test.yaml @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +suite: metrics +templates: + - server/metrics.yaml + - server/deployment.yaml + - networkpolicy.yaml +release: + name: mail + namespace: inbuxa +values: + - ../ci/defaults-values.yaml +tests: + - it: makes nothing by default + template: server/metrics.yaml + asserts: + - hasDocuments: + count: 0 + + - it: leaves the pods' metrics settings to the store by default + template: server/deployment.yaml + documentIndex: 0 + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: {name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE, value: /etc/inbuxa/metrics/password} + - notContains: + path: spec.template.spec.volumes + content: {name: metrics} + any: true + + - it: makes a password and a headless Service over every server pod + template: server/metrics.yaml + set: + server.metrics.prometheus.enabled: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: data.username + value: cHJvbWV0aGV1cw== + documentSelector: {path: kind, value: Secret} + - matchRegex: + path: data.password + pattern: ^[A-Za-z0-9+/=]{40,}$ + documentSelector: {path: kind, value: Secret} + - equal: + path: spec.clusterIP + value: None + documentSelector: {path: kind, value: Service} + - notExists: + path: spec.selector["inbuxa.org/group"] + documentSelector: {path: kind, value: Service} + + - it: uses an existing Secret instead of making one + template: server/metrics.yaml + set: + server.metrics.prometheus.enabled: true + server.metrics.prometheus.existingSecret: scrape + server.metrics.prometheus.serviceMonitor.enabled: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.endpoints[0].basicAuth.password + value: {name: scrape, key: password} + documentSelector: {path: kind, value: ServiceMonitor} + + - it: gives every pod the username and the password as a file + template: server/deployment.yaml + set: + server.metrics.prometheus.enabled: true + server.metrics.prometheus.username: scraper + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: {name: INBUXA_METRICS_PROMETHEUS_USERNAME, value: scraper} + - contains: + path: spec.template.spec.containers[0].env + content: {name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE, value: /etc/inbuxa/metrics/password} + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: {name: metrics, mountPath: /etc/inbuxa/metrics, readOnly: true} + - contains: + path: spec.template.spec.volumes + content: + name: metrics + secret: + secretName: mail-inbuxa-metrics + defaultMode: 0440 + items: [{key: password, path: password}] + + - it: scrapes each pod with basic auth and labels its group + template: server/metrics.yaml + set: + server.metrics.prometheus.enabled: true + server.metrics.prometheus.serviceMonitor.enabled: true + server.metrics.prometheus.serviceMonitor.labels: {release: prom} + documentSelector: {path: kind, value: ServiceMonitor} + asserts: + - equal: + path: metadata.labels.release + value: prom + - equal: + path: spec.podTargetLabels + value: [inbuxa.org/group] + - equal: + path: spec.endpoints[0].path + value: /metrics/prometheus + - equal: + path: spec.endpoints[0].basicAuth.username + value: {name: mail-inbuxa-metrics, key: username} + - equal: + path: spec.selector.matchLabels["inbuxa.org/metrics"] + value: "true" + + - it: needs scraping on for a ServiceMonitor + template: server/metrics.yaml + set: + server.metrics.prometheus.serviceMonitor.enabled: true + asserts: + - failedTemplate: + errorPattern: turn on enabled + + - it: needs the ServiceMonitor for the rules + template: server/metrics.yaml + set: + server.metrics.prometheus.enabled: true + server.metrics.prometheus.rules.enabled: true + asserts: + - failedTemplate: + errorPattern: they need serviceMonitor on + + - it: alerts on this release's pods, with the thresholds given + template: server/metrics.yaml + set: + server.metrics.prometheus.enabled: true + server.metrics.prometheus.serviceMonitor.enabled: true + server.metrics.prometheus.rules.enabled: true + server.metrics.prometheus.rules.queueOldestHours: 12 + server.metrics.prometheus.rules.certificateDays: 7 + documentSelector: {path: kind, value: PrometheusRule} + asserts: + - lengthEqual: + path: spec.groups[0].rules + count: 9 + - equal: + path: spec.groups[0].rules[0].expr + value: up{job="mail-inbuxa-metrics",namespace="inbuxa"} == 0 + - equal: + path: spec.groups[0].rules[1].expr + value: time() - (cluster_lease_renewed{job="mail-inbuxa-metrics",namespace="inbuxa"} > 0) > 300 + - equal: + path: spec.groups[0].rules[3].expr + value: time() - max(queue_oldest_message{job="mail-inbuxa-metrics",namespace="inbuxa",inbuxa_org_group="core"} > 0) > 43200 + - equal: + path: spec.groups[0].rules[5].expr + value: min(tls_earliest_expiry{job="mail-inbuxa-metrics",namespace="inbuxa"} > 0) - time() < 604800 + - equal: + path: spec.groups[0].rules[0].annotations.summary + value: "{{ $labels.pod }} doesn't answer the metrics scrape" + - matchRegex: + path: spec.groups[0].rules[7].expr + pattern: pod=~"mail-inbuxa-\(front\|mta\|core\)-\.\*",container="server" + + - it: leaves out the kube-state-metrics alerts when asked + template: server/metrics.yaml + set: + server.metrics.prometheus.enabled: true + server.metrics.prometheus.serviceMonitor.enabled: true + server.metrics.prometheus.rules.enabled: true + server.metrics.prometheus.rules.kubeStateMetrics: false + documentSelector: {path: kind, value: PrometheusRule} + asserts: + - lengthEqual: + path: spec.groups[0].rules + count: 7 + + - it: pushes OTLP from every pod, the headers from a file + template: server/deployment.yaml + documentIndex: 2 + set: + server.metrics.otlp.enabled: true + server.metrics.otlp.endpoint: http://otel.monitoring:4317 + server.metrics.otlp.protocol: grpc + server.metrics.otlp.intervalSeconds: 30 + server.metrics.otlp.headersSecret: otel + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: {name: OTEL_EXPORTER_OTLP_ENDPOINT, value: http://otel.monitoring:4317} + - contains: + path: spec.template.spec.containers[0].env + content: {name: OTEL_EXPORTER_OTLP_PROTOCOL, value: grpc} + - contains: + path: spec.template.spec.containers[0].env + content: {name: OTEL_METRIC_EXPORT_INTERVAL, value: "30000"} + - contains: + path: spec.template.spec.containers[0].env + content: + name: OTEL_RESOURCE_ATTRIBUTES + value: service.instance.id=$(OTEL_POD_NAME),k8s.pod.name=$(OTEL_POD_NAME),k8s.namespace.name=inbuxa,k8s.deployment.name=mail-inbuxa-core,inbuxa.group=core + - contains: + path: spec.template.spec.containers[0].env + content: {name: INBUXA_OTLP_HEADERS_FILE, value: /etc/inbuxa/otlp/headers} + - contains: + path: spec.template.spec.volumes + content: + name: otlp + secret: {secretName: otel, defaultMode: 0440, items: [{key: headers, path: headers}]} + + - it: needs a collector for OTLP + template: server/metrics.yaml + set: + server.metrics.otlp.enabled: true + asserts: + - failedTemplate: + errorPattern: name the collector + + - it: ships the dashboard for Grafana's sidecar + template: server/metrics.yaml + set: + server.metrics.grafanaDashboard.enabled: true + asserts: + - equal: + path: metadata.labels.grafana_dashboard + value: "1" + - matchRegex: + path: data["inbuxa.json"] + pattern: '"uid": "inbuxa-server"' + + - it: lets Prometheus through to the http port when that is narrowed + template: networkpolicy.yaml + set: + networkPolicy.httpFrom: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: ingress}} + networkPolicy.metricsFrom: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}} + documentSelector: {path: metadata.name, value: mail-inbuxa-server} + asserts: + - contains: + path: spec.ingress[1].from + content: {namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}} diff --git a/charts/inbuxa/values.schema.json b/charts/inbuxa/values.schema.json index ef97a49..216e60e 100644 --- a/charts/inbuxa/values.schema.json +++ b/charts/inbuxa/values.schema.json @@ -557,6 +557,137 @@ } } } + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "properties": { + "prometheus": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "username": { + "type": "string", + "pattern": "^[^:\\s]+$" + }, + "existingSecret": { + "type": "string" + }, + "serviceMonitor": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "interval": { + "type": "string", + "pattern": "^[0-9]+(ms|s|m|h)$" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "rules": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "kubeStateMetrics": { + "type": "boolean" + }, + "queueMessages": { + "type": "integer", + "minimum": 1 + }, + "queueOldestHours": { + "type": "number", + "exclusiveMinimum": 0 + }, + "deliveryFailureRatio": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "deliveryFailureMinimum": { + "type": "integer", + "minimum": 1 + }, + "certificateDays": { + "type": "number", + "exclusiveMinimum": 0 + }, + "storeLatencyMs": { + "type": "number", + "exclusiveMinimum": 0 + } + } + } + } + }, + "otlp": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "endpoint": { + "type": "string", + "pattern": "^$|^https?://" + }, + "protocol": { + "enum": [ + "http/protobuf", + "grpc" + ] + }, + "intervalSeconds": { + "type": "integer", + "minimum": 1 + }, + "headersSecret": { + "type": "string" + } + } + }, + "grafanaDashboard": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } } } }, @@ -782,6 +913,12 @@ }, "webmailInternet": { "type": "boolean" + }, + "metricsFrom": { + "type": "array", + "items": { + "type": "object" + } } } }, diff --git a/charts/inbuxa/values.yaml b/charts/inbuxa/values.yaml index 260608b..db56932 100644 --- a/charts/inbuxa/values.yaml +++ b/charts/inbuxa/values.yaml @@ -272,6 +272,69 @@ server: pop3s: 10995 sieve: 14190 + # Metrics, for Prometheus or an OpenTelemetry collector. Needs server + # 27.1-rc.34; an older server ignores these settings. + metrics: + prometheus: + # Every server pod serves /metrics/prometheus on its http port, behind + # basic auth. A headless Service, -metrics, lists them all. + enabled: false + username: prometheus + # A Secret with the keys username (the same as above) and password. + # Empty: the chart makes one, with a random password kept across + # upgrades. The pods read the password as a file. + existingSecret: "" + # A ServiceMonitor for the Prometheus Operator, scraping each pod + serviceMonitor: + enabled: false + interval: 30s + labels: {} + # PrometheusRules: alerts for a pod down, a cluster lease not renewed, + # a large or stuck queue, failing deliveries, a certificate about to + # expire and a slow data store. They read the pods' group from the + # label the ServiceMonitor adds, so they need it on. + rules: + enabled: false + labels: {} + # Alerts on restarts and readiness from kube-state-metrics, when the + # cluster has it + kubeStateMetrics: true + # Messages in the queue + queueMessages: 5000 + # Hours the oldest message may wait. A queue keeps retrying a + # message for days, so one that waits longer is worth a look. + queueOldestHours: 24 + # Share of deliveries that bounce, over half an hour, once at least + # deliveryFailureMinimum have + deliveryFailureRatio: 0.25 + deliveryFailureMinimum: 10 + # Days before the first certificate in use expires + certificateDays: 14 + # 99th-percentile data store read or write, in milliseconds + storeLatencyMs: 1000 + # Push metrics to an OpenTelemetry collector over OTLP. Only when the + # store has no OpenTelemetry metrics set: the console's setting wins. + # Every pod pushes its own, named by service.instance.id and + # k8s.pod.name; the collector adds them up. + otlp: + enabled: false + # The collector, e.g. http://otel-collector.monitoring:4318 + endpoint: "" + # http/protobuf or grpc + protocol: http/protobuf + intervalSeconds: 60 + # A Secret whose key headers holds headers to send, key=value pairs by + # commas or lines, for a collector that wants a token. The pods read it + # as a file. + headersSecret: "" + # A Grafana dashboard as a ConfigMap, for Grafana's dashboard sidecar + grafanaDashboard: + enabled: false + # The label the sidecar looks for + labels: + grafana_dashboard: "1" + annotations: {} + # The webmail, signing people in through the server's own page (OAuth). Off # by default: it needs its own address, and the server's publicUrl. webmail: @@ -365,6 +428,11 @@ networkPolicy: # matchLabels: {kubernetes.io/metadata.name: ingress-nginx} # Empty: from anywhere. The webmail always reaches the server. httpFrom: [] + # Prometheus, when httpFrom is set: it scrapes the server pods' http + # port. As NetworkPolicy peers, e.g. + # - namespaceSelector: + # matchLabels: {kubernetes.io/metadata.name: monitoring} + metricsFrom: [] # Where the server and its Jobs may connect, as NetworkPolicy egress rules, # besides DNS. Empty: anywhere, which a mail server needs to deliver mail # unless it hands everything to a relay. Narrow it to your data store,