Metrics: scraping, alert rules, OTLP and a Grafana dashboard (K-C10) #27
No files matched your search
@@ -78,6 +78,17 @@ runs, pinned by digest.
|
||||
status names the part and why. The server container keeps its last log
|
||||
lines as its termination message too. Needs server 27.1-rc.33: an older
|
||||
server doesn't know `--check`.
|
||||
- Metrics (`server.metrics`, off by default). `prometheus`: every server
|
||||
pod serves `/metrics/prometheus` behind a username and a password the pods
|
||||
read as a file, and a headless Service lists them all, with a
|
||||
ServiceMonitor that scrapes each pod and labels its group. `rules`:
|
||||
PrometheusRules for a pod down, a cluster lease not renewed, a large or
|
||||
stuck queue, deliveries bouncing, a certificate about to expire, a slow
|
||||
data store, and (from kube-state-metrics) restarts and flapping readiness.
|
||||
`otlp`: push to an OpenTelemetry collector, the token from a Secret read as
|
||||
a file. `grafanaDashboard`: a dashboard for Grafana's sidecar.
|
||||
`networkPolicy.metricsFrom` lets Prometheus through when `httpFrom` is
|
||||
narrowed. Needs server 27.1-rc.34.
|
||||
- `networkPolicy`, on by default: every pod starts from deny all, in and
|
||||
out, and gets only what it needs. The server takes mail on its listeners
|
||||
from anywhere and connects anywhere; `networkPolicy.httpFrom` and
|
||||
|
||||
@@ -59,6 +59,17 @@ server:
|
||||
mail:
|
||||
loadBalancerClass: example.org/lb
|
||||
annotations: {example.org/lb-pool: mail}
|
||||
metrics:
|
||||
prometheus:
|
||||
enabled: true
|
||||
serviceMonitor: {enabled: true, interval: 15s, labels: {release: kube-prometheus}}
|
||||
rules: {enabled: true, labels: {release: kube-prometheus}, certificateDays: 21}
|
||||
otlp:
|
||||
enabled: true
|
||||
endpoint: http://otel-collector.monitoring:4317
|
||||
protocol: grpc
|
||||
headersSecret: otel-headers
|
||||
grafanaDashboard: {enabled: true}
|
||||
seed:
|
||||
enabled: true
|
||||
request:
|
||||
@@ -103,6 +114,9 @@ networkPolicy:
|
||||
httpFrom:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: gateway}
|
||||
metricsFrom:
|
||||
- namespaceSelector:
|
||||
matchLabels: {kubernetes.io/metadata.name: monitoring}
|
||||
serverEgress:
|
||||
- to:
|
||||
- namespaceSelector:
|
||||
|
||||
@@ -0,0 +1,571 @@
|
||||
{
|
||||
"title": "inbuxa",
|
||||
"uid": "inbuxa-server",
|
||||
"tags": [
|
||||
"inbuxa",
|
||||
"mail"
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"refresh": "1m",
|
||||
"timezone": "browser",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "datasource",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"label": "Data source"
|
||||
},
|
||||
{
|
||||
"name": "namespace",
|
||||
"type": "query",
|
||||
"label": "Namespace",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"query": {
|
||||
"query": "label_values(server_memory, namespace)",
|
||||
"refId": "ns"
|
||||
},
|
||||
"refresh": 2
|
||||
},
|
||||
{
|
||||
"name": "job",
|
||||
"type": "query",
|
||||
"label": "Release",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"query": {
|
||||
"query": "label_values(server_memory{namespace=\"$namespace\"}, job)",
|
||||
"refId": "job"
|
||||
},
|
||||
"refresh": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Pods up",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(up{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "up",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Queued messages",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 6,
|
||||
"y": 0,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_count{job=\"$job\",namespace=\"$namespace\",inbuxa_org_group=\"core\"})",
|
||||
"legendFormat": "queued",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {},
|
||||
"description": "Counted from the store every five minutes by the core pod"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Oldest queued message",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "time() - max(queue_oldest_message{job=\"$job\",namespace=\"$namespace\",inbuxa_org_group=\"core\"} > 0)",
|
||||
"legendFormat": "age",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "First certificate expires in",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 18,
|
||||
"y": 0,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "min(tls_earliest_expiry{job=\"$job\",namespace=\"$namespace\"} > 0) - time()",
|
||||
"legendFormat": "expires",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Deliveries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 4,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(delivery_delivered{job=\"$job\",namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "delivered",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(delivery_dsn_temp_fail{job=\"$job\",namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "delayed (DSN)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "sum(rate(delivery_dsn_perm_fail{job=\"$job\",namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "bounced (DSN)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Messages received",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 4,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(message_ingest_time_count{job=\"$job\",namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "stored",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Open connections",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 12,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (inbuxa_org_group) (smtp_active_connections{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "smtp {{inbuxa_org_group}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum by (inbuxa_org_group) (imap_active_connections{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "imap {{inbuxa_org_group}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "sum by (inbuxa_org_group) (pop3_active_connections{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "pop3 {{inbuxa_org_group}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"expr": "sum by (inbuxa_org_group) (http_active_connections{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "http {{inbuxa_org_group}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "E",
|
||||
"expr": "sum by (inbuxa_org_group) (sieve_active_connections{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "sieve {{inbuxa_org_group}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "F",
|
||||
"expr": "sum by (inbuxa_org_group) (delivery_active_connections{job=\"$job\",namespace=\"$namespace\"})",
|
||||
"legendFormat": "delivery {{inbuxa_org_group}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Request time, 99th percentile",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 12,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(smtp_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "smtp",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(imap_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "imap",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "http",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Data store, 99th percentile",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 20,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_data_read_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "data read",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_data_write_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "data write",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_blob_read_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "blob read",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(store_blob_write_time_bucket{job=\"$job\",namespace=\"$namespace\"}[5m])))",
|
||||
"legendFormat": "blob write",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Cluster lease, seconds since renewed",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 20,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "time() - (cluster_lease_renewed{job=\"$job\",namespace=\"$namespace\"} > 0)",
|
||||
"legendFormat": "{{pod}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {},
|
||||
"description": "Each pod renews once a minute; over five minutes and the cluster counts it gone"
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Memory",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 28,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "server_memory{job=\"$job\",namespace=\"$namespace\"}",
|
||||
"legendFormat": "{{pod}}",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Failed sign-ins and bans",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 28,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(auth_failed{job=\"$job\",namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "failed sign-ins",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(security_authentication_ban{job=\"$job\",namespace=\"$namespace\"}[5m]))",
|
||||
"legendFormat": "bans",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${datasource}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
}
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
}
|
||||
}
|
||||
@@ -177,3 +177,8 @@ chart.
|
||||
{{- end }}
|
||||
{{- join "," $dirs }}
|
||||
{{- end }}
|
||||
|
||||
{{/* The Secret with the metrics scrape username and password. */}}
|
||||
{{- define "inbuxa.server.metricsSecret" -}}
|
||||
{{- default (printf "%s-metrics" (include "inbuxa.fullname" .)) .Values.server.metrics.prometheus.existingSecret -}}
|
||||
{{- end -}}
|
||||
@@ -9,6 +9,7 @@ all, in and out; the policies after it open only what each one needs:
|
||||
server its mail listeners and https from anywhere (networkPolicy.serverEgress
|
||||
anywhere; http from the webmail narrows it)
|
||||
and from networkPolicy.httpFrom
|
||||
and .metricsFrom
|
||||
seed, upgrade, nothing as the server
|
||||
egress-check
|
||||
webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80
|
||||
@@ -74,6 +75,9 @@ spec:
|
||||
{{- if $np.httpFrom }}
|
||||
from:
|
||||
{{- toYaml $np.httpFrom | nindent 8 }}
|
||||
{{- with $np.metricsFrom }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if $root.Values.webmail.enabled }}
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
|
||||
@@ -223,6 +223,34 @@ spec:
|
||||
- name: INBUXA_ADMIN_URL
|
||||
value: {{ trimSuffix "/" $root.Values.console.publicUrl | quote }}
|
||||
{{- end }}
|
||||
{{- with $server.metrics }}
|
||||
{{- if .prometheus.enabled }}
|
||||
# /metrics/prometheus, behind this username and password (K-C10)
|
||||
- name: INBUXA_METRICS_PROMETHEUS_USERNAME
|
||||
value: {{ .prometheus.username | quote }}
|
||||
- name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE
|
||||
value: /etc/inbuxa/metrics/password
|
||||
{{- end }}
|
||||
{{- if .otlp.enabled }}
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: {{ .otlp.endpoint | quote }}
|
||||
- name: OTEL_EXPORTER_OTLP_PROTOCOL
|
||||
value: {{ .otlp.protocol | quote }}
|
||||
- name: OTEL_METRIC_EXPORT_INTERVAL
|
||||
value: {{ mul (int .otlp.intervalSeconds) 1000 | quote }}
|
||||
- name: OTEL_POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: OTEL_RESOURCE_ATTRIBUTES
|
||||
value: {{ printf "service.instance.id=$(OTEL_POD_NAME),k8s.pod.name=$(OTEL_POD_NAME),k8s.namespace.name=%s,k8s.deployment.name=%s-%s,inbuxa.group=%s" $root.Release.Namespace (include "inbuxa.fullname" $root) $group $group | quote }}
|
||||
{{- if .otlp.headersSecret }}
|
||||
# A file, not the environment: the headers carry the token
|
||||
- name: INBUXA_OTLP_HEADERS_FILE
|
||||
value: /etc/inbuxa/otlp/headers
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with $g.extraEnv }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
@@ -293,6 +321,16 @@ spec:
|
||||
mountPath: /etc/inbuxa/tls/{{ $name }}
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if $server.metrics.prometheus.enabled }}
|
||||
- name: metrics
|
||||
mountPath: /etc/inbuxa/metrics
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if and $server.metrics.otlp.enabled $server.metrics.otlp.headersSecret }}
|
||||
- name: otlp
|
||||
mountPath: /etc/inbuxa/otlp
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
- name: data
|
||||
mountPath: /var/lib/inbuxa
|
||||
- name: tmp
|
||||
@@ -334,6 +372,24 @@ spec:
|
||||
- {key: tls.crt, path: tls.crt}
|
||||
- {key: tls.key, path: tls.key}
|
||||
{{- end }}
|
||||
{{- if $server.metrics.prometheus.enabled }}
|
||||
- name: metrics
|
||||
secret:
|
||||
secretName: {{ include "inbuxa.server.metricsSecret" $root }}
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- key: password
|
||||
path: password
|
||||
{{- end }}
|
||||
{{- if and $server.metrics.otlp.enabled $server.metrics.otlp.headersSecret }}
|
||||
- name: otlp
|
||||
secret:
|
||||
secretName: {{ $server.metrics.otlp.headersSecret }}
|
||||
defaultMode: 0440
|
||||
items:
|
||||
- key: headers
|
||||
path: headers
|
||||
{{- end }}
|
||||
# Scratch only: everything lasting is in the data store
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
{{/*
|
||||
SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
Metrics: the scrape password, a headless Service listing every server pod,
|
||||
a ServiceMonitor, PrometheusRules and a Grafana dashboard.
|
||||
|
||||
Each pod counts its own traffic, so Prometheus scrapes every pod, not a
|
||||
Service that picks one. The queue gauges are read from the core pod, which
|
||||
always counts the queue itself: another pod's queue count is only what it
|
||||
queued and delivered since the last count.
|
||||
*/}}
|
||||
{{- $root := . }}
|
||||
{{- $full := include "inbuxa.fullname" . }}
|
||||
{{- $m := .Values.server.metrics }}
|
||||
{{- $prom := $m.prometheus }}
|
||||
{{- if and (or $prom.serviceMonitor.enabled $prom.rules.enabled) (not $prom.enabled) }}
|
||||
{{- fail "server.metrics.prometheus: turn on enabled for the ServiceMonitor or the rules" }}
|
||||
{{- end }}
|
||||
{{- if and $prom.rules.enabled (not $prom.serviceMonitor.enabled) }}
|
||||
{{- fail "server.metrics.prometheus.rules: they need serviceMonitor on, which labels each pod's series with its group" }}
|
||||
{{- end }}
|
||||
{{- if and $m.otlp.enabled (not $m.otlp.endpoint) }}
|
||||
{{- fail "server.metrics.otlp.endpoint: name the collector, e.g. http://otel-collector.monitoring:4318" }}
|
||||
{{- end }}
|
||||
{{- if $prom.enabled }}
|
||||
{{- $secret := include "inbuxa.server.metricsSecret" . }}
|
||||
{{- if not $prom.existingSecret }}
|
||||
{{- /* Random once, then kept, so Prometheus and the pods agree across upgrades */}}
|
||||
{{- $existing := (lookup "v1" "Secret" .Release.Namespace $secret).data | default dict }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ $secret }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
type: Opaque
|
||||
data:
|
||||
username: {{ $prom.username | b64enc }}
|
||||
password: {{ index $existing "password" | default (randAlphaNum 40 | b64enc) }}
|
||||
---
|
||||
{{- end }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $full }}-metrics
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
inbuxa.org/metrics: "true"
|
||||
spec:
|
||||
# Headless: every pod is its own target
|
||||
clusterIP: None
|
||||
selector:
|
||||
{{- include "inbuxa.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
{{- if $prom.serviceMonitor.enabled }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ $full }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
{{- with $prom.serviceMonitor.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "inbuxa.selectorLabels" . | nindent 6 }}
|
||||
app.kubernetes.io/component: server
|
||||
inbuxa.org/metrics: "true"
|
||||
namespaceSelector:
|
||||
matchNames: [{{ .Release.Namespace }}]
|
||||
# inbuxa_org_group on every series: front, mta or core
|
||||
podTargetLabels: [inbuxa.org/group]
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics/prometheus
|
||||
interval: {{ $prom.serviceMonitor.interval }}
|
||||
basicAuth:
|
||||
username:
|
||||
name: {{ $secret }}
|
||||
key: username
|
||||
password:
|
||||
name: {{ $secret }}
|
||||
key: password
|
||||
{{- end }}
|
||||
{{- if $prom.rules.enabled }}
|
||||
{{- $r := $prom.rules }}
|
||||
{{- $sel := printf "job=\"%s-metrics\",namespace=\"%s\"" $full .Release.Namespace }}
|
||||
{{- $core := printf "%s,inbuxa_org_group=\"core\"" $sel }}
|
||||
{{- $pods := printf "namespace=\"%s\",pod=~\"%s-(front|mta|core)-.*\"" .Release.Namespace $full }}
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: {{ $full }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
{{- with $r.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
groups:
|
||||
- name: inbuxa.server
|
||||
rules:
|
||||
- alert: InbuxaServerDown
|
||||
expr: up{ {{- $sel -}} } == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: {{`"{{ $labels.pod }} doesn't answer the metrics scrape"`}}
|
||||
description: Prometheus has failed to scrape this server pod for five minutes.
|
||||
- alert: InbuxaClusterLeaseNotRenewed
|
||||
expr: time() - (cluster_lease_renewed{ {{- $sel -}} } > 0) > 300
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: {{`"{{ $labels.pod }} has stopped renewing its cluster lease"`}}
|
||||
description: >-
|
||||
The pod runs, but hasn't written its heartbeat to the data store
|
||||
for over five minutes, so the rest of the cluster sees it as
|
||||
gone. Its logs say why; usually the data store.
|
||||
- alert: InbuxaQueueLarge
|
||||
expr: max(queue_count{ {{- $core -}} }) > {{ int $r.queueMessages }}
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: {{`"{{ $value }} messages are waiting in the queue"`}}
|
||||
description: >-
|
||||
More than {{ int $r.queueMessages }} messages for half an hour.
|
||||
Delivery may be failing or slow; the console's queue says to
|
||||
where.
|
||||
- alert: InbuxaQueueMessageWaiting
|
||||
expr: time() - max(queue_oldest_message{ {{- $core -}} } > 0) > {{ printf "%.0f" (mulf $r.queueOldestHours 3600) }}
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: A message has been in the queue for over {{ $r.queueOldestHours }} hours
|
||||
description: >-
|
||||
The oldest queued message arrived {{`{{ $value | humanizeDuration }}`}}
|
||||
ago. The console's queue shows its recipients and the last
|
||||
error.
|
||||
- alert: InbuxaDeliveriesFailing
|
||||
expr: >-
|
||||
sum(rate(delivery_dsn_perm_fail{ {{- $sel -}} }[30m]))
|
||||
/ (sum(rate(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) + (sum(rate(delivery_delivered{ {{- $sel -}} }[30m])) or vector(0)))
|
||||
> {{ $r.deliveryFailureRatio }}
|
||||
and sum(increase(delivery_dsn_perm_fail{ {{- $sel -}} }[30m])) >= {{ int $r.deliveryFailureMinimum }}
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: {{`"{{ $value | humanizePercentage }} of deliveries are bouncing"`}}
|
||||
description: >-
|
||||
Over half an hour, more than {{ printf "%g" (mulf $r.deliveryFailureRatio 100) }}% of
|
||||
outbound deliveries ended in a bounce. Check whether the mta
|
||||
pods' address is listed, or a relay refuses them.
|
||||
- alert: InbuxaCertificateExpiring
|
||||
expr: min(tls_earliest_expiry{ {{- $sel -}} } > 0) - time() < {{ printf "%.0f" (mulf $r.certificateDays 86400) }}
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: {{`"A certificate the server uses expires in {{ $value | humanizeDuration }}"`}}
|
||||
description: >-
|
||||
Renewal should have replaced it by now. For ACME, the server's
|
||||
logs and the console's certificates page say why it hasn't; for
|
||||
a mounted Secret, cert-manager's Certificate status.
|
||||
- alert: InbuxaDataStoreSlow
|
||||
expr: >-
|
||||
histogram_quantile(0.99, sum by (le) (rate(store_data_read_time_bucket{ {{- $sel -}} }[10m]))) > {{ $r.storeLatencyMs }}
|
||||
or histogram_quantile(0.99, sum by (le) (rate(store_data_write_time_bucket{ {{- $sel -}} }[10m]))) > {{ $r.storeLatencyMs }}
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: {{`"Data store reads or writes take {{ $value }} ms at the 99th percentile"`}}
|
||||
description: Everything waits on the data store. Check its load and the network to it.
|
||||
{{- if $r.kubeStateMetrics }}
|
||||
- alert: InbuxaServerRestarting
|
||||
expr: increase(kube_pod_container_status_restarts_total{ {{- $pods -}} ,container="server"}[30m]) > 2
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: {{`"{{ $labels.pod }} restarted {{ $value }} times in half an hour"`}}
|
||||
description: The container's last log lines are its termination message (kubectl describe pod).
|
||||
- alert: InbuxaServerReadinessFlapping
|
||||
expr: changes(kube_pod_status_ready{ {{- $pods -}} ,condition="true"}[30m]) > 4
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: {{`"{{ $labels.pod }} keeps going in and out of ready"`}}
|
||||
description: A pod is ready while its data store answers; flapping readiness usually means the store is struggling.
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $m.grafanaDashboard.enabled }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $full }}-dashboard
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: server
|
||||
{{- with $m.grafanaDashboard.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with $m.grafanaDashboard.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
data:
|
||||
inbuxa.json: |-
|
||||
{{- .Files.Get "files/grafana-dashboard.json" | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,243 @@
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
suite: metrics
|
||||
templates:
|
||||
- server/metrics.yaml
|
||||
- server/deployment.yaml
|
||||
- networkpolicy.yaml
|
||||
release:
|
||||
name: mail
|
||||
namespace: inbuxa
|
||||
values:
|
||||
- ../ci/defaults-values.yaml
|
||||
tests:
|
||||
- it: makes nothing by default
|
||||
template: server/metrics.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
|
||||
- it: leaves the pods' metrics settings to the store by default
|
||||
template: server/deployment.yaml
|
||||
documentIndex: 0
|
||||
asserts:
|
||||
- notContains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE, value: /etc/inbuxa/metrics/password}
|
||||
- notContains:
|
||||
path: spec.template.spec.volumes
|
||||
content: {name: metrics}
|
||||
any: true
|
||||
|
||||
- it: makes a password and a headless Service over every server pod
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 2
|
||||
- equal:
|
||||
path: data.username
|
||||
value: cHJvbWV0aGV1cw==
|
||||
documentSelector: {path: kind, value: Secret}
|
||||
- matchRegex:
|
||||
path: data.password
|
||||
pattern: ^[A-Za-z0-9+/=]{40,}$
|
||||
documentSelector: {path: kind, value: Secret}
|
||||
- equal:
|
||||
path: spec.clusterIP
|
||||
value: None
|
||||
documentSelector: {path: kind, value: Service}
|
||||
- notExists:
|
||||
path: spec.selector["inbuxa.org/group"]
|
||||
documentSelector: {path: kind, value: Service}
|
||||
|
||||
- it: uses an existing Secret instead of making one
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
server.metrics.prometheus.existingSecret: scrape
|
||||
server.metrics.prometheus.serviceMonitor.enabled: true
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 2
|
||||
- equal:
|
||||
path: spec.endpoints[0].basicAuth.password
|
||||
value: {name: scrape, key: password}
|
||||
documentSelector: {path: kind, value: ServiceMonitor}
|
||||
|
||||
- it: gives every pod the username and the password as a file
|
||||
template: server/deployment.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
server.metrics.prometheus.username: scraper
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: INBUXA_METRICS_PROMETHEUS_USERNAME, value: scraper}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: INBUXA_METRICS_PROMETHEUS_SECRET_FILE, value: /etc/inbuxa/metrics/password}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
content: {name: metrics, mountPath: /etc/inbuxa/metrics, readOnly: true}
|
||||
- contains:
|
||||
path: spec.template.spec.volumes
|
||||
content:
|
||||
name: metrics
|
||||
secret:
|
||||
secretName: mail-inbuxa-metrics
|
||||
defaultMode: 0440
|
||||
items: [{key: password, path: password}]
|
||||
|
||||
- it: scrapes each pod with basic auth and labels its group
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
server.metrics.prometheus.serviceMonitor.enabled: true
|
||||
server.metrics.prometheus.serviceMonitor.labels: {release: prom}
|
||||
documentSelector: {path: kind, value: ServiceMonitor}
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.labels.release
|
||||
value: prom
|
||||
- equal:
|
||||
path: spec.podTargetLabels
|
||||
value: [inbuxa.org/group]
|
||||
- equal:
|
||||
path: spec.endpoints[0].path
|
||||
value: /metrics/prometheus
|
||||
- equal:
|
||||
path: spec.endpoints[0].basicAuth.username
|
||||
value: {name: mail-inbuxa-metrics, key: username}
|
||||
- equal:
|
||||
path: spec.selector.matchLabels["inbuxa.org/metrics"]
|
||||
value: "true"
|
||||
|
||||
- it: needs scraping on for a ServiceMonitor
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.serviceMonitor.enabled: true
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: turn on enabled
|
||||
|
||||
- it: needs the ServiceMonitor for the rules
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
server.metrics.prometheus.rules.enabled: true
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: they need serviceMonitor on
|
||||
|
||||
- it: alerts on this release's pods, with the thresholds given
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
server.metrics.prometheus.serviceMonitor.enabled: true
|
||||
server.metrics.prometheus.rules.enabled: true
|
||||
server.metrics.prometheus.rules.queueOldestHours: 12
|
||||
server.metrics.prometheus.rules.certificateDays: 7
|
||||
documentSelector: {path: kind, value: PrometheusRule}
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.groups[0].rules
|
||||
count: 9
|
||||
- equal:
|
||||
path: spec.groups[0].rules[0].expr
|
||||
value: up{job="mail-inbuxa-metrics",namespace="inbuxa"} == 0
|
||||
- equal:
|
||||
path: spec.groups[0].rules[1].expr
|
||||
value: time() - (cluster_lease_renewed{job="mail-inbuxa-metrics",namespace="inbuxa"} > 0) > 300
|
||||
- equal:
|
||||
path: spec.groups[0].rules[3].expr
|
||||
value: time() - max(queue_oldest_message{job="mail-inbuxa-metrics",namespace="inbuxa",inbuxa_org_group="core"} > 0) > 43200
|
||||
- equal:
|
||||
path: spec.groups[0].rules[5].expr
|
||||
value: min(tls_earliest_expiry{job="mail-inbuxa-metrics",namespace="inbuxa"} > 0) - time() < 604800
|
||||
- equal:
|
||||
path: spec.groups[0].rules[0].annotations.summary
|
||||
value: "{{ $labels.pod }} doesn't answer the metrics scrape"
|
||||
- matchRegex:
|
||||
path: spec.groups[0].rules[7].expr
|
||||
pattern: pod=~"mail-inbuxa-\(front\|mta\|core\)-\.\*",container="server"
|
||||
|
||||
- it: leaves out the kube-state-metrics alerts when asked
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.prometheus.enabled: true
|
||||
server.metrics.prometheus.serviceMonitor.enabled: true
|
||||
server.metrics.prometheus.rules.enabled: true
|
||||
server.metrics.prometheus.rules.kubeStateMetrics: false
|
||||
documentSelector: {path: kind, value: PrometheusRule}
|
||||
asserts:
|
||||
- lengthEqual:
|
||||
path: spec.groups[0].rules
|
||||
count: 7
|
||||
|
||||
- it: pushes OTLP from every pod, the headers from a file
|
||||
template: server/deployment.yaml
|
||||
documentIndex: 2
|
||||
set:
|
||||
server.metrics.otlp.enabled: true
|
||||
server.metrics.otlp.endpoint: http://otel.monitoring:4317
|
||||
server.metrics.otlp.protocol: grpc
|
||||
server.metrics.otlp.intervalSeconds: 30
|
||||
server.metrics.otlp.headersSecret: otel
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: OTEL_EXPORTER_OTLP_ENDPOINT, value: http://otel.monitoring:4317}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: OTEL_EXPORTER_OTLP_PROTOCOL, value: grpc}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: OTEL_METRIC_EXPORT_INTERVAL, value: "30000"}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: OTEL_RESOURCE_ATTRIBUTES
|
||||
value: service.instance.id=$(OTEL_POD_NAME),k8s.pod.name=$(OTEL_POD_NAME),k8s.namespace.name=inbuxa,k8s.deployment.name=mail-inbuxa-core,inbuxa.group=core
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: INBUXA_OTLP_HEADERS_FILE, value: /etc/inbuxa/otlp/headers}
|
||||
- contains:
|
||||
path: spec.template.spec.volumes
|
||||
content:
|
||||
name: otlp
|
||||
secret: {secretName: otel, defaultMode: 0440, items: [{key: headers, path: headers}]}
|
||||
|
||||
- it: needs a collector for OTLP
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.otlp.enabled: true
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorPattern: name the collector
|
||||
|
||||
- it: ships the dashboard for Grafana's sidecar
|
||||
template: server/metrics.yaml
|
||||
set:
|
||||
server.metrics.grafanaDashboard.enabled: true
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.labels.grafana_dashboard
|
||||
value: "1"
|
||||
- matchRegex:
|
||||
path: data["inbuxa.json"]
|
||||
pattern: '"uid": "inbuxa-server"'
|
||||
|
||||
- it: lets Prometheus through to the http port when that is narrowed
|
||||
template: networkpolicy.yaml
|
||||
set:
|
||||
networkPolicy.httpFrom:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: ingress}}
|
||||
networkPolicy.metricsFrom:
|
||||
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}
|
||||
documentSelector: {path: metadata.name, value: mail-inbuxa-server}
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.ingress[1].from
|
||||
content: {namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: monitoring}}}
|
||||
@@ -557,6 +557,137 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"prometheus": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"username": {
|
||||
"type": "string",
|
||||
"pattern": "^[^:\\s]+$"
|
||||
},
|
||||
"existingSecret": {
|
||||
"type": "string"
|
||||
},
|
||||
"serviceMonitor": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"interval": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9]+(ms|s|m|h)$"
|
||||
},
|
||||
"labels": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"rules": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"labels": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"kubeStateMetrics": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"queueMessages": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"queueOldestHours": {
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0
|
||||
},
|
||||
"deliveryFailureRatio": {
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"deliveryFailureMinimum": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"certificateDays": {
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0
|
||||
},
|
||||
"storeLatencyMs": {
|
||||
"type": "number",
|
||||
"exclusiveMinimum": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"otlp": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"endpoint": {
|
||||
"type": "string",
|
||||
"pattern": "^$|^https?://"
|
||||
},
|
||||
"protocol": {
|
||||
"enum": [
|
||||
"http/protobuf",
|
||||
"grpc"
|
||||
]
|
||||
},
|
||||
"intervalSeconds": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
},
|
||||
"headersSecret": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"grafanaDashboard": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"labels": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"annotations": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -782,6 +913,12 @@
|
||||
},
|
||||
"webmailInternet": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"metricsFrom": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -272,6 +272,69 @@ server:
|
||||
pop3s: 10995
|
||||
sieve: 14190
|
||||
|
||||
# Metrics, for Prometheus or an OpenTelemetry collector. Needs server
|
||||
# 27.1-rc.34; an older server ignores these settings.
|
||||
metrics:
|
||||
prometheus:
|
||||
# Every server pod serves /metrics/prometheus on its http port, behind
|
||||
# basic auth. A headless Service, <release>-metrics, lists them all.
|
||||
enabled: false
|
||||
username: prometheus
|
||||
# A Secret with the keys username (the same as above) and password.
|
||||
# Empty: the chart makes one, with a random password kept across
|
||||
# upgrades. The pods read the password as a file.
|
||||
existingSecret: ""
|
||||
# A ServiceMonitor for the Prometheus Operator, scraping each pod
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
interval: 30s
|
||||
labels: {}
|
||||
# PrometheusRules: alerts for a pod down, a cluster lease not renewed,
|
||||
# a large or stuck queue, failing deliveries, a certificate about to
|
||||
# expire and a slow data store. They read the pods' group from the
|
||||
# label the ServiceMonitor adds, so they need it on.
|
||||
rules:
|
||||
enabled: false
|
||||
labels: {}
|
||||
# Alerts on restarts and readiness from kube-state-metrics, when the
|
||||
# cluster has it
|
||||
kubeStateMetrics: true
|
||||
# Messages in the queue
|
||||
queueMessages: 5000
|
||||
# Hours the oldest message may wait. A queue keeps retrying a
|
||||
# message for days, so one that waits longer is worth a look.
|
||||
queueOldestHours: 24
|
||||
# Share of deliveries that bounce, over half an hour, once at least
|
||||
# deliveryFailureMinimum have
|
||||
deliveryFailureRatio: 0.25
|
||||
deliveryFailureMinimum: 10
|
||||
# Days before the first certificate in use expires
|
||||
certificateDays: 14
|
||||
# 99th-percentile data store read or write, in milliseconds
|
||||
storeLatencyMs: 1000
|
||||
# Push metrics to an OpenTelemetry collector over OTLP. Only when the
|
||||
# store has no OpenTelemetry metrics set: the console's setting wins.
|
||||
# Every pod pushes its own, named by service.instance.id and
|
||||
# k8s.pod.name; the collector adds them up.
|
||||
otlp:
|
||||
enabled: false
|
||||
# The collector, e.g. http://otel-collector.monitoring:4318
|
||||
endpoint: ""
|
||||
# http/protobuf or grpc
|
||||
protocol: http/protobuf
|
||||
intervalSeconds: 60
|
||||
# A Secret whose key headers holds headers to send, key=value pairs by
|
||||
# commas or lines, for a collector that wants a token. The pods read it
|
||||
# as a file.
|
||||
headersSecret: ""
|
||||
# A Grafana dashboard as a ConfigMap, for Grafana's dashboard sidecar
|
||||
grafanaDashboard:
|
||||
enabled: false
|
||||
# The label the sidecar looks for
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations: {}
|
||||
|
||||
# The webmail, signing people in through the server's own page (OAuth). Off
|
||||
# by default: it needs its own address, and the server's publicUrl.
|
||||
webmail:
|
||||
@@ -365,6 +428,11 @@ networkPolicy:
|
||||
# matchLabels: {kubernetes.io/metadata.name: ingress-nginx}
|
||||
# Empty: from anywhere. The webmail always reaches the server.
|
||||
httpFrom: []
|
||||
# Prometheus, when httpFrom is set: it scrapes the server pods' http
|
||||
# port. As NetworkPolicy peers, e.g.
|
||||
# - namespaceSelector:
|
||||
# matchLabels: {kubernetes.io/metadata.name: monitoring}
|
||||
metricsFrom: []
|
||||
# Where the server and its Jobs may connect, as NetworkPolicy egress rules,
|
||||
# besides DNS. Empty: anywhere, which a mail server needs to deliver mail
|
||||
# unless it hands everything to a relay. Narrow it to your data store,
|
||||
|
||||
Reference in new issue
Block a user