From 1fce1c4e7bc651b22c413935cc74b99a0a88f1a2 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 13:05:44 -0700 Subject: [PATCH] Backups and store checks on a schedule (K-C11) --- CHANGELOG.md | 8 + charts/inbuxa/ci/everything-values.yaml | 9 + charts/inbuxa/templates/networkpolicy.yaml | 10 +- charts/inbuxa/templates/server/cronjobs.yaml | 188 +++++++++++++++++++ charts/inbuxa/templates/server/metrics.yaml | 27 +++ charts/inbuxa/tests/cronjobs_test.yaml | 142 ++++++++++++++ charts/inbuxa/values.schema.json | 111 +++++++++++ charts/inbuxa/values.yaml | 52 +++++ 8 files changed, 544 insertions(+), 3 deletions(-) create mode 100644 charts/inbuxa/templates/server/cronjobs.yaml create mode 100644 charts/inbuxa/tests/cronjobs_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index f82a2a9..91dee26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,14 @@ runs, pinned by digest. status names the part and why. The server container keeps its last log lines as its termination message too. Needs server 27.1-rc.33: an older server doesn't know `--check`. +- Backups and store checks on a schedule. `server.backup` exports the + whole store nightly to an S3 bucket (`inbuxa --export s3://...`), streamed + while the server runs, into a folder per export with `export.json` + written last; `inbuxa --import s3://bucket/prefix` restores the newest + finished one. `server.storeCheck` runs a quick check nightly and a full + one weekly; a check that finds problems fails its Job. With the metrics + rules and kube-state-metrics, InbuxaBackupMissing and + InbuxaStoreCheckFailed alert on them. Needs server 27.1-rc.34. - Metrics (`server.metrics`, off by default). `prometheus`: every server pod serves `/metrics/prometheus` behind a username and a password the pods read as a file, and a headless Service lists them all, with a diff --git a/charts/inbuxa/ci/everything-values.yaml b/charts/inbuxa/ci/everything-values.yaml index 38e8a90..67ccb19 100644 --- a/charts/inbuxa/ci/everything-values.yaml +++ b/charts/inbuxa/ci/everything-values.yaml @@ -59,6 +59,15 @@ server: mail: loadBalancerClass: example.org/lb annotations: {example.org/lb-pool: mail} + backup: + enabled: true + url: s3://mail-backups/inbuxa + endpoint: http://minio.storage.svc:9000 + existingSecret: inbuxa-backup-keys + timeZone: America/Phoenix + storeCheck: + quick: {enabled: true} + full: {enabled: true} metrics: prometheus: enabled: true diff --git a/charts/inbuxa/templates/networkpolicy.yaml b/charts/inbuxa/templates/networkpolicy.yaml index 07d65e7..e2cd0c8 100644 --- a/charts/inbuxa/templates/networkpolicy.yaml +++ b/charts/inbuxa/templates/networkpolicy.yaml @@ -11,7 +11,9 @@ all, in and out; the policies after it open only what each one needs: and from networkPolicy.httpFrom and .metricsFrom seed, upgrade, nothing as the server - egress-check + egress-check, + backup, + store-check webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80 and 443 anywhere, for remote images and calendars (networkPolicy.webmailInternet) @@ -91,7 +93,9 @@ spec: --- # The seed and upgrade Jobs reach the data store as the server does; the # upgrade Job reaches the Kubernetes API too, and the egress check the API -# and server.egress.checkUrl. Nothing connects to them. +# and server.egress.checkUrl. The backup and store checks reach the stores, +# and the backup its bucket (when networkPolicy.serverEgress narrows the +# way out, it has to name the bucket too). Nothing connects to them. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -104,7 +108,7 @@ spec: matchLabels: {{- include "inbuxa.selectorLabels" $root | nindent 6 }} matchExpressions: - - {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check]} + - {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check, backup, store-check]} policyTypes: [Ingress, Egress] egress: {{- include "inbuxa.networkPolicy.serverEgress" $root | nindent 4 }} diff --git a/charts/inbuxa/templates/server/cronjobs.yaml b/charts/inbuxa/templates/server/cronjobs.yaml new file mode 100644 index 0000000..88f88a1 --- /dev/null +++ b/charts/inbuxa/templates/server/cronjobs.yaml @@ -0,0 +1,188 @@ +{{/* +SPDX-FileCopyrightText: 2026 Coffey Labs LLC +SPDX-License-Identifier: Apache-2.0 + +Scheduled one-shots on the server image, against the store the server pods +use: a nightly export to S3 (server.backup) and store checks +(server.storeCheck). Each run is the server binary with one flag, the +server's config.json and its Secrets; it opens no ports and nothing +connects to it. Never two of one kind at once (concurrencyPolicy Forbid), +and the server itself runs one store check at a time: a check that finds +another running exits 2. +*/}} +{{- $root := . }} +{{- $server := .Values.server }} +{{- $full := include "inbuxa.fullname" . }} +{{- $backup := $server.backup }} +{{- $check := $server.storeCheck }} +{{- $jobs := list }} +{{- if $backup.enabled }} +{{- if not $backup.url }} +{{- fail "server.backup.url: name the bucket, as s3://bucket/prefix" }} +{{- end }} +{{- $jobs = append $jobs (dict "name" "backup" "component" "backup" "schedule" $backup.schedule "timeZone" $backup.timeZone "timeout" $backup.timeoutSeconds "backoff" $backup.backoffLimit "ok" $backup.successfulJobsHistoryLimit "failed" $backup.failedJobsHistoryLimit "resources" $backup.resources "args" (list "--export" $backup.url)) }} +{{- end }} +{{- range $mode := list "quick" "full" }} +{{- $c := index $check $mode }} +{{- if $c.enabled }} +{{- /* Never again on its own: exit 3 is an answer (problems found), not a failure to retry */}} +{{- $jobs = append $jobs (dict "name" (printf "store-check-%s" $mode) "component" "store-check" "schedule" $c.schedule "timeZone" $check.timeZone "timeout" $c.timeoutSeconds "backoff" 0 "ok" $check.successfulJobsHistoryLimit "failed" $check.failedJobsHistoryLimit "resources" $check.resources "args" (list "--store-check" $mode)) }} +{{- end }} +{{- end }} +{{- range $i, $job := $jobs }} +{{- if $i }} +--- +{{- end }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $full }}-{{ $job.name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: {{ $job.component }} + annotations: + ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it" + ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing" + ignore-check.kube-linter.io/restart-policy: "Never: a failed run is a new Job, up to backoffLimit; a store check is never rerun" + {{- if eq $job.name "backup" }} + ignore-check.kube-linter.io/env-var-secret: "INBUXA_EXPORT_S3_SECRET_KEY_FILE holds a path; the key is a mounted file" + {{- end }} +spec: + schedule: {{ $job.schedule | quote }} + {{- with $job.timeZone }} + timeZone: {{ . | quote }} + {{- end }} + concurrencyPolicy: Forbid + # A run missed by more than an hour (the controller was down) waits for + # the next one + startingDeadlineSeconds: 3600 + successfulJobsHistoryLimit: {{ $job.ok }} + failedJobsHistoryLimit: {{ $job.failed }} + jobTemplate: + metadata: + labels: + {{- include "inbuxa.labels" $root | nindent 8 }} + app.kubernetes.io/component: {{ $job.component }} + spec: + backoffLimit: {{ $job.backoff }} + activeDeadlineSeconds: {{ $job.timeout }} + template: + metadata: + labels: + {{- include "inbuxa.labels" $root | nindent 12 }} + app.kubernetes.io/component: {{ $job.component }} + spec: + {{- if and (eq $job.name "backup") $backup.serviceAccountName }} + # The bucket's keys from the pod's role (IRSA, Pod Identity) + serviceAccountName: {{ $backup.serviceAccountName }} + {{- else }} + automountServiceAccountToken: false + {{- end }} + enableServiceLinks: false + restartPolicy: Never + {{- with $root.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $server.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + containers: + - name: {{ $job.component }} + image: {{ include "inbuxa.server.image" $root }} + imagePullPolicy: {{ $server.image.pullPolicy }} + args: + {{- range $job.args }} + - {{ . | quote }} + {{- end }} + - "--config" + - "/etc/inbuxa/config.json" + env: + - name: INBUXA_HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- if eq $job.name "backup" }} + {{- with $backup.endpoint }} + - name: INBUXA_EXPORT_S3_ENDPOINT + value: {{ . | quote }} + {{- end }} + - name: INBUXA_EXPORT_S3_REGION + value: {{ $backup.region | quote }} + {{- if $backup.existingSecret }} + - name: INBUXA_EXPORT_S3_ACCESS_KEY_FILE + value: /etc/inbuxa/backup-s3/access-key + - name: INBUXA_EXPORT_S3_SECRET_KEY_FILE + value: /etc/inbuxa/backup-s3/secret-key + {{- end }} + {{- end }} + resources: + {{- toYaml (default $server.resources $job.resources) | nindent 16 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: config + mountPath: /etc/inbuxa/config.json + subPath: config.json + readOnly: true + {{- if $server.secretMounts }} + - name: secrets + mountPath: /etc/inbuxa/secrets + readOnly: true + {{- end }} + {{- if and (eq $job.name "backup") $backup.existingSecret }} + - name: backup-s3 + mountPath: /etc/inbuxa/backup-s3 + readOnly: true + {{- end }} + - name: data + mountPath: /var/lib/inbuxa + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: {{ $full }}-server + {{- if $server.secretMounts }} + - name: secrets + projected: + defaultMode: 0440 + sources: + {{- range $server.secretMounts }} + - secret: + name: {{ .secretName }} + items: + {{- toYaml .items | nindent 24 }} + {{- end }} + {{- end }} + {{- if and (eq $job.name "backup") $backup.existingSecret }} + - name: backup-s3 + secret: + secretName: {{ $backup.existingSecret }} + defaultMode: 0440 + items: + - {key: access-key, path: access-key} + - {key: secret-key, path: secret-key} + {{- end }} + # Scratch only: the export streams to the bucket + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} +{{- end }} diff --git a/charts/inbuxa/templates/server/metrics.yaml b/charts/inbuxa/templates/server/metrics.yaml index e5d80cd..3571e05 100644 --- a/charts/inbuxa/templates/server/metrics.yaml +++ b/charts/inbuxa/templates/server/metrics.yaml @@ -210,6 +210,33 @@ spec: annotations: summary: {{`"{{ $labels.pod }} keeps going in and out of ready"`}} description: A pod is ready while its data store answers; flapping readiness usually means the store is struggling. + {{- if $root.Values.server.backup.enabled }} + - alert: InbuxaBackupMissing + expr: >- + time() - max(kube_cronjob_status_last_successful_time{namespace="{{ .Release.Namespace }}",cronjob="{{ $full }}-backup"} + or kube_cronjob_created{namespace="{{ .Release.Namespace }}",cronjob="{{ $full }}-backup"}) + > {{ printf "%.0f" (mulf $r.backupHours 3600) }} + for: 10m + labels: + severity: critical + annotations: + summary: No export of the store has finished for {{ $r.backupHours }} hours + description: >- + The backup CronJob's runs have failed, or not started. Its last + Job's log says why (kubectl logs job/...). + {{- end }} + {{- if or $root.Values.server.storeCheck.quick.enabled $root.Values.server.storeCheck.full.enabled }} + - alert: InbuxaStoreCheckFailed + expr: kube_job_status_failed{namespace="{{ .Release.Namespace }}",job_name=~"{{ $full }}-store-check-.*"} > 0 + labels: + severity: warning + annotations: + summary: {{`"Store check {{ $labels.job_name }} found problems, or couldn't finish"`}} + description: >- + Its log has the summary; the console's Store checks page lists + the problems. Delete the Job once it has been looked at: the + alert lasts while the failed Job does. + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/inbuxa/tests/cronjobs_test.yaml b/charts/inbuxa/tests/cronjobs_test.yaml new file mode 100644 index 0000000..2145074 --- /dev/null +++ b/charts/inbuxa/tests/cronjobs_test.yaml @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +suite: scheduled jobs +templates: + - server/cronjobs.yaml + - server/metrics.yaml + - networkpolicy.yaml +release: + name: mail + namespace: inbuxa +values: + - ../ci/defaults-values.yaml +tests: + - it: schedules nothing by default + template: server/cronjobs.yaml + asserts: + - hasDocuments: + count: 0 + + - it: needs a bucket for the backup + template: server/cronjobs.yaml + set: + server.backup.enabled: true + asserts: + - failedTemplate: + errorPattern: name the bucket + + - it: exports to the bucket nightly, keys from a Secret as files + template: server/cronjobs.yaml + set: + server.backup.enabled: true + server.backup.url: s3://mail-backups/inbuxa + server.backup.endpoint: http://minio.storage:9000 + server.backup.existingSecret: backup-keys + server.backup.timeZone: America/Phoenix + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.name + value: mail-inbuxa-backup + - equal: + path: spec.schedule + value: "30 2 * * *" + - equal: + path: spec.timeZone + value: America/Phoenix + - equal: + path: spec.concurrencyPolicy + value: Forbid + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].args + value: [--export, s3://mail-backups/inbuxa, --config, /etc/inbuxa/config.json] + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: {name: INBUXA_EXPORT_S3_ENDPOINT, value: http://minio.storage:9000} + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: {name: INBUXA_EXPORT_S3_SECRET_KEY_FILE, value: /etc/inbuxa/backup-s3/secret-key} + - contains: + path: spec.jobTemplate.spec.template.spec.volumes + content: + name: backup-s3 + secret: + secretName: backup-keys + defaultMode: 0440 + items: [{key: access-key, path: access-key}, {key: secret-key, path: secret-key}] + - equal: + path: spec.jobTemplate.spec.template.spec.volumes[0].configMap.name + value: mail-inbuxa-server + - equal: + path: spec.jobTemplate.spec.template.spec.automountServiceAccountToken + value: false + + - it: takes the bucket's keys from the pod's role when no Secret is named + template: server/cronjobs.yaml + set: + server.backup.enabled: true + server.backup.url: s3://mail-backups + server.backup.serviceAccountName: inbuxa-backup + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.serviceAccountName + value: inbuxa-backup + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: {name: INBUXA_EXPORT_S3_ACCESS_KEY_FILE, value: /etc/inbuxa/backup-s3/access-key} + + - it: checks the store quick nightly and full weekly, never retrying a check + template: server/cronjobs.yaml + set: + server.storeCheck.quick.enabled: true + server.storeCheck.full.enabled: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].args + value: [--store-check, quick, --config, /etc/inbuxa/config.json] + documentIndex: 0 + - equal: + path: spec.schedule + value: "15 4 * * 0" + documentIndex: 1 + - equal: + path: spec.jobTemplate.spec.backoffLimit + value: 0 + documentIndex: 1 + - equal: + path: spec.jobTemplate.spec.activeDeadlineSeconds + value: 172800 + documentIndex: 1 + + - it: lets the scheduled jobs out as the server, and nothing in + template: networkpolicy.yaml + documentSelector: {path: metadata.name, value: mail-inbuxa-jobs} + asserts: + - contains: + path: spec.podSelector.matchExpressions + content: {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check, backup, store-check]} + + - it: alerts when no backup finishes, and when a store check fails + template: server/metrics.yaml + set: + server.backup.enabled: true + server.backup.url: s3://mail-backups + server.storeCheck.quick.enabled: true + server.metrics.prometheus.enabled: true + server.metrics.prometheus.serviceMonitor.enabled: true + server.metrics.prometheus.rules.enabled: true + server.metrics.prometheus.rules.backupHours: 30 + documentSelector: {path: kind, value: PrometheusRule} + asserts: + - lengthEqual: + path: spec.groups[0].rules + count: 11 + - matchRegex: + path: spec.groups[0].rules[9].expr + pattern: cronjob="mail-inbuxa-backup"\}\)\s+> 108000$ + - equal: + path: spec.groups[0].rules[10].expr + value: kube_job_status_failed{namespace="inbuxa",job_name=~"mail-inbuxa-store-check-.*"} > 0 diff --git a/charts/inbuxa/values.schema.json b/charts/inbuxa/values.schema.json index 216e60e..ec63d4c 100644 --- a/charts/inbuxa/values.schema.json +++ b/charts/inbuxa/values.schema.json @@ -635,6 +635,10 @@ "storeLatencyMs": { "type": "number", "exclusiveMinimum": 0 + }, + "backupHours": { + "type": "number", + "exclusiveMinimum": 0 } } } @@ -688,6 +692,113 @@ } } } + }, + "backup": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "schedule": { + "type": "string", + "minLength": 1 + }, + "timeZone": { + "type": "string" + }, + "url": { + "type": "string", + "pattern": "^$|^s3://[^/]+(/.*)?$" + }, + "endpoint": { + "type": "string", + "pattern": "^$|^https?://" + }, + "region": { + "type": "string", + "minLength": 1 + }, + "existingSecret": { + "type": "string" + }, + "serviceAccountName": { + "type": "string" + }, + "timeoutSeconds": { + "type": "integer", + "minimum": 60 + }, + "backoffLimit": { + "type": "integer", + "minimum": 0 + }, + "successfulJobsHistoryLimit": { + "type": "integer", + "minimum": 0 + }, + "failedJobsHistoryLimit": { + "type": "integer", + "minimum": 0 + }, + "resources": { + "type": "object" + } + } + }, + "storeCheck": { + "type": "object", + "additionalProperties": false, + "properties": { + "quick": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "schedule": { + "type": "string", + "minLength": 1 + }, + "timeoutSeconds": { + "type": "integer", + "minimum": 60 + } + } + }, + "full": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "schedule": { + "type": "string", + "minLength": 1 + }, + "timeoutSeconds": { + "type": "integer", + "minimum": 60 + } + } + }, + "timeZone": { + "type": "string" + }, + "successfulJobsHistoryLimit": { + "type": "integer", + "minimum": 0 + }, + "failedJobsHistoryLimit": { + "type": "integer", + "minimum": 0 + }, + "resources": { + "type": "object" + } + } } } }, diff --git a/charts/inbuxa/values.yaml b/charts/inbuxa/values.yaml index db56932..699d2cf 100644 --- a/charts/inbuxa/values.yaml +++ b/charts/inbuxa/values.yaml @@ -272,6 +272,55 @@ server: pop3s: 10995 sieve: 14190 + # A nightly export of the whole store to an S3 bucket (`inbuxa --export + # s3://...`), streamed, while the server runs. Each goes in a new folder + # named for the time, with export.json written last: an export without it + # didn't finish. Nothing here deletes old exports: give the bucket a + # lifecycle rule. Restore into a new, empty store with + # `inbuxa --import s3://bucket/prefix` (the newest finished export) before + # any server pod starts. Needs server 27.1-rc.34. + backup: + enabled: false + # Cron, in timeZone (the controller's, UTC on most clusters, when empty) + schedule: "30 2 * * *" + timeZone: "" + # s3://bucket/prefix + url: "" + # An S3-compatible service (MinIO, Garage, Ceph), addressed path-style; + # empty for AWS + endpoint: "" + region: us-east-1 + # A Secret with the keys under access-key and secret-key. Empty: the + # AWS defaults (the pod's role, through serviceAccountName) + existingSecret: "" + serviceAccountName: "" + # A run longer than this is stopped (and fails) + timeoutSeconds: 21600 + # Tries per night: a failed try leaves an unfinished folder behind + backoffLimit: 1 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + resources: {} + + # Store checks on a schedule (`inbuxa --store-check`): quick finds blobs + # the store names but doesn't have; full also reads every blob and checks + # its hash, paced to 20 MiB/s. Each is saved with the console's other store + # checks, and a check that finds problems fails its Job (exit 3), so + # kube-state-metrics, and the alerts, see it. Needs server 27.1-rc.34. + storeCheck: + quick: + enabled: false + schedule: "15 3 * * *" + timeoutSeconds: 21600 + full: + enabled: false + schedule: "15 4 * * 0" + timeoutSeconds: 172800 + timeZone: "" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + resources: {} + # Metrics, for Prometheus or an OpenTelemetry collector. Needs server # 27.1-rc.34; an older server ignores these settings. metrics: @@ -312,6 +361,9 @@ server: certificateDays: 14 # 99th-percentile data store read or write, in milliseconds storeLatencyMs: 1000 + # With backup on and kubeStateMetrics: hours since the last export + # that finished + backupHours: 36 # Push metrics to an OpenTelemetry collector over OTLP. Only when the # store has no OpenTelemetry metrics set: the console's setting wins. # Every pod pushes its own, named by service.instance.id and -- 2.54.0