Backups and store checks on a schedule (K-C11) #28

Merged
jcoffey-dev merged 1 commits from feat/scheduled-jobs into main 2026-11-19 21:08:46 +00:00
8 changed files with 544 additions and 3 deletions

No files matched your search

+8
View File
@@ -78,6 +78,14 @@ runs, pinned by digest.
status names the part and why. The server container keeps its last log
lines as its termination message too. Needs server 27.1-rc.33: an older
server doesn't know `--check`.
- Backups and store checks on a schedule. `server.backup` exports the
whole store nightly to an S3 bucket (`inbuxa --export s3://...`), streamed
while the server runs, into a folder per export with `export.json`
written last; `inbuxa --import s3://bucket/prefix` restores the newest
finished one. `server.storeCheck` runs a quick check nightly and a full
one weekly; a check that finds problems fails its Job. With the metrics
rules and kube-state-metrics, InbuxaBackupMissing and
InbuxaStoreCheckFailed alert on them. Needs server 27.1-rc.34.
- Metrics (`server.metrics`, off by default). `prometheus`: every server
pod serves `/metrics/prometheus` behind a username and a password the pods
read as a file, and a headless Service lists them all, with a
+9
View File
@@ -59,6 +59,15 @@ server:
mail:
loadBalancerClass: example.org/lb
annotations: {example.org/lb-pool: mail}
backup:
enabled: true
url: s3://mail-backups/inbuxa
endpoint: http://minio.storage.svc:9000
existingSecret: inbuxa-backup-keys
timeZone: America/Phoenix
storeCheck:
quick: {enabled: true}
full: {enabled: true}
metrics:
prometheus:
enabled: true
+7 -3
View File
@@ -11,7 +11,9 @@ all, in and out; the policies after it open only what each one needs:
and from networkPolicy.httpFrom
and .metricsFrom
seed, upgrade, nothing as the server
egress-check
egress-check,
backup,
store-check
webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80
and 443 anywhere, for remote images
and calendars (networkPolicy.webmailInternet)
@@ -91,7 +93,9 @@ spec:
---
# The seed and upgrade Jobs reach the data store as the server does; the
# upgrade Job reaches the Kubernetes API too, and the egress check the API
# and server.egress.checkUrl. Nothing connects to them.
# and server.egress.checkUrl. The backup and store checks reach the stores,
# and the backup its bucket (when networkPolicy.serverEgress narrows the
# way out, it has to name the bucket too). Nothing connects to them.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
@@ -104,7 +108,7 @@ spec:
matchLabels:
{{- include "inbuxa.selectorLabels" $root | nindent 6 }}
matchExpressions:
- {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check]}
- {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check, backup, store-check]}
policyTypes: [Ingress, Egress]
egress:
{{- include "inbuxa.networkPolicy.serverEgress" $root | nindent 4 }}
@@ -0,0 +1,188 @@
{{/*
SPDX-FileCopyrightText: 2026 Coffey Labs LLC
SPDX-License-Identifier: Apache-2.0
Scheduled one-shots on the server image, against the store the server pods
use: a nightly export to S3 (server.backup) and store checks
(server.storeCheck). Each run is the server binary with one flag, the
server's config.json and its Secrets; it opens no ports and nothing
connects to it. Never two of one kind at once (concurrencyPolicy Forbid),
and the server itself runs one store check at a time: a check that finds
another running exits 2.
*/}}
{{- $root := . }}
{{- $server := .Values.server }}
{{- $full := include "inbuxa.fullname" . }}
{{- $backup := $server.backup }}
{{- $check := $server.storeCheck }}
{{- $jobs := list }}
{{- if $backup.enabled }}
{{- if not $backup.url }}
{{- fail "server.backup.url: name the bucket, as s3://bucket/prefix" }}
{{- end }}
{{- $jobs = append $jobs (dict "name" "backup" "component" "backup" "schedule" $backup.schedule "timeZone" $backup.timeZone "timeout" $backup.timeoutSeconds "backoff" $backup.backoffLimit "ok" $backup.successfulJobsHistoryLimit "failed" $backup.failedJobsHistoryLimit "resources" $backup.resources "args" (list "--export" $backup.url)) }}
{{- end }}
{{- range $mode := list "quick" "full" }}
{{- $c := index $check $mode }}
{{- if $c.enabled }}
{{- /* Never again on its own: exit 3 is an answer (problems found), not a failure to retry */}}
{{- $jobs = append $jobs (dict "name" (printf "store-check-%s" $mode) "component" "store-check" "schedule" $c.schedule "timeZone" $check.timeZone "timeout" $c.timeoutSeconds "backoff" 0 "ok" $check.successfulJobsHistoryLimit "failed" $check.failedJobsHistoryLimit "resources" $check.resources "args" (list "--store-check" $mode)) }}
{{- end }}
{{- end }}
{{- range $i, $job := $jobs }}
{{- if $i }}
---
{{- end }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ $full }}-{{ $job.name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: {{ $job.component }}
annotations:
ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it"
ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing"
ignore-check.kube-linter.io/restart-policy: "Never: a failed run is a new Job, up to backoffLimit; a store check is never rerun"
{{- if eq $job.name "backup" }}
ignore-check.kube-linter.io/env-var-secret: "INBUXA_EXPORT_S3_SECRET_KEY_FILE holds a path; the key is a mounted file"
{{- end }}
spec:
schedule: {{ $job.schedule | quote }}
{{- with $job.timeZone }}
timeZone: {{ . | quote }}
{{- end }}
concurrencyPolicy: Forbid
# A run missed by more than an hour (the controller was down) waits for
# the next one
startingDeadlineSeconds: 3600
successfulJobsHistoryLimit: {{ $job.ok }}
failedJobsHistoryLimit: {{ $job.failed }}
jobTemplate:
metadata:
labels:
{{- include "inbuxa.labels" $root | nindent 8 }}
app.kubernetes.io/component: {{ $job.component }}
spec:
backoffLimit: {{ $job.backoff }}
activeDeadlineSeconds: {{ $job.timeout }}
template:
metadata:
labels:
{{- include "inbuxa.labels" $root | nindent 12 }}
app.kubernetes.io/component: {{ $job.component }}
spec:
{{- if and (eq $job.name "backup") $backup.serviceAccountName }}
# The bucket's keys from the pod's role (IRSA, Pod Identity)
serviceAccountName: {{ $backup.serviceAccountName }}
{{- else }}
automountServiceAccountToken: false
{{- end }}
enableServiceLinks: false
restartPolicy: Never
{{- with $root.Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with $server.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with $server.tolerations }}
tolerations:
{{- toYaml . | nindent 12 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 2000
runAsGroup: 2000
fsGroup: 2000
seccompProfile:
type: RuntimeDefault
containers:
- name: {{ $job.component }}
image: {{ include "inbuxa.server.image" $root }}
imagePullPolicy: {{ $server.image.pullPolicy }}
args:
{{- range $job.args }}
- {{ . | quote }}
{{- end }}
- "--config"
- "/etc/inbuxa/config.json"
env:
- name: INBUXA_HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
{{- if eq $job.name "backup" }}
{{- with $backup.endpoint }}
- name: INBUXA_EXPORT_S3_ENDPOINT
value: {{ . | quote }}
{{- end }}
- name: INBUXA_EXPORT_S3_REGION
value: {{ $backup.region | quote }}
{{- if $backup.existingSecret }}
- name: INBUXA_EXPORT_S3_ACCESS_KEY_FILE
value: /etc/inbuxa/backup-s3/access-key
- name: INBUXA_EXPORT_S3_SECRET_KEY_FILE
value: /etc/inbuxa/backup-s3/secret-key
{{- end }}
{{- end }}
resources:
{{- toYaml (default $server.resources $job.resources) | nindent 16 }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
volumeMounts:
- name: config
mountPath: /etc/inbuxa/config.json
subPath: config.json
readOnly: true
{{- if $server.secretMounts }}
- name: secrets
mountPath: /etc/inbuxa/secrets
readOnly: true
{{- end }}
{{- if and (eq $job.name "backup") $backup.existingSecret }}
- name: backup-s3
mountPath: /etc/inbuxa/backup-s3
readOnly: true
{{- end }}
- name: data
mountPath: /var/lib/inbuxa
- name: tmp
mountPath: /tmp
volumes:
- name: config
configMap:
name: {{ $full }}-server
{{- if $server.secretMounts }}
- name: secrets
projected:
defaultMode: 0440
sources:
{{- range $server.secretMounts }}
- secret:
name: {{ .secretName }}
items:
{{- toYaml .items | nindent 24 }}
{{- end }}
{{- end }}
{{- if and (eq $job.name "backup") $backup.existingSecret }}
- name: backup-s3
secret:
secretName: {{ $backup.existingSecret }}
defaultMode: 0440
items:
- {key: access-key, path: access-key}
- {key: secret-key, path: secret-key}
{{- end }}
# Scratch only: the export streams to the bucket
- name: data
emptyDir: {}
- name: tmp
emptyDir: {}
{{- end }}
@@ -210,6 +210,33 @@ spec:
annotations:
summary: {{`"{{ $labels.pod }} keeps going in and out of ready"`}}
description: A pod is ready while its data store answers; flapping readiness usually means the store is struggling.
{{- if $root.Values.server.backup.enabled }}
- alert: InbuxaBackupMissing
expr: >-
time() - max(kube_cronjob_status_last_successful_time{namespace="{{ .Release.Namespace }}",cronjob="{{ $full }}-backup"}
or kube_cronjob_created{namespace="{{ .Release.Namespace }}",cronjob="{{ $full }}-backup"})
> {{ printf "%.0f" (mulf $r.backupHours 3600) }}
for: 10m
labels:
severity: critical
annotations:
summary: No export of the store has finished for {{ $r.backupHours }} hours
description: >-
The backup CronJob's runs have failed, or not started. Its last
Job's log says why (kubectl logs job/...).
{{- end }}
{{- if or $root.Values.server.storeCheck.quick.enabled $root.Values.server.storeCheck.full.enabled }}
- alert: InbuxaStoreCheckFailed
expr: kube_job_status_failed{namespace="{{ .Release.Namespace }}",job_name=~"{{ $full }}-store-check-.*"} > 0
labels:
severity: warning
annotations:
summary: {{`"Store check {{ $labels.job_name }} found problems, or couldn't finish"`}}
description: >-
Its log has the summary; the console's Store checks page lists
the problems. Delete the Job once it has been looked at: the
alert lasts while the failed Job does.
{{- end }}
{{- end }}
{{- end }}
{{- end }}
+142
View File
@@ -0,0 +1,142 @@
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
suite: scheduled jobs
templates:
- server/cronjobs.yaml
- server/metrics.yaml
- networkpolicy.yaml
release:
name: mail
namespace: inbuxa
values:
- ../ci/defaults-values.yaml
tests:
- it: schedules nothing by default
template: server/cronjobs.yaml
asserts:
- hasDocuments:
count: 0
- it: needs a bucket for the backup
template: server/cronjobs.yaml
set:
server.backup.enabled: true
asserts:
- failedTemplate:
errorPattern: name the bucket
- it: exports to the bucket nightly, keys from a Secret as files
template: server/cronjobs.yaml
set:
server.backup.enabled: true
server.backup.url: s3://mail-backups/inbuxa
server.backup.endpoint: http://minio.storage:9000
server.backup.existingSecret: backup-keys
server.backup.timeZone: America/Phoenix
asserts:
- hasDocuments:
count: 1
- equal:
path: metadata.name
value: mail-inbuxa-backup
- equal:
path: spec.schedule
value: "30 2 * * *"
- equal:
path: spec.timeZone
value: America/Phoenix
- equal:
path: spec.concurrencyPolicy
value: Forbid
- equal:
path: spec.jobTemplate.spec.template.spec.containers[0].args
value: [--export, s3://mail-backups/inbuxa, --config, /etc/inbuxa/config.json]
- contains:
path: spec.jobTemplate.spec.template.spec.containers[0].env
content: {name: INBUXA_EXPORT_S3_ENDPOINT, value: http://minio.storage:9000}
- contains:
path: spec.jobTemplate.spec.template.spec.containers[0].env
content: {name: INBUXA_EXPORT_S3_SECRET_KEY_FILE, value: /etc/inbuxa/backup-s3/secret-key}
- contains:
path: spec.jobTemplate.spec.template.spec.volumes
content:
name: backup-s3
secret:
secretName: backup-keys
defaultMode: 0440
items: [{key: access-key, path: access-key}, {key: secret-key, path: secret-key}]
- equal:
path: spec.jobTemplate.spec.template.spec.volumes[0].configMap.name
value: mail-inbuxa-server
- equal:
path: spec.jobTemplate.spec.template.spec.automountServiceAccountToken
value: false
- it: takes the bucket's keys from the pod's role when no Secret is named
template: server/cronjobs.yaml
set:
server.backup.enabled: true
server.backup.url: s3://mail-backups
server.backup.serviceAccountName: inbuxa-backup
asserts:
- equal:
path: spec.jobTemplate.spec.template.spec.serviceAccountName
value: inbuxa-backup
- notContains:
path: spec.jobTemplate.spec.template.spec.containers[0].env
content: {name: INBUXA_EXPORT_S3_ACCESS_KEY_FILE, value: /etc/inbuxa/backup-s3/access-key}
- it: checks the store quick nightly and full weekly, never retrying a check
template: server/cronjobs.yaml
set:
server.storeCheck.quick.enabled: true
server.storeCheck.full.enabled: true
asserts:
- hasDocuments:
count: 2
- equal:
path: spec.jobTemplate.spec.template.spec.containers[0].args
value: [--store-check, quick, --config, /etc/inbuxa/config.json]
documentIndex: 0
- equal:
path: spec.schedule
value: "15 4 * * 0"
documentIndex: 1
- equal:
path: spec.jobTemplate.spec.backoffLimit
value: 0
documentIndex: 1
- equal:
path: spec.jobTemplate.spec.activeDeadlineSeconds
value: 172800
documentIndex: 1
- it: lets the scheduled jobs out as the server, and nothing in
template: networkpolicy.yaml
documentSelector: {path: metadata.name, value: mail-inbuxa-jobs}
asserts:
- contains:
path: spec.podSelector.matchExpressions
content: {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check, backup, store-check]}
- it: alerts when no backup finishes, and when a store check fails
template: server/metrics.yaml
set:
server.backup.enabled: true
server.backup.url: s3://mail-backups
server.storeCheck.quick.enabled: true
server.metrics.prometheus.enabled: true
server.metrics.prometheus.serviceMonitor.enabled: true
server.metrics.prometheus.rules.enabled: true
server.metrics.prometheus.rules.backupHours: 30
documentSelector: {path: kind, value: PrometheusRule}
asserts:
- lengthEqual:
path: spec.groups[0].rules
count: 11
- matchRegex:
path: spec.groups[0].rules[9].expr
pattern: cronjob="mail-inbuxa-backup"\}\)\s+> 108000$
- equal:
path: spec.groups[0].rules[10].expr
value: kube_job_status_failed{namespace="inbuxa",job_name=~"mail-inbuxa-store-check-.*"} > 0
+111
View File
@@ -635,6 +635,10 @@
"storeLatencyMs": {
"type": "number",
"exclusiveMinimum": 0
},
"backupHours": {
"type": "number",
"exclusiveMinimum": 0
}
}
}
@@ -688,6 +692,113 @@
}
}
}
},
"backup": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"schedule": {
"type": "string",
"minLength": 1
},
"timeZone": {
"type": "string"
},
"url": {
"type": "string",
"pattern": "^$|^s3://[^/]+(/.*)?$"
},
"endpoint": {
"type": "string",
"pattern": "^$|^https?://"
},
"region": {
"type": "string",
"minLength": 1
},
"existingSecret": {
"type": "string"
},
"serviceAccountName": {
"type": "string"
},
"timeoutSeconds": {
"type": "integer",
"minimum": 60
},
"backoffLimit": {
"type": "integer",
"minimum": 0
},
"successfulJobsHistoryLimit": {
"type": "integer",
"minimum": 0
},
"failedJobsHistoryLimit": {
"type": "integer",
"minimum": 0
},
"resources": {
"type": "object"
}
}
},
"storeCheck": {
"type": "object",
"additionalProperties": false,
"properties": {
"quick": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"schedule": {
"type": "string",
"minLength": 1
},
"timeoutSeconds": {
"type": "integer",
"minimum": 60
}
}
},
"full": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"schedule": {
"type": "string",
"minLength": 1
},
"timeoutSeconds": {
"type": "integer",
"minimum": 60
}
}
},
"timeZone": {
"type": "string"
},
"successfulJobsHistoryLimit": {
"type": "integer",
"minimum": 0
},
"failedJobsHistoryLimit": {
"type": "integer",
"minimum": 0
},
"resources": {
"type": "object"
}
}
}
}
},
+52
View File
@@ -272,6 +272,55 @@ server:
pop3s: 10995
sieve: 14190
# A nightly export of the whole store to an S3 bucket (`inbuxa --export
# s3://...`), streamed, while the server runs. Each goes in a new folder
# named for the time, with export.json written last: an export without it
# didn't finish. Nothing here deletes old exports: give the bucket a
# lifecycle rule. Restore into a new, empty store with
# `inbuxa --import s3://bucket/prefix` (the newest finished export) before
# any server pod starts. Needs server 27.1-rc.34.
backup:
enabled: false
# Cron, in timeZone (the controller's, UTC on most clusters, when empty)
schedule: "30 2 * * *"
timeZone: ""
# s3://bucket/prefix
url: ""
# An S3-compatible service (MinIO, Garage, Ceph), addressed path-style;
# empty for AWS
endpoint: ""
region: us-east-1
# A Secret with the keys under access-key and secret-key. Empty: the
# AWS defaults (the pod's role, through serviceAccountName)
existingSecret: ""
serviceAccountName: ""
# A run longer than this is stopped (and fails)
timeoutSeconds: 21600
# Tries per night: a failed try leaves an unfinished folder behind
backoffLimit: 1
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
resources: {}
# Store checks on a schedule (`inbuxa --store-check`): quick finds blobs
# the store names but doesn't have; full also reads every blob and checks
# its hash, paced to 20 MiB/s. Each is saved with the console's other store
# checks, and a check that finds problems fails its Job (exit 3), so
# kube-state-metrics, and the alerts, see it. Needs server 27.1-rc.34.
storeCheck:
quick:
enabled: false
schedule: "15 3 * * *"
timeoutSeconds: 21600
full:
enabled: false
schedule: "15 4 * * 0"
timeoutSeconds: 172800
timeZone: ""
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
resources: {}
# Metrics, for Prometheus or an OpenTelemetry collector. Needs server
# 27.1-rc.34; an older server ignores these settings.
metrics:
@@ -312,6 +361,9 @@ server:
certificateDays: 14
# 99th-percentile data store read or write, in milliseconds
storeLatencyMs: 1000
# With backup on and kubeStateMetrics: hours since the last export
# that finished
backupHours: 36
# Push metrics to an OpenTelemetry collector over OTLP. Only when the
# store has no OpenTelemetry metrics set: the console's setting wins.
# Every pod pushes its own, named by service.instance.id and