server.upgrade: the store's upgrade before the rollout (K-C9) #17

Merged
jcoffey-dev merged 1 commits from feat/upgrade-hooks into main 2026-11-18 20:47:16 +00:00
6 changed files with 432 additions and 0 deletions

No files matched your search

+6
View File
@@ -27,3 +27,9 @@ runs, pinned by digest.
- `server.seed`: the new store's setup (domain, administrator, blob store,
coordinator, roles), applied once on install by a pre-install Job running
the server's `--seed`, before any server pod starts.
- `server.upgrade`: before an upgrade or rollback rolls out, a Job on the
new image asks the server what it does to the store (`--upgrade-plan`).
A layout change waits for `upgrade.acknowledgeLayout`. When a layout step
can't run with older pods up, the server Deployments are scaled to 0
first, which a rolling update can't do. Then `--upgrade-store` upgrades
the store.
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# The chart's pre-upgrade run, on the new server image, before Helm rolls
# out the new pods. It asks the server what the upgrade does to the store
# (--upgrade-plan), refuses a layout change nobody acknowledged, stops the
# server pods when a step needs every node stopped, and upgrades the store
# (--upgrade-store). Only sh, sed and curl: the image has nothing more.
#
# From the Job: RELEASE, NAMESPACE, DEPLOYMENTS (space-separated),
# ACKNOWLEDGE_LAYOUT, STOP_SERVERS (true/false), STOP_WAIT_SECONDS.
set -u
config=/etc/inbuxa/config.json
# The server logs to stdout too; the plan is the one line of JSON
out=$(inbuxa --upgrade-plan --config "$config")
code=$?
plan=$(printf '%s\n' "$out" | grep '^{' | tail -n 1)
echo "Plan: $plan"
case $code in
0)
echo "The store needs no layout change: nothing to do before the rollout."
exit 0
;;
3 | 4) ;;
2)
echo "A newer release has changed this store's layout. This release can't run on it:" \
"upgrade to that release, or restore the backup taken before it." >&2
exit 1
;;
*)
echo "The server couldn't plan the upgrade (exit $code); its message is above." >&2
exit 1
;;
esac
from=$(printf '%s' "$plan" | sed -n 's/.*"store":\([0-9]*\).*/\1/p')
target=$(printf '%s' "$plan" | sed -n 's/.*"target":\([0-9]*\).*/\1/p')
if [ "${ACKNOWLEDGE_LAYOUT:-0}" != "$target" ]; then
cat >&2 <<MSG
This upgrade changes the store's layout from version $from to $target, which
older releases can't read: going back means restoring a backup taken before
it. Take one now, then upgrade again with
--set server.upgrade.acknowledgeLayout=$target
Nothing has been changed.
MSG
exit 1
fi
if [ "$code" = 4 ]; then
if [ "${STOP_SERVERS:-true}" != true ]; then
echo "A layout step needs every server pod stopped, and stopServers is off:" \
"the upgrade waits for them to stop, and fails if they don't."
else
echo "A layout step needs every server pod stopped: scaling $DEPLOYMENTS to 0. Mail is down until the rollout."
api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"
sa=/var/run/secrets/kubernetes.io/serviceaccount
token=$(cat "$sa/token")
kube() {
curl -fsS --cacert "$sa/ca.crt" -H "Authorization: Bearer $token" "$@"
}
for deployment in $DEPLOYMENTS; do
kube -X PATCH -H 'Content-Type: application/merge-patch+json' \
-d '{"spec":{"replicas":0}}' \
"$api/apis/apps/v1/namespaces/$NAMESPACE/deployments/$deployment/scale" >/dev/null \
|| { echo "Couldn't scale $deployment to 0." >&2; exit 1; }
done
selector="app.kubernetes.io/instance%3D$RELEASE,app.kubernetes.io/component%3Dserver"
waited=0
# The API server pretty-prints for curl: "items": [] with a space
until kube "$api/api/v1/namespaces/$NAMESPACE/pods?labelSelector=$selector" | grep -Eq '"items": *\[ *\]'; do
if [ "$waited" -ge "${STOP_WAIT_SECONDS:-300}" ]; then
echo "The server pods didn't stop within ${STOP_WAIT_SECONDS:-300}s." >&2
exit 1
fi
sleep 5
waited=$((waited + 5))
done
echo "Every server pod has stopped (${waited}s)."
fi
fi
exec inbuxa --upgrade-store --config "$config"
+206
View File
@@ -0,0 +1,206 @@
{{/*
SPDX-FileCopyrightText: 2026 Coffey Labs LLC
SPDX-License-Identifier: Apache-2.0
Before an upgrade rolls new pods out: a Job on the new image asks the server
what the upgrade does to the store, refuses a layout change nobody
acknowledged, stops the server pods when a layout step needs every node
stopped (a rolling update would stall there), and upgrades the store. The
script is files/upgrade.sh.
The rights to scale the server Deployments belong to a ServiceAccount that
exists only while the hook runs. On a rollback the same run refuses a store
the older release can't read.
*/}}
{{- $root := . }}
{{- $server := .Values.server }}
{{- $upgrade := $server.upgrade }}
{{- if $upgrade.enabled }}
{{- $name := printf "%s-upgrade" (include "inbuxa.fullname" $root) }}
{{- $deployments := list }}
{{- range $group := list "front" "mta" "core" }}
{{- if (index $server.groups $group).enabled }}
{{- $deployments = append $deployments (printf "%s-%s" (include "inbuxa.fullname" $root) $group) }}
{{- end }}
{{- end }}
{{- $hook := dict "helm.sh/hook" "pre-upgrade,pre-rollback" "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
{{- if $upgrade.stopServers }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: upgrade
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: upgrade
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
rules:
# Scale this release's server Deployments, and nothing else
- apiGroups: ["apps"]
resources: ["deployments/scale"]
resourceNames:
{{- toYaml $deployments | nindent 6 }}
verbs: ["get", "patch"]
# See when their pods are gone
- apiGroups: [""]
resources: ["pods"]
verbs: ["list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: upgrade
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $name }}
subjects:
- kind: ServiceAccount
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
---
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: upgrade
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $hook) | nindent 4 }}
data:
config.json: {{ toJson $server.dataStore | quote }}
upgrade.sh: |
{{- $root.Files.Get "files/upgrade.sh" | nindent 4 }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: upgrade
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "0") $hook) | nindent 4 }}
ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it"
ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing"
ignore-check.kube-linter.io/restart-policy: "Never on purpose: a failed upgrade is for a person to look at, not to retry"
spec:
backoffLimit: 0
activeDeadlineSeconds: {{ $upgrade.timeoutSeconds }}
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
{{- include "inbuxa.labels" $root | nindent 8 }}
app.kubernetes.io/component: upgrade
spec:
{{- if $upgrade.stopServers }}
serviceAccountName: {{ $name }}
automountServiceAccountToken: true
{{- else }}
automountServiceAccountToken: false
{{- end }}
enableServiceLinks: false
restartPolicy: Never
{{- with $root.Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 2000
runAsGroup: 2000
fsGroup: 2000
seccompProfile:
type: RuntimeDefault
containers:
- name: upgrade
image: {{ include "inbuxa.server.image" $root }}
imagePullPolicy: {{ $server.image.pullPolicy }}
command: ["/bin/sh", "/etc/inbuxa/upgrade/upgrade.sh"]
env:
- name: INBUXA_HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: RELEASE
value: {{ $root.Release.Name | quote }}
- name: NAMESPACE
value: {{ $root.Release.Namespace | quote }}
- name: DEPLOYMENTS
value: {{ join " " $deployments | quote }}
- name: ACKNOWLEDGE_LAYOUT
value: {{ $upgrade.acknowledgeLayout | quote }}
- name: STOP_SERVERS
value: {{ $upgrade.stopServers | quote }}
- name: STOP_WAIT_SECONDS
value: {{ add $server.terminationGracePeriodSeconds 180 | quote }}
resources:
{{- toYaml (default $server.resources $upgrade.resources) | nindent 12 }}
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
volumeMounts:
- name: hook
mountPath: /etc/inbuxa/config.json
subPath: config.json
readOnly: true
- name: hook
mountPath: /etc/inbuxa/upgrade/upgrade.sh
subPath: upgrade.sh
readOnly: true
{{- if $server.secretMounts }}
- name: secrets
mountPath: /etc/inbuxa/secrets
readOnly: true
{{- end }}
- name: data
mountPath: /var/lib/inbuxa
- name: tmp
mountPath: /tmp
volumes:
- name: hook
configMap:
name: {{ $name }}
{{- if $server.secretMounts }}
- name: secrets
projected:
defaultMode: 0440
sources:
{{- range $server.secretMounts }}
- secret:
name: {{ .secretName }}
items:
{{- toYaml .items | nindent 20 }}
{{- end }}
{{- end }}
- name: data
emptyDir: {}
- name: tmp
emptyDir: {}
{{- end }}
+90
View File
@@ -0,0 +1,90 @@
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
suite: upgrade
templates:
- server/upgrade.yaml
release:
name: mail
namespace: inbuxa
values:
- ../ci/defaults-values.yaml
tests:
- it: runs before an upgrade or a rollback, the rights first
asserts:
- hasDocuments:
count: 5
- equal:
path: metadata.annotations["helm.sh/hook"]
value: pre-upgrade,pre-rollback
- equal:
path: metadata.annotations["helm.sh/hook-weight"]
value: "-10"
documentIndex: 1
- equal:
path: metadata.annotations["helm.sh/hook-weight"]
value: "0"
documentIndex: 4
- it: may scale this release's server Deployments, and nothing else
documentIndex: 1
asserts:
- equal:
path: rules[0]
value:
apiGroups: [apps]
resources: [deployments/scale]
resourceNames: [mail-inbuxa-front, mail-inbuxa-mta, mail-inbuxa-core]
verbs: [get, patch]
- lengthEqual:
path: rules
count: 2
- it: tells the script the release, the acknowledgement and the groups
set:
server.upgrade.acknowledgeLayout: 3
server.groups.mta.enabled: false
server.services.mail.groups: [front]
documentIndex: 4
asserts:
- contains:
path: spec.template.spec.containers[0].env
content: {name: ACKNOWLEDGE_LAYOUT, value: "3"}
- contains:
path: spec.template.spec.containers[0].env
content: {name: DEPLOYMENTS, value: mail-inbuxa-front mail-inbuxa-core}
- contains:
path: spec.template.spec.containers[0].env
content: {name: STOP_WAIT_SECONDS, value: "300"}
- equal:
path: spec.template.spec.serviceAccountName
value: mail-inbuxa-upgrade
- it: carries the script from files/upgrade.sh
documentIndex: 3
asserts:
- matchRegex:
path: data["upgrade.sh"]
pattern: inbuxa --upgrade-plan --config
- exists:
path: data["config.json"]
- it: needs no rights when it mustn't stop the servers
set:
server.upgrade.stopServers: false
asserts:
- hasDocuments:
count: 2
- equal:
path: spec.template.spec.automountServiceAccountToken
value: false
documentIndex: 1
- notExists:
path: spec.template.spec.serviceAccountName
documentIndex: 1
- it: can be turned off
set:
server.upgrade.enabled: false
asserts:
- hasDocuments:
count: 0
+24
View File
@@ -235,6 +235,30 @@
}
}
},
"upgrade": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"type": "boolean"
},
"acknowledgeLayout": {
"description": "The store layout version an upgrade may move the store to; 0 for none.",
"type": "integer",
"minimum": 0
},
"stopServers": {
"type": "boolean"
},
"timeoutSeconds": {
"type": "integer",
"minimum": 60
},
"resources": {
"type": "object"
}
}
},
"publicUrl": {
"type": "string",
"pattern": "^(https?://[^/\\s]+)?$"
+17
View File
@@ -59,6 +59,23 @@ server:
timeoutSeconds: 600
resources: {}
# Before an upgrade rolls new pods out, a Job on the new image asks the
# server what the upgrade does to the store, and upgrades it. Give
# `helm upgrade` a --timeout long enough for it: 20m covers a layout step
# that has to wait for the old pods' heartbeats to run out.
upgrade:
enabled: true
# A change to the store's layout can't be undone without a backup taken
# before it. The upgrade refuses one until this names the layout version
# it moves to (the refusal says which): 3 for the 27.1 crossing.
acknowledgeLayout: 0
# When a layout step needs every server pod stopped, scale the server
# Deployments to 0 first (mail is down until the rollout). Off: the
# upgrade waits for them to stop, and fails if they don't.
stopServers: true
timeoutSeconds: 1800
resources: {}
# Where browsers and clients reach the server, as https://host. Sign-in
# pages and OAuth metadata name it.
publicUrl: ""