server.upgrade: the store's upgrade before the rollout (K-C9) #17
No files matched your search
@@ -27,3 +27,9 @@ runs, pinned by digest.
|
||||
- `server.seed`: the new store's setup (domain, administrator, blob store,
|
||||
coordinator, roles), applied once on install by a pre-install Job running
|
||||
the server's `--seed`, before any server pod starts.
|
||||
- `server.upgrade`: before an upgrade or rollback rolls out, a Job on the
|
||||
new image asks the server what it does to the store (`--upgrade-plan`).
|
||||
A layout change waits for `upgrade.acknowledgeLayout`. When a layout step
|
||||
can't run with older pods up, the server Deployments are scaled to 0
|
||||
first, which a rolling update can't do. Then `--upgrade-store` upgrades
|
||||
the store.
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The chart's pre-upgrade run, on the new server image, before Helm rolls
|
||||
# out the new pods. It asks the server what the upgrade does to the store
|
||||
# (--upgrade-plan), refuses a layout change nobody acknowledged, stops the
|
||||
# server pods when a step needs every node stopped, and upgrades the store
|
||||
# (--upgrade-store). Only sh, sed and curl: the image has nothing more.
|
||||
#
|
||||
# From the Job: RELEASE, NAMESPACE, DEPLOYMENTS (space-separated),
|
||||
# ACKNOWLEDGE_LAYOUT, STOP_SERVERS (true/false), STOP_WAIT_SECONDS.
|
||||
set -u
|
||||
|
||||
config=/etc/inbuxa/config.json
|
||||
|
||||
# The server logs to stdout too; the plan is the one line of JSON
|
||||
out=$(inbuxa --upgrade-plan --config "$config")
|
||||
code=$?
|
||||
plan=$(printf '%s\n' "$out" | grep '^{' | tail -n 1)
|
||||
echo "Plan: $plan"
|
||||
|
||||
case $code in
|
||||
0)
|
||||
echo "The store needs no layout change: nothing to do before the rollout."
|
||||
exit 0
|
||||
;;
|
||||
3 | 4) ;;
|
||||
2)
|
||||
echo "A newer release has changed this store's layout. This release can't run on it:" \
|
||||
"upgrade to that release, or restore the backup taken before it." >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "The server couldn't plan the upgrade (exit $code); its message is above." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
from=$(printf '%s' "$plan" | sed -n 's/.*"store":\([0-9]*\).*/\1/p')
|
||||
target=$(printf '%s' "$plan" | sed -n 's/.*"target":\([0-9]*\).*/\1/p')
|
||||
|
||||
if [ "${ACKNOWLEDGE_LAYOUT:-0}" != "$target" ]; then
|
||||
cat >&2 <<MSG
|
||||
This upgrade changes the store's layout from version $from to $target, which
|
||||
older releases can't read: going back means restoring a backup taken before
|
||||
it. Take one now, then upgrade again with
|
||||
|
||||
--set server.upgrade.acknowledgeLayout=$target
|
||||
|
||||
Nothing has been changed.
|
||||
MSG
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$code" = 4 ]; then
|
||||
if [ "${STOP_SERVERS:-true}" != true ]; then
|
||||
echo "A layout step needs every server pod stopped, and stopServers is off:" \
|
||||
"the upgrade waits for them to stop, and fails if they don't."
|
||||
else
|
||||
echo "A layout step needs every server pod stopped: scaling $DEPLOYMENTS to 0. Mail is down until the rollout."
|
||||
api="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"
|
||||
sa=/var/run/secrets/kubernetes.io/serviceaccount
|
||||
token=$(cat "$sa/token")
|
||||
kube() {
|
||||
curl -fsS --cacert "$sa/ca.crt" -H "Authorization: Bearer $token" "$@"
|
||||
}
|
||||
for deployment in $DEPLOYMENTS; do
|
||||
kube -X PATCH -H 'Content-Type: application/merge-patch+json' \
|
||||
-d '{"spec":{"replicas":0}}' \
|
||||
"$api/apis/apps/v1/namespaces/$NAMESPACE/deployments/$deployment/scale" >/dev/null \
|
||||
|| { echo "Couldn't scale $deployment to 0." >&2; exit 1; }
|
||||
done
|
||||
selector="app.kubernetes.io/instance%3D$RELEASE,app.kubernetes.io/component%3Dserver"
|
||||
waited=0
|
||||
# The API server pretty-prints for curl: "items": [] with a space
|
||||
until kube "$api/api/v1/namespaces/$NAMESPACE/pods?labelSelector=$selector" | grep -Eq '"items": *\[ *\]'; do
|
||||
if [ "$waited" -ge "${STOP_WAIT_SECONDS:-300}" ]; then
|
||||
echo "The server pods didn't stop within ${STOP_WAIT_SECONDS:-300}s." >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
waited=$((waited + 5))
|
||||
done
|
||||
echo "Every server pod has stopped (${waited}s)."
|
||||
fi
|
||||
fi
|
||||
|
||||
exec inbuxa --upgrade-store --config "$config"
|
||||
@@ -0,0 +1,206 @@
|
||||
{{/*
|
||||
SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
Before an upgrade rolls new pods out: a Job on the new image asks the server
|
||||
what the upgrade does to the store, refuses a layout change nobody
|
||||
acknowledged, stops the server pods when a layout step needs every node
|
||||
stopped (a rolling update would stall there), and upgrades the store. The
|
||||
script is files/upgrade.sh.
|
||||
|
||||
The rights to scale the server Deployments belong to a ServiceAccount that
|
||||
exists only while the hook runs. On a rollback the same run refuses a store
|
||||
the older release can't read.
|
||||
*/}}
|
||||
{{- $root := . }}
|
||||
{{- $server := .Values.server }}
|
||||
{{- $upgrade := $server.upgrade }}
|
||||
{{- if $upgrade.enabled }}
|
||||
{{- $name := printf "%s-upgrade" (include "inbuxa.fullname" $root) }}
|
||||
{{- $deployments := list }}
|
||||
{{- range $group := list "front" "mta" "core" }}
|
||||
{{- if (index $server.groups $group).enabled }}
|
||||
{{- $deployments = append $deployments (printf "%s-%s" (include "inbuxa.fullname" $root) $group) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $hook := dict "helm.sh/hook" "pre-upgrade,pre-rollback" "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }}
|
||||
{{- if $upgrade.stopServers }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $root.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" $root | nindent 4 }}
|
||||
app.kubernetes.io/component: upgrade
|
||||
annotations:
|
||||
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
|
||||
automountServiceAccountToken: false
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $root.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" $root | nindent 4 }}
|
||||
app.kubernetes.io/component: upgrade
|
||||
annotations:
|
||||
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
|
||||
rules:
|
||||
# Scale this release's server Deployments, and nothing else
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments/scale"]
|
||||
resourceNames:
|
||||
{{- toYaml $deployments | nindent 6 }}
|
||||
verbs: ["get", "patch"]
|
||||
# See when their pods are gone
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["list"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $root.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" $root | nindent 4 }}
|
||||
app.kubernetes.io/component: upgrade
|
||||
annotations:
|
||||
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: {{ $name }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ $name }}
|
||||
namespace: {{ $root.Release.Namespace }}
|
||||
---
|
||||
{{- end }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $root.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" $root | nindent 4 }}
|
||||
app.kubernetes.io/component: upgrade
|
||||
annotations:
|
||||
{{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $hook) | nindent 4 }}
|
||||
data:
|
||||
config.json: {{ toJson $server.dataStore | quote }}
|
||||
upgrade.sh: |
|
||||
{{- $root.Files.Get "files/upgrade.sh" | nindent 4 }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
namespace: {{ $root.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "inbuxa.labels" $root | nindent 4 }}
|
||||
app.kubernetes.io/component: upgrade
|
||||
annotations:
|
||||
{{- toYaml (merge (dict "helm.sh/hook-weight" "0") $hook) | nindent 4 }}
|
||||
ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it"
|
||||
ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing"
|
||||
ignore-check.kube-linter.io/restart-policy: "Never on purpose: a failed upgrade is for a person to look at, not to retry"
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
activeDeadlineSeconds: {{ $upgrade.timeoutSeconds }}
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "inbuxa.labels" $root | nindent 8 }}
|
||||
app.kubernetes.io/component: upgrade
|
||||
spec:
|
||||
{{- if $upgrade.stopServers }}
|
||||
serviceAccountName: {{ $name }}
|
||||
automountServiceAccountToken: true
|
||||
{{- else }}
|
||||
automountServiceAccountToken: false
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
restartPolicy: Never
|
||||
{{- with $root.Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 2000
|
||||
runAsGroup: 2000
|
||||
fsGroup: 2000
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: upgrade
|
||||
image: {{ include "inbuxa.server.image" $root }}
|
||||
imagePullPolicy: {{ $server.image.pullPolicy }}
|
||||
command: ["/bin/sh", "/etc/inbuxa/upgrade/upgrade.sh"]
|
||||
env:
|
||||
- name: INBUXA_HOSTNAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: RELEASE
|
||||
value: {{ $root.Release.Name | quote }}
|
||||
- name: NAMESPACE
|
||||
value: {{ $root.Release.Namespace | quote }}
|
||||
- name: DEPLOYMENTS
|
||||
value: {{ join " " $deployments | quote }}
|
||||
- name: ACKNOWLEDGE_LAYOUT
|
||||
value: {{ $upgrade.acknowledgeLayout | quote }}
|
||||
- name: STOP_SERVERS
|
||||
value: {{ $upgrade.stopServers | quote }}
|
||||
- name: STOP_WAIT_SECONDS
|
||||
value: {{ add $server.terminationGracePeriodSeconds 180 | quote }}
|
||||
resources:
|
||||
{{- toYaml (default $server.resources $upgrade.resources) | nindent 12 }}
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
volumeMounts:
|
||||
- name: hook
|
||||
mountPath: /etc/inbuxa/config.json
|
||||
subPath: config.json
|
||||
readOnly: true
|
||||
- name: hook
|
||||
mountPath: /etc/inbuxa/upgrade/upgrade.sh
|
||||
subPath: upgrade.sh
|
||||
readOnly: true
|
||||
{{- if $server.secretMounts }}
|
||||
- name: secrets
|
||||
mountPath: /etc/inbuxa/secrets
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
- name: data
|
||||
mountPath: /var/lib/inbuxa
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: hook
|
||||
configMap:
|
||||
name: {{ $name }}
|
||||
{{- if $server.secretMounts }}
|
||||
- name: secrets
|
||||
projected:
|
||||
defaultMode: 0440
|
||||
sources:
|
||||
{{- range $server.secretMounts }}
|
||||
- secret:
|
||||
name: {{ .secretName }}
|
||||
items:
|
||||
{{- toYaml .items | nindent 20 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
suite: upgrade
|
||||
templates:
|
||||
- server/upgrade.yaml
|
||||
release:
|
||||
name: mail
|
||||
namespace: inbuxa
|
||||
values:
|
||||
- ../ci/defaults-values.yaml
|
||||
tests:
|
||||
- it: runs before an upgrade or a rollback, the rights first
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 5
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/hook"]
|
||||
value: pre-upgrade,pre-rollback
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/hook-weight"]
|
||||
value: "-10"
|
||||
documentIndex: 1
|
||||
- equal:
|
||||
path: metadata.annotations["helm.sh/hook-weight"]
|
||||
value: "0"
|
||||
documentIndex: 4
|
||||
|
||||
- it: may scale this release's server Deployments, and nothing else
|
||||
documentIndex: 1
|
||||
asserts:
|
||||
- equal:
|
||||
path: rules[0]
|
||||
value:
|
||||
apiGroups: [apps]
|
||||
resources: [deployments/scale]
|
||||
resourceNames: [mail-inbuxa-front, mail-inbuxa-mta, mail-inbuxa-core]
|
||||
verbs: [get, patch]
|
||||
- lengthEqual:
|
||||
path: rules
|
||||
count: 2
|
||||
|
||||
- it: tells the script the release, the acknowledgement and the groups
|
||||
set:
|
||||
server.upgrade.acknowledgeLayout: 3
|
||||
server.groups.mta.enabled: false
|
||||
server.services.mail.groups: [front]
|
||||
documentIndex: 4
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: ACKNOWLEDGE_LAYOUT, value: "3"}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: DEPLOYMENTS, value: mail-inbuxa-front mail-inbuxa-core}
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content: {name: STOP_WAIT_SECONDS, value: "300"}
|
||||
- equal:
|
||||
path: spec.template.spec.serviceAccountName
|
||||
value: mail-inbuxa-upgrade
|
||||
|
||||
- it: carries the script from files/upgrade.sh
|
||||
documentIndex: 3
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["upgrade.sh"]
|
||||
pattern: inbuxa --upgrade-plan --config
|
||||
- exists:
|
||||
path: data["config.json"]
|
||||
|
||||
- it: needs no rights when it mustn't stop the servers
|
||||
set:
|
||||
server.upgrade.stopServers: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 2
|
||||
- equal:
|
||||
path: spec.template.spec.automountServiceAccountToken
|
||||
value: false
|
||||
documentIndex: 1
|
||||
- notExists:
|
||||
path: spec.template.spec.serviceAccountName
|
||||
documentIndex: 1
|
||||
|
||||
- it: can be turned off
|
||||
set:
|
||||
server.upgrade.enabled: false
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
@@ -235,6 +235,30 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"upgrade": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"acknowledgeLayout": {
|
||||
"description": "The store layout version an upgrade may move the store to; 0 for none.",
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"stopServers": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"timeoutSeconds": {
|
||||
"type": "integer",
|
||||
"minimum": 60
|
||||
},
|
||||
"resources": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"publicUrl": {
|
||||
"type": "string",
|
||||
"pattern": "^(https?://[^/\\s]+)?$"
|
||||
|
||||
@@ -59,6 +59,23 @@ server:
|
||||
timeoutSeconds: 600
|
||||
resources: {}
|
||||
|
||||
# Before an upgrade rolls new pods out, a Job on the new image asks the
|
||||
# server what the upgrade does to the store, and upgrades it. Give
|
||||
# `helm upgrade` a --timeout long enough for it: 20m covers a layout step
|
||||
# that has to wait for the old pods' heartbeats to run out.
|
||||
upgrade:
|
||||
enabled: true
|
||||
# A change to the store's layout can't be undone without a backup taken
|
||||
# before it. The upgrade refuses one until this names the layout version
|
||||
# it moves to (the refusal says which): 3 for the 27.1 crossing.
|
||||
acknowledgeLayout: 0
|
||||
# When a layout step needs every server pod stopped, scale the server
|
||||
# Deployments to 0 first (mail is down until the rollout). Off: the
|
||||
# upgrade waits for them to stop, and fails if they don't.
|
||||
stopServers: true
|
||||
timeoutSeconds: 1800
|
||||
resources: {}
|
||||
|
||||
# Where browsers and clients reach the server, as https://host. Sign-in
|
||||
# pages and OAuth metadata name it.
|
||||
publicUrl: ""
|
||||
|
||||
Reference in new issue
Block a user