diff --git a/CHANGELOG.md b/CHANGELOG.md index e966664..094a689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,3 +27,9 @@ runs, pinned by digest. - `server.seed`: the new store's setup (domain, administrator, blob store, coordinator, roles), applied once on install by a pre-install Job running the server's `--seed`, before any server pod starts. +- `server.upgrade`: before an upgrade or rollback rolls out, a Job on the + new image asks the server what it does to the store (`--upgrade-plan`). + A layout change waits for `upgrade.acknowledgeLayout`. When a layout step + can't run with older pods up, the server Deployments are scaled to 0 + first, which a rolling update can't do. Then `--upgrade-store` upgrades + the store. diff --git a/charts/inbuxa/files/upgrade.sh b/charts/inbuxa/files/upgrade.sh new file mode 100755 index 0000000..2102f90 --- /dev/null +++ b/charts/inbuxa/files/upgrade.sh @@ -0,0 +1,89 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# The chart's pre-upgrade run, on the new server image, before Helm rolls +# out the new pods. It asks the server what the upgrade does to the store +# (--upgrade-plan), refuses a layout change nobody acknowledged, stops the +# server pods when a step needs every node stopped, and upgrades the store +# (--upgrade-store). Only sh, sed and curl: the image has nothing more. +# +# From the Job: RELEASE, NAMESPACE, DEPLOYMENTS (space-separated), +# ACKNOWLEDGE_LAYOUT, STOP_SERVERS (true/false), STOP_WAIT_SECONDS. +set -u + +config=/etc/inbuxa/config.json + +# The server logs to stdout too; the plan is the one line of JSON +out=$(inbuxa --upgrade-plan --config "$config") +code=$? +plan=$(printf '%s\n' "$out" | grep '^{' | tail -n 1) +echo "Plan: $plan" + +case $code in + 0) + echo "The store needs no layout change: nothing to do before the rollout." + exit 0 + ;; + 3 | 4) ;; + 2) + echo "A newer release has changed this store's layout. This release can't run on it:" \ + "upgrade to that release, or restore the backup taken before it." >&2 + exit 1 + ;; + *) + echo "The server couldn't plan the upgrade (exit $code); its message is above." >&2 + exit 1 + ;; +esac + +from=$(printf '%s' "$plan" | sed -n 's/.*"store":\([0-9]*\).*/\1/p') +target=$(printf '%s' "$plan" | sed -n 's/.*"target":\([0-9]*\).*/\1/p') + +if [ "${ACKNOWLEDGE_LAYOUT:-0}" != "$target" ]; then + cat >&2 </dev/null \ + || { echo "Couldn't scale $deployment to 0." >&2; exit 1; } + done + selector="app.kubernetes.io/instance%3D$RELEASE,app.kubernetes.io/component%3Dserver" + waited=0 + # The API server pretty-prints for curl: "items": [] with a space + until kube "$api/api/v1/namespaces/$NAMESPACE/pods?labelSelector=$selector" | grep -Eq '"items": *\[ *\]'; do + if [ "$waited" -ge "${STOP_WAIT_SECONDS:-300}" ]; then + echo "The server pods didn't stop within ${STOP_WAIT_SECONDS:-300}s." >&2 + exit 1 + fi + sleep 5 + waited=$((waited + 5)) + done + echo "Every server pod has stopped (${waited}s)." + fi +fi + +exec inbuxa --upgrade-store --config "$config" diff --git a/charts/inbuxa/templates/server/upgrade.yaml b/charts/inbuxa/templates/server/upgrade.yaml new file mode 100644 index 0000000..650a0de --- /dev/null +++ b/charts/inbuxa/templates/server/upgrade.yaml @@ -0,0 +1,206 @@ +{{/* +SPDX-FileCopyrightText: 2026 Coffey Labs LLC +SPDX-License-Identifier: Apache-2.0 + +Before an upgrade rolls new pods out: a Job on the new image asks the server +what the upgrade does to the store, refuses a layout change nobody +acknowledged, stops the server pods when a layout step needs every node +stopped (a rolling update would stall there), and upgrades the store. The +script is files/upgrade.sh. + +The rights to scale the server Deployments belong to a ServiceAccount that +exists only while the hook runs. On a rollback the same run refuses a store +the older release can't read. +*/}} +{{- $root := . }} +{{- $server := .Values.server }} +{{- $upgrade := $server.upgrade }} +{{- if $upgrade.enabled }} +{{- $name := printf "%s-upgrade" (include "inbuxa.fullname" $root) }} +{{- $deployments := list }} +{{- range $group := list "front" "mta" "core" }} +{{- if (index $server.groups $group).enabled }} +{{- $deployments = append $deployments (printf "%s-%s" (include "inbuxa.fullname" $root) $group) }} +{{- end }} +{{- end }} +{{- $hook := dict "helm.sh/hook" "pre-upgrade,pre-rollback" "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }} +{{- if $upgrade.stopServers }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: upgrade + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: upgrade + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }} +rules: + # Scale this release's server Deployments, and nothing else + - apiGroups: ["apps"] + resources: ["deployments/scale"] + resourceNames: + {{- toYaml $deployments | nindent 6 }} + verbs: ["get", "patch"] + # See when their pods are gone + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: upgrade + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} +--- +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: upgrade + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $hook) | nindent 4 }} +data: + config.json: {{ toJson $server.dataStore | quote }} + upgrade.sh: | + {{- $root.Files.Get "files/upgrade.sh" | nindent 4 }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: upgrade + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "0") $hook) | nindent 4 }} + ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it" + ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing" + ignore-check.kube-linter.io/restart-policy: "Never on purpose: a failed upgrade is for a person to look at, not to retry" +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ $upgrade.timeoutSeconds }} + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + {{- include "inbuxa.labels" $root | nindent 8 }} + app.kubernetes.io/component: upgrade + spec: + {{- if $upgrade.stopServers }} + serviceAccountName: {{ $name }} + automountServiceAccountToken: true + {{- else }} + automountServiceAccountToken: false + {{- end }} + enableServiceLinks: false + restartPolicy: Never + {{- with $root.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + fsGroup: 2000 + seccompProfile: + type: RuntimeDefault + containers: + - name: upgrade + image: {{ include "inbuxa.server.image" $root }} + imagePullPolicy: {{ $server.image.pullPolicy }} + command: ["/bin/sh", "/etc/inbuxa/upgrade/upgrade.sh"] + env: + - name: INBUXA_HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: RELEASE + value: {{ $root.Release.Name | quote }} + - name: NAMESPACE + value: {{ $root.Release.Namespace | quote }} + - name: DEPLOYMENTS + value: {{ join " " $deployments | quote }} + - name: ACKNOWLEDGE_LAYOUT + value: {{ $upgrade.acknowledgeLayout | quote }} + - name: STOP_SERVERS + value: {{ $upgrade.stopServers | quote }} + - name: STOP_WAIT_SECONDS + value: {{ add $server.terminationGracePeriodSeconds 180 | quote }} + resources: + {{- toYaml (default $server.resources $upgrade.resources) | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: hook + mountPath: /etc/inbuxa/config.json + subPath: config.json + readOnly: true + - name: hook + mountPath: /etc/inbuxa/upgrade/upgrade.sh + subPath: upgrade.sh + readOnly: true + {{- if $server.secretMounts }} + - name: secrets + mountPath: /etc/inbuxa/secrets + readOnly: true + {{- end }} + - name: data + mountPath: /var/lib/inbuxa + - name: tmp + mountPath: /tmp + volumes: + - name: hook + configMap: + name: {{ $name }} + {{- if $server.secretMounts }} + - name: secrets + projected: + defaultMode: 0440 + sources: + {{- range $server.secretMounts }} + - secret: + name: {{ .secretName }} + items: + {{- toYaml .items | nindent 20 }} + {{- end }} + {{- end }} + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} +{{- end }} diff --git a/charts/inbuxa/tests/upgrade_test.yaml b/charts/inbuxa/tests/upgrade_test.yaml new file mode 100644 index 0000000..7d86f9b --- /dev/null +++ b/charts/inbuxa/tests/upgrade_test.yaml @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +suite: upgrade +templates: + - server/upgrade.yaml +release: + name: mail + namespace: inbuxa +values: + - ../ci/defaults-values.yaml +tests: + - it: runs before an upgrade or a rollback, the rights first + asserts: + - hasDocuments: + count: 5 + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-upgrade,pre-rollback + - equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "-10" + documentIndex: 1 + - equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "0" + documentIndex: 4 + + - it: may scale this release's server Deployments, and nothing else + documentIndex: 1 + asserts: + - equal: + path: rules[0] + value: + apiGroups: [apps] + resources: [deployments/scale] + resourceNames: [mail-inbuxa-front, mail-inbuxa-mta, mail-inbuxa-core] + verbs: [get, patch] + - lengthEqual: + path: rules + count: 2 + + - it: tells the script the release, the acknowledgement and the groups + set: + server.upgrade.acknowledgeLayout: 3 + server.groups.mta.enabled: false + server.services.mail.groups: [front] + documentIndex: 4 + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: {name: ACKNOWLEDGE_LAYOUT, value: "3"} + - contains: + path: spec.template.spec.containers[0].env + content: {name: DEPLOYMENTS, value: mail-inbuxa-front mail-inbuxa-core} + - contains: + path: spec.template.spec.containers[0].env + content: {name: STOP_WAIT_SECONDS, value: "300"} + - equal: + path: spec.template.spec.serviceAccountName + value: mail-inbuxa-upgrade + + - it: carries the script from files/upgrade.sh + documentIndex: 3 + asserts: + - matchRegex: + path: data["upgrade.sh"] + pattern: inbuxa --upgrade-plan --config + - exists: + path: data["config.json"] + + - it: needs no rights when it mustn't stop the servers + set: + server.upgrade.stopServers: false + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + documentIndex: 1 + - notExists: + path: spec.template.spec.serviceAccountName + documentIndex: 1 + + - it: can be turned off + set: + server.upgrade.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/inbuxa/values.schema.json b/charts/inbuxa/values.schema.json index 3bf674f..e84c57e 100644 --- a/charts/inbuxa/values.schema.json +++ b/charts/inbuxa/values.schema.json @@ -235,6 +235,30 @@ } } }, + "upgrade": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "acknowledgeLayout": { + "description": "The store layout version an upgrade may move the store to; 0 for none.", + "type": "integer", + "minimum": 0 + }, + "stopServers": { + "type": "boolean" + }, + "timeoutSeconds": { + "type": "integer", + "minimum": 60 + }, + "resources": { + "type": "object" + } + } + }, "publicUrl": { "type": "string", "pattern": "^(https?://[^/\\s]+)?$" diff --git a/charts/inbuxa/values.yaml b/charts/inbuxa/values.yaml index 9f6cdb9..f8280e2 100644 --- a/charts/inbuxa/values.yaml +++ b/charts/inbuxa/values.yaml @@ -59,6 +59,23 @@ server: timeoutSeconds: 600 resources: {} + # Before an upgrade rolls new pods out, a Job on the new image asks the + # server what the upgrade does to the store, and upgrades it. Give + # `helm upgrade` a --timeout long enough for it: 20m covers a layout step + # that has to wait for the old pods' heartbeats to run out. + upgrade: + enabled: true + # A change to the store's layout can't be undone without a backup taken + # before it. The upgrade refuses one until this names the layout version + # it moves to (the refusal says which): 3 for the 27.1 crossing. + acknowledgeLayout: 0 + # When a layout step needs every server pod stopped, scale the server + # Deployments to 0 first (mail is down until the rollout). Off: the + # upgrade waits for them to stop, and fails if they don't. + stopServers: true + timeoutSeconds: 1800 + resources: {} + # Where browsers and clients reach the server, as https://host. Sign-in # pages and OAuth metadata name it. publicUrl: ""