Files
inbuxa-k8s-lab/lab/chart/rolling-upgrade-test.sh
jcoffey-dev ad65f0d10a Upgrade tests with the published images (K-L5)
rolling-upgrade-test.sh: release N-1 to N from ghcr.io, pinned by digest,
with mail flowing the whole way (mailflow.py, a message a second, retried
as an MTA would) and every accepted message checked to arrive exactly
once. 27.1-rc.33 to rc.34: rolled in 19s, 43 accepted, 43 arrived, one
connection refused while a pod went away and retried.

crossing-test.sh: a store made and filled by 2026.11.18.3 (layout 1),
upgraded by the chart to 27.1-rc.34: refused until the layout change is
acknowledged, then crossed in 194s (1 to 2 to 3), mailbox digests equal.

Fixes on the way: fresh-store.sh's pods and seed-job.yaml carry the
chart's labels, so its NetworkPolicies cover them as the Kyverno policy
requires; fresh-store.sh and seed-store.sh take PULL for published images;
mailtest.py and mailbox-digest.py send from a sender domain of their own
(the default throttle allows 25 an hour per sender domain and recipient),
and mailbox-digest.py one message per session, paced.
2026-11-19 21:07:40 -07:00

90 lines
3.6 KiB
Bash
Executable File

#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L5, rolling: the published server images, release N-1 to release N,
# with mail flowing the whole way. On the lab's seeded install:
#
# 1. helm upgrade to FROM (ghcr.io, pinned by digest); mail works
# 2. mail starts flowing (mailflow.py: a message a second, retried like
# an MTA would when a pod goes away)
# 3. helm upgrade to TO: the upgrade hook finds no layout change, so the
# Deployments roll
# 4. every server pod runs TO's digest, mail still works, and every
# message accepted during the roll arrived exactly once
#
# FROM=27.1-rc.33 TO=27.1-rc.34 lab/chart/rolling-upgrade-test.sh
#
# Leaves the lab on TO. Needs docker (to read the digests) and the kind
# nodes to reach ghcr.io.
set -euo pipefail
cd "$(dirname "$0")"
FROM="${FROM:?the release to start from, e.g. 27.1-rc.33}"
TO="${TO:?the release to upgrade to, e.g. 27.1-rc.34}"
IMAGE=ghcr.io/inbuxa/inbuxa-server
K=(kubectl --context kind-inbuxa-lab)
H=(helm --kube-context kind-inbuxa-lab -n inbuxa)
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
WORK="$(mktemp -d)"
FLOW=""
trap '[ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null; rm -rf "$WORK"' EXIT
fail() { echo "!! $*" >&2; exit 1; }
digest() { docker buildx imagetools inspect "$IMAGE:$1" --format '{{json .Manifest.Digest}}' | tr -d '"'; }
upgrade() { # digest, then helm's own flags
local d="$1"; shift
"${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml \
--set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \
--set server.image.digest="$d" --wait --timeout 20m "$@"
}
# Every server pod is ready and runs exactly this digest, once the old ones
# have finished terminating (helm's --wait returns before they have)
on_digest() {
local want="$1" got
for _ in $(seq 1 60); do
got="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server \
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{" "}{.status.containerStatuses[0].ready}{"\n"}{end}')"
if [ "$(grep -c . <<<"$got")" -ge 5 ] && ! grep -qv "@$want true\$" <<<"$got"; then return 0; fi
sleep 3
done
echo "$got"; return 1
}
hook_log() { "${K[@]}" -n inbuxa logs job/inbuxa-upgrade 2>/dev/null | grep -v '^20..-' || true; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
mail() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 mailtest.py "$(lb)"; }
from_d="$(digest "$FROM")"; to_d="$(digest "$TO")"
[[ "$from_d" =~ ^sha256: && "$to_d" =~ ^sha256: ]] || fail "couldn't read the digests of $FROM and $TO"
echo "$FROM = $from_d"
echo "$TO = $to_d"
echo "== 1. on $FROM"
start=$(date +%s)
upgrade "$from_d" >/dev/null
echo " in $(( $(date +%s) - start ))s"
on_digest "$from_d" || fail "not every server pod runs $FROM"
mail
echo "== 2. mail flowing"
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" \
python3 mailflow.py "$(lb)" > "$WORK/flow.log" 2>&1 &
FLOW=$!
sleep 20
echo "== 3. rolling to $TO"
start=$(date +%s)
upgrade "$to_d" >/dev/null
echo " in $(( $(date +%s) - start ))s; the hook said:"; hook_log | sed 's/^/ /'
sleep 20
touch "$WORK/stop"
flow=0; wait "$FLOW" || flow=$?; FLOW=""
sed 's/^/ /' "$WORK/flow.log"
echo "== 4. on $TO"
on_digest "$to_d" || fail "not every server pod runs $TO"
mail
[ "$flow" = 0 ] || fail "mail accepted during the roll was lost or doubled"
echo "== $FROM to $TO: rolled with mail flowing, nothing lost"