Upgrade tests with the published images (K-L5) #30

Merged
jcoffey-dev merged 1 commits from feat/upgrade-real-images into main 2026-11-20 04:10:09 +00:00
8 changed files with 323 additions and 7 deletions

No files matched your search

+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L5, crossing: a store made and filled by a published 2026.x server, then
# upgraded by the chart to a 27.1 release, with the layout change the
# crossing brings:
#
# 1. a fresh store set up by OLD (fresh-store.sh), and OLD taking mail
# through the chart's own mail Service, as one pod with the chart's
# labels (OLD predates the chart: it runs as the lab's spike pods do)
# 2. the mailbox digests: every mailbox's count and a hash over its messages
# 3. helm upgrade to TO without acknowledging the layout change: refused,
# the store untouched
# 4. acknowledged: the hook upgrades the store, the Deployments come up on
# TO, and the digests are what OLD left
#
# OLD=2026.11.18.3 TO=27.1-rc.34 lab/chart/crossing-test.sh
#
# WIPES the lab's inbuxa database, and leaves a store fresh-store.sh made.
# seed-store.sh and a helm upgrade put the lab back as the other tests want it.
set -euo pipefail
cd "$(dirname "$0")"
OLD="${OLD:?the 2026.x release to start from, e.g. 2026.11.18.3}"
TO="${TO:?the 27.1 release to cross to, e.g. 27.1-rc.34}"
MESSAGES="${MESSAGES:-15}"
IMAGE=ghcr.io/inbuxa/inbuxa-server
K=(kubectl --context kind-inbuxa-lab)
H=(helm --kube-context kind-inbuxa-lab -n inbuxa)
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
WORK="$(mktemp -d)"
trap '"${K[@]}" -n inbuxa delete pod inbuxa-old --ignore-not-found >/dev/null 2>&1; rm -rf "$WORK"' EXIT
fail() { echo "!! $*" >&2; exit 1; }
digest_of() { docker buildx imagetools inspect "$IMAGE:$1" --format '{{json .Manifest.Digest}}' | tr -d '"'; }
mark() { "${K[@]}" -n data exec inbuxa-pg-1 -c postgres -- psql -d inbuxa -At \
-c "select encode(v,'hex') from \"_\" where k = 'Vs'::bytea"; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
mailbox_digest() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$(lb)" "$@"; }
upgrade() {
"${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml --set server.seed.enabled=false \
--set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \
--set server.image.digest="$to_d" --wait --timeout 20m "$@"
}
hook_log() { "${K[@]}" -n inbuxa logs job/inbuxa-upgrade 2>/dev/null | grep -v '^20..-' || true; }
old_d="$(digest_of "$OLD")"; to_d="$(digest_of "$TO")"
[[ "$old_d" =~ ^sha256: && "$to_d" =~ ^sha256: ]] || fail "couldn't read the digests of $OLD and $TO"
echo "$OLD = $old_d"
echo "$TO = $to_d"
echo "== 1. a store made by $OLD"
IMAGE="$IMAGE@$old_d" PULL=IfNotPresent ./fresh-store.sh
echo " layout mark: $(mark)"
# OLD serving mail, behind the chart's Service and NetworkPolicies. The
# Service targets ports by name, so the pod names them as the chart's do.
PORTS="$("${K[@]}" -n inbuxa get deploy inbuxa-front -o jsonpath='{.spec.template.spec.containers[0].ports}')"
sed -n '/^ template:/,$p' ../spike/core.yaml | sed '1d' |
sed -e "s|^ ||" \
-e "s|metadata: {labels: {app: inbuxa-server, group: core}}|metadata: {name: inbuxa-old, namespace: inbuxa, labels: {app.kubernetes.io/name: inbuxa, app.kubernetes.io/instance: inbuxa, app.kubernetes.io/component: server, inbuxa.org/serves-mail: \"true\"}}|" \
-e "s|image: inbuxa-lab/server:rc27|image: $IMAGE@$old_d|" \
-e "s|imagePullPolicy: Never|imagePullPolicy: IfNotPresent\n ports: $PORTS|" \
-e '/topologySpreadConstraints:/,+1d' > "$WORK/old.yaml"
"${K[@]}" -n inbuxa delete pod inbuxa-old --ignore-not-found --wait >/dev/null
{ printf 'apiVersion: v1\nkind: Pod\n'; cat "$WORK/old.yaml"; } | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n inbuxa wait --for=condition=Ready pod/inbuxa-old --timeout=300s >/dev/null ||
{ "${K[@]}" -n inbuxa logs inbuxa-old | tail -20; fail "$OLD never got ready"; }
echo "== 2. mail into $OLD, and its digests"
for _ in $(seq 1 30); do timeout 5 bash -c "</dev/tcp/$(lb)/25" 2>/dev/null && break; sleep 2; done
mailbox_digest send "$MESSAGES" > "$WORK/before"
sed 's/^/ /' "$WORK/before"
grep -q . "$WORK/before" || fail "no mailboxes"
"${K[@]}" -n inbuxa delete pod inbuxa-old --wait >/dev/null
before_mark="$(mark)"
echo "== 3. to $TO, not acknowledged"
if upgrade >/dev/null 2>&1; then fail "the crossing went through unacknowledged"; fi
echo " refused; the hook said:"; hook_log | sed 's/^/ /'
[ "$(mark)" = "$before_mark" ] || fail "the refused upgrade changed the store"
echo "== 4. acknowledged"
start=$(date +%s)
upgrade --set server.upgrade.acknowledgeLayout=3 >/dev/null
echo " crossed in $(( $(date +%s) - start ))s; the hook said:"; hook_log | sed 's/^/ /'
[ "$(mark)" = 00000003 ] || fail "the store isn't at layout 3: $(mark)"
for _ in $(seq 1 60); do
got="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server \
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{" "}{.status.containerStatuses[0].ready}{"\n"}{end}')"
[ "$(grep -c . <<<"$got")" -ge 5 ] && ! grep -qv "@$to_d true\$" <<<"$got" && break
sleep 3
done
grep -qv "@$to_d true\$" <<<"$got" && fail "not every server pod runs $TO"
mailbox_digest > "$WORK/after"
diff "$WORK/before" "$WORK/after" >/dev/null ||
{ diff "$WORK/before" "$WORK/after" | sed 's/^/ /'; fail "the mail isn't what $OLD left"; }
echo " digests match"
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 mailtest.py "$(lb)"
echo "== $OLD to $TO: crossed, every message as it was"
+9 -1
View File
@@ -14,6 +14,8 @@ set -euo pipefail
cd "$(dirname "$0")"
K=(kubectl --context kind-inbuxa-lab)
IMAGE="${IMAGE:-inbuxa-lab/server:main}"
# Never for an image loaded into kind; IfNotPresent for a published one
PULL="${PULL:-Never}"
PORTS="${PORTS:-pop3s=off}"
DOMAIN="${DOMAIN:-lab.test}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
@@ -21,11 +23,17 @@ mkdir -p "$STATE" && chmod 700 "$STATE"
secret() { "${K[@]}" -n inbuxa get secret "$1" -o "jsonpath={.data.$2}" | base64 -d; }
# A one-off server pod from ../spike, on this image and these listener ports
# A one-off server pod from ../spike, on this image and these listener ports.
# It carries the chart's server labels, so the chart's deny-all and server
# NetworkPolicies cover it, as the lab's Kyverno policy requires.
LABELS='app.kubernetes.io/name: inbuxa, app.kubernetes.io/instance: inbuxa, app.kubernetes.io/component: server'
run_pod() {
local file="$1" name="$2"
"${K[@]}" -n inbuxa delete pod "$name" --ignore-not-found --wait >/dev/null
sed -e "s|image: inbuxa-lab/server:rc27|image: $IMAGE|" \
-e "s|imagePullPolicy: Never|imagePullPolicy: $PULL|" \
-e "s|^ name: $name\$| name: $name\n labels: {$LABELS}|" -e '/^ labels: {app: inbuxa-setup}$/d' \
-e "s| - {name: INBUXA_HOSTNAME, value: $name}| - {name: INBUXA_HOSTNAME, value: $name}\n - {name: INBUXA_LISTENER_PORTS, value: \"$PORTS\"}|" \
"../spike/$file" | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n inbuxa wait --for=condition=Ready "pod/$name" --timeout=180s >/dev/null
+8 -3
View File
@@ -51,10 +51,13 @@ if len(sys.argv) > 2 and sys.argv[2] == "send":
m = imap()
before = total(m)
run = uuid.uuid4().hex[:8]
with smtplib.SMTP(lb, 25, timeout=30) as s:
for i in range(count):
# A session per message: servers cap the messages one session may send
for i in range(count):
with smtplib.SMTP(lb, 25, timeout=30) as s:
msg = EmailMessage()
msg["From"] = "[email protected]"
# Its own sender domain: the default throttle allows 25 an hour
# from one sender domain to one recipient
msg["From"] = f"lab-sender@m{i}.{run}.sender.test"
msg["To"] = account
msg["Subject"] = f"backup-test {run} {i}"
msg["Date"] = formatdate()
@@ -63,6 +66,8 @@ if len(sys.argv) > 2 and sys.argv[2] == "send":
msg.add_attachment((f"line {i}: " + uuid.uuid4().hex + "\n") * (60 + 17 * i),
filename=f"notes-{i}.txt")
s.send_message(msg)
# Under the default 5 a second from one address
time.sleep(0.3)
for _ in range(60):
if total(m) >= before + count:
break
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""Mail flowing through an upgrade (K-L5): sends one message a second over
SMTP (25), each from its own sender domain, until STOP_FILE appears, as a sending MTA would, so a refused or
dropped connection is retried rather than lost. Then waits for every
message the server accepted to show up over IMAPS (993) and checks each
arrived exactly once.
ACCOUNT=user@domain PASSWORD_FILE=... STOP_FILE=... mailflow.py LB
Prints a summary line and exits 1 if anything accepted was lost or came
twice. The lab's certificate is self-signed, so TLS isn't verified.
"""
import imaplib, os, re, smtplib, ssl, sys, time, uuid
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
lb = sys.argv[1]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
stop = os.environ["STOP_FILE"]
run = uuid.uuid4().hex[:8]
tls = ssl.create_default_context()
tls.check_hostname = False
tls.verify_mode = ssl.CERT_NONE
def send(n):
msg = EmailMessage()
# A sender domain per message: the server's default throttle allows 25
# an hour from one sender domain to one recipient
msg["From"] = f"lab-sender@m{n}.{run}.sender.test"
msg["To"] = account
msg["Subject"] = f"mailflow {run} {n}"
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test")
msg.set_content(f"mailflow {run} message {n}\n")
with smtplib.SMTP(lb, 25, timeout=20) as s:
s.send_message(msg)
accepted, retries, started = 0, 0, time.time()
while not os.path.exists(stop):
n = accepted + 1
try:
send(n)
accepted = n
except (OSError, smtplib.SMTPException) as e:
# Refused, reset or a 4xx: a real MTA tries again later
retries += 1
print(f" retry {n}: {type(e).__name__}: {e}"[:160], flush=True)
time.sleep(2)
continue
time.sleep(1)
sent_for = time.time() - started
def count():
"""How many times each message number of this run is in the account."""
seen = {}
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30)
try:
m.login(account, password)
_, boxes = m.list()
for line in boxes:
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
if m.select(name, readonly=True)[0] != "OK":
continue
_, data = m.search(None, "SUBJECT", f'"mailflow {run}"')
ids = data[0].split()
if not ids:
continue
_, hdrs = m.fetch(b",".join(ids), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
for part in hdrs:
if isinstance(part, tuple):
mm = re.search(rf"mailflow {run} (\d+)", part[1].decode())
if mm:
seen[int(mm.group(1))] = seen.get(int(mm.group(1)), 0) + 1
finally:
try:
m.logout()
except Exception:
pass
return seen
deadline = time.time() + 180
while True:
try:
seen = count()
except (OSError, imaplib.IMAP4.error) as e:
print(f" imap: {e}"[:160], flush=True)
seen = {}
missing = [n for n in range(1, accepted + 1) if n not in seen]
if not missing or time.time() > deadline:
break
time.sleep(5)
twice = sorted(n for n, c in seen.items() if c > 1)
print(f"mailflow {run}: {accepted} accepted over {sent_for:.0f}s, {retries} retried, "
f"{accepted - len(missing)} arrived, {len(missing)} missing, {len(twice)} twice")
if missing:
print(f" missing: {missing[:20]}")
if twice:
print(f" twice: {twice[:20]}")
sys.exit(1 if missing or twice or accepted == 0 else 0)
+3 -1
View File
@@ -21,7 +21,9 @@ tls.verify_mode = ssl.CERT_NONE
msg = EmailMessage()
# A sender domain with no DNS at all: one that publishes DMARC (example.net
# does, p=reject) would send the message to Junk
msg["From"] = "[email protected]"
# ...and one of its own each run: the server's default throttle allows 25
# messages an hour from one sender domain to one recipient
msg["From"] = f"lab-sender@{token[:12]}.sender.test"
msg["To"] = account
msg["Subject"] = f"chart smoke {token}"
msg["Date"] = formatdate()
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L5, rolling: the published server images, release N-1 to release N,
# with mail flowing the whole way. On the lab's seeded install:
#
# 1. helm upgrade to FROM (ghcr.io, pinned by digest); mail works
# 2. mail starts flowing (mailflow.py: a message a second, retried like
# an MTA would when a pod goes away)
# 3. helm upgrade to TO: the upgrade hook finds no layout change, so the
# Deployments roll
# 4. every server pod runs TO's digest, mail still works, and every
# message accepted during the roll arrived exactly once
#
# FROM=27.1-rc.33 TO=27.1-rc.34 lab/chart/rolling-upgrade-test.sh
#
# Leaves the lab on TO. Needs docker (to read the digests) and the kind
# nodes to reach ghcr.io.
set -euo pipefail
cd "$(dirname "$0")"
FROM="${FROM:?the release to start from, e.g. 27.1-rc.33}"
TO="${TO:?the release to upgrade to, e.g. 27.1-rc.34}"
IMAGE=ghcr.io/inbuxa/inbuxa-server
K=(kubectl --context kind-inbuxa-lab)
H=(helm --kube-context kind-inbuxa-lab -n inbuxa)
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
WORK="$(mktemp -d)"
FLOW=""
trap '[ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null; rm -rf "$WORK"' EXIT
fail() { echo "!! $*" >&2; exit 1; }
digest() { docker buildx imagetools inspect "$IMAGE:$1" --format '{{json .Manifest.Digest}}' | tr -d '"'; }
upgrade() { # digest, then helm's own flags
local d="$1"; shift
"${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml \
--set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \
--set server.image.digest="$d" --wait --timeout 20m "$@"
}
# Every server pod is ready and runs exactly this digest, once the old ones
# have finished terminating (helm's --wait returns before they have)
on_digest() {
local want="$1" got
for _ in $(seq 1 60); do
got="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server \
-o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{" "}{.status.containerStatuses[0].ready}{"\n"}{end}')"
if [ "$(grep -c . <<<"$got")" -ge 5 ] && ! grep -qv "@$want true\$" <<<"$got"; then return 0; fi
sleep 3
done
echo "$got"; return 1
}
hook_log() { "${K[@]}" -n inbuxa logs job/inbuxa-upgrade 2>/dev/null | grep -v '^20..-' || true; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
mail() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 mailtest.py "$(lb)"; }
from_d="$(digest "$FROM")"; to_d="$(digest "$TO")"
[[ "$from_d" =~ ^sha256: && "$to_d" =~ ^sha256: ]] || fail "couldn't read the digests of $FROM and $TO"
echo "$FROM = $from_d"
echo "$TO = $to_d"
echo "== 1. on $FROM"
start=$(date +%s)
upgrade "$from_d" >/dev/null
echo " in $(( $(date +%s) - start ))s"
on_digest "$from_d" || fail "not every server pod runs $FROM"
mail
echo "== 2. mail flowing"
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" \
python3 mailflow.py "$(lb)" > "$WORK/flow.log" 2>&1 &
FLOW=$!
sleep 20
echo "== 3. rolling to $TO"
start=$(date +%s)
upgrade "$to_d" >/dev/null
echo " in $(( $(date +%s) - start ))s; the hook said:"; hook_log | sed 's/^/ /'
sleep 20
touch "$WORK/stop"
flow=0; wait "$FLOW" || flow=$?; FLOW=""
sed 's/^/ /' "$WORK/flow.log"
echo "== 4. on $TO"
on_digest "$to_d" || fail "not every server pod runs $TO"
mail
[ "$flow" = 0 ] || fail "mail accepted during the roll was lost or doubled"
echo "== $FROM to $TO: rolled with mail flowing, nothing lost"
+3 -1
View File
@@ -12,7 +12,9 @@ spec:
ttlSecondsAfterFinished: 3600
template:
metadata:
labels: {app: inbuxa-seed}
# The chart's seed labels: the chart's deny-all and jobs NetworkPolicies
# cover it, as the lab's Kyverno policy requires
labels: {app: inbuxa-seed, app.kubernetes.io/name: inbuxa, app.kubernetes.io/instance: inbuxa, app.kubernetes.io/component: seed}
spec:
restartPolicy: Never
automountServiceAccountToken: false
+3 -1
View File
@@ -11,6 +11,8 @@ set -euo pipefail
cd "$(dirname "$0")"
K=(kubectl --context kind-inbuxa-lab)
IMAGE="${IMAGE:-inbuxa-lab/server:main}"
# Never for an image loaded into kind; IfNotPresent for a published one
PULL="${PULL:-Never}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
mkdir -p "$STATE" && chmod 700 "$STATE"
@@ -41,7 +43,7 @@ rm -f "$STATE/s3-access-key"
echo "== the seed run"
"${K[@]}" -n inbuxa delete job inbuxa-seed --ignore-not-found --wait >/dev/null
sed "s|image: IMAGE|image: $IMAGE|" seed-job.yaml | "${K[@]}" apply -f - >/dev/null
sed -e "s|image: IMAGE|image: $IMAGE|" -e "s|imagePullPolicy: Never|imagePullPolicy: $PULL|" seed-job.yaml | "${K[@]}" apply -f - >/dev/null
if ! "${K[@]}" -n inbuxa wait --for=condition=Complete job/inbuxa-seed --timeout=300s >/dev/null 2>&1; then
"${K[@]}" -n inbuxa logs job/inbuxa-seed --tail=40 >&2 || true
echo "!! the seed run failed" >&2