Files
jcoffey-dev ea991b75fd
ci / check (pull_request) Successful in 40s
Chaos: a missing message's queue id, and what the server's logs say on it
When the chaos test finds an accepted message missing, it couldn't tell a
late delivery from a lost one: a PostgreSQL failover left one message
missing after 7 minutes with no retry. mailflow.py now keeps the queue id
from each 250 ("queued with id ...", hex) and prints it for each message
missing, and chaos-test.sh then shows the server's log lines for that id
(decimal there): delivered late, deferred and why, or never seen.
2026-11-20 16:44:18 -07:00

165 lines
6.3 KiB
Bash
Executable File

#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L4: faults with mail flowing. For each scenario, mailflow.py sends a
# message a second over SMTP (retrying like an MTA when refused), the fault
# happens, the lab recovers, and every message the server accepted has to
# arrive exactly once:
#
# pod-kill an mta pod and a front pod killed outright (no grace)
# drain a worker with server pods drained, then uncordoned
# pg-failover the PostgreSQL primary deleted: CloudNativePG promotes the
# replica
# nats-restart NATS restarted, one pod at a time
# s3-outage Garage frozen for 60 s (SIGSTOP; requests hang, as a
# stuck S3 does), then thawed
#
# lab/chart/chaos-test.sh
# SCENARIOS="pg-failover s3-outage" lab/chart/chaos-test.sh
#
# On the lab's seeded install; ends with chart/mailcheck.sh, so everything
# still works after the lot. The drain skips the nodes with the PostgreSQL
# primary and Garage (whose data is an emptyDir in the lab, so it can't
# move); a node it cordons, and a frozen Garage, are put back on any exit.
set -euo pipefail
cd "$(dirname "$0")"
. ../versions.env
KCTX="kind-$KIND_CLUSTER"
K=(kubectl --context "$KCTX")
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
SCENARIOS="${SCENARIOS:-pod-kill drain pg-failover nats-restart s3-outage}"
WORK="$(mktemp -d)"
FLOW="" CORDONED="" FROZEN=""
cleanup() {
[ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null || true
[ -n "$CORDONED" ] && "${K[@]}" uncordon "$CORDONED" >/dev/null 2>&1 || true
[ -n "$FROZEN" ] && docker exec "$FROZEN" sh -c 'kill -CONT $(pgrep -x garage)' 2>/dev/null || true
rm -rf "$WORK"
}
trap cleanup EXIT
fail() { echo "!! $*" >&2; exit 1; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
# The server back to full strength: every Deployment rolled out and ready
healthy() {
for d in front mta core; do
"${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=300s >/dev/null
done
}
data_ready() {
"${K[@]}" -n data wait cluster.postgresql.cnpg.io/inbuxa-pg --for=condition=Ready --timeout=600s >/dev/null
"${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null
"${K[@]}" -n data rollout status deploy/garage --timeout=120s >/dev/null
}
node_of() { "${K[@]}" -n "$1" get pod "$2" -o jsonpath='{.spec.nodeName}'; }
primary() { "${K[@]}" -n data get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}'; }
garage_node() { "${K[@]}" -n data get pod -l app=garage -o jsonpath='{.items[0].spec.nodeName}'; }
pod_kill() {
local mta front
mta="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-mta)"
front="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-front)"
echo " killing ${mta#pod/} and ${front#pod/}"
"${K[@]}" -n inbuxa delete "$mta" "$front" --grace-period=0 --force --wait=false >/dev/null 2>&1
sleep 5
healthy
}
drain() {
local avoid node
avoid="$(node_of data "$(primary)") $(garage_node)"
node="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \
| sort | uniq -c | sort -rn | awk '{print $2}' | grep -vxF -f <(tr ' ' '\n' <<<"$avoid") | head -1 || true)"
[ -n "$node" ] || { echo " no node to drain without the PostgreSQL primary or Garage; skipped"; return 0; }
echo " draining $node (not $avoid)"
CORDONED="$node"
"${K[@]}" drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout=300s >/dev/null 2>"$WORK/drain.err" \
|| { cat "$WORK/drain.err" >&2; return 1; }
healthy
"${K[@]}" uncordon "$node" >/dev/null
CORDONED=""
data_ready
}
pg_failover() {
local old new=""
old="$(primary)"
echo " deleting the primary, $old"
"${K[@]}" -n data delete pod "$old" --wait=false >/dev/null
for _ in $(seq 1 100); do
new="$(primary)"
[ -n "$new" ] && [ "$new" != "$old" ] && break
sleep 3
done
[ "$new" != "$old" ] || { echo " no new primary"; return 1; }
echo " new primary $new"
data_ready
healthy
}
nats_restart() {
"${K[@]}" -n data rollout restart statefulset/nats >/dev/null
"${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null
healthy
}
s3_outage() {
local node
node="$(garage_node)"
echo " freezing Garage on $node for 60 s"
FROZEN="$node"
docker exec "$node" sh -c 'kill -STOP $(pgrep -x garage)'
sleep 60
docker exec "$node" sh -c 'kill -CONT $(pgrep -x garage)'
FROZEN=""
echo " thawed"
healthy
}
declare -A RESULT TOOK
for sc in $SCENARIOS; do
fn="${sc//-/_}"
declare -F "$fn" >/dev/null || fail "no scenario $sc"
echo "== $sc"
start=$(date +%s)
rm -f "$WORK/stop"
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" WAIT=420 \
python3 mailflow.py "$(lb)" > "$WORK/$sc.log" 2>&1 &
FLOW=$!
sleep 15
fault=0
"$fn" || fault=$?
sleep 20
touch "$WORK/stop"
flow=0; wait "$FLOW" || flow=$?; FLOW=""
# The retries too: why each send was refused says what the fault did
sed 's/^/ /' "$WORK/$sc.log"
TOOK[$sc]=$(( $(date +%s) - start ))
# A message missing: what the server's logs say about it, by its queue id
# (delivered late, deferred and why, or nothing at all)
for id in $(sed -n 's/^ queue id [0-9]*: \([0-9][0-9]*\)$/\1/p' "$WORK/$sc.log"); do
echo " -- the server's logs on queue id $id"
"${K[@]}" -n inbuxa logs -l app.kubernetes.io/component=server --prefix --tail=-1 --since=30m 2>/dev/null \
| sed 's/\x1b\[[0-9;]*m//g' | grep -wF -- "$id" | tail -20 | cut -c1-400 | sed 's/^/ /' || true
done
if [ "$fault" != 0 ]; then RESULT[$sc]="FAIL (the fault or the recovery)"
elif [ "$flow" != 0 ]; then RESULT[$sc]="FAIL (mail lost or doubled)"
else RESULT[$sc]="ok"; fi
done
echo "== after it all: every client check"
after=0
PORTS="${PORTS:-18080}" STATE="$STATE" ./mailcheck.sh >"$WORK/after.log" 2>&1 || after=$?
grep -E '^ (ok|FAIL)|^== every' "$WORK/after.log" || tail -20 "$WORK/after.log"
echo "== chaos"
bad=0
for sc in $SCENARIOS; do
printf ' %-13s %-34s %ss\n' "$sc" "${RESULT[$sc]}" "${TOOK[$sc]}"
[ "${RESULT[$sc]}" = ok ] || bad=1
done
[ "$after" = 0 ] || { echo " the client checks failed afterwards"; bad=1; }
exit "$bad"