Chaos tests: five faults with mail flowing, nothing lost or doubled (K-L4) #35

Merged
jcoffey-dev merged 1 commits from feat/chaos into main 2026-11-20 07:18:47 +00:00
5 changed files with 220 additions and 3 deletions

No files matched your search

+39
View File
@@ -49,6 +49,7 @@ jobs:
runs-on: ubuntu-24.04
outputs:
kubernetes: ${{ steps.m.outputs.kubernetes }}
newest: ${{ steps.m.outputs.newest }}
runners: ${{ steps.m.outputs.runners }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -63,6 +64,7 @@ jobs:
RUNNERS: ${{ inputs.architectures || '["ubuntu-24.04", "ubuntu-24.04-arm"]' }}
run: |
echo "kubernetes=$(jq -c . lab/kubernetes.json)" >> "$GITHUB_OUTPUT"
echo "newest=$(jq -c '.[-1]' lab/kubernetes.json)" >> "$GITHUB_OUTPUT"
echo "runners=$(jq -c . <<<"$RUNNERS")" >> "$GITHUB_OUTPUT"
e2e:
@@ -108,3 +110,40 @@ jobs:
run: |
free -h | head -2
docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' || true
# The chaos tests (K-L4): the lab's own runs only (nightly, by hand, its
# pushes), not a chart branch's, which stays quick. Newest minor, both
# architectures.
chaos:
needs: matrix
if: ${{ inputs.chart_ref == '' }}
name: chaos (${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }})
strategy:
fail-fast: false
matrix:
runner: ${{ fromJSON(needs.matrix.outputs.runners) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 60
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: inbuxa/inbuxa-k8s-lab
ref: ${{ github.sha }}
path: inbuxa-k8s-lab
persist-credentials: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: inbuxa/inbuxa-charts
ref: ${{ vars.CHART_REF || 'main' }}
path: inbuxa-charts
persist-credentials: false
- name: Tools (pinned, checksummed)
run: |
inbuxa-k8s-lab/lab/tools.sh "$HOME/.local/bin"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Lab end to end, then chaos
env:
KIND_NODE_IMAGE: ${{ fromJSON(needs.matrix.outputs.newest).image }}
E2E_LABEL: chaos, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}
CHAOS: "1"
run: inbuxa-k8s-lab/lab/e2e.sh
+6
View File
@@ -41,6 +41,12 @@ CardDAV, the webmail signing in across both replicas and the console. It's
what `e2e.sh` runs after the install, and what to run after changing the
lab by hand.
`lab/chart/chaos-test.sh` breaks things with mail flowing, a message a
second, and checks that every message the server accepted arrived exactly
once: a server pod killed, a node drained, the PostgreSQL primary failed
over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest
Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`).
`lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB,
and the data layer (CloudNativePG, NATS, Garage). It adds the Helm
repositories it needs; the versions are pinned in `lab/versions.env`.
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L4: faults with mail flowing. For each scenario, mailflow.py sends a
# message a second over SMTP (retrying like an MTA when refused), the fault
# happens, the lab recovers, and every message the server accepted has to
# arrive exactly once:
#
# pod-kill an mta pod and a front pod killed outright (no grace)
# drain a worker with server pods drained, then uncordoned
# pg-failover the PostgreSQL primary deleted: CloudNativePG promotes the
# replica
# nats-restart NATS restarted, one pod at a time
# s3-outage Garage frozen for 60 s (SIGSTOP; requests hang, as a
# stuck S3 does), then thawed
#
# lab/chart/chaos-test.sh
# SCENARIOS="pg-failover s3-outage" lab/chart/chaos-test.sh
#
# On the lab's seeded install; ends with chart/mailcheck.sh, so everything
# still works after the lot. The drain skips the nodes with the PostgreSQL
# primary and Garage (whose data is an emptyDir in the lab, so it can't
# move); a node it cordons, and a frozen Garage, are put back on any exit.
set -euo pipefail
cd "$(dirname "$0")"
. ../versions.env
KCTX="kind-$KIND_CLUSTER"
K=(kubectl --context "$KCTX")
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
SCENARIOS="${SCENARIOS:-pod-kill drain pg-failover nats-restart s3-outage}"
WORK="$(mktemp -d)"
FLOW="" CORDONED="" FROZEN=""
cleanup() {
[ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null || true
[ -n "$CORDONED" ] && "${K[@]}" uncordon "$CORDONED" >/dev/null 2>&1 || true
[ -n "$FROZEN" ] && docker exec "$FROZEN" sh -c 'kill -CONT $(pgrep -x garage)' 2>/dev/null || true
rm -rf "$WORK"
}
trap cleanup EXIT
fail() { echo "!! $*" >&2; exit 1; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
# The server back to full strength: every Deployment rolled out and ready
healthy() {
for d in front mta core; do
"${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=300s >/dev/null
done
}
data_ready() {
"${K[@]}" -n data wait cluster.postgresql.cnpg.io/inbuxa-pg --for=condition=Ready --timeout=600s >/dev/null
"${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null
"${K[@]}" -n data rollout status deploy/garage --timeout=120s >/dev/null
}
node_of() { "${K[@]}" -n "$1" get pod "$2" -o jsonpath='{.spec.nodeName}'; }
primary() { "${K[@]}" -n data get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}'; }
garage_node() { "${K[@]}" -n data get pod -l app=garage -o jsonpath='{.items[0].spec.nodeName}'; }
pod_kill() {
local mta front
mta="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-mta)"
front="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-front)"
echo " killing ${mta#pod/} and ${front#pod/}"
"${K[@]}" -n inbuxa delete "$mta" "$front" --grace-period=0 --force --wait=false >/dev/null 2>&1
sleep 5
healthy
}
drain() {
local avoid node
avoid="$(node_of data "$(primary)") $(garage_node)"
node="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \
| sort | uniq -c | sort -rn | awk '{print $2}' | grep -vxF -f <(tr ' ' '\n' <<<"$avoid") | head -1 || true)"
[ -n "$node" ] || { echo " no node to drain without the PostgreSQL primary or Garage; skipped"; return 0; }
echo " draining $node (not $avoid)"
CORDONED="$node"
"${K[@]}" drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout=300s >/dev/null 2>"$WORK/drain.err" \
|| { cat "$WORK/drain.err" >&2; return 1; }
healthy
"${K[@]}" uncordon "$node" >/dev/null
CORDONED=""
data_ready
}
pg_failover() {
local old new=""
old="$(primary)"
echo " deleting the primary, $old"
"${K[@]}" -n data delete pod "$old" --wait=false >/dev/null
for _ in $(seq 1 100); do
new="$(primary)"
[ -n "$new" ] && [ "$new" != "$old" ] && break
sleep 3
done
[ "$new" != "$old" ] || { echo " no new primary"; return 1; }
echo " new primary $new"
data_ready
healthy
}
nats_restart() {
"${K[@]}" -n data rollout restart statefulset/nats >/dev/null
"${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null
healthy
}
s3_outage() {
local node
node="$(garage_node)"
echo " freezing Garage on $node for 60 s"
FROZEN="$node"
docker exec "$node" sh -c 'kill -STOP $(pgrep -x garage)'
sleep 60
docker exec "$node" sh -c 'kill -CONT $(pgrep -x garage)'
FROZEN=""
echo " thawed"
healthy
}
declare -A RESULT TOOK
for sc in $SCENARIOS; do
fn="${sc//-/_}"
declare -F "$fn" >/dev/null || fail "no scenario $sc"
echo "== $sc"
start=$(date +%s)
rm -f "$WORK/stop"
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" WAIT=420 \
python3 mailflow.py "$(lb)" > "$WORK/$sc.log" 2>&1 &
FLOW=$!
sleep 15
fault=0
"$fn" || fault=$?
sleep 20
touch "$WORK/stop"
flow=0; wait "$FLOW" || flow=$?; FLOW=""
# The retries too: why each send was refused says what the fault did
sed 's/^/ /' "$WORK/$sc.log"
TOOK[$sc]=$(( $(date +%s) - start ))
if [ "$fault" != 0 ]; then RESULT[$sc]="FAIL (the fault or the recovery)"
elif [ "$flow" != 0 ]; then RESULT[$sc]="FAIL (mail lost or doubled)"
else RESULT[$sc]="ok"; fi
done
echo "== after it all: every client check"
after=0
PORTS="${PORTS:-18080}" STATE="$STATE" ./mailcheck.sh >"$WORK/after.log" 2>&1 || after=$?
grep -E '^ (ok|FAIL)|^== every' "$WORK/after.log" || tail -20 "$WORK/after.log"
echo "== chaos"
bad=0
for sc in $SCENARIOS; do
printf ' %-13s %-34s %ss\n' "$sc" "${RESULT[$sc]}" "${TOOK[$sc]}"
[ "${RESULT[$sc]}" = ok ] || bad=1
done
[ "$after" = 0 ] || { echo " the client checks failed afterwards"; bad=1; }
exit "$bad"
+8 -2
View File
@@ -36,7 +36,12 @@ def send(n):
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test")
msg.set_content(f"mailflow {run} message {n}\n")
with smtplib.SMTP(lb, 25, timeout=20) as s:
# As an MTA waits (RFC 5321 4.5.3.2): a dead address fails fast, but a
# slow reply is waited for, up to minutes. Giving up early on the reply
# to the end of DATA and sending again is how a sender makes duplicates.
with smtplib.SMTP(timeout=20) as s:
s.connect(lb, 25)
s.sock.settimeout(300)
s.send_message(msg)
@@ -85,7 +90,8 @@ def count():
return seen
deadline = time.time() + 180
# WAIT: how long to wait for what was accepted (an outage can delay delivery)
deadline = time.time() + int(os.environ.get("WAIT", "180"))
while True:
try:
seen = count()
+10 -1
View File
@@ -10,7 +10,8 @@
# JMAP, SMTP out to an outside server, CalDAV, CardDAV, the webmail across
# both replicas and the console
#
# and how long each part took. CI runs this on a fresh runner; on the bench
# and how long each part took; CHAOS=1 adds the chaos tests after it. CI
# runs this on a fresh runner; on the bench
# it builds the lab from scratch too, so take the lab down first
# (lab/down.sh) unless you mean to reuse it.
#
@@ -97,5 +98,13 @@ s=$SECONDS
PORTS="$P" STATE="$STATE" "$here/chart/mailcheck.sh"
lap "mail (SMTP in, submission, IMAP, JMAP, SMTP out), CalDAV, CardDAV, webmail, console" "$s"
# CHAOS=1: then the faults, with mail flowing (chart/chaos-test.sh)
if [ -n "${CHAOS:-}" ]; then
echo "== chaos (chart/chaos-test.sh)"
s=$SECONDS
PORTS="$P" STATE="$STATE" "$here/chart/chaos-test.sh"
lap "chaos: pod kill, drain, PostgreSQL failover, NATS restart, S3 outage" "$s"
fi
lap "**total**"
echo "== passed in $(( SECONDS - t0 ))s"