From 7392b6031684838b8bfaad5c1df7cbff4a9a2f39 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 00:03:14 -0700 Subject: [PATCH] Chaos tests: five faults with mail flowing, nothing lost or doubled (K-L4) lab/chart/chaos-test.sh, with mailflow.py sending a message a second: an mta and a front pod killed outright, a worker drained (not the one with the PostgreSQL primary or Garage) and uncordoned, the PostgreSQL primary deleted for CloudNativePG to fail over, NATS restarted, and Garage frozen for 60 s with SIGSTOP. Every accepted message has to arrive exactly once; mailcheck.sh runs after the lot. mailflow.py now waits for replies as an MTA does (RFC 5321 4.5.3.2): 20 s to connect, 5 minutes for a reply. With 20 s for everything it gave up on the end-of-DATA reply while S3 was frozen and sent again, which made a duplicate; waiting, the server answers once. CI: a chaos job in e2e.yml, nightly, by hand and on the lab's pushes, newest minor, amd64 and arm64 (CHAOS=1 lab/e2e.sh). Not on chart branches, whose check stays quick. --- .github/workflows/e2e.yml | 39 ++++++++++ README.md | 6 ++ lab/chart/chaos-test.sh | 157 ++++++++++++++++++++++++++++++++++++++ lab/chart/mailflow.py | 10 ++- lab/e2e.sh | 11 ++- 5 files changed, 220 insertions(+), 3 deletions(-) create mode 100755 lab/chart/chaos-test.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ef8f7cc..42ef104 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -49,6 +49,7 @@ jobs: runs-on: ubuntu-24.04 outputs: kubernetes: ${{ steps.m.outputs.kubernetes }} + newest: ${{ steps.m.outputs.newest }} runners: ${{ steps.m.outputs.runners }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -63,6 +64,7 @@ jobs: RUNNERS: ${{ inputs.architectures || '["ubuntu-24.04", "ubuntu-24.04-arm"]' }} run: | echo "kubernetes=$(jq -c . lab/kubernetes.json)" >> "$GITHUB_OUTPUT" + echo "newest=$(jq -c '.[-1]' lab/kubernetes.json)" >> "$GITHUB_OUTPUT" echo "runners=$(jq -c . <<<"$RUNNERS")" >> "$GITHUB_OUTPUT" e2e: @@ -108,3 +110,40 @@ jobs: run: | free -h | head -2 docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' || true + + # The chaos tests (K-L4): the lab's own runs only (nightly, by hand, its + # pushes), not a chart branch's, which stays quick. Newest minor, both + # architectures. + chaos: + needs: matrix + if: ${{ inputs.chart_ref == '' }} + name: chaos (${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}) + strategy: + fail-fast: false + matrix: + runner: ${{ fromJSON(needs.matrix.outputs.runners) }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: inbuxa/inbuxa-k8s-lab + ref: ${{ github.sha }} + path: inbuxa-k8s-lab + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: inbuxa/inbuxa-charts + ref: ${{ vars.CHART_REF || 'main' }} + path: inbuxa-charts + persist-credentials: false + - name: Tools (pinned, checksummed) + run: | + inbuxa-k8s-lab/lab/tools.sh "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Lab end to end, then chaos + env: + KIND_NODE_IMAGE: ${{ fromJSON(needs.matrix.outputs.newest).image }} + E2E_LABEL: chaos, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }} + CHAOS: "1" + run: inbuxa-k8s-lab/lab/e2e.sh diff --git a/README.md b/README.md index dd05e62..c0c598c 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,12 @@ CardDAV, the webmail signing in across both replicas and the console. It's what `e2e.sh` runs after the install, and what to run after changing the lab by hand. +`lab/chart/chaos-test.sh` breaks things with mail flowing, a message a +second, and checks that every message the server accepted arrived exactly +once: a server pod killed, a node drained, the PostgreSQL primary failed +over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest +Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`). + `lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB, and the data layer (CloudNativePG, NATS, Garage). It adds the Helm repositories it needs; the versions are pinned in `lab/versions.env`. diff --git a/lab/chart/chaos-test.sh b/lab/chart/chaos-test.sh new file mode 100755 index 0000000..52189c1 --- /dev/null +++ b/lab/chart/chaos-test.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L4: faults with mail flowing. For each scenario, mailflow.py sends a +# message a second over SMTP (retrying like an MTA when refused), the fault +# happens, the lab recovers, and every message the server accepted has to +# arrive exactly once: +# +# pod-kill an mta pod and a front pod killed outright (no grace) +# drain a worker with server pods drained, then uncordoned +# pg-failover the PostgreSQL primary deleted: CloudNativePG promotes the +# replica +# nats-restart NATS restarted, one pod at a time +# s3-outage Garage frozen for 60 s (SIGSTOP; requests hang, as a +# stuck S3 does), then thawed +# +# lab/chart/chaos-test.sh +# SCENARIOS="pg-failover s3-outage" lab/chart/chaos-test.sh +# +# On the lab's seeded install; ends with chart/mailcheck.sh, so everything +# still works after the lot. The drain skips the nodes with the PostgreSQL +# primary and Garage (whose data is an emptyDir in the lab, so it can't +# move); a node it cordons, and a frozen Garage, are put back on any exit. +set -euo pipefail + +cd "$(dirname "$0")" +. ../versions.env +KCTX="kind-$KIND_CLUSTER" +K=(kubectl --context "$KCTX") +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +SCENARIOS="${SCENARIOS:-pod-kill drain pg-failover nats-restart s3-outage}" +WORK="$(mktemp -d)" +FLOW="" CORDONED="" FROZEN="" +cleanup() { + [ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null || true + [ -n "$CORDONED" ] && "${K[@]}" uncordon "$CORDONED" >/dev/null 2>&1 || true + [ -n "$FROZEN" ] && docker exec "$FROZEN" sh -c 'kill -CONT $(pgrep -x garage)' 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT +fail() { echo "!! $*" >&2; exit 1; } + +lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } +# The server back to full strength: every Deployment rolled out and ready +healthy() { + for d in front mta core; do + "${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=300s >/dev/null + done +} +data_ready() { + "${K[@]}" -n data wait cluster.postgresql.cnpg.io/inbuxa-pg --for=condition=Ready --timeout=600s >/dev/null + "${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null + "${K[@]}" -n data rollout status deploy/garage --timeout=120s >/dev/null +} +node_of() { "${K[@]}" -n "$1" get pod "$2" -o jsonpath='{.spec.nodeName}'; } +primary() { "${K[@]}" -n data get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}'; } +garage_node() { "${K[@]}" -n data get pod -l app=garage -o jsonpath='{.items[0].spec.nodeName}'; } + +pod_kill() { + local mta front + mta="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-mta)" + front="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-front)" + echo " killing ${mta#pod/} and ${front#pod/}" + "${K[@]}" -n inbuxa delete "$mta" "$front" --grace-period=0 --force --wait=false >/dev/null 2>&1 + sleep 5 + healthy +} + +drain() { + local avoid node + avoid="$(node_of data "$(primary)") $(garage_node)" + node="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \ + | sort | uniq -c | sort -rn | awk '{print $2}' | grep -vxF -f <(tr ' ' '\n' <<<"$avoid") | head -1 || true)" + [ -n "$node" ] || { echo " no node to drain without the PostgreSQL primary or Garage; skipped"; return 0; } + echo " draining $node (not $avoid)" + CORDONED="$node" + "${K[@]}" drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout=300s >/dev/null 2>"$WORK/drain.err" \ + || { cat "$WORK/drain.err" >&2; return 1; } + healthy + "${K[@]}" uncordon "$node" >/dev/null + CORDONED="" + data_ready +} + +pg_failover() { + local old new="" + old="$(primary)" + echo " deleting the primary, $old" + "${K[@]}" -n data delete pod "$old" --wait=false >/dev/null + for _ in $(seq 1 100); do + new="$(primary)" + [ -n "$new" ] && [ "$new" != "$old" ] && break + sleep 3 + done + [ "$new" != "$old" ] || { echo " no new primary"; return 1; } + echo " new primary $new" + data_ready + healthy +} + +nats_restart() { + "${K[@]}" -n data rollout restart statefulset/nats >/dev/null + "${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null + healthy +} + +s3_outage() { + local node + node="$(garage_node)" + echo " freezing Garage on $node for 60 s" + FROZEN="$node" + docker exec "$node" sh -c 'kill -STOP $(pgrep -x garage)' + sleep 60 + docker exec "$node" sh -c 'kill -CONT $(pgrep -x garage)' + FROZEN="" + echo " thawed" + healthy +} + +declare -A RESULT TOOK +for sc in $SCENARIOS; do + fn="${sc//-/_}" + declare -F "$fn" >/dev/null || fail "no scenario $sc" + echo "== $sc" + start=$(date +%s) + rm -f "$WORK/stop" + ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" WAIT=420 \ + python3 mailflow.py "$(lb)" > "$WORK/$sc.log" 2>&1 & + FLOW=$! + sleep 15 + fault=0 + "$fn" || fault=$? + sleep 20 + touch "$WORK/stop" + flow=0; wait "$FLOW" || flow=$?; FLOW="" + # The retries too: why each send was refused says what the fault did + sed 's/^/ /' "$WORK/$sc.log" + TOOK[$sc]=$(( $(date +%s) - start )) + if [ "$fault" != 0 ]; then RESULT[$sc]="FAIL (the fault or the recovery)" + elif [ "$flow" != 0 ]; then RESULT[$sc]="FAIL (mail lost or doubled)" + else RESULT[$sc]="ok"; fi +done + +echo "== after it all: every client check" +after=0 +PORTS="${PORTS:-18080}" STATE="$STATE" ./mailcheck.sh >"$WORK/after.log" 2>&1 || after=$? +grep -E '^ (ok|FAIL)|^== every' "$WORK/after.log" || tail -20 "$WORK/after.log" + +echo "== chaos" +bad=0 +for sc in $SCENARIOS; do + printf ' %-13s %-34s %ss\n' "$sc" "${RESULT[$sc]}" "${TOOK[$sc]}" + [ "${RESULT[$sc]}" = ok ] || bad=1 +done +[ "$after" = 0 ] || { echo " the client checks failed afterwards"; bad=1; } +exit "$bad" diff --git a/lab/chart/mailflow.py b/lab/chart/mailflow.py index 4bfd083..d4827bf 100755 --- a/lab/chart/mailflow.py +++ b/lab/chart/mailflow.py @@ -36,7 +36,12 @@ def send(n): msg["Date"] = formatdate() msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test") msg.set_content(f"mailflow {run} message {n}\n") - with smtplib.SMTP(lb, 25, timeout=20) as s: + # As an MTA waits (RFC 5321 4.5.3.2): a dead address fails fast, but a + # slow reply is waited for, up to minutes. Giving up early on the reply + # to the end of DATA and sending again is how a sender makes duplicates. + with smtplib.SMTP(timeout=20) as s: + s.connect(lb, 25) + s.sock.settimeout(300) s.send_message(msg) @@ -85,7 +90,8 @@ def count(): return seen -deadline = time.time() + 180 +# WAIT: how long to wait for what was accepted (an outage can delay delivery) +deadline = time.time() + int(os.environ.get("WAIT", "180")) while True: try: seen = count() diff --git a/lab/e2e.sh b/lab/e2e.sh index 0c2c47e..6dfa8e8 100755 --- a/lab/e2e.sh +++ b/lab/e2e.sh @@ -10,7 +10,8 @@ # JMAP, SMTP out to an outside server, CalDAV, CardDAV, the webmail across # both replicas and the console # -# and how long each part took. CI runs this on a fresh runner; on the bench +# and how long each part took; CHAOS=1 adds the chaos tests after it. CI +# runs this on a fresh runner; on the bench # it builds the lab from scratch too, so take the lab down first # (lab/down.sh) unless you mean to reuse it. # @@ -97,5 +98,13 @@ s=$SECONDS PORTS="$P" STATE="$STATE" "$here/chart/mailcheck.sh" lap "mail (SMTP in, submission, IMAP, JMAP, SMTP out), CalDAV, CardDAV, webmail, console" "$s" +# CHAOS=1: then the faults, with mail flowing (chart/chaos-test.sh) +if [ -n "${CHAOS:-}" ]; then + echo "== chaos (chart/chaos-test.sh)" + s=$SECONDS + PORTS="$P" STATE="$STATE" "$here/chart/chaos-test.sh" + lap "chaos: pod kill, drain, PostgreSQL failover, NATS restart, S3 outage" "$s" +fi + lap "**total**" echo "== passed in $(( SECONDS - t0 ))s" -- 2.54.0