Chaos tests: five faults with mail flowing, nothing lost or doubled (K-L4) #35
No files matched your search
@@ -49,6 +49,7 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
outputs:
|
||||
kubernetes: ${{ steps.m.outputs.kubernetes }}
|
||||
newest: ${{ steps.m.outputs.newest }}
|
||||
runners: ${{ steps.m.outputs.runners }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -63,6 +64,7 @@ jobs:
|
||||
RUNNERS: ${{ inputs.architectures || '["ubuntu-24.04", "ubuntu-24.04-arm"]' }}
|
||||
run: |
|
||||
echo "kubernetes=$(jq -c . lab/kubernetes.json)" >> "$GITHUB_OUTPUT"
|
||||
echo "newest=$(jq -c '.[-1]' lab/kubernetes.json)" >> "$GITHUB_OUTPUT"
|
||||
echo "runners=$(jq -c . <<<"$RUNNERS")" >> "$GITHUB_OUTPUT"
|
||||
|
||||
e2e:
|
||||
@@ -108,3 +110,40 @@ jobs:
|
||||
run: |
|
||||
free -h | head -2
|
||||
docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' || true
|
||||
|
||||
# The chaos tests (K-L4): the lab's own runs only (nightly, by hand, its
|
||||
# pushes), not a chart branch's, which stays quick. Newest minor, both
|
||||
# architectures.
|
||||
chaos:
|
||||
needs: matrix
|
||||
if: ${{ inputs.chart_ref == '' }}
|
||||
name: chaos (${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: ${{ fromJSON(needs.matrix.outputs.runners) }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: inbuxa/inbuxa-k8s-lab
|
||||
ref: ${{ github.sha }}
|
||||
path: inbuxa-k8s-lab
|
||||
persist-credentials: false
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: inbuxa/inbuxa-charts
|
||||
ref: ${{ vars.CHART_REF || 'main' }}
|
||||
path: inbuxa-charts
|
||||
persist-credentials: false
|
||||
- name: Tools (pinned, checksummed)
|
||||
run: |
|
||||
inbuxa-k8s-lab/lab/tools.sh "$HOME/.local/bin"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
- name: Lab end to end, then chaos
|
||||
env:
|
||||
KIND_NODE_IMAGE: ${{ fromJSON(needs.matrix.outputs.newest).image }}
|
||||
E2E_LABEL: chaos, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}
|
||||
CHAOS: "1"
|
||||
run: inbuxa-k8s-lab/lab/e2e.sh
|
||||
@@ -41,6 +41,12 @@ CardDAV, the webmail signing in across both replicas and the console. It's
|
||||
what `e2e.sh` runs after the install, and what to run after changing the
|
||||
lab by hand.
|
||||
|
||||
`lab/chart/chaos-test.sh` breaks things with mail flowing, a message a
|
||||
second, and checks that every message the server accepted arrived exactly
|
||||
once: a server pod killed, a node drained, the PostgreSQL primary failed
|
||||
over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest
|
||||
Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`).
|
||||
|
||||
`lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB,
|
||||
and the data layer (CloudNativePG, NATS, Garage). It adds the Helm
|
||||
repositories it needs; the versions are pinned in `lab/versions.env`.
|
||||
|
||||
Executable
+157
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-L4: faults with mail flowing. For each scenario, mailflow.py sends a
|
||||
# message a second over SMTP (retrying like an MTA when refused), the fault
|
||||
# happens, the lab recovers, and every message the server accepted has to
|
||||
# arrive exactly once:
|
||||
#
|
||||
# pod-kill an mta pod and a front pod killed outright (no grace)
|
||||
# drain a worker with server pods drained, then uncordoned
|
||||
# pg-failover the PostgreSQL primary deleted: CloudNativePG promotes the
|
||||
# replica
|
||||
# nats-restart NATS restarted, one pod at a time
|
||||
# s3-outage Garage frozen for 60 s (SIGSTOP; requests hang, as a
|
||||
# stuck S3 does), then thawed
|
||||
#
|
||||
# lab/chart/chaos-test.sh
|
||||
# SCENARIOS="pg-failover s3-outage" lab/chart/chaos-test.sh
|
||||
#
|
||||
# On the lab's seeded install; ends with chart/mailcheck.sh, so everything
|
||||
# still works after the lot. The drain skips the nodes with the PostgreSQL
|
||||
# primary and Garage (whose data is an emptyDir in the lab, so it can't
|
||||
# move); a node it cordons, and a frozen Garage, are put back on any exit.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
. ../versions.env
|
||||
KCTX="kind-$KIND_CLUSTER"
|
||||
K=(kubectl --context "$KCTX")
|
||||
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
|
||||
SCENARIOS="${SCENARIOS:-pod-kill drain pg-failover nats-restart s3-outage}"
|
||||
WORK="$(mktemp -d)"
|
||||
FLOW="" CORDONED="" FROZEN=""
|
||||
cleanup() {
|
||||
[ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null || true
|
||||
[ -n "$CORDONED" ] && "${K[@]}" uncordon "$CORDONED" >/dev/null 2>&1 || true
|
||||
[ -n "$FROZEN" ] && docker exec "$FROZEN" sh -c 'kill -CONT $(pgrep -x garage)' 2>/dev/null || true
|
||||
rm -rf "$WORK"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
fail() { echo "!! $*" >&2; exit 1; }
|
||||
|
||||
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
|
||||
# The server back to full strength: every Deployment rolled out and ready
|
||||
healthy() {
|
||||
for d in front mta core; do
|
||||
"${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=300s >/dev/null
|
||||
done
|
||||
}
|
||||
data_ready() {
|
||||
"${K[@]}" -n data wait cluster.postgresql.cnpg.io/inbuxa-pg --for=condition=Ready --timeout=600s >/dev/null
|
||||
"${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null
|
||||
"${K[@]}" -n data rollout status deploy/garage --timeout=120s >/dev/null
|
||||
}
|
||||
node_of() { "${K[@]}" -n "$1" get pod "$2" -o jsonpath='{.spec.nodeName}'; }
|
||||
primary() { "${K[@]}" -n data get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}'; }
|
||||
garage_node() { "${K[@]}" -n data get pod -l app=garage -o jsonpath='{.items[0].spec.nodeName}'; }
|
||||
|
||||
pod_kill() {
|
||||
local mta front
|
||||
mta="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-mta)"
|
||||
front="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,app.kubernetes.io/instance=inbuxa -o name | grep -m1 inbuxa-front)"
|
||||
echo " killing ${mta#pod/} and ${front#pod/}"
|
||||
"${K[@]}" -n inbuxa delete "$mta" "$front" --grace-period=0 --force --wait=false >/dev/null 2>&1
|
||||
sleep 5
|
||||
healthy
|
||||
}
|
||||
|
||||
drain() {
|
||||
local avoid node
|
||||
avoid="$(node_of data "$(primary)") $(garage_node)"
|
||||
node="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' \
|
||||
| sort | uniq -c | sort -rn | awk '{print $2}' | grep -vxF -f <(tr ' ' '\n' <<<"$avoid") | head -1 || true)"
|
||||
[ -n "$node" ] || { echo " no node to drain without the PostgreSQL primary or Garage; skipped"; return 0; }
|
||||
echo " draining $node (not $avoid)"
|
||||
CORDONED="$node"
|
||||
"${K[@]}" drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout=300s >/dev/null 2>"$WORK/drain.err" \
|
||||
|| { cat "$WORK/drain.err" >&2; return 1; }
|
||||
healthy
|
||||
"${K[@]}" uncordon "$node" >/dev/null
|
||||
CORDONED=""
|
||||
data_ready
|
||||
}
|
||||
|
||||
pg_failover() {
|
||||
local old new=""
|
||||
old="$(primary)"
|
||||
echo " deleting the primary, $old"
|
||||
"${K[@]}" -n data delete pod "$old" --wait=false >/dev/null
|
||||
for _ in $(seq 1 100); do
|
||||
new="$(primary)"
|
||||
[ -n "$new" ] && [ "$new" != "$old" ] && break
|
||||
sleep 3
|
||||
done
|
||||
[ "$new" != "$old" ] || { echo " no new primary"; return 1; }
|
||||
echo " new primary $new"
|
||||
data_ready
|
||||
healthy
|
||||
}
|
||||
|
||||
nats_restart() {
|
||||
"${K[@]}" -n data rollout restart statefulset/nats >/dev/null
|
||||
"${K[@]}" -n data rollout status statefulset/nats --timeout=300s >/dev/null
|
||||
healthy
|
||||
}
|
||||
|
||||
s3_outage() {
|
||||
local node
|
||||
node="$(garage_node)"
|
||||
echo " freezing Garage on $node for 60 s"
|
||||
FROZEN="$node"
|
||||
docker exec "$node" sh -c 'kill -STOP $(pgrep -x garage)'
|
||||
sleep 60
|
||||
docker exec "$node" sh -c 'kill -CONT $(pgrep -x garage)'
|
||||
FROZEN=""
|
||||
echo " thawed"
|
||||
healthy
|
||||
}
|
||||
|
||||
declare -A RESULT TOOK
|
||||
for sc in $SCENARIOS; do
|
||||
fn="${sc//-/_}"
|
||||
declare -F "$fn" >/dev/null || fail "no scenario $sc"
|
||||
echo "== $sc"
|
||||
start=$(date +%s)
|
||||
rm -f "$WORK/stop"
|
||||
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" WAIT=420 \
|
||||
python3 mailflow.py "$(lb)" > "$WORK/$sc.log" 2>&1 &
|
||||
FLOW=$!
|
||||
sleep 15
|
||||
fault=0
|
||||
"$fn" || fault=$?
|
||||
sleep 20
|
||||
touch "$WORK/stop"
|
||||
flow=0; wait "$FLOW" || flow=$?; FLOW=""
|
||||
# The retries too: why each send was refused says what the fault did
|
||||
sed 's/^/ /' "$WORK/$sc.log"
|
||||
TOOK[$sc]=$(( $(date +%s) - start ))
|
||||
if [ "$fault" != 0 ]; then RESULT[$sc]="FAIL (the fault or the recovery)"
|
||||
elif [ "$flow" != 0 ]; then RESULT[$sc]="FAIL (mail lost or doubled)"
|
||||
else RESULT[$sc]="ok"; fi
|
||||
done
|
||||
|
||||
echo "== after it all: every client check"
|
||||
after=0
|
||||
PORTS="${PORTS:-18080}" STATE="$STATE" ./mailcheck.sh >"$WORK/after.log" 2>&1 || after=$?
|
||||
grep -E '^ (ok|FAIL)|^== every' "$WORK/after.log" || tail -20 "$WORK/after.log"
|
||||
|
||||
echo "== chaos"
|
||||
bad=0
|
||||
for sc in $SCENARIOS; do
|
||||
printf ' %-13s %-34s %ss\n' "$sc" "${RESULT[$sc]}" "${TOOK[$sc]}"
|
||||
[ "${RESULT[$sc]}" = ok ] || bad=1
|
||||
done
|
||||
[ "$after" = 0 ] || { echo " the client checks failed afterwards"; bad=1; }
|
||||
exit "$bad"
|
||||
@@ -36,7 +36,12 @@ def send(n):
|
||||
msg["Date"] = formatdate()
|
||||
msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test")
|
||||
msg.set_content(f"mailflow {run} message {n}\n")
|
||||
with smtplib.SMTP(lb, 25, timeout=20) as s:
|
||||
# As an MTA waits (RFC 5321 4.5.3.2): a dead address fails fast, but a
|
||||
# slow reply is waited for, up to minutes. Giving up early on the reply
|
||||
# to the end of DATA and sending again is how a sender makes duplicates.
|
||||
with smtplib.SMTP(timeout=20) as s:
|
||||
s.connect(lb, 25)
|
||||
s.sock.settimeout(300)
|
||||
s.send_message(msg)
|
||||
|
||||
|
||||
@@ -85,7 +90,8 @@ def count():
|
||||
return seen
|
||||
|
||||
|
||||
deadline = time.time() + 180
|
||||
# WAIT: how long to wait for what was accepted (an outage can delay delivery)
|
||||
deadline = time.time() + int(os.environ.get("WAIT", "180"))
|
||||
while True:
|
||||
try:
|
||||
seen = count()
|
||||
|
||||
+10
-1
@@ -10,7 +10,8 @@
|
||||
# JMAP, SMTP out to an outside server, CalDAV, CardDAV, the webmail across
|
||||
# both replicas and the console
|
||||
#
|
||||
# and how long each part took. CI runs this on a fresh runner; on the bench
|
||||
# and how long each part took; CHAOS=1 adds the chaos tests after it. CI
|
||||
# runs this on a fresh runner; on the bench
|
||||
# it builds the lab from scratch too, so take the lab down first
|
||||
# (lab/down.sh) unless you mean to reuse it.
|
||||
#
|
||||
@@ -97,5 +98,13 @@ s=$SECONDS
|
||||
PORTS="$P" STATE="$STATE" "$here/chart/mailcheck.sh"
|
||||
lap "mail (SMTP in, submission, IMAP, JMAP, SMTP out), CalDAV, CardDAV, webmail, console" "$s"
|
||||
|
||||
# CHAOS=1: then the faults, with mail flowing (chart/chaos-test.sh)
|
||||
if [ -n "${CHAOS:-}" ]; then
|
||||
echo "== chaos (chart/chaos-test.sh)"
|
||||
s=$SECONDS
|
||||
PORTS="$P" STATE="$STATE" "$here/chart/chaos-test.sh"
|
||||
lap "chaos: pod kill, drain, PostgreSQL failover, NATS restart, S3 outage" "$s"
|
||||
fi
|
||||
|
||||
lap "**total**"
|
||||
echo "== passed in $(( SECONDS - t0 ))s"
|
||||
Reference in new issue
Block a user