Files
jcoffey-dev 315e84e242 The soak: a week of steady load and daily chaos (K-L7)
lab/soak/soak.sh start runs SOAK_DAYS (7) days on an installed chart as a
systemd user unit: soak-load.py from this host through the load balancer
(mail in every 10 s, out to the sink every minute, two IMAP clients in
IDLE), three webmail tabs in a pod (chart/webmail-tabs.py, now with a
summary every minute and signing in again when its session runs out),
chart/chaos-test.sh with one scenario a day, and soak-watch.py: the sink's
log, the tabs' summaries and every pod in the namespaces each minute,
day-N.md at each day's end and report.md for the run.

It passes with no message lost or doubled and no restart outside the
chaos windows. Every count survives the load generator restarting: what
was sent is saved before the next send, and an arrival is saved, with its
IMAP UID, before it's deleted.
2026-11-20 01:07:27 -07:00

183 lines
8.1 KiB
Bash
Executable File

#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L7: the soak. Steady load for SOAK_DAYS days (7) on the lab's seeded
# install, one chaos scenario a day, a report at the end of each day and
# one for the whole run.
#
# lab/soak/soak.sh start checks the install, then runs in the
# background (a systemd user unit)
# lab/soak/soak.sh status where it's up to, and today's numbers so far
# lab/soak/soak.sh stop
# lab/soak/soak.sh report the reports written so far
#
# The load (soak-load.py) comes from this host through the load balancer:
# mail in, mail out to the sink, IMAP IDLE. The webmail tabs
# (chart/webmail-tabs.py) run in a pod, as there's no ingress. Chaos is
# chart/chaos-test.sh with one scenario a day, CHAOS_AFTER seconds into the
# day (2 h). Reports and state go to SOAK_DIR; a run is resumed, not
# restarted, by `start` after a stop or a reboot, until it's done.
#
# A shorter run, to try the harness: SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300
#
# The unit stops when its user logs out unless lingering is on
# (`loginctl enable-linger`), which is the machine owner's call.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
. "$HERE/../versions.env"
export KCTX="${KCTX:-kind-$KIND_CLUSTER}"
K=(kubectl --context "$KCTX")
LAB_STATE="${LAB_STATE:-$HOME/.cache/inbuxa-lab}"
export STATE="${SOAK_DIR:-$LAB_STATE/soak}"
export SOAK_DAY="${SOAK_DAY:-86400}"
SOAK_DAYS="${SOAK_DAYS:-7}"
CHAOS_AFTER="${CHAOS_AFTER:-7200}"
SCENARIOS=(pod-kill drain pg-failover nats-restart s3-outage pod-kill drain)
UNIT=inbuxa-soak
fail() { echo "!! $*" >&2; exit 1; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
tabs_pod() {
"${K[@]}" create namespace lab-soak --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" label namespace lab-soak pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null
"${K[@]}" -n lab-soak create configmap soak-tabs --from-file="$HERE/../chart/webmail-tabs.py" \
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-soak create secret generic soak-tabs --from-file=password="$LAB_STATE/alice.password" \
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-soak apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: soak-tabs}
spec:
replicas: 1
selector: {matchLabels: {app: soak-tabs}}
template:
metadata: {labels: {app: soak-tabs}}
spec:
automountServiceAccountToken: false
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: tabs
image: $PG_IMAGE
imagePullPolicy: IfNotPresent
command: [python3, -u, /tabs/webmail-tabs.py, "http://inbuxa-webmail.inbuxa.svc:8080", "http://inbuxa-http.inbuxa.svc:8080"]
env:
- {name: ACCOUNT, value: [email protected]}
- {name: PASSWORD_FILE, value: /secret/password}
- {name: INTERVAL, value: "5"}
- {name: SUMMARY_EVERY, value: "60"}
resources: {requests: {cpu: 10m, memory: 32Mi}, limits: {memory: 128Mi}}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: tabs, mountPath: /tabs}
- {name: secret, mountPath: /secret}
volumes:
- {name: tabs, configMap: {name: soak-tabs}}
- {name: secret, secret: {secretName: soak-tabs, defaultMode: 0440}}
EOF
"${K[@]}" -n lab-soak rollout status deploy/soak-tabs --timeout=180s >/dev/null
}
start() {
systemctl --user is-active --quiet "$UNIT" && fail "already running: $0 status"
mkdir -p "$STATE"
if [ -f "$STATE/started" ] && [ -f "$STATE/report.md" ]; then
fail "the run in $STATE is finished; move it away to start another"
fi
if [ ! -f "$STATE/started" ]; then
echo "== every client check first (it also routes sink.test to the sink)"
PORTS="${PORTS:-18080}" STATE="$LAB_STATE" "$HERE/../chart/mailcheck.sh" >"$STATE/mailcheck.log" 2>&1 \
|| { tail -20 "$STATE/mailcheck.log"; fail "the install isn't healthy; not starting"; }
date +%s >"$STATE/started"
fi
echo "== the webmail tabs"
tabs_pod
systemd-run --user --unit "$UNIT" --collect --quiet \
--setenv=KCTX="$KCTX" --setenv=SOAK_DIR="$STATE" --setenv=SOAK_DAY="$SOAK_DAY" \
--setenv=SOAK_DAYS="$SOAK_DAYS" --setenv=CHAOS_AFTER="$CHAOS_AFTER" --setenv=LAB_STATE="$LAB_STATE" \
--setenv=PATH="$PATH" --setenv=HOME="$HOME" \
bash "$HERE/soak.sh" run
echo "== running: $SOAK_DAYS day(s) of ${SOAK_DAY}s; $0 status"
}
# The unit's body: never exits on a hiccup, only when the run is done
run() {
set +e
mkdir -p "$STATE"
local started load_pid="" day now since
started="$(cat "$STATE/started")"
stop_load() { [ -n "$load_pid" ] && kill "$load_pid" 2>/dev/null; wait "$load_pid" 2>/dev/null; }
trap 'stop_load; exit 0' TERM INT
while :; do
if [ -z "$load_pid" ] || ! kill -0 "$load_pid" 2>/dev/null; then
ACCOUNT=[email protected] PASSWORD_FILE="$LAB_STATE/alice.password" \
python3 "$HERE/soak-load.py" "$(lb)" >>"$STATE/load.log" 2>&1 &
load_pid=$!
fi
python3 "$HERE/soak-watch.py" collect >>"$STATE/watch.log" 2>&1
now=$(date +%s)
day=$(( (now - started) / SOAK_DAY + 1 ))
since=$(( (now - started) % SOAK_DAY ))
# Yesterday's report, once
if [ "$day" -gt 1 ] && [ ! -f "$STATE/day-$((day - 1)).md" ]; then
python3 "$HERE/soak-watch.py" report "$((day - 1))" >/dev/null 2>>"$STATE/watch.log"
fi
if [ "$day" -gt "$SOAK_DAYS" ]; then
python3 "$HERE/soak-watch.py" final >/dev/null 2>>"$STATE/watch.log"
stop_load
"${K[@]}" -n lab-soak scale deploy/soak-tabs --replicas=0 >/dev/null 2>&1
exit 0
fi
# Today's chaos, once
if [ "$since" -ge "$CHAOS_AFTER" ] && [ ! -f "$STATE/chaos-day-$day.log" ]; then
local sc="${SCENARIOS[$(( (day - 1) % ${#SCENARIOS[@]} ))]}" t0 rc
t0=$(date +%s)
printf '{"day": %d, "scenario": "%s", "start": %d}\n' "$day" "$sc" "$t0" >>"$STATE/chaos.jsonl"
SCENARIOS="$sc" STATE="$LAB_STATE" PORTS=18180 "$HERE/../chart/chaos-test.sh" >"$STATE/chaos-day-$day.log" 2>&1
rc=$?
python3 - "$STATE/chaos.jsonl" "$day" "$rc" "$(date +%s)" <<'PY'
import json, sys
path, day, rc, end = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
rows = [json.loads(l) for l in open(path) if l.strip()]
for r in rows:
if r["day"] == day and "end" not in r:
r["end"], r["result"] = end, "ok" if rc == 0 else f"FAILED (exit {rc}, see chaos-day-{day}.log)"
open(path, "w").write("".join(json.dumps(r) + "\n" for r in rows))
PY
fi
sleep 60
done
}
status() {
if systemctl --user is-active --quiet "$UNIT"; then echo "== running"; else echo "== not running"; fi
[ -f "$STATE/started" ] || { echo " no run in $STATE"; return 0; }
local started now; started="$(cat "$STATE/started")"; now=$(date +%s)
echo " day $(( (now - started) / SOAK_DAY + 1 )) of $SOAK_DAYS, $(( (now - started) % SOAK_DAY / 60 )) min in"
python3 - "$STATE" <<'PY'
import json, os, sys
d = sys.argv[1]
L = json.load(open(os.path.join(d, "load.json")))
W = json.load(open(os.path.join(d, "watch.json"))) if os.path.exists(os.path.join(d, "watch.json")) else {"out": {}, "tabs": {}}
i, o, idle = L["inbound"], L["outbound"], L["idle"]
print(f" in: {i['sent']} sent, {i['arrived']} arrived, {len(L['pending'])} pending, {i['twice']} twice")
print(f" out: {o['sent']} sent, {len(W['out'])} at the sink")
print(f" idle: {idle['notices']} notices, {idle['drops']} drops")
t = list(W["tabs"].values())
print(f" webmail: {sum(x.get('requests', 0) for x in t)} requests, {sum(x.get('failed', 0) for x in t)} failed")
PY
ls "$STATE"/day-*.md 2>/dev/null | sed 's/^/ /' || true
}
case "${1:-}" in
start) start ;;
run) run ;;
status) status ;;
stop) systemctl --user stop "$UNIT" 2>/dev/null || true; echo "== stopped (start resumes it)" ;;
report) cat "$STATE"/day-*.md "$STATE"/report.md 2>/dev/null || echo "no reports yet" ;;
*) sed -n '4,25p' "$0"; exit 2 ;;
esac