The soak harness (K-L7) and report.sh for RC testers (K-L9) #37

Merged
jcoffey-dev merged 3 commits from feat/soak-and-report into main 2026-11-20 09:20:59 +00:00
6 changed files with 967 additions and 3 deletions

No files matched your search

+25
View File
@@ -47,6 +47,31 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed
over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest
Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`).
`lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs
and checks no request failed and every live-update stream came back.
`lab/soak/soak.sh start` is the soak: a week (`SOAK_DAYS`) of steady load
on an installed chart, with a different chaos scenario each day. Mail in
every 10 s and out to the sink every minute, two IMAP clients in IDLE (all
from this host, through the load balancer), and three webmail tabs in a pod.
Each day ends with a report (`day-N.md` in `~/.cache/inbuxa-lab/soak`):
mail sent, arrived, late, lost or doubled, IDLE notices and drops, webmail
failures, and every pod restart or replacement, planned (during chaos) or
not. `report.md` sums up the run. It passes with no message lost or
doubled, no unplanned restart, and every chaos scenario recovered. It runs as a systemd user unit, so
`soak.sh status` and `stop` work from any shell, and `start` resumes a
stopped run. `SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300` tries it in an
hour.
`lab/report.sh` is for release-candidate testers on their own cluster: the
Kubernetes version and distribution, the nodes' OS, kernel, runtime and
architecture, the network plugin and load balancer, the Pod Security level
and anything it refused, the chart and image versions, each component's
pods, and any test results (`--results FILE`), as a block to paste into a
report. It only reads. Names, registries, addresses and hostnames are left
out, and the block is scrubbed of anything shaped like an IP address or a
hostname, unless `--include-names`.
`lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB,
and the data layer (CloudNativePG, NATS, Garage). It adds the Helm
repositories it needs; the versions are pinned in `lab/versions.env`.
+29 -3
View File
@@ -17,14 +17,21 @@ and prints one JSON line: requests made and failed (with the errors), and
per tab how its streams ended (cleanly, or cut off mid-stream) and the
longest wait for a stream to deliver again. A clean end is what the
browser takes in its stride; a failed request is what shows an error.
For the soak (K-L7): INTERVAL sets the time between requests (0.2 s);
SUMMARY_EVERY (seconds) prints that line every so often while running,
with "at" and the errors since the last one; with no STOP_FILE it runs
until stopped.
"""
import http.client, json, os, sys, threading, time, urllib.parse
webmail, server = sys.argv[1:3]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
stop = os.environ["STOP_FILE"]
stop = os.environ.get("STOP_FILE", "/nonexistent/stop")
TABS = int(os.environ.get("TABS", "3"))
INTERVAL = float(os.environ.get("INTERVAL", "0.2"))
SUMMARY_EVERY = float(os.environ.get("SUMMARY_EVERY", "0"))
def request(base, method, path, headers=None, body=None, timeout=30):
@@ -65,7 +72,7 @@ def sign_in():
session, mail_account = sign_in()
H = {"cookie": f"ihm_session={session}"}
lock = threading.Lock()
report = {"requests": 0, "failed": 0, "errors": [], "tabs": []}
report = {"requests": 0, "failed": 0, "signins": 1, "errors": [], "tabs": []}
def tab(i):
@@ -79,6 +86,10 @@ def tab(i):
conn = http.client.HTTPConnection(u.hostname, u.port, timeout=45)
conn.request("GET", "/api/events?types=*&closeafter=no&ping=30", headers=dict(H, accept="text/event-stream"))
res = conn.getresponse()
if res.status == 401: # signed out: the request loop signs in again
conn.close()
time.sleep(1)
continue
if res.status != 200:
raise RuntimeError(f"HTTP {res.status}")
me["streams"] += 1
@@ -107,19 +118,34 @@ for t in threads:
body = json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
"methodCalls": [["Mailbox/get", {"accountId": mail_account, "properties": ["name"]}, "0"]]})
next_summary = time.time() + SUMMARY_EVERY
while not os.path.exists(stop):
with lock:
report["requests"] += 1
try:
res, data = request(webmail, "POST", "/api/jmap", dict(H, **{"content-type": "application/json",
"x-requested-with": "ihasmail"}), body)
if res.status == 401:
# The session ran out (12 h by default): sign in again, as the
# page sends the browser back to; the tabs pick up the new cookie
session, mail_account = sign_in()
H["cookie"] = f"ihm_session={session}"
with lock:
report["signins"] += 1
report["requests"] -= 1
continue
if res.status != 200 or b"Mailbox/get" not in data:
raise RuntimeError(f"HTTP {res.status} {data[:80]!r}")
except Exception as e:
with lock:
report["failed"] += 1
report["errors"].append(f"request: {type(e).__name__}: {e}"[:160])
time.sleep(0.2)
time.sleep(INTERVAL)
if SUMMARY_EVERY and time.time() >= next_summary:
next_summary += SUMMARY_EVERY
with lock:
print(json.dumps(dict(report, at=time.time(), errors=report["errors"][:15])), flush=True)
report["errors"] = []
for t in threads:
t.join(timeout=50)
Executable
+173
View File
@@ -0,0 +1,173 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L9: what a release-candidate tester's cluster is, and how the chart
# did on it, as a block to paste into an issue or forum post.
#
# lab/report.sh [--context NAME] [-n NAMESPACE] [-r RELEASE]
# [--results FILE]... [--include-names]
#
# Reads, never changes: the Kubernetes version and distribution, the nodes'
# OS, kernel, runtime and architecture, the network plugin, the load
# balancer, the namespace's Pod Security level and anything it refused,
# the chart and image versions, each component's pods (ready, restarts),
# warning events, and the test results given with --results (the lab's
# e2e summary by default, if there is one).
#
# No names: nodes, the namespace, the release, registries, addresses and
# hostnames are left out, and the whole block is scrubbed of anything
# shaped like an IP address or a hostname before it's printed.
# --include-names keeps them, for a private report.
set -euo pipefail
CTX=() NS=inbuxa RELEASE=inbuxa NAMES=0 RESULTS=()
while [ $# -gt 0 ]; do
case "$1" in
--context) CTX=(--context "$2"); shift 2 ;;
-n|--namespace) NS="$2"; shift 2 ;;
-r|--release) RELEASE="$2"; shift 2 ;;
--results) RESULTS+=("$2"); shift 2 ;;
--include-names) NAMES=1; shift ;;
-h|--help) sed -n '4,21p' "$0"; exit 0 ;;
*) echo "unknown option $1 (--help)" >&2; exit 2 ;;
esac
done
if [ ${#RESULTS[@]} -eq 0 ] && [ -f "$HOME/.cache/inbuxa-lab/e2e-summary.md" ]; then
RESULTS=("$HOME/.cache/inbuxa-lab/e2e-summary.md")
fi
command -v kubectl >/dev/null || { echo "needs kubectl" >&2; exit 1; }
K=(kubectl "${CTX[@]}")
HELM_CTX=()
[ ${#CTX[@]} -gt 0 ] && HELM_CTX=(--kube-context "${CTX[1]}")
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT
"${K[@]}" version -o json >"$WORK/version.json" 2>/dev/null || echo '{}' >"$WORK/version.json"
"${K[@]}" get nodes -o json >"$WORK/nodes.json"
"${K[@]}" get pods -A -o json >"$WORK/pods.json"
"${K[@]}" get daemonsets -A -o json >"$WORK/ds.json"
"${K[@]}" get namespace "$NS" -o json >"$WORK/ns.json" 2>/dev/null || echo '{}' >"$WORK/ns.json"
"${K[@]}" -n "$NS" get services -o json >"$WORK/svc.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/svc.json"
"${K[@]}" -n "$NS" get events -o json >"$WORK/events.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/events.json"
if command -v helm >/dev/null; then
helm "${HELM_CTX[@]}" -n "$NS" list -o json >"$WORK/helm.json" 2>/dev/null || echo '[]' >"$WORK/helm.json"
else
echo '[]' >"$WORK/helm.json"
fi
: >"$WORK/results.md"
for f in "${RESULTS[@]}"; do { echo; echo "From \`$(basename "$f")\`:"; echo; cat "$f"; } >>"$WORK/results.md"; done
python3 - "$WORK" "$NS" "$RELEASE" "$NAMES" <<'PY'
import collections, json, os, re, sys
work, ns, release, names = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1"
J = lambda f: json.load(open(os.path.join(work, f)))
version, nodes, pods, ds = J("version.json"), J("nodes.json")["items"], J("pods.json")["items"], J("ds.json")["items"]
nsobj, svcs, events, helm = J("ns.json"), J("svc.json")["items"], J("events.json")["items"], J("helm.json")
server = version.get("serverVersion", {}).get("gitVersion", "?")
distro = next((d for pat, d in [(r"\+k3s", "k3s"), (r"\+rke2", "RKE2"), (r"-eks-", "EKS"), (r"-gke\.", "GKE"),
(r"\+vmware", "Tanzu"), (r"\+k0s", "k0s"), (r"-aks", "AKS")] if re.search(pat, server)), None)
images = " ".join(c.get("image", "") for p in pods for c in p["spec"].get("containers", []))
if not distro:
if any(n["metadata"].get("labels", {}).get("node.openshift.io/os_id") for n in nodes):
distro = "OpenShift"
elif re.search(r"kindest/|kindnet", images):
distro = "kind"
elif any("talos" in n["status"]["nodeInfo"].get("osImage", "").lower() for n in nodes):
distro = "Talos"
elif re.search(r"microk8s|calico.*snap", images):
distro = "MicroK8s"
else:
distro = "upstream or unrecognised"
def nodeinfo(n):
i = n["status"]["nodeInfo"]
roles = sorted(k.split("/", 1)[1] for k in n["metadata"].get("labels", {}) if k.startswith("node-role.kubernetes.io/")) or ["worker"]
return (",".join(roles), i.get("osImage", "?"), i.get("kernelVersion", "?"), i.get("containerRuntimeVersion", "?"), i.get("architecture", "?"))
node_rows = collections.Counter(nodeinfo(n) for n in nodes)
def present(*words):
hay = " ".join([p["metadata"]["name"] for p in pods] + [d["metadata"]["name"] for d in ds]).lower()
return [w for w in words if w.lower() in hay]
cni = present("cilium", "calico", "canal", "flannel", "kindnet", "weave", "antrea", "kube-router", "aws-node", "azure-cns", "ovnkube", "multus") or ["not recognised"]
lbs = present("metallb", "kube-vip", "svclb", "cilium-l2", "purelb", "openelb", "loxilb")
def short_image(image):
"""repository basename and tag or digest; no registry host."""
name, _, digest = image.partition("@")
repo, tag = (name.rsplit(":", 1) + [""])[:2] if ":" in name.rsplit("/", 1)[-1] else (name, "")
base = image if names else repo.rsplit("/", 1)[-1] + (f":{tag}" if tag else "") + (f"@{digest[:19]}…" if digest else "")
return base
ours = [p for p in pods if p["metadata"]["namespace"] == ns]
by_comp = collections.OrderedDict()
for p in sorted(ours, key=lambda p: p["metadata"].get("labels", {}).get("app.kubernetes.io/name", "") + p["metadata"]["name"]):
labels = p["metadata"].get("labels", {})
comp = labels.get("app.kubernetes.io/component", "?")
owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "")
# The server's role groups are Deployments named <release>-<group>
name = p["metadata"]["name"]
group = re.sub(r"-[a-z0-9]{6,10}-[a-z0-9]{5}$|-[a-z0-9]{5}$", "", name)
group = group[len(release) + 1:] if group.startswith(release + "-") and not names else group
key = (comp, group, owner)
statuses = p["status"].get("containerStatuses", []) or []
e = by_comp.setdefault(key, {"pods": 0, "ready": 0, "restarts": 0, "phase": collections.Counter(), "images": set(), "why": set()})
e["pods"] += 1
e["ready"] += all(c.get("ready") for c in statuses) and bool(statuses)
e["restarts"] += sum(c.get("restartCount", 0) for c in statuses)
e["phase"][p["status"].get("phase", "?")] += 1
e["images"].update(short_image(c["image"]) for c in p["spec"]["containers"])
for c in statuses:
r = (c.get("lastState", {}).get("terminated") or {}).get("reason")
if r:
e["why"].add(r)
psa = {k.rsplit("/", 1)[1]: v for k, v in nsobj.get("metadata", {}).get("labels", {}).items() if k.startswith("pod-security.kubernetes.io/")}
refused = [e for e in events if "violates PodSecurity" in e.get("message", "") or "forbidden" in e.get("message", "").lower()]
warnings = collections.Counter(e.get("reason", "?") for e in events if e.get("type") == "Warning")
lbsvc = [s for s in svcs if s["spec"].get("type") == "LoadBalancer"]
rel = next((h for h in helm if h.get("name") == release), {})
L = ["### inbuxa on Kubernetes: test report", ""]
L += ["| | |", "|---|---|",
f"| Kubernetes | {server} ({distro}) |",
f"| Nodes | {len(nodes)} |",
f"| Network plugin | {', '.join(cni)} |",
f"| Load balancer | {', '.join(lbs) or 'cloud or not recognised'}"
+ (f"; {len(lbsvc)} LoadBalancer service(s), class {', '.join(sorted({s['spec'].get('loadBalancerClass') or 'default' for s in lbsvc}))}, "
f"{sum(1 for s in lbsvc if s.get('status', {}).get('loadBalancer', {}).get('ingress'))} with an address" if lbsvc else "; no LoadBalancer service") + " |",
f"| Pod Security | {', '.join(f'{k}={v}' for k, v in sorted(psa.items())) or 'no labels on the namespace'}; "
f"{len(refused)} refusal event(s) |",
f"| Chart | {rel.get('chart', 'helm release not found')}, app {rel.get('app_version', '?')}, revision {rel.get('revision', '?')}, {rel.get('status', '?')} |",
""]
L += ["| Roles | OS | Kernel | Runtime | Arch | Nodes |", "|---|---|---|---|---|---|"]
L += [f"| {r} | {o} | {k} | {rt} | {a} | {c} |" for (r, o, k, rt, a), c in sorted(node_rows.items())]
L += ["", "| Component | Group | Pods | Ready | Restarts | Images |", "|---|---|---|---|---|---|"]
for (comp, group, owner), e in by_comp.items():
extra = f" ({', '.join(sorted(e['why']))})" if e["why"] else ""
phases = "" if set(e["phase"]) <= {"Running"} else " " + ", ".join(f"{v} {k}" for k, v in e["phase"].items())
L.append(f"| {comp} | {group} ({owner}) | {e['pods']}{phases} | {e['ready']} | {e['restarts']}{extra} | {', '.join(sorted(e['images']))} |")
if warnings:
L += ["", "Warning events in the namespace (kept about an hour): " + ", ".join(f"{k} ×{v}" for k, v in warnings.most_common())]
results = open(os.path.join(work, "results.md")).read().strip()
if results:
L += ["", "#### Test results", "", results]
text = "\n".join(L) + "\n"
if not names:
# Belt and braces: nothing shaped like an address or a hostname leaves
text = re.sub(r"\b(?:\d{1,3}\.){3}\d{1,3}(?:/\d+)?\b", "[ip]", text)
# IPv6 (and MACs): hex groups with "::", or five groups or more; a time
# like 12:34:56 has three
text = re.sub(r"(?<![\w:])[0-9a-f]{0,4}(?::[0-9a-f]{0,4}){2,7}(?![\w:])",
lambda m: "[ip]" if "::" in m.group(0) or m.group(0).count(":") >= 4 else m.group(0), text, flags=re.I)
# name.domain.tld; versions (1.2.3), file names and image tags don't match
text = re.sub(r"\b(?=[a-z0-9-]*[a-z][a-z0-9-]*\.)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\b(?![\w-])",
lambda m: m.group(0) if re.fullmatch(r"[\w-]+\.(md|json|yaml|sh|py|txt|log)", m.group(0)) else "[host]",
text, flags=re.I)
if ns not in ("inbuxa", "default"):
text = text.replace(ns, "[namespace]")
print(text)
PY
+272
View File
@@ -0,0 +1,272 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""The soak's steady mail load (K-L7), from outside the cluster through its
load balancer, as real clients reach it. Runs until killed, and picks up
where it left off when started again: everything it knows is in STATE.
ACCOUNT=user@domain PASSWORD_FILE=... STATE=dir soak-load.py LB
inbound a message every IN_EVERY s (10) over SMTP 25, each from its own
sender domain, retried like an MTA when refused
verify every 30 s, over IMAPS: which arrived (any mailbox), then
deleted, so the mailbox stays small; a number seen again after
that came twice
outbound a message every OUT_EVERY s (60) over submission 465 with AUTH,
to [email protected] ... [email protected] in turn (the server
allows 25 an hour from one sender domain to one recipient, and
[email protected] is left to chart/mailcheck.py); soak-watch.py
checks the sink received each once
idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're
told of new mail, and how often they drop. They watch the
mailbox the soak's mail lands in (verify finds it: unknown
senders' mail goes to Junk Mail), as a client watches INBOX
STATE/load.json is rewritten every 30 s; STATE/events.jsonl gets a line
for anything that went wrong. The lab's certificate is self-signed, so TLS
isn't verified.
"""
import imaplib, json, os, re, smtplib, socket, ssl, sys, threading, time, uuid
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
lb = sys.argv[1]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
state_dir = os.environ["STATE"]
IN_EVERY = float(os.environ.get("IN_EVERY", "10"))
OUT_EVERY = float(os.environ.get("OUT_EVERY", "60"))
IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2"))
SINK_SPREAD = int(os.environ.get("SINK_SPREAD", "10"))
tls = ssl.create_default_context()
tls.check_hostname = False
tls.verify_mode = ssl.CERT_NONE
lock = threading.RLock() # event() and save() are called with it held
STATE = os.path.join(state_dir, "load.json")
EVENTS = os.path.join(state_dir, "events.jsonl")
try:
S = json.load(open(STATE))
except (OSError, ValueError):
S = {"run": uuid.uuid4().hex[:8], "started": time.time(),
"inbound": {"next": 1, "sent": 0, "retries": 0, "arrived": 0, "twice": 0},
"pending": {}, "arrived_late": 0, "slowest_s": 0,
"outbound": {"next": 1, "sent": 0, "retries": 0}, "out_sent": {},
"idle": {"notices": 0, "drops": 0, "longest_quiet_s": 0},
"seen": {}, "starts": 0}
S["starts"] += 1
run = S["run"]
def event(kind, detail):
with lock:
with open(EVENTS, "a") as f:
f.write(json.dumps({"at": time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
def save():
with lock:
# A duplicate a day late is still caught; older arrivals are dropped
old = time.time() - 2 * 86400
S["seen"] = {n: v for n, v in S.get("seen", {}).items() if v[1] > old}
tmp = STATE + ".tmp"
with open(tmp, "w") as f:
json.dump(S, f)
os.replace(tmp, STATE)
def message(frm, to, subject):
msg = EmailMessage()
msg["From"], msg["To"], msg["Subject"] = frm, to, subject
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain=f"{run}.soak.test")
msg.set_content(subject + "\n")
return msg
def every(seconds, fn):
"""fn() every `seconds`, keeping to the schedule; fn handles its errors."""
nxt = time.time()
while True:
fn()
nxt += seconds
time.sleep(max(0, nxt - time.time()))
def inbound():
with lock:
n = S["inbound"]["next"]
# A sender domain per message: the default throttle allows 25 an hour
# from one sender domain to one recipient
msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}")
try:
# As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py
with smtplib.SMTP(timeout=20) as s:
s.connect(lb, 25)
s.sock.settimeout(300)
s.send_message(msg)
except (OSError, smtplib.SMTPException) as e:
with lock:
S["inbound"]["retries"] += 1
event("inbound-retry", f"{n}: {type(e).__name__}: {e}")
return # the same number goes again next time
with lock:
S["inbound"]["next"] = n + 1
S["inbound"]["sent"] += 1
S["pending"][str(n)] = time.time()
save() # at once: a restart must never send a number twice
def outbound():
with lock:
n = S["outbound"]["next"]
msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}")
try:
with smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) as s:
s.sock.settimeout(300)
s.login(account, password)
s.send_message(msg)
except (OSError, smtplib.SMTPException) as e:
with lock:
S["outbound"]["retries"] += 1
event("outbound-retry", f"{n}: {type(e).__name__}: {e}")
return
with lock:
S["outbound"]["next"] = n + 1
S["outbound"]["sent"] += 1
S["out_sent"][str(n)] = time.time()
save()
SUBJECT = re.compile(rf"soak {run} in (\d+)")
def verify():
try:
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=60)
except OSError as e:
event("verify", f"connect: {e}")
return
try:
m.login(account, password)
_, boxes = m.list()
for line in boxes:
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
if m.select(name)[0] != "OK":
continue
_, data = m.search(None, "SUBJECT", f'"soak {run} in"')
ids = data[0].split()
if not ids:
continue
_, hdrs = m.fetch(b",".join(ids), "(UID BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
now = time.time()
for part in hdrs:
if not isinstance(part, tuple):
continue
mm = SUBJECT.search(part[1].decode(errors="replace"))
uid = re.search(rb"UID (\d+)", part[0])
if not mm or not uid:
continue
n, copy = mm.group(1), f"{name}:{uid.group(1).decode()}"
with lock:
sent = S["pending"].pop(n, None)
if sent is not None:
S["inbound"]["arrived"] += 1
S["seen"][n] = [copy, now]
S["landed"] = name
took = now - sent
S["slowest_s"] = max(S["slowest_s"], round(took))
if took > 900:
S["arrived_late"] += 1
elif S["seen"].get(n, [None])[0] != copy:
S["inbound"]["twice"] += 1
S["seen"][n] = [copy, now]
event("twice", f"in {n} arrived again ({name})")
# else: this very copy, left by a delete that was cut short
# What arrived is on disk before it's deleted, so a restart in
# between neither loses it nor counts it twice
save()
m.store(b",".join(ids), "+FLAGS", "\\Deleted")
m.expunge()
except (OSError, imaplib.IMAP4.error) as e:
event("verify", f"{type(e).__name__}: {e}")
finally:
try:
m.logout()
except Exception:
pass
save()
def idle(i):
"""One client in IDLE, as a desktop mail app holds it; re-issued every
25 minutes (RFC 2177 asks for under 29)."""
while not S.get("landed"):
time.sleep(5)
last_notice = time.time()
while True:
sock = None
try:
raw = socket.create_connection((lb, 993), timeout=30)
sock = tls.wrap_socket(raw, server_hostname="mail.lab.test")
f = sock.makefile("rb")
def cmd(tag, text):
sock.sendall(f"{tag} {text}\r\n".encode())
while True:
line = f.readline()
if not line:
raise ConnectionError("closed")
if line.startswith(tag.encode() + b" "):
if b" OK" not in line[:12]:
raise ConnectionError(line.decode(errors="replace").strip())
return
f.readline() # greeting
cmd("a1", f'LOGIN "{account}" "{password}"')
cmd("a2", f"SELECT {S['landed']}")
n = 3
while True:
tag = f"a{n}"
n += 1
sock.sendall(f"{tag} IDLE\r\n".encode())
sock.settimeout(60)
renew = time.time() + 25 * 60
while time.time() < renew:
try:
line = f.readline()
except socket.timeout:
continue
if not line:
raise ConnectionError("closed in IDLE")
if line.endswith(b"EXISTS\r\n"):
now = time.time()
with lock:
S["idle"]["notices"] += 1
S["idle"]["longest_quiet_s"] = max(S["idle"]["longest_quiet_s"], round(now - last_notice))
last_notice = now
sock.sendall(b"DONE\r\n")
sock.settimeout(30)
while not f.readline().startswith(tag.encode() + b" "):
pass
except (OSError, ConnectionError, ssl.SSLError) as e:
with lock:
S["idle"]["drops"] += 1
event("idle-drop", f"client {i}: {type(e).__name__}: {e}")
finally:
if sock:
try:
sock.close()
except OSError:
pass
time.sleep(5)
save()
event("start", f"run {run}, start {S['starts']}")
for target in ([lambda: every(IN_EVERY, inbound), lambda: every(OUT_EVERY, outbound), lambda: every(30, verify)]
+ [(lambda i=i: idle(i)) for i in range(IDLE_CLIENTS)]):
threading.Thread(target=target, daemon=True).start()
while True:
time.sleep(30)
save()
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""What the soak (K-L7) sees, and its reports.
soak-watch.py collect once a minute, from soak.sh
soak-watch.py report DAY that day's report, STATE/day-DAY.md
soak-watch.py final the whole run, STATE/report.md
collect reads, with kubectl (KCTX):
- the sink's log: every message soak-load.py sent out, counted once per
delivery (by the sink's own receive time)
- the webmail tabs' summaries (chart/webmail-tabs.py, one pod or several
over the run, summed)
- every pod in the inbuxa, data, lab-sink and lab-soak namespaces: a pod
that's new, or a container that restarted, is an event, planned when
it falls in a chaos window (STATE/chaos.jsonl), unplanned otherwise
Everything is kept in STATE/watch.json, events in STATE/events.jsonl
beside soak-load.py's. Reports compare against the snapshot the previous
day's report took, so each day stands on its own.
"""
import json, os, re, subprocess, sys, time
STATE_DIR = os.environ["STATE"]
KCTX = os.environ["KCTX"]
DAY = int(os.environ.get("SOAK_DAY", "86400"))
NAMESPACES = ("inbuxa", "data", "lab-sink", "lab-soak")
WATCH = os.path.join(STATE_DIR, "watch.json")
EVENTS = os.path.join(STATE_DIR, "events.jsonl")
LOST_AFTER = int(os.environ.get("LOST_AFTER", "3600")) # not there an hour after it was sent
def kubectl(*args):
return subprocess.run(["kubectl", "--context", KCTX, *args], capture_output=True, text=True, timeout=120).stdout
def load(path, default):
try:
return json.load(open(path))
except (OSError, ValueError):
return default
def save(path, data):
tmp = path + ".tmp"
with open(tmp, "w") as f:
json.dump(data, f)
os.replace(tmp, path)
def event(kind, detail, at=None):
with open(EVENTS, "a") as f:
f.write(json.dumps({"at": at or time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
def new_lines(W, ns, selector, key):
"""Log lines of every pod matching `selector` since we last looked."""
out = []
for pod in kubectl("-n", ns, "get", "pods", "-l", selector, "-o", "name").split():
since = W["since"].get(f"{key}/{pod}")
args = ["-n", ns, "logs", pod, "--timestamps"] + (["--since-time", since] if since else [])
last = since
for line in kubectl(*args).splitlines():
ts, _, text = line.partition(" ")
if since and ts <= since:
continue # --since-time is inclusive, to the second
out.append((pod, text))
last = ts
if last:
W["since"][f"{key}/{pod}"] = last
return out
def collect():
W = load(WATCH, {"since": {}, "out": {}, "tabs": {}, "pods": {}, "first": True})
load_state = load(os.path.join(STATE_DIR, "load.json"), {})
run = load_state.get("run")
# The sink: each soak message that reached it
for _, text in new_lines(W, "lab-sink", "app=lab-sink", "sink"):
try:
rec = json.loads(text)
except ValueError:
continue
m = re.fullmatch(rf"soak {run} out (\d+)", rec.get("subject", ""))
if m:
W["out"][m.group(1)] = W["out"].get(m.group(1), 0) + 1
if W["out"][m.group(1)] == 2:
event("out-twice", f"out {m.group(1)} reached the sink again")
# The webmail tabs: the latest summary of each pod
for pod, text in new_lines(W, "lab-soak", "app=soak-tabs", "tabs"):
try:
rec = json.loads(text)
except ValueError:
continue
for e in rec.get("errors", []):
event("webmail", f"{pod.split('/')[-1]}: {e}", rec.get("at"))
rec.pop("errors", None)
W["tabs"][pod] = rec
# The pods
now = time.time()
seen = {}
for ns in NAMESPACES:
for p in json.loads(kubectl("-n", ns, "get", "pods", "-o", "json") or '{"items": []}')["items"]:
uid = p["metadata"]["uid"]
name = f"{ns}/{p['metadata']['name']}"
statuses = p.get("status", {}).get("containerStatuses", []) or []
restarts = sum(c.get("restartCount", 0) for c in statuses)
seen[uid] = {"name": name, "restarts": restarts}
before = W["pods"].get(uid)
if before is None and not W["first"]:
owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "none")
event("pod-new", f"{name} ({owner})", now)
elif before and restarts > before["restarts"]:
why = ", ".join(sorted({(c.get("lastState", {}).get("terminated") or {}).get("reason", "?")
for c in statuses if c.get("restartCount", 0)}))
event("restart", f"{name}: {restarts - before['restarts']} more ({why})", now)
for uid, p in W["pods"].items():
if uid not in seen:
event("pod-gone", p["name"], now)
W["pods"], W["first"] = seen, False
save(WATCH, W)
def counters():
"""Every running total, flattened, for day-to-day differences."""
L = load(os.path.join(STATE_DIR, "load.json"), {})
W = load(WATCH, {"out": {}, "tabs": {}})
tabs = list(W.get("tabs", {}).values())
c = {f"in_{k}": v for k, v in L.get("inbound", {}).items() if k != "next"}
c.update({f"out_{k}": v for k, v in L.get("outbound", {}).items() if k != "next"})
c.update({f"idle_{k}": v for k, v in L.get("idle", {}).items() if k != "longest_quiet_s"})
c["in_late"] = L.get("arrived_late", 0)
c["out_received"] = len(W.get("out", {}))
c["out_twice"] = sum(1 for v in W.get("out", {}).values() if v > 1)
c["load_starts"] = L.get("starts", 0)
for k in ("requests", "failed", "signins"):
c[f"web_{k}"] = sum(t.get(k, 0) for t in tabs)
for k in ("streams", "clean_ends", "cut_off"):
c[f"web_{k}"] = sum(x.get(k, 0) for t in tabs for x in t.get("tabs", []))
return c, L, W
def chaos_windows():
out = []
for line in open(os.path.join(STATE_DIR, "chaos.jsonl")) if os.path.exists(os.path.join(STATE_DIR, "chaos.jsonl")) else []:
try:
out.append(json.loads(line))
except ValueError:
pass
return out
def events_between(a, b):
out = []
if os.path.exists(EVENTS):
for line in open(EVENTS):
try:
e = json.loads(line)
except ValueError:
continue
if a <= e["at"] < b:
out.append(e)
return out
def in_chaos(at, windows):
# A minute's grace after, for what the collector notices late
return any(w["start"] <= at <= w.get("end", time.time()) + 120 for w in windows)
def day_numbers(day, started):
"""(this day's counters, the day's start and end)."""
c, L, W = counters()
snap = os.path.join(STATE_DIR, f"day-{day}.json")
prev = load(os.path.join(STATE_DIR, f"day-{day - 1}.json"), {}) if day > 1 else {}
# Taken once, at the day's end; a report written again uses it
if os.path.exists(snap):
c = load(snap, c)
else:
save(snap, c)
delta = {k: v - prev.get(k, 0) for k, v in c.items()}
return delta, c, L, W, started + (day - 1) * DAY, started + day * DAY
def missing(L, W, now):
lost_in = sorted(int(n) for n, t in L.get("pending", {}).items() if now - t > LOST_AFTER)
lost_out = sorted(int(n) for n, t in L.get("out_sent", {}).items()
if now - t > LOST_AFTER and n not in W.get("out", {}))
return lost_in, lost_out
def report(day):
started = float(open(os.path.join(STATE_DIR, "started")).read())
d, c, L, W, a, b = day_numbers(day, started)
now = time.time()
lost_in, lost_out = missing(L, W, now)
windows = [w for w in chaos_windows() if a <= w["start"] < b]
evs = events_between(a, b)
restarts = [e for e in evs if e["kind"] in ("restart", "pod-new")]
unplanned = [e for e in restarts if not in_chaos(e["at"], chaos_windows())]
twice = d.get("in_twice", 0) + d.get("out_twice", 0)
chaos_ok = all(w.get("result") == "ok" for w in windows)
ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok
fmt = lambda t: time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(t))
lines = [
f"# Soak day {day}: {'clean' if ok else 'NOT clean'}",
"",
f"{fmt(a)} to {fmt(b)}, run `{L.get('run')}`, context `{KCTX}`.",
"",
"| | |",
"|---|---|",
f"| Mail in | {d.get('in_sent', 0)} sent, {d.get('in_arrived', 0)} arrived, {d.get('in_late', 0)} after 15 min, "
f"{d.get('in_twice', 0)} twice, {d.get('in_retries', 0)} sends retried; slowest so far {L.get('slowest_s', 0)} s |",
f"| Mail out | {d.get('out_sent', 0)} sent, {d.get('out_received', 0)} reached the sink, {d.get('out_twice', 0)} twice, "
f"{d.get('out_retries', 0)} sends retried |",
f"| Lost so far | in: {len(lost_in)} {lost_in[:10] if lost_in else ''}, out: {len(lost_out)} {lost_out[:10] if lost_out else ''} "
f"(not there {LOST_AFTER // 60} min after sending) |",
f"| IMAP IDLE | {d.get('idle_notices', 0)} new-mail notices, {d.get('idle_drops', 0)} drops; "
f"longest quiet so far {L.get('idle', {}).get('longest_quiet_s', 0)} s |",
f"| Webmail tabs | {d.get('web_requests', 0)} requests, {d.get('web_failed', 0)} failed, "
f"{d.get('web_streams', 0)} streams ({d.get('web_cut_off', 0)} cut off), {d.get('web_signins', 0)} sign-ins |",
f"| Restarts | {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos |",
f"| Load generator | started {d.get('load_starts', 0)} more time(s) |",
"",
"## Chaos",
"",
]
lines += [f"- {w['scenario']}: {w.get('result', 'still running')} ({fmt(w['start'])})" for w in windows] or ["- none today"]
if unplanned:
lines += ["", "## Unplanned", ""] + [f"- {fmt(e['at'])} {e['kind']}: {e['detail']}" for e in unplanned[:30]]
kinds = {}
for e in evs:
kinds.setdefault(e["kind"], []).append(e)
if kinds:
lines += ["", "## Events", ""]
for k, es in sorted(kinds.items()):
lines.append(f"- {k}: {len(es)}; first: {es[0]['detail']}")
text = "\n".join(lines) + "\n"
open(os.path.join(STATE_DIR, f"day-{day}.md"), "w").write(text)
print(text)
return ok
def final():
started = float(open(os.path.join(STATE_DIR, "started")).read())
days = sorted(int(m.group(1)) for f in os.listdir(STATE_DIR) if (m := re.fullmatch(r"day-(\d+)\.md", f)))
c, L, W = counters()
lost_in, lost_out = missing(L, W, time.time())
windows = chaos_windows()
restarts = [e for e in events_between(started, time.time()) if e["kind"] in ("restart", "pod-new")]
unplanned = [e for e in restarts if not in_chaos(e["at"], windows)]
twice = c.get("in_twice", 0) + c.get("out_twice", 0)
chaos_ok = all(w.get("result") == "ok" for w in windows)
ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok
lines = [f"# Soak: {'PASSED' if ok else 'FAILED'}", "",
f"{len(days)} day(s) from {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(started))}, context `{KCTX}`.", "",
f"- Mail in: {c.get('in_sent', 0)} sent, {c.get('in_arrived', 0)} arrived, {len(lost_in)} lost, {c.get('in_twice', 0)} twice",
f"- Mail out: {c.get('out_sent', 0)} sent, {c.get('out_received', 0)} reached the sink, {len(lost_out)} lost, {c.get('out_twice', 0)} twice",
f"- IMAP IDLE: {c.get('idle_notices', 0)} notices, {c.get('idle_drops', 0)} drops",
f"- Webmail: {c.get('web_requests', 0)} requests, {c.get('web_failed', 0)} failed, {c.get('web_cut_off', 0)} streams cut off",
f"- Restarts: {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos",
f"- Chaos: " + ", ".join(f"{w['scenario']} {w.get('result', '?')}" for w in windows), "",
"| Day | |", "|---|---|"]
for dnum in days:
first = open(os.path.join(STATE_DIR, f"day-{dnum}.md")).readline().strip("# \n")
lines.append(f"| {dnum} | {first} |")
text = "\n".join(lines) + "\n"
open(os.path.join(STATE_DIR, "report.md"), "w").write(text)
print(text)
return ok
if __name__ == "__main__":
what = sys.argv[1]
if what == "collect":
collect()
elif what == "report":
sys.exit(0 if report(int(sys.argv[2])) else 1)
elif what == "final":
sys.exit(0 if final() else 1)
else:
sys.exit(__doc__)
+182
View File
@@ -0,0 +1,182 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L7: the soak. Steady load for SOAK_DAYS days (7) on the lab's seeded
# install, one chaos scenario a day, a report at the end of each day and
# one for the whole run.
#
# lab/soak/soak.sh start checks the install, then runs in the
# background (a systemd user unit)
# lab/soak/soak.sh status where it's up to, and today's numbers so far
# lab/soak/soak.sh stop
# lab/soak/soak.sh report the reports written so far
#
# The load (soak-load.py) comes from this host through the load balancer:
# mail in, mail out to the sink, IMAP IDLE. The webmail tabs
# (chart/webmail-tabs.py) run in a pod, as there's no ingress. Chaos is
# chart/chaos-test.sh with one scenario a day, CHAOS_AFTER seconds into the
# day (2 h). Reports and state go to SOAK_DIR; a run is resumed, not
# restarted, by `start` after a stop or a reboot, until it's done.
#
# A shorter run, to try the harness: SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300
#
# The unit stops when its user logs out unless lingering is on
# (`loginctl enable-linger`), which is the machine owner's call.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
. "$HERE/../versions.env"
export KCTX="${KCTX:-kind-$KIND_CLUSTER}"
K=(kubectl --context "$KCTX")
LAB_STATE="${LAB_STATE:-$HOME/.cache/inbuxa-lab}"
export STATE="${SOAK_DIR:-$LAB_STATE/soak}"
export SOAK_DAY="${SOAK_DAY:-86400}"
SOAK_DAYS="${SOAK_DAYS:-7}"
CHAOS_AFTER="${CHAOS_AFTER:-7200}"
SCENARIOS=(pod-kill drain pg-failover nats-restart s3-outage pod-kill drain)
UNIT=inbuxa-soak
fail() { echo "!! $*" >&2; exit 1; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
tabs_pod() {
"${K[@]}" create namespace lab-soak --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" label namespace lab-soak pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null
"${K[@]}" -n lab-soak create configmap soak-tabs --from-file="$HERE/../chart/webmail-tabs.py" \
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-soak create secret generic soak-tabs --from-file=password="$LAB_STATE/alice.password" \
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-soak apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: soak-tabs}
spec:
replicas: 1
selector: {matchLabels: {app: soak-tabs}}
template:
metadata: {labels: {app: soak-tabs}}
spec:
automountServiceAccountToken: false
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: tabs
image: $PG_IMAGE
imagePullPolicy: IfNotPresent
command: [python3, -u, /tabs/webmail-tabs.py, "http://inbuxa-webmail.inbuxa.svc:8080", "http://inbuxa-http.inbuxa.svc:8080"]
env:
- {name: ACCOUNT, value: [email protected]}
- {name: PASSWORD_FILE, value: /secret/password}
- {name: INTERVAL, value: "5"}
- {name: SUMMARY_EVERY, value: "60"}
resources: {requests: {cpu: 10m, memory: 32Mi}, limits: {memory: 128Mi}}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: tabs, mountPath: /tabs}
- {name: secret, mountPath: /secret}
volumes:
- {name: tabs, configMap: {name: soak-tabs}}
- {name: secret, secret: {secretName: soak-tabs, defaultMode: 0440}}
EOF
"${K[@]}" -n lab-soak rollout status deploy/soak-tabs --timeout=180s >/dev/null
}
start() {
systemctl --user is-active --quiet "$UNIT" && fail "already running: $0 status"
mkdir -p "$STATE"
if [ -f "$STATE/started" ] && [ -f "$STATE/report.md" ]; then
fail "the run in $STATE is finished; move it away to start another"
fi
if [ ! -f "$STATE/started" ]; then
echo "== every client check first (it also routes sink.test to the sink)"
PORTS="${PORTS:-18080}" STATE="$LAB_STATE" "$HERE/../chart/mailcheck.sh" >"$STATE/mailcheck.log" 2>&1 \
|| { tail -20 "$STATE/mailcheck.log"; fail "the install isn't healthy; not starting"; }
date +%s >"$STATE/started"
fi
echo "== the webmail tabs"
tabs_pod
systemd-run --user --unit "$UNIT" --collect --quiet \
--setenv=KCTX="$KCTX" --setenv=SOAK_DIR="$STATE" --setenv=SOAK_DAY="$SOAK_DAY" \
--setenv=SOAK_DAYS="$SOAK_DAYS" --setenv=CHAOS_AFTER="$CHAOS_AFTER" --setenv=LAB_STATE="$LAB_STATE" \
--setenv=PATH="$PATH" --setenv=HOME="$HOME" \
bash "$HERE/soak.sh" run
echo "== running: $SOAK_DAYS day(s) of ${SOAK_DAY}s; $0 status"
}
# The unit's body: never exits on a hiccup, only when the run is done
run() {
set +e
mkdir -p "$STATE"
local started load_pid="" day now since
started="$(cat "$STATE/started")"
stop_load() { [ -n "$load_pid" ] && kill "$load_pid" 2>/dev/null; wait "$load_pid" 2>/dev/null; }
trap 'stop_load; exit 0' TERM INT
while :; do
if [ -z "$load_pid" ] || ! kill -0 "$load_pid" 2>/dev/null; then
ACCOUNT=[email protected] PASSWORD_FILE="$LAB_STATE/alice.password" \
python3 "$HERE/soak-load.py" "$(lb)" >>"$STATE/load.log" 2>&1 &
load_pid=$!
fi
python3 "$HERE/soak-watch.py" collect >>"$STATE/watch.log" 2>&1
now=$(date +%s)
day=$(( (now - started) / SOAK_DAY + 1 ))
since=$(( (now - started) % SOAK_DAY ))
# Yesterday's report, once
if [ "$day" -gt 1 ] && [ ! -f "$STATE/day-$((day - 1)).md" ]; then
python3 "$HERE/soak-watch.py" report "$((day - 1))" >/dev/null 2>>"$STATE/watch.log"
fi
if [ "$day" -gt "$SOAK_DAYS" ]; then
python3 "$HERE/soak-watch.py" final >/dev/null 2>>"$STATE/watch.log"
stop_load
"${K[@]}" -n lab-soak scale deploy/soak-tabs --replicas=0 >/dev/null 2>&1
exit 0
fi
# Today's chaos, once
if [ "$since" -ge "$CHAOS_AFTER" ] && [ ! -f "$STATE/chaos-day-$day.log" ]; then
local sc="${SCENARIOS[$(( (day - 1) % ${#SCENARIOS[@]} ))]}" t0 rc
t0=$(date +%s)
printf '{"day": %d, "scenario": "%s", "start": %d}\n' "$day" "$sc" "$t0" >>"$STATE/chaos.jsonl"
SCENARIOS="$sc" STATE="$LAB_STATE" PORTS=18180 "$HERE/../chart/chaos-test.sh" >"$STATE/chaos-day-$day.log" 2>&1
rc=$?
python3 - "$STATE/chaos.jsonl" "$day" "$rc" "$(date +%s)" <<'PY'
import json, sys
path, day, rc, end = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
rows = [json.loads(l) for l in open(path) if l.strip()]
for r in rows:
if r["day"] == day and "end" not in r:
r["end"], r["result"] = end, "ok" if rc == 0 else f"FAILED (exit {rc}, see chaos-day-{day}.log)"
open(path, "w").write("".join(json.dumps(r) + "\n" for r in rows))
PY
fi
sleep 60
done
}
status() {
if systemctl --user is-active --quiet "$UNIT"; then echo "== running"; else echo "== not running"; fi
[ -f "$STATE/started" ] || { echo " no run in $STATE"; return 0; }
local started now; started="$(cat "$STATE/started")"; now=$(date +%s)
echo " day $(( (now - started) / SOAK_DAY + 1 )) of $SOAK_DAYS, $(( (now - started) % SOAK_DAY / 60 )) min in"
python3 - "$STATE" <<'PY'
import json, os, sys
d = sys.argv[1]
L = json.load(open(os.path.join(d, "load.json")))
W = json.load(open(os.path.join(d, "watch.json"))) if os.path.exists(os.path.join(d, "watch.json")) else {"out": {}, "tabs": {}}
i, o, idle = L["inbound"], L["outbound"], L["idle"]
print(f" in: {i['sent']} sent, {i['arrived']} arrived, {len(L['pending'])} pending, {i['twice']} twice")
print(f" out: {o['sent']} sent, {len(W['out'])} at the sink")
print(f" idle: {idle['notices']} notices, {idle['drops']} drops")
t = list(W["tabs"].values())
print(f" webmail: {sum(x.get('requests', 0) for x in t)} requests, {sum(x.get('failed', 0) for x in t)} failed")
PY
ls "$STATE"/day-*.md 2>/dev/null | sed 's/^/ /' || true
}
case "${1:-}" in
start) start ;;
run) run ;;
status) status ;;
stop) systemctl --user stop "$UNIT" 2>/dev/null || true; echo "== stopped (start resumes it)" ;;
report) cat "$STATE"/day-*.md "$STATE"/report.md 2>/dev/null || echo "no reports yet" ;;
*) sed -n '4,25p' "$0"; exit 2 ;;
esac