The soak harness (K-L7) and report.sh for RC testers (K-L9) #37
No files matched your search
@@ -47,6 +47,31 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed
|
||||
over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest
|
||||
Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`).
|
||||
|
||||
`lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs
|
||||
and checks no request failed and every live-update stream came back.
|
||||
|
||||
`lab/soak/soak.sh start` is the soak: a week (`SOAK_DAYS`) of steady load
|
||||
on an installed chart, with a different chaos scenario each day. Mail in
|
||||
every 10 s and out to the sink every minute, two IMAP clients in IDLE (all
|
||||
from this host, through the load balancer), and three webmail tabs in a pod.
|
||||
Each day ends with a report (`day-N.md` in `~/.cache/inbuxa-lab/soak`):
|
||||
mail sent, arrived, late, lost or doubled, IDLE notices and drops, webmail
|
||||
failures, and every pod restart or replacement, planned (during chaos) or
|
||||
not. `report.md` sums up the run. It passes with no message lost or
|
||||
doubled, no unplanned restart, and every chaos scenario recovered. It runs as a systemd user unit, so
|
||||
`soak.sh status` and `stop` work from any shell, and `start` resumes a
|
||||
stopped run. `SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300` tries it in an
|
||||
hour.
|
||||
|
||||
`lab/report.sh` is for release-candidate testers on their own cluster: the
|
||||
Kubernetes version and distribution, the nodes' OS, kernel, runtime and
|
||||
architecture, the network plugin and load balancer, the Pod Security level
|
||||
and anything it refused, the chart and image versions, each component's
|
||||
pods, and any test results (`--results FILE`), as a block to paste into a
|
||||
report. It only reads. Names, registries, addresses and hostnames are left
|
||||
out, and the block is scrubbed of anything shaped like an IP address or a
|
||||
hostname, unless `--include-names`.
|
||||
|
||||
`lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB,
|
||||
and the data layer (CloudNativePG, NATS, Garage). It adds the Helm
|
||||
repositories it needs; the versions are pinned in `lab/versions.env`.
|
||||
|
||||
@@ -17,14 +17,21 @@ and prints one JSON line: requests made and failed (with the errors), and
|
||||
per tab how its streams ended (cleanly, or cut off mid-stream) and the
|
||||
longest wait for a stream to deliver again. A clean end is what the
|
||||
browser takes in its stride; a failed request is what shows an error.
|
||||
|
||||
For the soak (K-L7): INTERVAL sets the time between requests (0.2 s);
|
||||
SUMMARY_EVERY (seconds) prints that line every so often while running,
|
||||
with "at" and the errors since the last one; with no STOP_FILE it runs
|
||||
until stopped.
|
||||
"""
|
||||
import http.client, json, os, sys, threading, time, urllib.parse
|
||||
|
||||
webmail, server = sys.argv[1:3]
|
||||
account = os.environ["ACCOUNT"]
|
||||
password = open(os.environ["PASSWORD_FILE"]).read().strip()
|
||||
stop = os.environ["STOP_FILE"]
|
||||
stop = os.environ.get("STOP_FILE", "/nonexistent/stop")
|
||||
TABS = int(os.environ.get("TABS", "3"))
|
||||
INTERVAL = float(os.environ.get("INTERVAL", "0.2"))
|
||||
SUMMARY_EVERY = float(os.environ.get("SUMMARY_EVERY", "0"))
|
||||
|
||||
|
||||
def request(base, method, path, headers=None, body=None, timeout=30):
|
||||
@@ -65,7 +72,7 @@ def sign_in():
|
||||
session, mail_account = sign_in()
|
||||
H = {"cookie": f"ihm_session={session}"}
|
||||
lock = threading.Lock()
|
||||
report = {"requests": 0, "failed": 0, "errors": [], "tabs": []}
|
||||
report = {"requests": 0, "failed": 0, "signins": 1, "errors": [], "tabs": []}
|
||||
|
||||
|
||||
def tab(i):
|
||||
@@ -79,6 +86,10 @@ def tab(i):
|
||||
conn = http.client.HTTPConnection(u.hostname, u.port, timeout=45)
|
||||
conn.request("GET", "/api/events?types=*&closeafter=no&ping=30", headers=dict(H, accept="text/event-stream"))
|
||||
res = conn.getresponse()
|
||||
if res.status == 401: # signed out: the request loop signs in again
|
||||
conn.close()
|
||||
time.sleep(1)
|
||||
continue
|
||||
if res.status != 200:
|
||||
raise RuntimeError(f"HTTP {res.status}")
|
||||
me["streams"] += 1
|
||||
@@ -107,19 +118,34 @@ for t in threads:
|
||||
|
||||
body = json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": [["Mailbox/get", {"accountId": mail_account, "properties": ["name"]}, "0"]]})
|
||||
next_summary = time.time() + SUMMARY_EVERY
|
||||
while not os.path.exists(stop):
|
||||
with lock:
|
||||
report["requests"] += 1
|
||||
try:
|
||||
res, data = request(webmail, "POST", "/api/jmap", dict(H, **{"content-type": "application/json",
|
||||
"x-requested-with": "ihasmail"}), body)
|
||||
if res.status == 401:
|
||||
# The session ran out (12 h by default): sign in again, as the
|
||||
# page sends the browser back to; the tabs pick up the new cookie
|
||||
session, mail_account = sign_in()
|
||||
H["cookie"] = f"ihm_session={session}"
|
||||
with lock:
|
||||
report["signins"] += 1
|
||||
report["requests"] -= 1
|
||||
continue
|
||||
if res.status != 200 or b"Mailbox/get" not in data:
|
||||
raise RuntimeError(f"HTTP {res.status} {data[:80]!r}")
|
||||
except Exception as e:
|
||||
with lock:
|
||||
report["failed"] += 1
|
||||
report["errors"].append(f"request: {type(e).__name__}: {e}"[:160])
|
||||
time.sleep(0.2)
|
||||
time.sleep(INTERVAL)
|
||||
if SUMMARY_EVERY and time.time() >= next_summary:
|
||||
next_summary += SUMMARY_EVERY
|
||||
with lock:
|
||||
print(json.dumps(dict(report, at=time.time(), errors=report["errors"][:15])), flush=True)
|
||||
report["errors"] = []
|
||||
|
||||
for t in threads:
|
||||
t.join(timeout=50)
|
||||
|
||||
Executable
+173
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-L9: what a release-candidate tester's cluster is, and how the chart
|
||||
# did on it, as a block to paste into an issue or forum post.
|
||||
#
|
||||
# lab/report.sh [--context NAME] [-n NAMESPACE] [-r RELEASE]
|
||||
# [--results FILE]... [--include-names]
|
||||
#
|
||||
# Reads, never changes: the Kubernetes version and distribution, the nodes'
|
||||
# OS, kernel, runtime and architecture, the network plugin, the load
|
||||
# balancer, the namespace's Pod Security level and anything it refused,
|
||||
# the chart and image versions, each component's pods (ready, restarts),
|
||||
# warning events, and the test results given with --results (the lab's
|
||||
# e2e summary by default, if there is one).
|
||||
#
|
||||
# No names: nodes, the namespace, the release, registries, addresses and
|
||||
# hostnames are left out, and the whole block is scrubbed of anything
|
||||
# shaped like an IP address or a hostname before it's printed.
|
||||
# --include-names keeps them, for a private report.
|
||||
set -euo pipefail
|
||||
|
||||
CTX=() NS=inbuxa RELEASE=inbuxa NAMES=0 RESULTS=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--context) CTX=(--context "$2"); shift 2 ;;
|
||||
-n|--namespace) NS="$2"; shift 2 ;;
|
||||
-r|--release) RELEASE="$2"; shift 2 ;;
|
||||
--results) RESULTS+=("$2"); shift 2 ;;
|
||||
--include-names) NAMES=1; shift ;;
|
||||
-h|--help) sed -n '4,21p' "$0"; exit 0 ;;
|
||||
*) echo "unknown option $1 (--help)" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
if [ ${#RESULTS[@]} -eq 0 ] && [ -f "$HOME/.cache/inbuxa-lab/e2e-summary.md" ]; then
|
||||
RESULTS=("$HOME/.cache/inbuxa-lab/e2e-summary.md")
|
||||
fi
|
||||
command -v kubectl >/dev/null || { echo "needs kubectl" >&2; exit 1; }
|
||||
K=(kubectl "${CTX[@]}")
|
||||
HELM_CTX=()
|
||||
[ ${#CTX[@]} -gt 0 ] && HELM_CTX=(--kube-context "${CTX[1]}")
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
"${K[@]}" version -o json >"$WORK/version.json" 2>/dev/null || echo '{}' >"$WORK/version.json"
|
||||
"${K[@]}" get nodes -o json >"$WORK/nodes.json"
|
||||
"${K[@]}" get pods -A -o json >"$WORK/pods.json"
|
||||
"${K[@]}" get daemonsets -A -o json >"$WORK/ds.json"
|
||||
"${K[@]}" get namespace "$NS" -o json >"$WORK/ns.json" 2>/dev/null || echo '{}' >"$WORK/ns.json"
|
||||
"${K[@]}" -n "$NS" get services -o json >"$WORK/svc.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/svc.json"
|
||||
"${K[@]}" -n "$NS" get events -o json >"$WORK/events.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/events.json"
|
||||
if command -v helm >/dev/null; then
|
||||
helm "${HELM_CTX[@]}" -n "$NS" list -o json >"$WORK/helm.json" 2>/dev/null || echo '[]' >"$WORK/helm.json"
|
||||
else
|
||||
echo '[]' >"$WORK/helm.json"
|
||||
fi
|
||||
: >"$WORK/results.md"
|
||||
for f in "${RESULTS[@]}"; do { echo; echo "From \`$(basename "$f")\`:"; echo; cat "$f"; } >>"$WORK/results.md"; done
|
||||
|
||||
python3 - "$WORK" "$NS" "$RELEASE" "$NAMES" <<'PY'
|
||||
import collections, json, os, re, sys
|
||||
|
||||
work, ns, release, names = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1"
|
||||
J = lambda f: json.load(open(os.path.join(work, f)))
|
||||
version, nodes, pods, ds = J("version.json"), J("nodes.json")["items"], J("pods.json")["items"], J("ds.json")["items"]
|
||||
nsobj, svcs, events, helm = J("ns.json"), J("svc.json")["items"], J("events.json")["items"], J("helm.json")
|
||||
|
||||
server = version.get("serverVersion", {}).get("gitVersion", "?")
|
||||
distro = next((d for pat, d in [(r"\+k3s", "k3s"), (r"\+rke2", "RKE2"), (r"-eks-", "EKS"), (r"-gke\.", "GKE"),
|
||||
(r"\+vmware", "Tanzu"), (r"\+k0s", "k0s"), (r"-aks", "AKS")] if re.search(pat, server)), None)
|
||||
images = " ".join(c.get("image", "") for p in pods for c in p["spec"].get("containers", []))
|
||||
if not distro:
|
||||
if any(n["metadata"].get("labels", {}).get("node.openshift.io/os_id") for n in nodes):
|
||||
distro = "OpenShift"
|
||||
elif re.search(r"kindest/|kindnet", images):
|
||||
distro = "kind"
|
||||
elif any("talos" in n["status"]["nodeInfo"].get("osImage", "").lower() for n in nodes):
|
||||
distro = "Talos"
|
||||
elif re.search(r"microk8s|calico.*snap", images):
|
||||
distro = "MicroK8s"
|
||||
else:
|
||||
distro = "upstream or unrecognised"
|
||||
|
||||
def nodeinfo(n):
|
||||
i = n["status"]["nodeInfo"]
|
||||
roles = sorted(k.split("/", 1)[1] for k in n["metadata"].get("labels", {}) if k.startswith("node-role.kubernetes.io/")) or ["worker"]
|
||||
return (",".join(roles), i.get("osImage", "?"), i.get("kernelVersion", "?"), i.get("containerRuntimeVersion", "?"), i.get("architecture", "?"))
|
||||
node_rows = collections.Counter(nodeinfo(n) for n in nodes)
|
||||
|
||||
def present(*words):
|
||||
hay = " ".join([p["metadata"]["name"] for p in pods] + [d["metadata"]["name"] for d in ds]).lower()
|
||||
return [w for w in words if w.lower() in hay]
|
||||
cni = present("cilium", "calico", "canal", "flannel", "kindnet", "weave", "antrea", "kube-router", "aws-node", "azure-cns", "ovnkube", "multus") or ["not recognised"]
|
||||
lbs = present("metallb", "kube-vip", "svclb", "cilium-l2", "purelb", "openelb", "loxilb")
|
||||
|
||||
def short_image(image):
|
||||
"""repository basename and tag or digest; no registry host."""
|
||||
name, _, digest = image.partition("@")
|
||||
repo, tag = (name.rsplit(":", 1) + [""])[:2] if ":" in name.rsplit("/", 1)[-1] else (name, "")
|
||||
base = image if names else repo.rsplit("/", 1)[-1] + (f":{tag}" if tag else "") + (f"@{digest[:19]}…" if digest else "")
|
||||
return base
|
||||
|
||||
ours = [p for p in pods if p["metadata"]["namespace"] == ns]
|
||||
by_comp = collections.OrderedDict()
|
||||
for p in sorted(ours, key=lambda p: p["metadata"].get("labels", {}).get("app.kubernetes.io/name", "") + p["metadata"]["name"]):
|
||||
labels = p["metadata"].get("labels", {})
|
||||
comp = labels.get("app.kubernetes.io/component", "?")
|
||||
owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "")
|
||||
# The server's role groups are Deployments named <release>-<group>
|
||||
name = p["metadata"]["name"]
|
||||
group = re.sub(r"-[a-z0-9]{6,10}-[a-z0-9]{5}$|-[a-z0-9]{5}$", "", name)
|
||||
group = group[len(release) + 1:] if group.startswith(release + "-") and not names else group
|
||||
key = (comp, group, owner)
|
||||
statuses = p["status"].get("containerStatuses", []) or []
|
||||
e = by_comp.setdefault(key, {"pods": 0, "ready": 0, "restarts": 0, "phase": collections.Counter(), "images": set(), "why": set()})
|
||||
e["pods"] += 1
|
||||
e["ready"] += all(c.get("ready") for c in statuses) and bool(statuses)
|
||||
e["restarts"] += sum(c.get("restartCount", 0) for c in statuses)
|
||||
e["phase"][p["status"].get("phase", "?")] += 1
|
||||
e["images"].update(short_image(c["image"]) for c in p["spec"]["containers"])
|
||||
for c in statuses:
|
||||
r = (c.get("lastState", {}).get("terminated") or {}).get("reason")
|
||||
if r:
|
||||
e["why"].add(r)
|
||||
|
||||
psa = {k.rsplit("/", 1)[1]: v for k, v in nsobj.get("metadata", {}).get("labels", {}).items() if k.startswith("pod-security.kubernetes.io/")}
|
||||
refused = [e for e in events if "violates PodSecurity" in e.get("message", "") or "forbidden" in e.get("message", "").lower()]
|
||||
warnings = collections.Counter(e.get("reason", "?") for e in events if e.get("type") == "Warning")
|
||||
lbsvc = [s for s in svcs if s["spec"].get("type") == "LoadBalancer"]
|
||||
rel = next((h for h in helm if h.get("name") == release), {})
|
||||
|
||||
L = ["### inbuxa on Kubernetes: test report", ""]
|
||||
L += ["| | |", "|---|---|",
|
||||
f"| Kubernetes | {server} ({distro}) |",
|
||||
f"| Nodes | {len(nodes)} |",
|
||||
f"| Network plugin | {', '.join(cni)} |",
|
||||
f"| Load balancer | {', '.join(lbs) or 'cloud or not recognised'}"
|
||||
+ (f"; {len(lbsvc)} LoadBalancer service(s), class {', '.join(sorted({s['spec'].get('loadBalancerClass') or 'default' for s in lbsvc}))}, "
|
||||
f"{sum(1 for s in lbsvc if s.get('status', {}).get('loadBalancer', {}).get('ingress'))} with an address" if lbsvc else "; no LoadBalancer service") + " |",
|
||||
f"| Pod Security | {', '.join(f'{k}={v}' for k, v in sorted(psa.items())) or 'no labels on the namespace'}; "
|
||||
f"{len(refused)} refusal event(s) |",
|
||||
f"| Chart | {rel.get('chart', 'helm release not found')}, app {rel.get('app_version', '?')}, revision {rel.get('revision', '?')}, {rel.get('status', '?')} |",
|
||||
""]
|
||||
L += ["| Roles | OS | Kernel | Runtime | Arch | Nodes |", "|---|---|---|---|---|---|"]
|
||||
L += [f"| {r} | {o} | {k} | {rt} | {a} | {c} |" for (r, o, k, rt, a), c in sorted(node_rows.items())]
|
||||
L += ["", "| Component | Group | Pods | Ready | Restarts | Images |", "|---|---|---|---|---|---|"]
|
||||
for (comp, group, owner), e in by_comp.items():
|
||||
extra = f" ({', '.join(sorted(e['why']))})" if e["why"] else ""
|
||||
phases = "" if set(e["phase"]) <= {"Running"} else " " + ", ".join(f"{v} {k}" for k, v in e["phase"].items())
|
||||
L.append(f"| {comp} | {group} ({owner}) | {e['pods']}{phases} | {e['ready']} | {e['restarts']}{extra} | {', '.join(sorted(e['images']))} |")
|
||||
if warnings:
|
||||
L += ["", "Warning events in the namespace (kept about an hour): " + ", ".join(f"{k} ×{v}" for k, v in warnings.most_common())]
|
||||
results = open(os.path.join(work, "results.md")).read().strip()
|
||||
if results:
|
||||
L += ["", "#### Test results", "", results]
|
||||
text = "\n".join(L) + "\n"
|
||||
|
||||
if not names:
|
||||
# Belt and braces: nothing shaped like an address or a hostname leaves
|
||||
text = re.sub(r"\b(?:\d{1,3}\.){3}\d{1,3}(?:/\d+)?\b", "[ip]", text)
|
||||
# IPv6 (and MACs): hex groups with "::", or five groups or more; a time
|
||||
# like 12:34:56 has three
|
||||
text = re.sub(r"(?<![\w:])[0-9a-f]{0,4}(?::[0-9a-f]{0,4}){2,7}(?![\w:])",
|
||||
lambda m: "[ip]" if "::" in m.group(0) or m.group(0).count(":") >= 4 else m.group(0), text, flags=re.I)
|
||||
# name.domain.tld; versions (1.2.3), file names and image tags don't match
|
||||
text = re.sub(r"\b(?=[a-z0-9-]*[a-z][a-z0-9-]*\.)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\b(?![\w-])",
|
||||
lambda m: m.group(0) if re.fullmatch(r"[\w-]+\.(md|json|yaml|sh|py|txt|log)", m.group(0)) else "[host]",
|
||||
text, flags=re.I)
|
||||
if ns not in ("inbuxa", "default"):
|
||||
text = text.replace(ns, "[namespace]")
|
||||
print(text)
|
||||
PY
|
||||
Executable
+272
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The soak's steady mail load (K-L7), from outside the cluster through its
|
||||
load balancer, as real clients reach it. Runs until killed, and picks up
|
||||
where it left off when started again: everything it knows is in STATE.
|
||||
|
||||
ACCOUNT=user@domain PASSWORD_FILE=... STATE=dir soak-load.py LB
|
||||
|
||||
inbound a message every IN_EVERY s (10) over SMTP 25, each from its own
|
||||
sender domain, retried like an MTA when refused
|
||||
verify every 30 s, over IMAPS: which arrived (any mailbox), then
|
||||
deleted, so the mailbox stays small; a number seen again after
|
||||
that came twice
|
||||
outbound a message every OUT_EVERY s (60) over submission 465 with AUTH,
|
||||
to [email protected] ... [email protected] in turn (the server
|
||||
allows 25 an hour from one sender domain to one recipient, and
|
||||
[email protected] is left to chart/mailcheck.py); soak-watch.py
|
||||
checks the sink received each once
|
||||
idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're
|
||||
told of new mail, and how often they drop. They watch the
|
||||
mailbox the soak's mail lands in (verify finds it: unknown
|
||||
senders' mail goes to Junk Mail), as a client watches INBOX
|
||||
|
||||
STATE/load.json is rewritten every 30 s; STATE/events.jsonl gets a line
|
||||
for anything that went wrong. The lab's certificate is self-signed, so TLS
|
||||
isn't verified.
|
||||
"""
|
||||
import imaplib, json, os, re, smtplib, socket, ssl, sys, threading, time, uuid
|
||||
from email.message import EmailMessage
|
||||
from email.utils import formatdate, make_msgid
|
||||
|
||||
lb = sys.argv[1]
|
||||
account = os.environ["ACCOUNT"]
|
||||
password = open(os.environ["PASSWORD_FILE"]).read().strip()
|
||||
state_dir = os.environ["STATE"]
|
||||
IN_EVERY = float(os.environ.get("IN_EVERY", "10"))
|
||||
OUT_EVERY = float(os.environ.get("OUT_EVERY", "60"))
|
||||
IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2"))
|
||||
SINK_SPREAD = int(os.environ.get("SINK_SPREAD", "10"))
|
||||
tls = ssl.create_default_context()
|
||||
tls.check_hostname = False
|
||||
tls.verify_mode = ssl.CERT_NONE
|
||||
lock = threading.RLock() # event() and save() are called with it held
|
||||
STATE = os.path.join(state_dir, "load.json")
|
||||
EVENTS = os.path.join(state_dir, "events.jsonl")
|
||||
|
||||
try:
|
||||
S = json.load(open(STATE))
|
||||
except (OSError, ValueError):
|
||||
S = {"run": uuid.uuid4().hex[:8], "started": time.time(),
|
||||
"inbound": {"next": 1, "sent": 0, "retries": 0, "arrived": 0, "twice": 0},
|
||||
"pending": {}, "arrived_late": 0, "slowest_s": 0,
|
||||
"outbound": {"next": 1, "sent": 0, "retries": 0}, "out_sent": {},
|
||||
"idle": {"notices": 0, "drops": 0, "longest_quiet_s": 0},
|
||||
"seen": {}, "starts": 0}
|
||||
S["starts"] += 1
|
||||
run = S["run"]
|
||||
|
||||
|
||||
def event(kind, detail):
|
||||
with lock:
|
||||
with open(EVENTS, "a") as f:
|
||||
f.write(json.dumps({"at": time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
|
||||
|
||||
|
||||
def save():
|
||||
with lock:
|
||||
# A duplicate a day late is still caught; older arrivals are dropped
|
||||
old = time.time() - 2 * 86400
|
||||
S["seen"] = {n: v for n, v in S.get("seen", {}).items() if v[1] > old}
|
||||
tmp = STATE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(S, f)
|
||||
os.replace(tmp, STATE)
|
||||
|
||||
|
||||
def message(frm, to, subject):
|
||||
msg = EmailMessage()
|
||||
msg["From"], msg["To"], msg["Subject"] = frm, to, subject
|
||||
msg["Date"] = formatdate()
|
||||
msg["Message-ID"] = make_msgid(domain=f"{run}.soak.test")
|
||||
msg.set_content(subject + "\n")
|
||||
return msg
|
||||
|
||||
|
||||
def every(seconds, fn):
|
||||
"""fn() every `seconds`, keeping to the schedule; fn handles its errors."""
|
||||
nxt = time.time()
|
||||
while True:
|
||||
fn()
|
||||
nxt += seconds
|
||||
time.sleep(max(0, nxt - time.time()))
|
||||
|
||||
|
||||
def inbound():
|
||||
with lock:
|
||||
n = S["inbound"]["next"]
|
||||
# A sender domain per message: the default throttle allows 25 an hour
|
||||
# from one sender domain to one recipient
|
||||
msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}")
|
||||
try:
|
||||
# As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py
|
||||
with smtplib.SMTP(timeout=20) as s:
|
||||
s.connect(lb, 25)
|
||||
s.sock.settimeout(300)
|
||||
s.send_message(msg)
|
||||
except (OSError, smtplib.SMTPException) as e:
|
||||
with lock:
|
||||
S["inbound"]["retries"] += 1
|
||||
event("inbound-retry", f"{n}: {type(e).__name__}: {e}")
|
||||
return # the same number goes again next time
|
||||
with lock:
|
||||
S["inbound"]["next"] = n + 1
|
||||
S["inbound"]["sent"] += 1
|
||||
S["pending"][str(n)] = time.time()
|
||||
save() # at once: a restart must never send a number twice
|
||||
|
||||
|
||||
def outbound():
|
||||
with lock:
|
||||
n = S["outbound"]["next"]
|
||||
msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}")
|
||||
try:
|
||||
with smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) as s:
|
||||
s.sock.settimeout(300)
|
||||
s.login(account, password)
|
||||
s.send_message(msg)
|
||||
except (OSError, smtplib.SMTPException) as e:
|
||||
with lock:
|
||||
S["outbound"]["retries"] += 1
|
||||
event("outbound-retry", f"{n}: {type(e).__name__}: {e}")
|
||||
return
|
||||
with lock:
|
||||
S["outbound"]["next"] = n + 1
|
||||
S["outbound"]["sent"] += 1
|
||||
S["out_sent"][str(n)] = time.time()
|
||||
save()
|
||||
|
||||
|
||||
SUBJECT = re.compile(rf"soak {run} in (\d+)")
|
||||
|
||||
|
||||
def verify():
|
||||
try:
|
||||
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=60)
|
||||
except OSError as e:
|
||||
event("verify", f"connect: {e}")
|
||||
return
|
||||
try:
|
||||
m.login(account, password)
|
||||
_, boxes = m.list()
|
||||
for line in boxes:
|
||||
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
|
||||
if m.select(name)[0] != "OK":
|
||||
continue
|
||||
_, data = m.search(None, "SUBJECT", f'"soak {run} in"')
|
||||
ids = data[0].split()
|
||||
if not ids:
|
||||
continue
|
||||
_, hdrs = m.fetch(b",".join(ids), "(UID BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
|
||||
now = time.time()
|
||||
for part in hdrs:
|
||||
if not isinstance(part, tuple):
|
||||
continue
|
||||
mm = SUBJECT.search(part[1].decode(errors="replace"))
|
||||
uid = re.search(rb"UID (\d+)", part[0])
|
||||
if not mm or not uid:
|
||||
continue
|
||||
n, copy = mm.group(1), f"{name}:{uid.group(1).decode()}"
|
||||
with lock:
|
||||
sent = S["pending"].pop(n, None)
|
||||
if sent is not None:
|
||||
S["inbound"]["arrived"] += 1
|
||||
S["seen"][n] = [copy, now]
|
||||
S["landed"] = name
|
||||
took = now - sent
|
||||
S["slowest_s"] = max(S["slowest_s"], round(took))
|
||||
if took > 900:
|
||||
S["arrived_late"] += 1
|
||||
elif S["seen"].get(n, [None])[0] != copy:
|
||||
S["inbound"]["twice"] += 1
|
||||
S["seen"][n] = [copy, now]
|
||||
event("twice", f"in {n} arrived again ({name})")
|
||||
# else: this very copy, left by a delete that was cut short
|
||||
# What arrived is on disk before it's deleted, so a restart in
|
||||
# between neither loses it nor counts it twice
|
||||
save()
|
||||
m.store(b",".join(ids), "+FLAGS", "\\Deleted")
|
||||
m.expunge()
|
||||
except (OSError, imaplib.IMAP4.error) as e:
|
||||
event("verify", f"{type(e).__name__}: {e}")
|
||||
finally:
|
||||
try:
|
||||
m.logout()
|
||||
except Exception:
|
||||
pass
|
||||
save()
|
||||
|
||||
|
||||
def idle(i):
|
||||
"""One client in IDLE, as a desktop mail app holds it; re-issued every
|
||||
25 minutes (RFC 2177 asks for under 29)."""
|
||||
while not S.get("landed"):
|
||||
time.sleep(5)
|
||||
last_notice = time.time()
|
||||
while True:
|
||||
sock = None
|
||||
try:
|
||||
raw = socket.create_connection((lb, 993), timeout=30)
|
||||
sock = tls.wrap_socket(raw, server_hostname="mail.lab.test")
|
||||
f = sock.makefile("rb")
|
||||
|
||||
def cmd(tag, text):
|
||||
sock.sendall(f"{tag} {text}\r\n".encode())
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
raise ConnectionError("closed")
|
||||
if line.startswith(tag.encode() + b" "):
|
||||
if b" OK" not in line[:12]:
|
||||
raise ConnectionError(line.decode(errors="replace").strip())
|
||||
return
|
||||
|
||||
f.readline() # greeting
|
||||
cmd("a1", f'LOGIN "{account}" "{password}"')
|
||||
cmd("a2", f"SELECT {S['landed']}")
|
||||
n = 3
|
||||
while True:
|
||||
tag = f"a{n}"
|
||||
n += 1
|
||||
sock.sendall(f"{tag} IDLE\r\n".encode())
|
||||
sock.settimeout(60)
|
||||
renew = time.time() + 25 * 60
|
||||
while time.time() < renew:
|
||||
try:
|
||||
line = f.readline()
|
||||
except socket.timeout:
|
||||
continue
|
||||
if not line:
|
||||
raise ConnectionError("closed in IDLE")
|
||||
if line.endswith(b"EXISTS\r\n"):
|
||||
now = time.time()
|
||||
with lock:
|
||||
S["idle"]["notices"] += 1
|
||||
S["idle"]["longest_quiet_s"] = max(S["idle"]["longest_quiet_s"], round(now - last_notice))
|
||||
last_notice = now
|
||||
sock.sendall(b"DONE\r\n")
|
||||
sock.settimeout(30)
|
||||
while not f.readline().startswith(tag.encode() + b" "):
|
||||
pass
|
||||
except (OSError, ConnectionError, ssl.SSLError) as e:
|
||||
with lock:
|
||||
S["idle"]["drops"] += 1
|
||||
event("idle-drop", f"client {i}: {type(e).__name__}: {e}")
|
||||
finally:
|
||||
if sock:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
save()
|
||||
event("start", f"run {run}, start {S['starts']}")
|
||||
for target in ([lambda: every(IN_EVERY, inbound), lambda: every(OUT_EVERY, outbound), lambda: every(30, verify)]
|
||||
+ [(lambda i=i: idle(i)) for i in range(IDLE_CLIENTS)]):
|
||||
threading.Thread(target=target, daemon=True).start()
|
||||
while True:
|
||||
time.sleep(30)
|
||||
save()
|
||||
Executable
+286
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""What the soak (K-L7) sees, and its reports.
|
||||
|
||||
soak-watch.py collect once a minute, from soak.sh
|
||||
soak-watch.py report DAY that day's report, STATE/day-DAY.md
|
||||
soak-watch.py final the whole run, STATE/report.md
|
||||
|
||||
collect reads, with kubectl (KCTX):
|
||||
- the sink's log: every message soak-load.py sent out, counted once per
|
||||
delivery (by the sink's own receive time)
|
||||
- the webmail tabs' summaries (chart/webmail-tabs.py, one pod or several
|
||||
over the run, summed)
|
||||
- every pod in the inbuxa, data, lab-sink and lab-soak namespaces: a pod
|
||||
that's new, or a container that restarted, is an event, planned when
|
||||
it falls in a chaos window (STATE/chaos.jsonl), unplanned otherwise
|
||||
|
||||
Everything is kept in STATE/watch.json, events in STATE/events.jsonl
|
||||
beside soak-load.py's. Reports compare against the snapshot the previous
|
||||
day's report took, so each day stands on its own.
|
||||
"""
|
||||
import json, os, re, subprocess, sys, time
|
||||
|
||||
STATE_DIR = os.environ["STATE"]
|
||||
KCTX = os.environ["KCTX"]
|
||||
DAY = int(os.environ.get("SOAK_DAY", "86400"))
|
||||
NAMESPACES = ("inbuxa", "data", "lab-sink", "lab-soak")
|
||||
WATCH = os.path.join(STATE_DIR, "watch.json")
|
||||
EVENTS = os.path.join(STATE_DIR, "events.jsonl")
|
||||
LOST_AFTER = int(os.environ.get("LOST_AFTER", "3600")) # not there an hour after it was sent
|
||||
|
||||
|
||||
def kubectl(*args):
|
||||
return subprocess.run(["kubectl", "--context", KCTX, *args], capture_output=True, text=True, timeout=120).stdout
|
||||
|
||||
|
||||
def load(path, default):
|
||||
try:
|
||||
return json.load(open(path))
|
||||
except (OSError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def save(path, data):
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(data, f)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def event(kind, detail, at=None):
|
||||
with open(EVENTS, "a") as f:
|
||||
f.write(json.dumps({"at": at or time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
|
||||
|
||||
|
||||
def new_lines(W, ns, selector, key):
|
||||
"""Log lines of every pod matching `selector` since we last looked."""
|
||||
out = []
|
||||
for pod in kubectl("-n", ns, "get", "pods", "-l", selector, "-o", "name").split():
|
||||
since = W["since"].get(f"{key}/{pod}")
|
||||
args = ["-n", ns, "logs", pod, "--timestamps"] + (["--since-time", since] if since else [])
|
||||
last = since
|
||||
for line in kubectl(*args).splitlines():
|
||||
ts, _, text = line.partition(" ")
|
||||
if since and ts <= since:
|
||||
continue # --since-time is inclusive, to the second
|
||||
out.append((pod, text))
|
||||
last = ts
|
||||
if last:
|
||||
W["since"][f"{key}/{pod}"] = last
|
||||
return out
|
||||
|
||||
|
||||
def collect():
|
||||
W = load(WATCH, {"since": {}, "out": {}, "tabs": {}, "pods": {}, "first": True})
|
||||
load_state = load(os.path.join(STATE_DIR, "load.json"), {})
|
||||
run = load_state.get("run")
|
||||
|
||||
# The sink: each soak message that reached it
|
||||
for _, text in new_lines(W, "lab-sink", "app=lab-sink", "sink"):
|
||||
try:
|
||||
rec = json.loads(text)
|
||||
except ValueError:
|
||||
continue
|
||||
m = re.fullmatch(rf"soak {run} out (\d+)", rec.get("subject", ""))
|
||||
if m:
|
||||
W["out"][m.group(1)] = W["out"].get(m.group(1), 0) + 1
|
||||
if W["out"][m.group(1)] == 2:
|
||||
event("out-twice", f"out {m.group(1)} reached the sink again")
|
||||
|
||||
# The webmail tabs: the latest summary of each pod
|
||||
for pod, text in new_lines(W, "lab-soak", "app=soak-tabs", "tabs"):
|
||||
try:
|
||||
rec = json.loads(text)
|
||||
except ValueError:
|
||||
continue
|
||||
for e in rec.get("errors", []):
|
||||
event("webmail", f"{pod.split('/')[-1]}: {e}", rec.get("at"))
|
||||
rec.pop("errors", None)
|
||||
W["tabs"][pod] = rec
|
||||
|
||||
# The pods
|
||||
now = time.time()
|
||||
seen = {}
|
||||
for ns in NAMESPACES:
|
||||
for p in json.loads(kubectl("-n", ns, "get", "pods", "-o", "json") or '{"items": []}')["items"]:
|
||||
uid = p["metadata"]["uid"]
|
||||
name = f"{ns}/{p['metadata']['name']}"
|
||||
statuses = p.get("status", {}).get("containerStatuses", []) or []
|
||||
restarts = sum(c.get("restartCount", 0) for c in statuses)
|
||||
seen[uid] = {"name": name, "restarts": restarts}
|
||||
before = W["pods"].get(uid)
|
||||
if before is None and not W["first"]:
|
||||
owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "none")
|
||||
event("pod-new", f"{name} ({owner})", now)
|
||||
elif before and restarts > before["restarts"]:
|
||||
why = ", ".join(sorted({(c.get("lastState", {}).get("terminated") or {}).get("reason", "?")
|
||||
for c in statuses if c.get("restartCount", 0)}))
|
||||
event("restart", f"{name}: {restarts - before['restarts']} more ({why})", now)
|
||||
for uid, p in W["pods"].items():
|
||||
if uid not in seen:
|
||||
event("pod-gone", p["name"], now)
|
||||
W["pods"], W["first"] = seen, False
|
||||
save(WATCH, W)
|
||||
|
||||
|
||||
def counters():
|
||||
"""Every running total, flattened, for day-to-day differences."""
|
||||
L = load(os.path.join(STATE_DIR, "load.json"), {})
|
||||
W = load(WATCH, {"out": {}, "tabs": {}})
|
||||
tabs = list(W.get("tabs", {}).values())
|
||||
c = {f"in_{k}": v for k, v in L.get("inbound", {}).items() if k != "next"}
|
||||
c.update({f"out_{k}": v for k, v in L.get("outbound", {}).items() if k != "next"})
|
||||
c.update({f"idle_{k}": v for k, v in L.get("idle", {}).items() if k != "longest_quiet_s"})
|
||||
c["in_late"] = L.get("arrived_late", 0)
|
||||
c["out_received"] = len(W.get("out", {}))
|
||||
c["out_twice"] = sum(1 for v in W.get("out", {}).values() if v > 1)
|
||||
c["load_starts"] = L.get("starts", 0)
|
||||
for k in ("requests", "failed", "signins"):
|
||||
c[f"web_{k}"] = sum(t.get(k, 0) for t in tabs)
|
||||
for k in ("streams", "clean_ends", "cut_off"):
|
||||
c[f"web_{k}"] = sum(x.get(k, 0) for t in tabs for x in t.get("tabs", []))
|
||||
return c, L, W
|
||||
|
||||
|
||||
def chaos_windows():
|
||||
out = []
|
||||
for line in open(os.path.join(STATE_DIR, "chaos.jsonl")) if os.path.exists(os.path.join(STATE_DIR, "chaos.jsonl")) else []:
|
||||
try:
|
||||
out.append(json.loads(line))
|
||||
except ValueError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def events_between(a, b):
|
||||
out = []
|
||||
if os.path.exists(EVENTS):
|
||||
for line in open(EVENTS):
|
||||
try:
|
||||
e = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if a <= e["at"] < b:
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
|
||||
def in_chaos(at, windows):
|
||||
# A minute's grace after, for what the collector notices late
|
||||
return any(w["start"] <= at <= w.get("end", time.time()) + 120 for w in windows)
|
||||
|
||||
|
||||
def day_numbers(day, started):
|
||||
"""(this day's counters, the day's start and end)."""
|
||||
c, L, W = counters()
|
||||
snap = os.path.join(STATE_DIR, f"day-{day}.json")
|
||||
prev = load(os.path.join(STATE_DIR, f"day-{day - 1}.json"), {}) if day > 1 else {}
|
||||
# Taken once, at the day's end; a report written again uses it
|
||||
if os.path.exists(snap):
|
||||
c = load(snap, c)
|
||||
else:
|
||||
save(snap, c)
|
||||
delta = {k: v - prev.get(k, 0) for k, v in c.items()}
|
||||
return delta, c, L, W, started + (day - 1) * DAY, started + day * DAY
|
||||
|
||||
|
||||
def missing(L, W, now):
|
||||
lost_in = sorted(int(n) for n, t in L.get("pending", {}).items() if now - t > LOST_AFTER)
|
||||
lost_out = sorted(int(n) for n, t in L.get("out_sent", {}).items()
|
||||
if now - t > LOST_AFTER and n not in W.get("out", {}))
|
||||
return lost_in, lost_out
|
||||
|
||||
|
||||
def report(day):
|
||||
started = float(open(os.path.join(STATE_DIR, "started")).read())
|
||||
d, c, L, W, a, b = day_numbers(day, started)
|
||||
now = time.time()
|
||||
lost_in, lost_out = missing(L, W, now)
|
||||
windows = [w for w in chaos_windows() if a <= w["start"] < b]
|
||||
evs = events_between(a, b)
|
||||
restarts = [e for e in evs if e["kind"] in ("restart", "pod-new")]
|
||||
unplanned = [e for e in restarts if not in_chaos(e["at"], chaos_windows())]
|
||||
twice = d.get("in_twice", 0) + d.get("out_twice", 0)
|
||||
chaos_ok = all(w.get("result") == "ok" for w in windows)
|
||||
ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok
|
||||
fmt = lambda t: time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(t))
|
||||
lines = [
|
||||
f"# Soak day {day}: {'clean' if ok else 'NOT clean'}",
|
||||
"",
|
||||
f"{fmt(a)} to {fmt(b)}, run `{L.get('run')}`, context `{KCTX}`.",
|
||||
"",
|
||||
"| | |",
|
||||
"|---|---|",
|
||||
f"| Mail in | {d.get('in_sent', 0)} sent, {d.get('in_arrived', 0)} arrived, {d.get('in_late', 0)} after 15 min, "
|
||||
f"{d.get('in_twice', 0)} twice, {d.get('in_retries', 0)} sends retried; slowest so far {L.get('slowest_s', 0)} s |",
|
||||
f"| Mail out | {d.get('out_sent', 0)} sent, {d.get('out_received', 0)} reached the sink, {d.get('out_twice', 0)} twice, "
|
||||
f"{d.get('out_retries', 0)} sends retried |",
|
||||
f"| Lost so far | in: {len(lost_in)} {lost_in[:10] if lost_in else ''}, out: {len(lost_out)} {lost_out[:10] if lost_out else ''} "
|
||||
f"(not there {LOST_AFTER // 60} min after sending) |",
|
||||
f"| IMAP IDLE | {d.get('idle_notices', 0)} new-mail notices, {d.get('idle_drops', 0)} drops; "
|
||||
f"longest quiet so far {L.get('idle', {}).get('longest_quiet_s', 0)} s |",
|
||||
f"| Webmail tabs | {d.get('web_requests', 0)} requests, {d.get('web_failed', 0)} failed, "
|
||||
f"{d.get('web_streams', 0)} streams ({d.get('web_cut_off', 0)} cut off), {d.get('web_signins', 0)} sign-ins |",
|
||||
f"| Restarts | {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos |",
|
||||
f"| Load generator | started {d.get('load_starts', 0)} more time(s) |",
|
||||
"",
|
||||
"## Chaos",
|
||||
"",
|
||||
]
|
||||
lines += [f"- {w['scenario']}: {w.get('result', 'still running')} ({fmt(w['start'])})" for w in windows] or ["- none today"]
|
||||
if unplanned:
|
||||
lines += ["", "## Unplanned", ""] + [f"- {fmt(e['at'])} {e['kind']}: {e['detail']}" for e in unplanned[:30]]
|
||||
kinds = {}
|
||||
for e in evs:
|
||||
kinds.setdefault(e["kind"], []).append(e)
|
||||
if kinds:
|
||||
lines += ["", "## Events", ""]
|
||||
for k, es in sorted(kinds.items()):
|
||||
lines.append(f"- {k}: {len(es)}; first: {es[0]['detail']}")
|
||||
text = "\n".join(lines) + "\n"
|
||||
open(os.path.join(STATE_DIR, f"day-{day}.md"), "w").write(text)
|
||||
print(text)
|
||||
return ok
|
||||
|
||||
|
||||
def final():
|
||||
started = float(open(os.path.join(STATE_DIR, "started")).read())
|
||||
days = sorted(int(m.group(1)) for f in os.listdir(STATE_DIR) if (m := re.fullmatch(r"day-(\d+)\.md", f)))
|
||||
c, L, W = counters()
|
||||
lost_in, lost_out = missing(L, W, time.time())
|
||||
windows = chaos_windows()
|
||||
restarts = [e for e in events_between(started, time.time()) if e["kind"] in ("restart", "pod-new")]
|
||||
unplanned = [e for e in restarts if not in_chaos(e["at"], windows)]
|
||||
twice = c.get("in_twice", 0) + c.get("out_twice", 0)
|
||||
chaos_ok = all(w.get("result") == "ok" for w in windows)
|
||||
ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok
|
||||
lines = [f"# Soak: {'PASSED' if ok else 'FAILED'}", "",
|
||||
f"{len(days)} day(s) from {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(started))}, context `{KCTX}`.", "",
|
||||
f"- Mail in: {c.get('in_sent', 0)} sent, {c.get('in_arrived', 0)} arrived, {len(lost_in)} lost, {c.get('in_twice', 0)} twice",
|
||||
f"- Mail out: {c.get('out_sent', 0)} sent, {c.get('out_received', 0)} reached the sink, {len(lost_out)} lost, {c.get('out_twice', 0)} twice",
|
||||
f"- IMAP IDLE: {c.get('idle_notices', 0)} notices, {c.get('idle_drops', 0)} drops",
|
||||
f"- Webmail: {c.get('web_requests', 0)} requests, {c.get('web_failed', 0)} failed, {c.get('web_cut_off', 0)} streams cut off",
|
||||
f"- Restarts: {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos",
|
||||
f"- Chaos: " + ", ".join(f"{w['scenario']} {w.get('result', '?')}" for w in windows), "",
|
||||
"| Day | |", "|---|---|"]
|
||||
for dnum in days:
|
||||
first = open(os.path.join(STATE_DIR, f"day-{dnum}.md")).readline().strip("# \n")
|
||||
lines.append(f"| {dnum} | {first} |")
|
||||
text = "\n".join(lines) + "\n"
|
||||
open(os.path.join(STATE_DIR, "report.md"), "w").write(text)
|
||||
print(text)
|
||||
return ok
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
what = sys.argv[1]
|
||||
if what == "collect":
|
||||
collect()
|
||||
elif what == "report":
|
||||
sys.exit(0 if report(int(sys.argv[2])) else 1)
|
||||
elif what == "final":
|
||||
sys.exit(0 if final() else 1)
|
||||
else:
|
||||
sys.exit(__doc__)
|
||||
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-L7: the soak. Steady load for SOAK_DAYS days (7) on the lab's seeded
|
||||
# install, one chaos scenario a day, a report at the end of each day and
|
||||
# one for the whole run.
|
||||
#
|
||||
# lab/soak/soak.sh start checks the install, then runs in the
|
||||
# background (a systemd user unit)
|
||||
# lab/soak/soak.sh status where it's up to, and today's numbers so far
|
||||
# lab/soak/soak.sh stop
|
||||
# lab/soak/soak.sh report the reports written so far
|
||||
#
|
||||
# The load (soak-load.py) comes from this host through the load balancer:
|
||||
# mail in, mail out to the sink, IMAP IDLE. The webmail tabs
|
||||
# (chart/webmail-tabs.py) run in a pod, as there's no ingress. Chaos is
|
||||
# chart/chaos-test.sh with one scenario a day, CHAOS_AFTER seconds into the
|
||||
# day (2 h). Reports and state go to SOAK_DIR; a run is resumed, not
|
||||
# restarted, by `start` after a stop or a reboot, until it's done.
|
||||
#
|
||||
# A shorter run, to try the harness: SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300
|
||||
#
|
||||
# The unit stops when its user logs out unless lingering is on
|
||||
# (`loginctl enable-linger`), which is the machine owner's call.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
. "$HERE/../versions.env"
|
||||
export KCTX="${KCTX:-kind-$KIND_CLUSTER}"
|
||||
K=(kubectl --context "$KCTX")
|
||||
LAB_STATE="${LAB_STATE:-$HOME/.cache/inbuxa-lab}"
|
||||
export STATE="${SOAK_DIR:-$LAB_STATE/soak}"
|
||||
export SOAK_DAY="${SOAK_DAY:-86400}"
|
||||
SOAK_DAYS="${SOAK_DAYS:-7}"
|
||||
CHAOS_AFTER="${CHAOS_AFTER:-7200}"
|
||||
SCENARIOS=(pod-kill drain pg-failover nats-restart s3-outage pod-kill drain)
|
||||
UNIT=inbuxa-soak
|
||||
fail() { echo "!! $*" >&2; exit 1; }
|
||||
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
|
||||
|
||||
tabs_pod() {
|
||||
"${K[@]}" create namespace lab-soak --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" label namespace lab-soak pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null
|
||||
"${K[@]}" -n lab-soak create configmap soak-tabs --from-file="$HERE/../chart/webmail-tabs.py" \
|
||||
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" -n lab-soak create secret generic soak-tabs --from-file=password="$LAB_STATE/alice.password" \
|
||||
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" -n lab-soak apply -f - >/dev/null <<EOF
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata: {name: soak-tabs}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector: {matchLabels: {app: soak-tabs}}
|
||||
template:
|
||||
metadata: {labels: {app: soak-tabs}}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
|
||||
containers:
|
||||
- name: tabs
|
||||
image: $PG_IMAGE
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [python3, -u, /tabs/webmail-tabs.py, "http://inbuxa-webmail.inbuxa.svc:8080", "http://inbuxa-http.inbuxa.svc:8080"]
|
||||
env:
|
||||
- {name: ACCOUNT, value: [email protected]}
|
||||
- {name: PASSWORD_FILE, value: /secret/password}
|
||||
- {name: INTERVAL, value: "5"}
|
||||
- {name: SUMMARY_EVERY, value: "60"}
|
||||
resources: {requests: {cpu: 10m, memory: 32Mi}, limits: {memory: 128Mi}}
|
||||
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
|
||||
volumeMounts:
|
||||
- {name: tabs, mountPath: /tabs}
|
||||
- {name: secret, mountPath: /secret}
|
||||
volumes:
|
||||
- {name: tabs, configMap: {name: soak-tabs}}
|
||||
- {name: secret, secret: {secretName: soak-tabs, defaultMode: 0440}}
|
||||
EOF
|
||||
"${K[@]}" -n lab-soak rollout status deploy/soak-tabs --timeout=180s >/dev/null
|
||||
}
|
||||
|
||||
start() {
|
||||
systemctl --user is-active --quiet "$UNIT" && fail "already running: $0 status"
|
||||
mkdir -p "$STATE"
|
||||
if [ -f "$STATE/started" ] && [ -f "$STATE/report.md" ]; then
|
||||
fail "the run in $STATE is finished; move it away to start another"
|
||||
fi
|
||||
if [ ! -f "$STATE/started" ]; then
|
||||
echo "== every client check first (it also routes sink.test to the sink)"
|
||||
PORTS="${PORTS:-18080}" STATE="$LAB_STATE" "$HERE/../chart/mailcheck.sh" >"$STATE/mailcheck.log" 2>&1 \
|
||||
|| { tail -20 "$STATE/mailcheck.log"; fail "the install isn't healthy; not starting"; }
|
||||
date +%s >"$STATE/started"
|
||||
fi
|
||||
echo "== the webmail tabs"
|
||||
tabs_pod
|
||||
systemd-run --user --unit "$UNIT" --collect --quiet \
|
||||
--setenv=KCTX="$KCTX" --setenv=SOAK_DIR="$STATE" --setenv=SOAK_DAY="$SOAK_DAY" \
|
||||
--setenv=SOAK_DAYS="$SOAK_DAYS" --setenv=CHAOS_AFTER="$CHAOS_AFTER" --setenv=LAB_STATE="$LAB_STATE" \
|
||||
--setenv=PATH="$PATH" --setenv=HOME="$HOME" \
|
||||
bash "$HERE/soak.sh" run
|
||||
echo "== running: $SOAK_DAYS day(s) of ${SOAK_DAY}s; $0 status"
|
||||
}
|
||||
|
||||
# The unit's body: never exits on a hiccup, only when the run is done
|
||||
run() {
|
||||
set +e
|
||||
mkdir -p "$STATE"
|
||||
local started load_pid="" day now since
|
||||
started="$(cat "$STATE/started")"
|
||||
stop_load() { [ -n "$load_pid" ] && kill "$load_pid" 2>/dev/null; wait "$load_pid" 2>/dev/null; }
|
||||
trap 'stop_load; exit 0' TERM INT
|
||||
while :; do
|
||||
if [ -z "$load_pid" ] || ! kill -0 "$load_pid" 2>/dev/null; then
|
||||
ACCOUNT=[email protected] PASSWORD_FILE="$LAB_STATE/alice.password" \
|
||||
python3 "$HERE/soak-load.py" "$(lb)" >>"$STATE/load.log" 2>&1 &
|
||||
load_pid=$!
|
||||
fi
|
||||
python3 "$HERE/soak-watch.py" collect >>"$STATE/watch.log" 2>&1
|
||||
|
||||
now=$(date +%s)
|
||||
day=$(( (now - started) / SOAK_DAY + 1 ))
|
||||
since=$(( (now - started) % SOAK_DAY ))
|
||||
# Yesterday's report, once
|
||||
if [ "$day" -gt 1 ] && [ ! -f "$STATE/day-$((day - 1)).md" ]; then
|
||||
python3 "$HERE/soak-watch.py" report "$((day - 1))" >/dev/null 2>>"$STATE/watch.log"
|
||||
fi
|
||||
if [ "$day" -gt "$SOAK_DAYS" ]; then
|
||||
python3 "$HERE/soak-watch.py" final >/dev/null 2>>"$STATE/watch.log"
|
||||
stop_load
|
||||
"${K[@]}" -n lab-soak scale deploy/soak-tabs --replicas=0 >/dev/null 2>&1
|
||||
exit 0
|
||||
fi
|
||||
# Today's chaos, once
|
||||
if [ "$since" -ge "$CHAOS_AFTER" ] && [ ! -f "$STATE/chaos-day-$day.log" ]; then
|
||||
local sc="${SCENARIOS[$(( (day - 1) % ${#SCENARIOS[@]} ))]}" t0 rc
|
||||
t0=$(date +%s)
|
||||
printf '{"day": %d, "scenario": "%s", "start": %d}\n' "$day" "$sc" "$t0" >>"$STATE/chaos.jsonl"
|
||||
SCENARIOS="$sc" STATE="$LAB_STATE" PORTS=18180 "$HERE/../chart/chaos-test.sh" >"$STATE/chaos-day-$day.log" 2>&1
|
||||
rc=$?
|
||||
python3 - "$STATE/chaos.jsonl" "$day" "$rc" "$(date +%s)" <<'PY'
|
||||
import json, sys
|
||||
path, day, rc, end = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
|
||||
rows = [json.loads(l) for l in open(path) if l.strip()]
|
||||
for r in rows:
|
||||
if r["day"] == day and "end" not in r:
|
||||
r["end"], r["result"] = end, "ok" if rc == 0 else f"FAILED (exit {rc}, see chaos-day-{day}.log)"
|
||||
open(path, "w").write("".join(json.dumps(r) + "\n" for r in rows))
|
||||
PY
|
||||
fi
|
||||
sleep 60
|
||||
done
|
||||
}
|
||||
|
||||
status() {
|
||||
if systemctl --user is-active --quiet "$UNIT"; then echo "== running"; else echo "== not running"; fi
|
||||
[ -f "$STATE/started" ] || { echo " no run in $STATE"; return 0; }
|
||||
local started now; started="$(cat "$STATE/started")"; now=$(date +%s)
|
||||
echo " day $(( (now - started) / SOAK_DAY + 1 )) of $SOAK_DAYS, $(( (now - started) % SOAK_DAY / 60 )) min in"
|
||||
python3 - "$STATE" <<'PY'
|
||||
import json, os, sys
|
||||
d = sys.argv[1]
|
||||
L = json.load(open(os.path.join(d, "load.json")))
|
||||
W = json.load(open(os.path.join(d, "watch.json"))) if os.path.exists(os.path.join(d, "watch.json")) else {"out": {}, "tabs": {}}
|
||||
i, o, idle = L["inbound"], L["outbound"], L["idle"]
|
||||
print(f" in: {i['sent']} sent, {i['arrived']} arrived, {len(L['pending'])} pending, {i['twice']} twice")
|
||||
print(f" out: {o['sent']} sent, {len(W['out'])} at the sink")
|
||||
print(f" idle: {idle['notices']} notices, {idle['drops']} drops")
|
||||
t = list(W["tabs"].values())
|
||||
print(f" webmail: {sum(x.get('requests', 0) for x in t)} requests, {sum(x.get('failed', 0) for x in t)} failed")
|
||||
PY
|
||||
ls "$STATE"/day-*.md 2>/dev/null | sed 's/^/ /' || true
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
start) start ;;
|
||||
run) run ;;
|
||||
status) status ;;
|
||||
stop) systemctl --user stop "$UNIT" 2>/dev/null || true; echo "== stopped (start resumes it)" ;;
|
||||
report) cat "$STATE"/day-*.md "$STATE"/report.md 2>/dev/null || echo "no reports yet" ;;
|
||||
*) sed -n '4,25p' "$0"; exit 2 ;;
|
||||
esac
|
||||
Reference in new issue
Block a user