From 315e84e242b08a7f7a3046d079a1a9d8d9639ada Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 01:07:27 -0700 Subject: [PATCH 1/3] The soak: a week of steady load and daily chaos (K-L7) lab/soak/soak.sh start runs SOAK_DAYS (7) days on an installed chart as a systemd user unit: soak-load.py from this host through the load balancer (mail in every 10 s, out to the sink every minute, two IMAP clients in IDLE), three webmail tabs in a pod (chart/webmail-tabs.py, now with a summary every minute and signing in again when its session runs out), chart/chaos-test.sh with one scenario a day, and soak-watch.py: the sink's log, the tabs' summaries and every pod in the namespaces each minute, day-N.md at each day's end and report.md for the run. It passes with no message lost or doubled and no restart outside the chaos windows. Every count survives the load generator restarting: what was sent is saved before the next send, and an arrival is saved, with its IMAP UID, before it's deleted. --- lab/chart/webmail-tabs.py | 32 ++++- lab/soak/soak-load.py | 269 ++++++++++++++++++++++++++++++++++++ lab/soak/soak-watch.py | 284 ++++++++++++++++++++++++++++++++++++++ lab/soak/soak.sh | 182 ++++++++++++++++++++++++ 4 files changed, 764 insertions(+), 3 deletions(-) create mode 100755 lab/soak/soak-load.py create mode 100755 lab/soak/soak-watch.py create mode 100755 lab/soak/soak.sh diff --git a/lab/chart/webmail-tabs.py b/lab/chart/webmail-tabs.py index d9dade2..7bfd214 100755 --- a/lab/chart/webmail-tabs.py +++ b/lab/chart/webmail-tabs.py @@ -17,14 +17,21 @@ and prints one JSON line: requests made and failed (with the errors), and per tab how its streams ended (cleanly, or cut off mid-stream) and the longest wait for a stream to deliver again. A clean end is what the browser takes in its stride; a failed request is what shows an error. + +For the soak (K-L7): INTERVAL sets the time between requests (0.2 s); +SUMMARY_EVERY (seconds) prints that line every so often while running, +with "at" and the errors since the last one; with no STOP_FILE it runs +until stopped. """ import http.client, json, os, sys, threading, time, urllib.parse webmail, server = sys.argv[1:3] account = os.environ["ACCOUNT"] password = open(os.environ["PASSWORD_FILE"]).read().strip() -stop = os.environ["STOP_FILE"] +stop = os.environ.get("STOP_FILE", "/nonexistent/stop") TABS = int(os.environ.get("TABS", "3")) +INTERVAL = float(os.environ.get("INTERVAL", "0.2")) +SUMMARY_EVERY = float(os.environ.get("SUMMARY_EVERY", "0")) def request(base, method, path, headers=None, body=None, timeout=30): @@ -65,7 +72,7 @@ def sign_in(): session, mail_account = sign_in() H = {"cookie": f"ihm_session={session}"} lock = threading.Lock() -report = {"requests": 0, "failed": 0, "errors": [], "tabs": []} +report = {"requests": 0, "failed": 0, "signins": 1, "errors": [], "tabs": []} def tab(i): @@ -79,6 +86,10 @@ def tab(i): conn = http.client.HTTPConnection(u.hostname, u.port, timeout=45) conn.request("GET", "/api/events?types=*&closeafter=no&ping=30", headers=dict(H, accept="text/event-stream")) res = conn.getresponse() + if res.status == 401: # signed out: the request loop signs in again + conn.close() + time.sleep(1) + continue if res.status != 200: raise RuntimeError(f"HTTP {res.status}") me["streams"] += 1 @@ -107,19 +118,34 @@ for t in threads: body = json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], "methodCalls": [["Mailbox/get", {"accountId": mail_account, "properties": ["name"]}, "0"]]}) +next_summary = time.time() + SUMMARY_EVERY while not os.path.exists(stop): with lock: report["requests"] += 1 try: res, data = request(webmail, "POST", "/api/jmap", dict(H, **{"content-type": "application/json", "x-requested-with": "ihasmail"}), body) + if res.status == 401: + # The session ran out (12 h by default): sign in again, as the + # page sends the browser back to; the tabs pick up the new cookie + session, mail_account = sign_in() + H["cookie"] = f"ihm_session={session}" + with lock: + report["signins"] += 1 + report["requests"] -= 1 + continue if res.status != 200 or b"Mailbox/get" not in data: raise RuntimeError(f"HTTP {res.status} {data[:80]!r}") except Exception as e: with lock: report["failed"] += 1 report["errors"].append(f"request: {type(e).__name__}: {e}"[:160]) - time.sleep(0.2) + time.sleep(INTERVAL) + if SUMMARY_EVERY and time.time() >= next_summary: + next_summary += SUMMARY_EVERY + with lock: + print(json.dumps(dict(report, at=time.time(), errors=report["errors"][:15])), flush=True) + report["errors"] = [] for t in threads: t.join(timeout=50) diff --git a/lab/soak/soak-load.py b/lab/soak/soak-load.py new file mode 100755 index 0000000..bd1136f --- /dev/null +++ b/lab/soak/soak-load.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""The soak's steady mail load (K-L7), from outside the cluster through its +load balancer, as real clients reach it. Runs until killed, and picks up +where it left off when started again: everything it knows is in STATE. + + ACCOUNT=user@domain PASSWORD_FILE=... STATE=dir soak-load.py LB + + inbound a message every IN_EVERY s (10) over SMTP 25, each from its own + sender domain, retried like an MTA when refused + verify every 30 s, over IMAPS: which arrived (any mailbox), then + deleted, so the mailbox stays small; a number seen again after + that came twice + outbound a message every OUT_EVERY s (60) over submission 465 with AUTH, + to probe@sink.test; soak.sh checks the sink received each once + idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're + told of new mail, and how often they drop. They watch the + mailbox the soak's mail lands in (verify finds it: unknown + senders' mail goes to Junk Mail), as a client watches INBOX + +STATE/load.json is rewritten every 30 s; STATE/events.jsonl gets a line +for anything that went wrong. The lab's certificate is self-signed, so TLS +isn't verified. +""" +import imaplib, json, os, re, smtplib, socket, ssl, sys, threading, time, uuid +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +lb = sys.argv[1] +account = os.environ["ACCOUNT"] +password = open(os.environ["PASSWORD_FILE"]).read().strip() +state_dir = os.environ["STATE"] +IN_EVERY = float(os.environ.get("IN_EVERY", "10")) +OUT_EVERY = float(os.environ.get("OUT_EVERY", "60")) +IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2")) +SINK = os.environ.get("SINK_RCPT", "probe@sink.test") +tls = ssl.create_default_context() +tls.check_hostname = False +tls.verify_mode = ssl.CERT_NONE +lock = threading.RLock() # event() and save() are called with it held +STATE = os.path.join(state_dir, "load.json") +EVENTS = os.path.join(state_dir, "events.jsonl") + +try: + S = json.load(open(STATE)) +except (OSError, ValueError): + S = {"run": uuid.uuid4().hex[:8], "started": time.time(), + "inbound": {"next": 1, "sent": 0, "retries": 0, "arrived": 0, "twice": 0}, + "pending": {}, "arrived_late": 0, "slowest_s": 0, + "outbound": {"next": 1, "sent": 0, "retries": 0}, "out_sent": {}, + "idle": {"notices": 0, "drops": 0, "longest_quiet_s": 0}, + "seen": {}, "starts": 0} +S["starts"] += 1 +run = S["run"] + + +def event(kind, detail): + with lock: + with open(EVENTS, "a") as f: + f.write(json.dumps({"at": time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n") + + +def save(): + with lock: + # A duplicate a day late is still caught; older arrivals are dropped + old = time.time() - 2 * 86400 + S["seen"] = {n: v for n, v in S.get("seen", {}).items() if v[1] > old} + tmp = STATE + ".tmp" + with open(tmp, "w") as f: + json.dump(S, f) + os.replace(tmp, STATE) + + +def message(frm, to, subject): + msg = EmailMessage() + msg["From"], msg["To"], msg["Subject"] = frm, to, subject + msg["Date"] = formatdate() + msg["Message-ID"] = make_msgid(domain=f"{run}.soak.test") + msg.set_content(subject + "\n") + return msg + + +def every(seconds, fn): + """fn() every `seconds`, keeping to the schedule; fn handles its errors.""" + nxt = time.time() + while True: + fn() + nxt += seconds + time.sleep(max(0, nxt - time.time())) + + +def inbound(): + with lock: + n = S["inbound"]["next"] + # A sender domain per message: the default throttle allows 25 an hour + # from one sender domain to one recipient + msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}") + try: + # As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py + with smtplib.SMTP(timeout=20) as s: + s.connect(lb, 25) + s.sock.settimeout(300) + s.send_message(msg) + except (OSError, smtplib.SMTPException) as e: + with lock: + S["inbound"]["retries"] += 1 + event("inbound-retry", f"{n}: {type(e).__name__}: {e}") + return # the same number goes again next time + with lock: + S["inbound"]["next"] = n + 1 + S["inbound"]["sent"] += 1 + S["pending"][str(n)] = time.time() + save() # at once: a restart must never send a number twice + + +def outbound(): + with lock: + n = S["outbound"]["next"] + msg = message(account, SINK, f"soak {run} out {n}") + try: + with smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) as s: + s.sock.settimeout(300) + s.login(account, password) + s.send_message(msg) + except (OSError, smtplib.SMTPException) as e: + with lock: + S["outbound"]["retries"] += 1 + event("outbound-retry", f"{n}: {type(e).__name__}: {e}") + return + with lock: + S["outbound"]["next"] = n + 1 + S["outbound"]["sent"] += 1 + S["out_sent"][str(n)] = time.time() + save() + + +SUBJECT = re.compile(rf"soak {run} in (\d+)") + + +def verify(): + try: + m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=60) + except OSError as e: + event("verify", f"connect: {e}") + return + try: + m.login(account, password) + _, boxes = m.list() + for line in boxes: + name = line.decode().rsplit(' "/" ', 1)[-1].strip() + if m.select(name)[0] != "OK": + continue + _, data = m.search(None, "SUBJECT", f'"soak {run} in"') + ids = data[0].split() + if not ids: + continue + _, hdrs = m.fetch(b",".join(ids), "(UID BODY.PEEK[HEADER.FIELDS (SUBJECT)])") + now = time.time() + for part in hdrs: + if not isinstance(part, tuple): + continue + mm = SUBJECT.search(part[1].decode(errors="replace")) + uid = re.search(rb"UID (\d+)", part[0]) + if not mm or not uid: + continue + n, copy = mm.group(1), f"{name}:{uid.group(1).decode()}" + with lock: + sent = S["pending"].pop(n, None) + if sent is not None: + S["inbound"]["arrived"] += 1 + S["seen"][n] = [copy, now] + S["landed"] = name + took = now - sent + S["slowest_s"] = max(S["slowest_s"], round(took)) + if took > 900: + S["arrived_late"] += 1 + elif S["seen"].get(n, [None])[0] != copy: + S["inbound"]["twice"] += 1 + S["seen"][n] = [copy, now] + event("twice", f"in {n} arrived again ({name})") + # else: this very copy, left by a delete that was cut short + # What arrived is on disk before it's deleted, so a restart in + # between neither loses it nor counts it twice + save() + m.store(b",".join(ids), "+FLAGS", "\\Deleted") + m.expunge() + except (OSError, imaplib.IMAP4.error) as e: + event("verify", f"{type(e).__name__}: {e}") + finally: + try: + m.logout() + except Exception: + pass + save() + + +def idle(i): + """One client in IDLE, as a desktop mail app holds it; re-issued every + 25 minutes (RFC 2177 asks for under 29).""" + while not S.get("landed"): + time.sleep(5) + last_notice = time.time() + while True: + sock = None + try: + raw = socket.create_connection((lb, 993), timeout=30) + sock = tls.wrap_socket(raw, server_hostname="mail.lab.test") + f = sock.makefile("rb") + + def cmd(tag, text): + sock.sendall(f"{tag} {text}\r\n".encode()) + while True: + line = f.readline() + if not line: + raise ConnectionError("closed") + if line.startswith(tag.encode() + b" "): + if b" OK" not in line[:12]: + raise ConnectionError(line.decode(errors="replace").strip()) + return + + f.readline() # greeting + cmd("a1", f'LOGIN "{account}" "{password}"') + cmd("a2", f"SELECT {S['landed']}") + n = 3 + while True: + tag = f"a{n}" + n += 1 + sock.sendall(f"{tag} IDLE\r\n".encode()) + sock.settimeout(60) + renew = time.time() + 25 * 60 + while time.time() < renew: + try: + line = f.readline() + except socket.timeout: + continue + if not line: + raise ConnectionError("closed in IDLE") + if line.endswith(b"EXISTS\r\n"): + now = time.time() + with lock: + S["idle"]["notices"] += 1 + S["idle"]["longest_quiet_s"] = max(S["idle"]["longest_quiet_s"], round(now - last_notice)) + last_notice = now + sock.sendall(b"DONE\r\n") + sock.settimeout(30) + while not f.readline().startswith(tag.encode() + b" "): + pass + except (OSError, ConnectionError, ssl.SSLError) as e: + with lock: + S["idle"]["drops"] += 1 + event("idle-drop", f"client {i}: {type(e).__name__}: {e}") + finally: + if sock: + try: + sock.close() + except OSError: + pass + time.sleep(5) + + +save() +event("start", f"run {run}, start {S['starts']}") +for target in ([lambda: every(IN_EVERY, inbound), lambda: every(OUT_EVERY, outbound), lambda: every(30, verify)] + + [(lambda i=i: idle(i)) for i in range(IDLE_CLIENTS)]): + threading.Thread(target=target, daemon=True).start() +while True: + time.sleep(30) + save() diff --git a/lab/soak/soak-watch.py b/lab/soak/soak-watch.py new file mode 100755 index 0000000..2a0a899 --- /dev/null +++ b/lab/soak/soak-watch.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""What the soak (K-L7) sees, and its reports. + + soak-watch.py collect once a minute, from soak.sh + soak-watch.py report DAY that day's report, STATE/day-DAY.md + soak-watch.py final the whole run, STATE/report.md + +collect reads, with kubectl (KCTX): + - the sink's log: every message soak-load.py sent out, counted once per + delivery (by the sink's own receive time) + - the webmail tabs' summaries (chart/webmail-tabs.py, one pod or several + over the run, summed) + - every pod in the inbuxa, data, lab-sink and lab-soak namespaces: a pod + that's new, or a container that restarted, is an event, planned when + it falls in a chaos window (STATE/chaos.jsonl), unplanned otherwise + +Everything is kept in STATE/watch.json, events in STATE/events.jsonl +beside soak-load.py's. Reports compare against the snapshot the previous +day's report took, so each day stands on its own. +""" +import json, os, re, subprocess, sys, time + +STATE_DIR = os.environ["STATE"] +KCTX = os.environ["KCTX"] +DAY = int(os.environ.get("SOAK_DAY", "86400")) +NAMESPACES = ("inbuxa", "data", "lab-sink", "lab-soak") +WATCH = os.path.join(STATE_DIR, "watch.json") +EVENTS = os.path.join(STATE_DIR, "events.jsonl") +LOST_AFTER = int(os.environ.get("LOST_AFTER", "3600")) # not there an hour after it was sent + + +def kubectl(*args): + return subprocess.run(["kubectl", "--context", KCTX, *args], capture_output=True, text=True, timeout=120).stdout + + +def load(path, default): + try: + return json.load(open(path)) + except (OSError, ValueError): + return default + + +def save(path, data): + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(data, f) + os.replace(tmp, path) + + +def event(kind, detail, at=None): + with open(EVENTS, "a") as f: + f.write(json.dumps({"at": at or time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n") + + +def new_lines(W, ns, selector, key): + """Log lines of every pod matching `selector` since we last looked.""" + out = [] + for pod in kubectl("-n", ns, "get", "pods", "-l", selector, "-o", "name").split(): + since = W["since"].get(f"{key}/{pod}") + args = ["-n", ns, "logs", pod, "--timestamps"] + (["--since-time", since] if since else []) + last = since + for line in kubectl(*args).splitlines(): + ts, _, text = line.partition(" ") + if since and ts <= since: + continue # --since-time is inclusive, to the second + out.append((pod, text)) + last = ts + if last: + W["since"][f"{key}/{pod}"] = last + return out + + +def collect(): + W = load(WATCH, {"since": {}, "out": {}, "tabs": {}, "pods": {}, "first": True}) + load_state = load(os.path.join(STATE_DIR, "load.json"), {}) + run = load_state.get("run") + + # The sink: each soak message that reached it + for _, text in new_lines(W, "lab-sink", "app=lab-sink", "sink"): + try: + rec = json.loads(text) + except ValueError: + continue + m = re.fullmatch(rf"soak {run} out (\d+)", rec.get("subject", "")) + if m: + W["out"][m.group(1)] = W["out"].get(m.group(1), 0) + 1 + if W["out"][m.group(1)] == 2: + event("out-twice", f"out {m.group(1)} reached the sink again") + + # The webmail tabs: the latest summary of each pod + for pod, text in new_lines(W, "lab-soak", "app=soak-tabs", "tabs"): + try: + rec = json.loads(text) + except ValueError: + continue + for e in rec.get("errors", []): + event("webmail", f"{pod.split('/')[-1]}: {e}", rec.get("at")) + rec.pop("errors", None) + W["tabs"][pod] = rec + + # The pods + now = time.time() + seen = {} + for ns in NAMESPACES: + for p in json.loads(kubectl("-n", ns, "get", "pods", "-o", "json") or '{"items": []}')["items"]: + uid = p["metadata"]["uid"] + name = f"{ns}/{p['metadata']['name']}" + statuses = p.get("status", {}).get("containerStatuses", []) or [] + restarts = sum(c.get("restartCount", 0) for c in statuses) + seen[uid] = {"name": name, "restarts": restarts} + before = W["pods"].get(uid) + if before is None and not W["first"]: + owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "none") + event("pod-new", f"{name} ({owner})", now) + elif before and restarts > before["restarts"]: + why = ", ".join(sorted({(c.get("lastState", {}).get("terminated") or {}).get("reason", "?") + for c in statuses if c.get("restartCount", 0)})) + event("restart", f"{name}: {restarts - before['restarts']} more ({why})", now) + for uid, p in W["pods"].items(): + if uid not in seen: + event("pod-gone", p["name"], now) + W["pods"], W["first"] = seen, False + save(WATCH, W) + + +def counters(): + """Every running total, flattened, for day-to-day differences.""" + L = load(os.path.join(STATE_DIR, "load.json"), {}) + W = load(WATCH, {"out": {}, "tabs": {}}) + tabs = list(W.get("tabs", {}).values()) + c = {f"in_{k}": v for k, v in L.get("inbound", {}).items() if k != "next"} + c.update({f"out_{k}": v for k, v in L.get("outbound", {}).items() if k != "next"}) + c.update({f"idle_{k}": v for k, v in L.get("idle", {}).items() if k != "longest_quiet_s"}) + c["in_late"] = L.get("arrived_late", 0) + c["out_received"] = len(W.get("out", {})) + c["out_twice"] = sum(1 for v in W.get("out", {}).values() if v > 1) + c["load_starts"] = L.get("starts", 0) + for k in ("requests", "failed", "signins"): + c[f"web_{k}"] = sum(t.get(k, 0) for t in tabs) + for k in ("streams", "clean_ends", "cut_off"): + c[f"web_{k}"] = sum(x.get(k, 0) for t in tabs for x in t.get("tabs", [])) + return c, L, W + + +def chaos_windows(): + out = [] + for line in open(os.path.join(STATE_DIR, "chaos.jsonl")) if os.path.exists(os.path.join(STATE_DIR, "chaos.jsonl")) else []: + try: + out.append(json.loads(line)) + except ValueError: + pass + return out + + +def events_between(a, b): + out = [] + if os.path.exists(EVENTS): + for line in open(EVENTS): + try: + e = json.loads(line) + except ValueError: + continue + if a <= e["at"] < b: + out.append(e) + return out + + +def in_chaos(at, windows): + # A minute's grace after, for what the collector notices late + return any(w["start"] <= at <= w.get("end", time.time()) + 120 for w in windows) + + +def day_numbers(day, started): + """(this day's counters, the day's start and end).""" + c, L, W = counters() + snap = os.path.join(STATE_DIR, f"day-{day}.json") + prev = load(os.path.join(STATE_DIR, f"day-{day - 1}.json"), {}) if day > 1 else {} + # Taken once, at the day's end; a report written again uses it + if os.path.exists(snap): + c = load(snap, c) + else: + save(snap, c) + delta = {k: v - prev.get(k, 0) for k, v in c.items()} + return delta, c, L, W, started + (day - 1) * DAY, started + day * DAY + + +def missing(L, W, now): + lost_in = sorted(int(n) for n, t in L.get("pending", {}).items() if now - t > LOST_AFTER) + lost_out = sorted(int(n) for n, t in L.get("out_sent", {}).items() + if now - t > LOST_AFTER and n not in W.get("out", {})) + return lost_in, lost_out + + +def report(day): + started = float(open(os.path.join(STATE_DIR, "started")).read()) + d, c, L, W, a, b = day_numbers(day, started) + now = time.time() + lost_in, lost_out = missing(L, W, now) + windows = [w for w in chaos_windows() if a <= w["start"] < b] + evs = events_between(a, b) + restarts = [e for e in evs if e["kind"] in ("restart", "pod-new")] + unplanned = [e for e in restarts if not in_chaos(e["at"], chaos_windows())] + twice = d.get("in_twice", 0) + d.get("out_twice", 0) + ok = not lost_in and not lost_out and not twice and not unplanned + fmt = lambda t: time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(t)) + lines = [ + f"# Soak day {day}: {'clean' if ok else 'NOT clean'}", + "", + f"{fmt(a)} to {fmt(b)}, run `{L.get('run')}`, context `{KCTX}`.", + "", + "| | |", + "|---|---|", + f"| Mail in | {d.get('in_sent', 0)} sent, {d.get('in_arrived', 0)} arrived, {d.get('in_late', 0)} after 15 min, " + f"{d.get('in_twice', 0)} twice, {d.get('in_retries', 0)} sends retried; slowest so far {L.get('slowest_s', 0)} s |", + f"| Mail out | {d.get('out_sent', 0)} sent, {d.get('out_received', 0)} reached the sink, {d.get('out_twice', 0)} twice, " + f"{d.get('out_retries', 0)} sends retried |", + f"| Lost so far | in: {len(lost_in)} {lost_in[:10] if lost_in else ''}, out: {len(lost_out)} {lost_out[:10] if lost_out else ''} " + f"(not there {LOST_AFTER // 60} min after sending) |", + f"| IMAP IDLE | {d.get('idle_notices', 0)} new-mail notices, {d.get('idle_drops', 0)} drops; " + f"longest quiet so far {L.get('idle', {}).get('longest_quiet_s', 0)} s |", + f"| Webmail tabs | {d.get('web_requests', 0)} requests, {d.get('web_failed', 0)} failed, " + f"{d.get('web_streams', 0)} streams ({d.get('web_cut_off', 0)} cut off), {d.get('web_signins', 0)} sign-ins |", + f"| Restarts | {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos |", + f"| Load generator | started {d.get('load_starts', 0)} more time(s) |", + "", + "## Chaos", + "", + ] + lines += [f"- {w['scenario']}: {w.get('result', 'still running')} ({fmt(w['start'])})" for w in windows] or ["- none today"] + if unplanned: + lines += ["", "## Unplanned", ""] + [f"- {fmt(e['at'])} {e['kind']}: {e['detail']}" for e in unplanned[:30]] + kinds = {} + for e in evs: + kinds.setdefault(e["kind"], []).append(e) + if kinds: + lines += ["", "## Events", ""] + for k, es in sorted(kinds.items()): + lines.append(f"- {k}: {len(es)}; first: {es[0]['detail']}") + text = "\n".join(lines) + "\n" + open(os.path.join(STATE_DIR, f"day-{day}.md"), "w").write(text) + print(text) + return ok + + +def final(): + started = float(open(os.path.join(STATE_DIR, "started")).read()) + days = sorted(int(m.group(1)) for f in os.listdir(STATE_DIR) if (m := re.fullmatch(r"day-(\d+)\.md", f))) + c, L, W = counters() + lost_in, lost_out = missing(L, W, time.time()) + windows = chaos_windows() + restarts = [e for e in events_between(started, time.time()) if e["kind"] in ("restart", "pod-new")] + unplanned = [e for e in restarts if not in_chaos(e["at"], windows)] + twice = c.get("in_twice", 0) + c.get("out_twice", 0) + ok = not lost_in and not lost_out and not twice and not unplanned + lines = [f"# Soak: {'PASSED' if ok else 'FAILED'}", "", + f"{len(days)} day(s) from {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(started))}, context `{KCTX}`.", "", + f"- Mail in: {c.get('in_sent', 0)} sent, {c.get('in_arrived', 0)} arrived, {len(lost_in)} lost, {c.get('in_twice', 0)} twice", + f"- Mail out: {c.get('out_sent', 0)} sent, {c.get('out_received', 0)} reached the sink, {len(lost_out)} lost, {c.get('out_twice', 0)} twice", + f"- IMAP IDLE: {c.get('idle_notices', 0)} notices, {c.get('idle_drops', 0)} drops", + f"- Webmail: {c.get('web_requests', 0)} requests, {c.get('web_failed', 0)} failed, {c.get('web_cut_off', 0)} streams cut off", + f"- Restarts: {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos", + f"- Chaos: " + ", ".join(f"{w['scenario']} {w.get('result', '?')}" for w in windows), "", + "| Day | |", "|---|---|"] + for dnum in days: + first = open(os.path.join(STATE_DIR, f"day-{dnum}.md")).readline().strip("# \n") + lines.append(f"| {dnum} | {first} |") + text = "\n".join(lines) + "\n" + open(os.path.join(STATE_DIR, "report.md"), "w").write(text) + print(text) + return ok + + +if __name__ == "__main__": + what = sys.argv[1] + if what == "collect": + collect() + elif what == "report": + sys.exit(0 if report(int(sys.argv[2])) else 1) + elif what == "final": + sys.exit(0 if final() else 1) + else: + sys.exit(__doc__) diff --git a/lab/soak/soak.sh b/lab/soak/soak.sh new file mode 100755 index 0000000..41d3cc3 --- /dev/null +++ b/lab/soak/soak.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L7: the soak. Steady load for SOAK_DAYS days (7) on the lab's seeded +# install, one chaos scenario a day, a report at the end of each day and +# one for the whole run. +# +# lab/soak/soak.sh start checks the install, then runs in the +# background (a systemd user unit) +# lab/soak/soak.sh status where it's up to, and today's numbers so far +# lab/soak/soak.sh stop +# lab/soak/soak.sh report the reports written so far +# +# The load (soak-load.py) comes from this host through the load balancer: +# mail in, mail out to the sink, IMAP IDLE. The webmail tabs +# (chart/webmail-tabs.py) run in a pod, as there's no ingress. Chaos is +# chart/chaos-test.sh with one scenario a day, CHAOS_AFTER seconds into the +# day (2 h). Reports and state go to SOAK_DIR; a run is resumed, not +# restarted, by `start` after a stop or a reboot, until it's done. +# +# A shorter run, to try the harness: SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300 +# +# The unit stops when its user logs out unless lingering is on +# (`loginctl enable-linger`), which is the machine owner's call. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +. "$HERE/../versions.env" +export KCTX="${KCTX:-kind-$KIND_CLUSTER}" +K=(kubectl --context "$KCTX") +LAB_STATE="${LAB_STATE:-$HOME/.cache/inbuxa-lab}" +export STATE="${SOAK_DIR:-$LAB_STATE/soak}" +export SOAK_DAY="${SOAK_DAY:-86400}" +SOAK_DAYS="${SOAK_DAYS:-7}" +CHAOS_AFTER="${CHAOS_AFTER:-7200}" +SCENARIOS=(pod-kill drain pg-failover nats-restart s3-outage pod-kill drain) +UNIT=inbuxa-soak +fail() { echo "!! $*" >&2; exit 1; } +lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } + +tabs_pod() { + "${K[@]}" create namespace lab-soak --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" label namespace lab-soak pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null + "${K[@]}" -n lab-soak create configmap soak-tabs --from-file="$HERE/../chart/webmail-tabs.py" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" -n lab-soak create secret generic soak-tabs --from-file=password="$LAB_STATE/alice.password" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" -n lab-soak apply -f - >/dev/null </dev/null +} + +start() { + systemctl --user is-active --quiet "$UNIT" && fail "already running: $0 status" + mkdir -p "$STATE" + if [ -f "$STATE/started" ] && [ -f "$STATE/report.md" ]; then + fail "the run in $STATE is finished; move it away to start another" + fi + if [ ! -f "$STATE/started" ]; then + echo "== every client check first (it also routes sink.test to the sink)" + PORTS="${PORTS:-18080}" STATE="$LAB_STATE" "$HERE/../chart/mailcheck.sh" >"$STATE/mailcheck.log" 2>&1 \ + || { tail -20 "$STATE/mailcheck.log"; fail "the install isn't healthy; not starting"; } + date +%s >"$STATE/started" + fi + echo "== the webmail tabs" + tabs_pod + systemd-run --user --unit "$UNIT" --collect --quiet \ + --setenv=KCTX="$KCTX" --setenv=SOAK_DIR="$STATE" --setenv=SOAK_DAY="$SOAK_DAY" \ + --setenv=SOAK_DAYS="$SOAK_DAYS" --setenv=CHAOS_AFTER="$CHAOS_AFTER" --setenv=LAB_STATE="$LAB_STATE" \ + --setenv=PATH="$PATH" --setenv=HOME="$HOME" \ + bash "$HERE/soak.sh" run + echo "== running: $SOAK_DAYS day(s) of ${SOAK_DAY}s; $0 status" +} + +# The unit's body: never exits on a hiccup, only when the run is done +run() { + set +e + mkdir -p "$STATE" + local started load_pid="" day now since + started="$(cat "$STATE/started")" + stop_load() { [ -n "$load_pid" ] && kill "$load_pid" 2>/dev/null; wait "$load_pid" 2>/dev/null; } + trap 'stop_load; exit 0' TERM INT + while :; do + if [ -z "$load_pid" ] || ! kill -0 "$load_pid" 2>/dev/null; then + ACCOUNT=alice@lab.test PASSWORD_FILE="$LAB_STATE/alice.password" \ + python3 "$HERE/soak-load.py" "$(lb)" >>"$STATE/load.log" 2>&1 & + load_pid=$! + fi + python3 "$HERE/soak-watch.py" collect >>"$STATE/watch.log" 2>&1 + + now=$(date +%s) + day=$(( (now - started) / SOAK_DAY + 1 )) + since=$(( (now - started) % SOAK_DAY )) + # Yesterday's report, once + if [ "$day" -gt 1 ] && [ ! -f "$STATE/day-$((day - 1)).md" ]; then + python3 "$HERE/soak-watch.py" report "$((day - 1))" >/dev/null 2>>"$STATE/watch.log" + fi + if [ "$day" -gt "$SOAK_DAYS" ]; then + python3 "$HERE/soak-watch.py" final >/dev/null 2>>"$STATE/watch.log" + stop_load + "${K[@]}" -n lab-soak scale deploy/soak-tabs --replicas=0 >/dev/null 2>&1 + exit 0 + fi + # Today's chaos, once + if [ "$since" -ge "$CHAOS_AFTER" ] && [ ! -f "$STATE/chaos-day-$day.log" ]; then + local sc="${SCENARIOS[$(( (day - 1) % ${#SCENARIOS[@]} ))]}" t0 rc + t0=$(date +%s) + printf '{"day": %d, "scenario": "%s", "start": %d}\n' "$day" "$sc" "$t0" >>"$STATE/chaos.jsonl" + SCENARIOS="$sc" STATE="$LAB_STATE" PORTS=18180 "$HERE/../chart/chaos-test.sh" >"$STATE/chaos-day-$day.log" 2>&1 + rc=$? + python3 - "$STATE/chaos.jsonl" "$day" "$rc" "$(date +%s)" <<'PY' +import json, sys +path, day, rc, end = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]) +rows = [json.loads(l) for l in open(path) if l.strip()] +for r in rows: + if r["day"] == day and "end" not in r: + r["end"], r["result"] = end, "ok" if rc == 0 else f"FAILED (exit {rc}, see chaos-day-{day}.log)" +open(path, "w").write("".join(json.dumps(r) + "\n" for r in rows)) +PY + fi + sleep 60 + done +} + +status() { + if systemctl --user is-active --quiet "$UNIT"; then echo "== running"; else echo "== not running"; fi + [ -f "$STATE/started" ] || { echo " no run in $STATE"; return 0; } + local started now; started="$(cat "$STATE/started")"; now=$(date +%s) + echo " day $(( (now - started) / SOAK_DAY + 1 )) of $SOAK_DAYS, $(( (now - started) % SOAK_DAY / 60 )) min in" + python3 - "$STATE" <<'PY' +import json, os, sys +d = sys.argv[1] +L = json.load(open(os.path.join(d, "load.json"))) +W = json.load(open(os.path.join(d, "watch.json"))) if os.path.exists(os.path.join(d, "watch.json")) else {"out": {}, "tabs": {}} +i, o, idle = L["inbound"], L["outbound"], L["idle"] +print(f" in: {i['sent']} sent, {i['arrived']} arrived, {len(L['pending'])} pending, {i['twice']} twice") +print(f" out: {o['sent']} sent, {len(W['out'])} at the sink") +print(f" idle: {idle['notices']} notices, {idle['drops']} drops") +t = list(W["tabs"].values()) +print(f" webmail: {sum(x.get('requests', 0) for x in t)} requests, {sum(x.get('failed', 0) for x in t)} failed") +PY + ls "$STATE"/day-*.md 2>/dev/null | sed 's/^/ /' || true +} + +case "${1:-}" in + start) start ;; + run) run ;; + status) status ;; + stop) systemctl --user stop "$UNIT" 2>/dev/null || true; echo "== stopped (start resumes it)" ;; + report) cat "$STATE"/day-*.md "$STATE"/report.md 2>/dev/null || echo "no reports yet" ;; + *) sed -n '4,25p' "$0"; exit 2 ;; +esac -- 2.54.0 From 25243e84fb9359fa1e4b4bd64f266198aee99d7d Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 01:07:31 -0700 Subject: [PATCH 2/3] report.sh for release-candidate testers (K-L9) Reads the cluster and the release and prints a block to paste into a report: Kubernetes version and distribution, nodes' OS, kernel, runtime and architecture, network plugin, load balancer, Pod Security level and refusals, chart and image versions, each component's pods, warning events and any test results (--results). Never changes anything. No names by default: nodes, registries, addresses and hostnames are left out, and the block is scrubbed of anything shaped like an IPv4 or IPv6 address, a MAC or a hostname (times, versions and file names survive). --include-names keeps them. README covers the soak and the report. --- README.md | 25 ++++++++ lab/report.sh | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100755 lab/report.sh diff --git a/README.md b/README.md index c0c598c..1e26eec 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,31 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`). +`lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs +and checks no request failed and every live-update stream came back. + +`lab/soak/soak.sh start` is the soak: a week (`SOAK_DAYS`) of steady load +on an installed chart, with a different chaos scenario each day. Mail in +every 10 s and out to the sink every minute, two IMAP clients in IDLE (all +from this host, through the load balancer), and three webmail tabs in a pod. +Each day ends with a report (`day-N.md` in `~/.cache/inbuxa-lab/soak`): +mail sent, arrived, late, lost or doubled, IDLE notices and drops, webmail +failures, and every pod restart or replacement, planned (during chaos) or +not. `report.md` sums up the run. It passes with no message lost or +doubled and no unplanned restart. It runs as a systemd user unit, so +`soak.sh status` and `stop` work from any shell, and `start` resumes a +stopped run. `SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300` tries it in an +hour. + +`lab/report.sh` is for release-candidate testers on their own cluster: the +Kubernetes version and distribution, the nodes' OS, kernel, runtime and +architecture, the network plugin and load balancer, the Pod Security level +and anything it refused, the chart and image versions, each component's +pods, and any test results (`--results FILE`), as a block to paste into a +report. It only reads. Names, registries, addresses and hostnames are left +out, and the block is scrubbed of anything shaped like an IP address or a +hostname, unless `--include-names`. + `lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB, and the data layer (CloudNativePG, NATS, Garage). It adds the Helm repositories it needs; the versions are pinned in `lab/versions.env`. diff --git a/lab/report.sh b/lab/report.sh new file mode 100755 index 0000000..29569d7 --- /dev/null +++ b/lab/report.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L9: what a release-candidate tester's cluster is, and how the chart +# did on it, as a block to paste into an issue or forum post. +# +# lab/report.sh [--context NAME] [-n NAMESPACE] [-r RELEASE] +# [--results FILE]... [--include-names] +# +# Reads, never changes: the Kubernetes version and distribution, the nodes' +# OS, kernel, runtime and architecture, the network plugin, the load +# balancer, the namespace's Pod Security level and anything it refused, +# the chart and image versions, each component's pods (ready, restarts), +# warning events, and the test results given with --results (the lab's +# e2e summary by default, if there is one). +# +# No names: nodes, the namespace, the release, registries, addresses and +# hostnames are left out, and the whole block is scrubbed of anything +# shaped like an IP address or a hostname before it's printed. +# --include-names keeps them, for a private report. +set -euo pipefail + +CTX=() NS=inbuxa RELEASE=inbuxa NAMES=0 RESULTS=() +while [ $# -gt 0 ]; do + case "$1" in + --context) CTX=(--context "$2"); shift 2 ;; + -n|--namespace) NS="$2"; shift 2 ;; + -r|--release) RELEASE="$2"; shift 2 ;; + --results) RESULTS+=("$2"); shift 2 ;; + --include-names) NAMES=1; shift ;; + -h|--help) sed -n '4,21p' "$0"; exit 0 ;; + *) echo "unknown option $1 (--help)" >&2; exit 2 ;; + esac +done +if [ ${#RESULTS[@]} -eq 0 ] && [ -f "$HOME/.cache/inbuxa-lab/e2e-summary.md" ]; then + RESULTS=("$HOME/.cache/inbuxa-lab/e2e-summary.md") +fi +command -v kubectl >/dev/null || { echo "needs kubectl" >&2; exit 1; } +K=(kubectl "${CTX[@]}") +HELM_CTX=() +[ ${#CTX[@]} -gt 0 ] && HELM_CTX=(--kube-context "${CTX[1]}") + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +"${K[@]}" version -o json >"$WORK/version.json" 2>/dev/null || echo '{}' >"$WORK/version.json" +"${K[@]}" get nodes -o json >"$WORK/nodes.json" +"${K[@]}" get pods -A -o json >"$WORK/pods.json" +"${K[@]}" get daemonsets -A -o json >"$WORK/ds.json" +"${K[@]}" get namespace "$NS" -o json >"$WORK/ns.json" 2>/dev/null || echo '{}' >"$WORK/ns.json" +"${K[@]}" -n "$NS" get services -o json >"$WORK/svc.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/svc.json" +"${K[@]}" -n "$NS" get events -o json >"$WORK/events.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/events.json" +if command -v helm >/dev/null; then + helm "${HELM_CTX[@]}" -n "$NS" list -o json >"$WORK/helm.json" 2>/dev/null || echo '[]' >"$WORK/helm.json" +else + echo '[]' >"$WORK/helm.json" +fi +: >"$WORK/results.md" +for f in "${RESULTS[@]}"; do { echo; echo "From \`$(basename "$f")\`:"; echo; cat "$f"; } >>"$WORK/results.md"; done + +python3 - "$WORK" "$NS" "$RELEASE" "$NAMES" <<'PY' +import collections, json, os, re, sys + +work, ns, release, names = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1" +J = lambda f: json.load(open(os.path.join(work, f))) +version, nodes, pods, ds = J("version.json"), J("nodes.json")["items"], J("pods.json")["items"], J("ds.json")["items"] +nsobj, svcs, events, helm = J("ns.json"), J("svc.json")["items"], J("events.json")["items"], J("helm.json") + +server = version.get("serverVersion", {}).get("gitVersion", "?") +distro = next((d for pat, d in [(r"\+k3s", "k3s"), (r"\+rke2", "RKE2"), (r"-eks-", "EKS"), (r"-gke\.", "GKE"), + (r"\+vmware", "Tanzu"), (r"\+k0s", "k0s"), (r"-aks", "AKS")] if re.search(pat, server)), None) +images = " ".join(c.get("image", "") for p in pods for c in p["spec"].get("containers", [])) +if not distro: + if any(n["metadata"].get("labels", {}).get("node.openshift.io/os_id") for n in nodes): + distro = "OpenShift" + elif re.search(r"kindest/|kindnet", images): + distro = "kind" + elif any("talos" in n["status"]["nodeInfo"].get("osImage", "").lower() for n in nodes): + distro = "Talos" + elif re.search(r"microk8s|calico.*snap", images): + distro = "MicroK8s" + else: + distro = "upstream or unrecognised" + +def nodeinfo(n): + i = n["status"]["nodeInfo"] + roles = sorted(k.split("/", 1)[1] for k in n["metadata"].get("labels", {}) if k.startswith("node-role.kubernetes.io/")) or ["worker"] + return (",".join(roles), i.get("osImage", "?"), i.get("kernelVersion", "?"), i.get("containerRuntimeVersion", "?"), i.get("architecture", "?")) +node_rows = collections.Counter(nodeinfo(n) for n in nodes) + +def present(*words): + hay = " ".join([p["metadata"]["name"] for p in pods] + [d["metadata"]["name"] for d in ds]).lower() + return [w for w in words if w.lower() in hay] +cni = present("cilium", "calico", "canal", "flannel", "kindnet", "weave", "antrea", "kube-router", "aws-node", "azure-cns", "ovnkube", "multus") or ["not recognised"] +lbs = present("metallb", "kube-vip", "svclb", "cilium-l2", "purelb", "openelb", "loxilb") + +def short_image(image): + """repository basename and tag or digest; no registry host.""" + name, _, digest = image.partition("@") + repo, tag = (name.rsplit(":", 1) + [""])[:2] if ":" in name.rsplit("/", 1)[-1] else (name, "") + base = image if names else repo.rsplit("/", 1)[-1] + (f":{tag}" if tag else "") + (f"@{digest[:19]}…" if digest else "") + return base + +ours = [p for p in pods if p["metadata"]["namespace"] == ns] +by_comp = collections.OrderedDict() +for p in sorted(ours, key=lambda p: p["metadata"].get("labels", {}).get("app.kubernetes.io/name", "") + p["metadata"]["name"]): + labels = p["metadata"].get("labels", {}) + comp = labels.get("app.kubernetes.io/component", "?") + owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "") + # The server's role groups are Deployments named - + name = p["metadata"]["name"] + group = re.sub(r"-[a-z0-9]{6,10}-[a-z0-9]{5}$|-[a-z0-9]{5}$", "", name) + group = group[len(release) + 1:] if group.startswith(release + "-") and not names else group + key = (comp, group, owner) + statuses = p["status"].get("containerStatuses", []) or [] + e = by_comp.setdefault(key, {"pods": 0, "ready": 0, "restarts": 0, "phase": collections.Counter(), "images": set(), "why": set()}) + e["pods"] += 1 + e["ready"] += all(c.get("ready") for c in statuses) and bool(statuses) + e["restarts"] += sum(c.get("restartCount", 0) for c in statuses) + e["phase"][p["status"].get("phase", "?")] += 1 + e["images"].update(short_image(c["image"]) for c in p["spec"]["containers"]) + for c in statuses: + r = (c.get("lastState", {}).get("terminated") or {}).get("reason") + if r: + e["why"].add(r) + +psa = {k.rsplit("/", 1)[1]: v for k, v in nsobj.get("metadata", {}).get("labels", {}).items() if k.startswith("pod-security.kubernetes.io/")} +refused = [e for e in events if "violates PodSecurity" in e.get("message", "") or "forbidden" in e.get("message", "").lower()] +warnings = collections.Counter(e.get("reason", "?") for e in events if e.get("type") == "Warning") +lbsvc = [s for s in svcs if s["spec"].get("type") == "LoadBalancer"] +rel = next((h for h in helm if h.get("name") == release), {}) + +L = ["### inbuxa on Kubernetes: test report", ""] +L += ["| | |", "|---|---|", + f"| Kubernetes | {server} ({distro}) |", + f"| Nodes | {len(nodes)} |", + f"| Network plugin | {', '.join(cni)} |", + f"| Load balancer | {', '.join(lbs) or 'cloud or not recognised'}" + + (f"; {len(lbsvc)} LoadBalancer service(s), class {', '.join(sorted({s['spec'].get('loadBalancerClass') or 'default' for s in lbsvc}))}, " + f"{sum(1 for s in lbsvc if s.get('status', {}).get('loadBalancer', {}).get('ingress'))} with an address" if lbsvc else "; no LoadBalancer service") + " |", + f"| Pod Security | {', '.join(f'{k}={v}' for k, v in sorted(psa.items())) or 'no labels on the namespace'}; " + f"{len(refused)} refusal event(s) |", + f"| Chart | {rel.get('chart', 'helm release not found')}, app {rel.get('app_version', '?')}, revision {rel.get('revision', '?')}, {rel.get('status', '?')} |", + ""] +L += ["| Roles | OS | Kernel | Runtime | Arch | Nodes |", "|---|---|---|---|---|---|"] +L += [f"| {r} | {o} | {k} | {rt} | {a} | {c} |" for (r, o, k, rt, a), c in sorted(node_rows.items())] +L += ["", "| Component | Group | Pods | Ready | Restarts | Images |", "|---|---|---|---|---|---|"] +for (comp, group, owner), e in by_comp.items(): + extra = f" ({', '.join(sorted(e['why']))})" if e["why"] else "" + phases = "" if set(e["phase"]) <= {"Running"} else " " + ", ".join(f"{v} {k}" for k, v in e["phase"].items()) + L.append(f"| {comp} | {group} ({owner}) | {e['pods']}{phases} | {e['ready']} | {e['restarts']}{extra} | {', '.join(sorted(e['images']))} |") +if warnings: + L += ["", "Warning events in the namespace (kept about an hour): " + ", ".join(f"{k} ×{v}" for k, v in warnings.most_common())] +results = open(os.path.join(work, "results.md")).read().strip() +if results: + L += ["", "#### Test results", "", results] +text = "\n".join(L) + "\n" + +if not names: + # Belt and braces: nothing shaped like an address or a hostname leaves + text = re.sub(r"\b(?:\d{1,3}\.){3}\d{1,3}(?:/\d+)?\b", "[ip]", text) + # IPv6 (and MACs): hex groups with "::", or five groups or more; a time + # like 12:34:56 has three + text = re.sub(r"(?= 4 else m.group(0), text, flags=re.I) + # name.domain.tld; versions (1.2.3), file names and image tags don't match + text = re.sub(r"\b(?=[a-z0-9-]*[a-z][a-z0-9-]*\.)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\b(?![\w-])", + lambda m: m.group(0) if re.fullmatch(r"[\w-]+\.(md|json|yaml|sh|py|txt|log)", m.group(0)) else "[host]", + text, flags=re.I) + if ns not in ("inbuxa", "default"): + text = text.replace(ns, "[namespace]") +print(text) +PY -- 2.54.0 From 07add6a662a52fc783bb0ee202ef14f10339498b Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 02:07:24 -0700 Subject: [PATCH 3/3] Soak: stay under the server's sending limit; a failed chaos scenario fails the run The trial (two 30-minute days) found both. Outbound mail went to one address, the same one mailcheck.py uses, and ran into the default limit of 25 an hour from one sender domain to one recipient; the drain day's check afterwards was refused for it. Outbound now goes to soak-0 ... soak-9@sink.test in turn. And the run read PASSED with that day's chaos FAILED: a day is clean, and a run passes, only when every scenario recovered. --- README.md | 2 +- lab/soak/soak-load.py | 9 ++++++--- lab/soak/soak-watch.py | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1e26eec..e8e72a8 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Each day ends with a report (`day-N.md` in `~/.cache/inbuxa-lab/soak`): mail sent, arrived, late, lost or doubled, IDLE notices and drops, webmail failures, and every pod restart or replacement, planned (during chaos) or not. `report.md` sums up the run. It passes with no message lost or -doubled and no unplanned restart. It runs as a systemd user unit, so +doubled, no unplanned restart, and every chaos scenario recovered. It runs as a systemd user unit, so `soak.sh status` and `stop` work from any shell, and `start` resumes a stopped run. `SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300` tries it in an hour. diff --git a/lab/soak/soak-load.py b/lab/soak/soak-load.py index bd1136f..e348d24 100755 --- a/lab/soak/soak-load.py +++ b/lab/soak/soak-load.py @@ -13,7 +13,10 @@ where it left off when started again: everything it knows is in STATE. deleted, so the mailbox stays small; a number seen again after that came twice outbound a message every OUT_EVERY s (60) over submission 465 with AUTH, - to probe@sink.test; soak.sh checks the sink received each once + to soak-0@sink.test ... soak-9@sink.test in turn (the server + allows 25 an hour from one sender domain to one recipient, and + probe@sink.test is left to chart/mailcheck.py); soak-watch.py + checks the sink received each once idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're told of new mail, and how often they drop. They watch the mailbox the soak's mail lands in (verify finds it: unknown @@ -34,7 +37,7 @@ state_dir = os.environ["STATE"] IN_EVERY = float(os.environ.get("IN_EVERY", "10")) OUT_EVERY = float(os.environ.get("OUT_EVERY", "60")) IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2")) -SINK = os.environ.get("SINK_RCPT", "probe@sink.test") +SINK_SPREAD = int(os.environ.get("SINK_SPREAD", "10")) tls = ssl.create_default_context() tls.check_hostname = False tls.verify_mode = ssl.CERT_NONE @@ -117,7 +120,7 @@ def inbound(): def outbound(): with lock: n = S["outbound"]["next"] - msg = message(account, SINK, f"soak {run} out {n}") + msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}") try: with smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) as s: s.sock.settimeout(300) diff --git a/lab/soak/soak-watch.py b/lab/soak/soak-watch.py index 2a0a899..8d6c56f 100755 --- a/lab/soak/soak-watch.py +++ b/lab/soak/soak-watch.py @@ -203,7 +203,8 @@ def report(day): restarts = [e for e in evs if e["kind"] in ("restart", "pod-new")] unplanned = [e for e in restarts if not in_chaos(e["at"], chaos_windows())] twice = d.get("in_twice", 0) + d.get("out_twice", 0) - ok = not lost_in and not lost_out and not twice and not unplanned + chaos_ok = all(w.get("result") == "ok" for w in windows) + ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok fmt = lambda t: time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(t)) lines = [ f"# Soak day {day}: {'clean' if ok else 'NOT clean'}", @@ -253,7 +254,8 @@ def final(): restarts = [e for e in events_between(started, time.time()) if e["kind"] in ("restart", "pod-new")] unplanned = [e for e in restarts if not in_chaos(e["at"], windows)] twice = c.get("in_twice", 0) + c.get("out_twice", 0) - ok = not lost_in and not lost_out and not twice and not unplanned + chaos_ok = all(w.get("result") == "ok" for w in windows) + ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok lines = [f"# Soak: {'PASSED' if ok else 'FAILED'}", "", f"{len(days)} day(s) from {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(started))}, context `{KCTX}`.", "", f"- Mail in: {c.get('in_sent', 0)} sent, {c.get('in_arrived', 0)} arrived, {len(lost_in)} lost, {c.get('in_twice', 0)} twice", -- 2.54.0