diff --git a/README.md b/README.md index c0c598c..e8e72a8 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,31 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`). +`lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs +and checks no request failed and every live-update stream came back. + +`lab/soak/soak.sh start` is the soak: a week (`SOAK_DAYS`) of steady load +on an installed chart, with a different chaos scenario each day. Mail in +every 10 s and out to the sink every minute, two IMAP clients in IDLE (all +from this host, through the load balancer), and three webmail tabs in a pod. +Each day ends with a report (`day-N.md` in `~/.cache/inbuxa-lab/soak`): +mail sent, arrived, late, lost or doubled, IDLE notices and drops, webmail +failures, and every pod restart or replacement, planned (during chaos) or +not. `report.md` sums up the run. It passes with no message lost or +doubled, no unplanned restart, and every chaos scenario recovered. It runs as a systemd user unit, so +`soak.sh status` and `stop` work from any shell, and `start` resumes a +stopped run. `SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300` tries it in an +hour. + +`lab/report.sh` is for release-candidate testers on their own cluster: the +Kubernetes version and distribution, the nodes' OS, kernel, runtime and +architecture, the network plugin and load balancer, the Pod Security level +and anything it refused, the chart and image versions, each component's +pods, and any test results (`--results FILE`), as a block to paste into a +report. It only reads. Names, registries, addresses and hostnames are left +out, and the block is scrubbed of anything shaped like an IP address or a +hostname, unless `--include-names`. + `lab/up.sh` is just the cluster: kind, Kyverno and the policies, MetalLB, and the data layer (CloudNativePG, NATS, Garage). It adds the Helm repositories it needs; the versions are pinned in `lab/versions.env`. diff --git a/lab/chart/webmail-tabs.py b/lab/chart/webmail-tabs.py index d9dade2..7bfd214 100755 --- a/lab/chart/webmail-tabs.py +++ b/lab/chart/webmail-tabs.py @@ -17,14 +17,21 @@ and prints one JSON line: requests made and failed (with the errors), and per tab how its streams ended (cleanly, or cut off mid-stream) and the longest wait for a stream to deliver again. A clean end is what the browser takes in its stride; a failed request is what shows an error. + +For the soak (K-L7): INTERVAL sets the time between requests (0.2 s); +SUMMARY_EVERY (seconds) prints that line every so often while running, +with "at" and the errors since the last one; with no STOP_FILE it runs +until stopped. """ import http.client, json, os, sys, threading, time, urllib.parse webmail, server = sys.argv[1:3] account = os.environ["ACCOUNT"] password = open(os.environ["PASSWORD_FILE"]).read().strip() -stop = os.environ["STOP_FILE"] +stop = os.environ.get("STOP_FILE", "/nonexistent/stop") TABS = int(os.environ.get("TABS", "3")) +INTERVAL = float(os.environ.get("INTERVAL", "0.2")) +SUMMARY_EVERY = float(os.environ.get("SUMMARY_EVERY", "0")) def request(base, method, path, headers=None, body=None, timeout=30): @@ -65,7 +72,7 @@ def sign_in(): session, mail_account = sign_in() H = {"cookie": f"ihm_session={session}"} lock = threading.Lock() -report = {"requests": 0, "failed": 0, "errors": [], "tabs": []} +report = {"requests": 0, "failed": 0, "signins": 1, "errors": [], "tabs": []} def tab(i): @@ -79,6 +86,10 @@ def tab(i): conn = http.client.HTTPConnection(u.hostname, u.port, timeout=45) conn.request("GET", "/api/events?types=*&closeafter=no&ping=30", headers=dict(H, accept="text/event-stream")) res = conn.getresponse() + if res.status == 401: # signed out: the request loop signs in again + conn.close() + time.sleep(1) + continue if res.status != 200: raise RuntimeError(f"HTTP {res.status}") me["streams"] += 1 @@ -107,19 +118,34 @@ for t in threads: body = json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], "methodCalls": [["Mailbox/get", {"accountId": mail_account, "properties": ["name"]}, "0"]]}) +next_summary = time.time() + SUMMARY_EVERY while not os.path.exists(stop): with lock: report["requests"] += 1 try: res, data = request(webmail, "POST", "/api/jmap", dict(H, **{"content-type": "application/json", "x-requested-with": "ihasmail"}), body) + if res.status == 401: + # The session ran out (12 h by default): sign in again, as the + # page sends the browser back to; the tabs pick up the new cookie + session, mail_account = sign_in() + H["cookie"] = f"ihm_session={session}" + with lock: + report["signins"] += 1 + report["requests"] -= 1 + continue if res.status != 200 or b"Mailbox/get" not in data: raise RuntimeError(f"HTTP {res.status} {data[:80]!r}") except Exception as e: with lock: report["failed"] += 1 report["errors"].append(f"request: {type(e).__name__}: {e}"[:160]) - time.sleep(0.2) + time.sleep(INTERVAL) + if SUMMARY_EVERY and time.time() >= next_summary: + next_summary += SUMMARY_EVERY + with lock: + print(json.dumps(dict(report, at=time.time(), errors=report["errors"][:15])), flush=True) + report["errors"] = [] for t in threads: t.join(timeout=50) diff --git a/lab/report.sh b/lab/report.sh new file mode 100755 index 0000000..29569d7 --- /dev/null +++ b/lab/report.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L9: what a release-candidate tester's cluster is, and how the chart +# did on it, as a block to paste into an issue or forum post. +# +# lab/report.sh [--context NAME] [-n NAMESPACE] [-r RELEASE] +# [--results FILE]... [--include-names] +# +# Reads, never changes: the Kubernetes version and distribution, the nodes' +# OS, kernel, runtime and architecture, the network plugin, the load +# balancer, the namespace's Pod Security level and anything it refused, +# the chart and image versions, each component's pods (ready, restarts), +# warning events, and the test results given with --results (the lab's +# e2e summary by default, if there is one). +# +# No names: nodes, the namespace, the release, registries, addresses and +# hostnames are left out, and the whole block is scrubbed of anything +# shaped like an IP address or a hostname before it's printed. +# --include-names keeps them, for a private report. +set -euo pipefail + +CTX=() NS=inbuxa RELEASE=inbuxa NAMES=0 RESULTS=() +while [ $# -gt 0 ]; do + case "$1" in + --context) CTX=(--context "$2"); shift 2 ;; + -n|--namespace) NS="$2"; shift 2 ;; + -r|--release) RELEASE="$2"; shift 2 ;; + --results) RESULTS+=("$2"); shift 2 ;; + --include-names) NAMES=1; shift ;; + -h|--help) sed -n '4,21p' "$0"; exit 0 ;; + *) echo "unknown option $1 (--help)" >&2; exit 2 ;; + esac +done +if [ ${#RESULTS[@]} -eq 0 ] && [ -f "$HOME/.cache/inbuxa-lab/e2e-summary.md" ]; then + RESULTS=("$HOME/.cache/inbuxa-lab/e2e-summary.md") +fi +command -v kubectl >/dev/null || { echo "needs kubectl" >&2; exit 1; } +K=(kubectl "${CTX[@]}") +HELM_CTX=() +[ ${#CTX[@]} -gt 0 ] && HELM_CTX=(--kube-context "${CTX[1]}") + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +"${K[@]}" version -o json >"$WORK/version.json" 2>/dev/null || echo '{}' >"$WORK/version.json" +"${K[@]}" get nodes -o json >"$WORK/nodes.json" +"${K[@]}" get pods -A -o json >"$WORK/pods.json" +"${K[@]}" get daemonsets -A -o json >"$WORK/ds.json" +"${K[@]}" get namespace "$NS" -o json >"$WORK/ns.json" 2>/dev/null || echo '{}' >"$WORK/ns.json" +"${K[@]}" -n "$NS" get services -o json >"$WORK/svc.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/svc.json" +"${K[@]}" -n "$NS" get events -o json >"$WORK/events.json" 2>/dev/null || echo '{"items":[]}' >"$WORK/events.json" +if command -v helm >/dev/null; then + helm "${HELM_CTX[@]}" -n "$NS" list -o json >"$WORK/helm.json" 2>/dev/null || echo '[]' >"$WORK/helm.json" +else + echo '[]' >"$WORK/helm.json" +fi +: >"$WORK/results.md" +for f in "${RESULTS[@]}"; do { echo; echo "From \`$(basename "$f")\`:"; echo; cat "$f"; } >>"$WORK/results.md"; done + +python3 - "$WORK" "$NS" "$RELEASE" "$NAMES" <<'PY' +import collections, json, os, re, sys + +work, ns, release, names = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] == "1" +J = lambda f: json.load(open(os.path.join(work, f))) +version, nodes, pods, ds = J("version.json"), J("nodes.json")["items"], J("pods.json")["items"], J("ds.json")["items"] +nsobj, svcs, events, helm = J("ns.json"), J("svc.json")["items"], J("events.json")["items"], J("helm.json") + +server = version.get("serverVersion", {}).get("gitVersion", "?") +distro = next((d for pat, d in [(r"\+k3s", "k3s"), (r"\+rke2", "RKE2"), (r"-eks-", "EKS"), (r"-gke\.", "GKE"), + (r"\+vmware", "Tanzu"), (r"\+k0s", "k0s"), (r"-aks", "AKS")] if re.search(pat, server)), None) +images = " ".join(c.get("image", "") for p in pods for c in p["spec"].get("containers", [])) +if not distro: + if any(n["metadata"].get("labels", {}).get("node.openshift.io/os_id") for n in nodes): + distro = "OpenShift" + elif re.search(r"kindest/|kindnet", images): + distro = "kind" + elif any("talos" in n["status"]["nodeInfo"].get("osImage", "").lower() for n in nodes): + distro = "Talos" + elif re.search(r"microk8s|calico.*snap", images): + distro = "MicroK8s" + else: + distro = "upstream or unrecognised" + +def nodeinfo(n): + i = n["status"]["nodeInfo"] + roles = sorted(k.split("/", 1)[1] for k in n["metadata"].get("labels", {}) if k.startswith("node-role.kubernetes.io/")) or ["worker"] + return (",".join(roles), i.get("osImage", "?"), i.get("kernelVersion", "?"), i.get("containerRuntimeVersion", "?"), i.get("architecture", "?")) +node_rows = collections.Counter(nodeinfo(n) for n in nodes) + +def present(*words): + hay = " ".join([p["metadata"]["name"] for p in pods] + [d["metadata"]["name"] for d in ds]).lower() + return [w for w in words if w.lower() in hay] +cni = present("cilium", "calico", "canal", "flannel", "kindnet", "weave", "antrea", "kube-router", "aws-node", "azure-cns", "ovnkube", "multus") or ["not recognised"] +lbs = present("metallb", "kube-vip", "svclb", "cilium-l2", "purelb", "openelb", "loxilb") + +def short_image(image): + """repository basename and tag or digest; no registry host.""" + name, _, digest = image.partition("@") + repo, tag = (name.rsplit(":", 1) + [""])[:2] if ":" in name.rsplit("/", 1)[-1] else (name, "") + base = image if names else repo.rsplit("/", 1)[-1] + (f":{tag}" if tag else "") + (f"@{digest[:19]}…" if digest else "") + return base + +ours = [p for p in pods if p["metadata"]["namespace"] == ns] +by_comp = collections.OrderedDict() +for p in sorted(ours, key=lambda p: p["metadata"].get("labels", {}).get("app.kubernetes.io/name", "") + p["metadata"]["name"]): + labels = p["metadata"].get("labels", {}) + comp = labels.get("app.kubernetes.io/component", "?") + owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "") + # The server's role groups are Deployments named - + name = p["metadata"]["name"] + group = re.sub(r"-[a-z0-9]{6,10}-[a-z0-9]{5}$|-[a-z0-9]{5}$", "", name) + group = group[len(release) + 1:] if group.startswith(release + "-") and not names else group + key = (comp, group, owner) + statuses = p["status"].get("containerStatuses", []) or [] + e = by_comp.setdefault(key, {"pods": 0, "ready": 0, "restarts": 0, "phase": collections.Counter(), "images": set(), "why": set()}) + e["pods"] += 1 + e["ready"] += all(c.get("ready") for c in statuses) and bool(statuses) + e["restarts"] += sum(c.get("restartCount", 0) for c in statuses) + e["phase"][p["status"].get("phase", "?")] += 1 + e["images"].update(short_image(c["image"]) for c in p["spec"]["containers"]) + for c in statuses: + r = (c.get("lastState", {}).get("terminated") or {}).get("reason") + if r: + e["why"].add(r) + +psa = {k.rsplit("/", 1)[1]: v for k, v in nsobj.get("metadata", {}).get("labels", {}).items() if k.startswith("pod-security.kubernetes.io/")} +refused = [e for e in events if "violates PodSecurity" in e.get("message", "") or "forbidden" in e.get("message", "").lower()] +warnings = collections.Counter(e.get("reason", "?") for e in events if e.get("type") == "Warning") +lbsvc = [s for s in svcs if s["spec"].get("type") == "LoadBalancer"] +rel = next((h for h in helm if h.get("name") == release), {}) + +L = ["### inbuxa on Kubernetes: test report", ""] +L += ["| | |", "|---|---|", + f"| Kubernetes | {server} ({distro}) |", + f"| Nodes | {len(nodes)} |", + f"| Network plugin | {', '.join(cni)} |", + f"| Load balancer | {', '.join(lbs) or 'cloud or not recognised'}" + + (f"; {len(lbsvc)} LoadBalancer service(s), class {', '.join(sorted({s['spec'].get('loadBalancerClass') or 'default' for s in lbsvc}))}, " + f"{sum(1 for s in lbsvc if s.get('status', {}).get('loadBalancer', {}).get('ingress'))} with an address" if lbsvc else "; no LoadBalancer service") + " |", + f"| Pod Security | {', '.join(f'{k}={v}' for k, v in sorted(psa.items())) or 'no labels on the namespace'}; " + f"{len(refused)} refusal event(s) |", + f"| Chart | {rel.get('chart', 'helm release not found')}, app {rel.get('app_version', '?')}, revision {rel.get('revision', '?')}, {rel.get('status', '?')} |", + ""] +L += ["| Roles | OS | Kernel | Runtime | Arch | Nodes |", "|---|---|---|---|---|---|"] +L += [f"| {r} | {o} | {k} | {rt} | {a} | {c} |" for (r, o, k, rt, a), c in sorted(node_rows.items())] +L += ["", "| Component | Group | Pods | Ready | Restarts | Images |", "|---|---|---|---|---|---|"] +for (comp, group, owner), e in by_comp.items(): + extra = f" ({', '.join(sorted(e['why']))})" if e["why"] else "" + phases = "" if set(e["phase"]) <= {"Running"} else " " + ", ".join(f"{v} {k}" for k, v in e["phase"].items()) + L.append(f"| {comp} | {group} ({owner}) | {e['pods']}{phases} | {e['ready']} | {e['restarts']}{extra} | {', '.join(sorted(e['images']))} |") +if warnings: + L += ["", "Warning events in the namespace (kept about an hour): " + ", ".join(f"{k} ×{v}" for k, v in warnings.most_common())] +results = open(os.path.join(work, "results.md")).read().strip() +if results: + L += ["", "#### Test results", "", results] +text = "\n".join(L) + "\n" + +if not names: + # Belt and braces: nothing shaped like an address or a hostname leaves + text = re.sub(r"\b(?:\d{1,3}\.){3}\d{1,3}(?:/\d+)?\b", "[ip]", text) + # IPv6 (and MACs): hex groups with "::", or five groups or more; a time + # like 12:34:56 has three + text = re.sub(r"(?= 4 else m.group(0), text, flags=re.I) + # name.domain.tld; versions (1.2.3), file names and image tags don't match + text = re.sub(r"\b(?=[a-z0-9-]*[a-z][a-z0-9-]*\.)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,}\b(?![\w-])", + lambda m: m.group(0) if re.fullmatch(r"[\w-]+\.(md|json|yaml|sh|py|txt|log)", m.group(0)) else "[host]", + text, flags=re.I) + if ns not in ("inbuxa", "default"): + text = text.replace(ns, "[namespace]") +print(text) +PY diff --git a/lab/soak/soak-load.py b/lab/soak/soak-load.py new file mode 100755 index 0000000..e348d24 --- /dev/null +++ b/lab/soak/soak-load.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""The soak's steady mail load (K-L7), from outside the cluster through its +load balancer, as real clients reach it. Runs until killed, and picks up +where it left off when started again: everything it knows is in STATE. + + ACCOUNT=user@domain PASSWORD_FILE=... STATE=dir soak-load.py LB + + inbound a message every IN_EVERY s (10) over SMTP 25, each from its own + sender domain, retried like an MTA when refused + verify every 30 s, over IMAPS: which arrived (any mailbox), then + deleted, so the mailbox stays small; a number seen again after + that came twice + outbound a message every OUT_EVERY s (60) over submission 465 with AUTH, + to soak-0@sink.test ... soak-9@sink.test in turn (the server + allows 25 an hour from one sender domain to one recipient, and + probe@sink.test is left to chart/mailcheck.py); soak-watch.py + checks the sink received each once + idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're + told of new mail, and how often they drop. They watch the + mailbox the soak's mail lands in (verify finds it: unknown + senders' mail goes to Junk Mail), as a client watches INBOX + +STATE/load.json is rewritten every 30 s; STATE/events.jsonl gets a line +for anything that went wrong. The lab's certificate is self-signed, so TLS +isn't verified. +""" +import imaplib, json, os, re, smtplib, socket, ssl, sys, threading, time, uuid +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +lb = sys.argv[1] +account = os.environ["ACCOUNT"] +password = open(os.environ["PASSWORD_FILE"]).read().strip() +state_dir = os.environ["STATE"] +IN_EVERY = float(os.environ.get("IN_EVERY", "10")) +OUT_EVERY = float(os.environ.get("OUT_EVERY", "60")) +IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2")) +SINK_SPREAD = int(os.environ.get("SINK_SPREAD", "10")) +tls = ssl.create_default_context() +tls.check_hostname = False +tls.verify_mode = ssl.CERT_NONE +lock = threading.RLock() # event() and save() are called with it held +STATE = os.path.join(state_dir, "load.json") +EVENTS = os.path.join(state_dir, "events.jsonl") + +try: + S = json.load(open(STATE)) +except (OSError, ValueError): + S = {"run": uuid.uuid4().hex[:8], "started": time.time(), + "inbound": {"next": 1, "sent": 0, "retries": 0, "arrived": 0, "twice": 0}, + "pending": {}, "arrived_late": 0, "slowest_s": 0, + "outbound": {"next": 1, "sent": 0, "retries": 0}, "out_sent": {}, + "idle": {"notices": 0, "drops": 0, "longest_quiet_s": 0}, + "seen": {}, "starts": 0} +S["starts"] += 1 +run = S["run"] + + +def event(kind, detail): + with lock: + with open(EVENTS, "a") as f: + f.write(json.dumps({"at": time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n") + + +def save(): + with lock: + # A duplicate a day late is still caught; older arrivals are dropped + old = time.time() - 2 * 86400 + S["seen"] = {n: v for n, v in S.get("seen", {}).items() if v[1] > old} + tmp = STATE + ".tmp" + with open(tmp, "w") as f: + json.dump(S, f) + os.replace(tmp, STATE) + + +def message(frm, to, subject): + msg = EmailMessage() + msg["From"], msg["To"], msg["Subject"] = frm, to, subject + msg["Date"] = formatdate() + msg["Message-ID"] = make_msgid(domain=f"{run}.soak.test") + msg.set_content(subject + "\n") + return msg + + +def every(seconds, fn): + """fn() every `seconds`, keeping to the schedule; fn handles its errors.""" + nxt = time.time() + while True: + fn() + nxt += seconds + time.sleep(max(0, nxt - time.time())) + + +def inbound(): + with lock: + n = S["inbound"]["next"] + # A sender domain per message: the default throttle allows 25 an hour + # from one sender domain to one recipient + msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}") + try: + # As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py + with smtplib.SMTP(timeout=20) as s: + s.connect(lb, 25) + s.sock.settimeout(300) + s.send_message(msg) + except (OSError, smtplib.SMTPException) as e: + with lock: + S["inbound"]["retries"] += 1 + event("inbound-retry", f"{n}: {type(e).__name__}: {e}") + return # the same number goes again next time + with lock: + S["inbound"]["next"] = n + 1 + S["inbound"]["sent"] += 1 + S["pending"][str(n)] = time.time() + save() # at once: a restart must never send a number twice + + +def outbound(): + with lock: + n = S["outbound"]["next"] + msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}") + try: + with smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) as s: + s.sock.settimeout(300) + s.login(account, password) + s.send_message(msg) + except (OSError, smtplib.SMTPException) as e: + with lock: + S["outbound"]["retries"] += 1 + event("outbound-retry", f"{n}: {type(e).__name__}: {e}") + return + with lock: + S["outbound"]["next"] = n + 1 + S["outbound"]["sent"] += 1 + S["out_sent"][str(n)] = time.time() + save() + + +SUBJECT = re.compile(rf"soak {run} in (\d+)") + + +def verify(): + try: + m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=60) + except OSError as e: + event("verify", f"connect: {e}") + return + try: + m.login(account, password) + _, boxes = m.list() + for line in boxes: + name = line.decode().rsplit(' "/" ', 1)[-1].strip() + if m.select(name)[0] != "OK": + continue + _, data = m.search(None, "SUBJECT", f'"soak {run} in"') + ids = data[0].split() + if not ids: + continue + _, hdrs = m.fetch(b",".join(ids), "(UID BODY.PEEK[HEADER.FIELDS (SUBJECT)])") + now = time.time() + for part in hdrs: + if not isinstance(part, tuple): + continue + mm = SUBJECT.search(part[1].decode(errors="replace")) + uid = re.search(rb"UID (\d+)", part[0]) + if not mm or not uid: + continue + n, copy = mm.group(1), f"{name}:{uid.group(1).decode()}" + with lock: + sent = S["pending"].pop(n, None) + if sent is not None: + S["inbound"]["arrived"] += 1 + S["seen"][n] = [copy, now] + S["landed"] = name + took = now - sent + S["slowest_s"] = max(S["slowest_s"], round(took)) + if took > 900: + S["arrived_late"] += 1 + elif S["seen"].get(n, [None])[0] != copy: + S["inbound"]["twice"] += 1 + S["seen"][n] = [copy, now] + event("twice", f"in {n} arrived again ({name})") + # else: this very copy, left by a delete that was cut short + # What arrived is on disk before it's deleted, so a restart in + # between neither loses it nor counts it twice + save() + m.store(b",".join(ids), "+FLAGS", "\\Deleted") + m.expunge() + except (OSError, imaplib.IMAP4.error) as e: + event("verify", f"{type(e).__name__}: {e}") + finally: + try: + m.logout() + except Exception: + pass + save() + + +def idle(i): + """One client in IDLE, as a desktop mail app holds it; re-issued every + 25 minutes (RFC 2177 asks for under 29).""" + while not S.get("landed"): + time.sleep(5) + last_notice = time.time() + while True: + sock = None + try: + raw = socket.create_connection((lb, 993), timeout=30) + sock = tls.wrap_socket(raw, server_hostname="mail.lab.test") + f = sock.makefile("rb") + + def cmd(tag, text): + sock.sendall(f"{tag} {text}\r\n".encode()) + while True: + line = f.readline() + if not line: + raise ConnectionError("closed") + if line.startswith(tag.encode() + b" "): + if b" OK" not in line[:12]: + raise ConnectionError(line.decode(errors="replace").strip()) + return + + f.readline() # greeting + cmd("a1", f'LOGIN "{account}" "{password}"') + cmd("a2", f"SELECT {S['landed']}") + n = 3 + while True: + tag = f"a{n}" + n += 1 + sock.sendall(f"{tag} IDLE\r\n".encode()) + sock.settimeout(60) + renew = time.time() + 25 * 60 + while time.time() < renew: + try: + line = f.readline() + except socket.timeout: + continue + if not line: + raise ConnectionError("closed in IDLE") + if line.endswith(b"EXISTS\r\n"): + now = time.time() + with lock: + S["idle"]["notices"] += 1 + S["idle"]["longest_quiet_s"] = max(S["idle"]["longest_quiet_s"], round(now - last_notice)) + last_notice = now + sock.sendall(b"DONE\r\n") + sock.settimeout(30) + while not f.readline().startswith(tag.encode() + b" "): + pass + except (OSError, ConnectionError, ssl.SSLError) as e: + with lock: + S["idle"]["drops"] += 1 + event("idle-drop", f"client {i}: {type(e).__name__}: {e}") + finally: + if sock: + try: + sock.close() + except OSError: + pass + time.sleep(5) + + +save() +event("start", f"run {run}, start {S['starts']}") +for target in ([lambda: every(IN_EVERY, inbound), lambda: every(OUT_EVERY, outbound), lambda: every(30, verify)] + + [(lambda i=i: idle(i)) for i in range(IDLE_CLIENTS)]): + threading.Thread(target=target, daemon=True).start() +while True: + time.sleep(30) + save() diff --git a/lab/soak/soak-watch.py b/lab/soak/soak-watch.py new file mode 100755 index 0000000..8d6c56f --- /dev/null +++ b/lab/soak/soak-watch.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""What the soak (K-L7) sees, and its reports. + + soak-watch.py collect once a minute, from soak.sh + soak-watch.py report DAY that day's report, STATE/day-DAY.md + soak-watch.py final the whole run, STATE/report.md + +collect reads, with kubectl (KCTX): + - the sink's log: every message soak-load.py sent out, counted once per + delivery (by the sink's own receive time) + - the webmail tabs' summaries (chart/webmail-tabs.py, one pod or several + over the run, summed) + - every pod in the inbuxa, data, lab-sink and lab-soak namespaces: a pod + that's new, or a container that restarted, is an event, planned when + it falls in a chaos window (STATE/chaos.jsonl), unplanned otherwise + +Everything is kept in STATE/watch.json, events in STATE/events.jsonl +beside soak-load.py's. Reports compare against the snapshot the previous +day's report took, so each day stands on its own. +""" +import json, os, re, subprocess, sys, time + +STATE_DIR = os.environ["STATE"] +KCTX = os.environ["KCTX"] +DAY = int(os.environ.get("SOAK_DAY", "86400")) +NAMESPACES = ("inbuxa", "data", "lab-sink", "lab-soak") +WATCH = os.path.join(STATE_DIR, "watch.json") +EVENTS = os.path.join(STATE_DIR, "events.jsonl") +LOST_AFTER = int(os.environ.get("LOST_AFTER", "3600")) # not there an hour after it was sent + + +def kubectl(*args): + return subprocess.run(["kubectl", "--context", KCTX, *args], capture_output=True, text=True, timeout=120).stdout + + +def load(path, default): + try: + return json.load(open(path)) + except (OSError, ValueError): + return default + + +def save(path, data): + tmp = path + ".tmp" + with open(tmp, "w") as f: + json.dump(data, f) + os.replace(tmp, path) + + +def event(kind, detail, at=None): + with open(EVENTS, "a") as f: + f.write(json.dumps({"at": at or time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n") + + +def new_lines(W, ns, selector, key): + """Log lines of every pod matching `selector` since we last looked.""" + out = [] + for pod in kubectl("-n", ns, "get", "pods", "-l", selector, "-o", "name").split(): + since = W["since"].get(f"{key}/{pod}") + args = ["-n", ns, "logs", pod, "--timestamps"] + (["--since-time", since] if since else []) + last = since + for line in kubectl(*args).splitlines(): + ts, _, text = line.partition(" ") + if since and ts <= since: + continue # --since-time is inclusive, to the second + out.append((pod, text)) + last = ts + if last: + W["since"][f"{key}/{pod}"] = last + return out + + +def collect(): + W = load(WATCH, {"since": {}, "out": {}, "tabs": {}, "pods": {}, "first": True}) + load_state = load(os.path.join(STATE_DIR, "load.json"), {}) + run = load_state.get("run") + + # The sink: each soak message that reached it + for _, text in new_lines(W, "lab-sink", "app=lab-sink", "sink"): + try: + rec = json.loads(text) + except ValueError: + continue + m = re.fullmatch(rf"soak {run} out (\d+)", rec.get("subject", "")) + if m: + W["out"][m.group(1)] = W["out"].get(m.group(1), 0) + 1 + if W["out"][m.group(1)] == 2: + event("out-twice", f"out {m.group(1)} reached the sink again") + + # The webmail tabs: the latest summary of each pod + for pod, text in new_lines(W, "lab-soak", "app=soak-tabs", "tabs"): + try: + rec = json.loads(text) + except ValueError: + continue + for e in rec.get("errors", []): + event("webmail", f"{pod.split('/')[-1]}: {e}", rec.get("at")) + rec.pop("errors", None) + W["tabs"][pod] = rec + + # The pods + now = time.time() + seen = {} + for ns in NAMESPACES: + for p in json.loads(kubectl("-n", ns, "get", "pods", "-o", "json") or '{"items": []}')["items"]: + uid = p["metadata"]["uid"] + name = f"{ns}/{p['metadata']['name']}" + statuses = p.get("status", {}).get("containerStatuses", []) or [] + restarts = sum(c.get("restartCount", 0) for c in statuses) + seen[uid] = {"name": name, "restarts": restarts} + before = W["pods"].get(uid) + if before is None and not W["first"]: + owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "none") + event("pod-new", f"{name} ({owner})", now) + elif before and restarts > before["restarts"]: + why = ", ".join(sorted({(c.get("lastState", {}).get("terminated") or {}).get("reason", "?") + for c in statuses if c.get("restartCount", 0)})) + event("restart", f"{name}: {restarts - before['restarts']} more ({why})", now) + for uid, p in W["pods"].items(): + if uid not in seen: + event("pod-gone", p["name"], now) + W["pods"], W["first"] = seen, False + save(WATCH, W) + + +def counters(): + """Every running total, flattened, for day-to-day differences.""" + L = load(os.path.join(STATE_DIR, "load.json"), {}) + W = load(WATCH, {"out": {}, "tabs": {}}) + tabs = list(W.get("tabs", {}).values()) + c = {f"in_{k}": v for k, v in L.get("inbound", {}).items() if k != "next"} + c.update({f"out_{k}": v for k, v in L.get("outbound", {}).items() if k != "next"}) + c.update({f"idle_{k}": v for k, v in L.get("idle", {}).items() if k != "longest_quiet_s"}) + c["in_late"] = L.get("arrived_late", 0) + c["out_received"] = len(W.get("out", {})) + c["out_twice"] = sum(1 for v in W.get("out", {}).values() if v > 1) + c["load_starts"] = L.get("starts", 0) + for k in ("requests", "failed", "signins"): + c[f"web_{k}"] = sum(t.get(k, 0) for t in tabs) + for k in ("streams", "clean_ends", "cut_off"): + c[f"web_{k}"] = sum(x.get(k, 0) for t in tabs for x in t.get("tabs", [])) + return c, L, W + + +def chaos_windows(): + out = [] + for line in open(os.path.join(STATE_DIR, "chaos.jsonl")) if os.path.exists(os.path.join(STATE_DIR, "chaos.jsonl")) else []: + try: + out.append(json.loads(line)) + except ValueError: + pass + return out + + +def events_between(a, b): + out = [] + if os.path.exists(EVENTS): + for line in open(EVENTS): + try: + e = json.loads(line) + except ValueError: + continue + if a <= e["at"] < b: + out.append(e) + return out + + +def in_chaos(at, windows): + # A minute's grace after, for what the collector notices late + return any(w["start"] <= at <= w.get("end", time.time()) + 120 for w in windows) + + +def day_numbers(day, started): + """(this day's counters, the day's start and end).""" + c, L, W = counters() + snap = os.path.join(STATE_DIR, f"day-{day}.json") + prev = load(os.path.join(STATE_DIR, f"day-{day - 1}.json"), {}) if day > 1 else {} + # Taken once, at the day's end; a report written again uses it + if os.path.exists(snap): + c = load(snap, c) + else: + save(snap, c) + delta = {k: v - prev.get(k, 0) for k, v in c.items()} + return delta, c, L, W, started + (day - 1) * DAY, started + day * DAY + + +def missing(L, W, now): + lost_in = sorted(int(n) for n, t in L.get("pending", {}).items() if now - t > LOST_AFTER) + lost_out = sorted(int(n) for n, t in L.get("out_sent", {}).items() + if now - t > LOST_AFTER and n not in W.get("out", {})) + return lost_in, lost_out + + +def report(day): + started = float(open(os.path.join(STATE_DIR, "started")).read()) + d, c, L, W, a, b = day_numbers(day, started) + now = time.time() + lost_in, lost_out = missing(L, W, now) + windows = [w for w in chaos_windows() if a <= w["start"] < b] + evs = events_between(a, b) + restarts = [e for e in evs if e["kind"] in ("restart", "pod-new")] + unplanned = [e for e in restarts if not in_chaos(e["at"], chaos_windows())] + twice = d.get("in_twice", 0) + d.get("out_twice", 0) + chaos_ok = all(w.get("result") == "ok" for w in windows) + ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok + fmt = lambda t: time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(t)) + lines = [ + f"# Soak day {day}: {'clean' if ok else 'NOT clean'}", + "", + f"{fmt(a)} to {fmt(b)}, run `{L.get('run')}`, context `{KCTX}`.", + "", + "| | |", + "|---|---|", + f"| Mail in | {d.get('in_sent', 0)} sent, {d.get('in_arrived', 0)} arrived, {d.get('in_late', 0)} after 15 min, " + f"{d.get('in_twice', 0)} twice, {d.get('in_retries', 0)} sends retried; slowest so far {L.get('slowest_s', 0)} s |", + f"| Mail out | {d.get('out_sent', 0)} sent, {d.get('out_received', 0)} reached the sink, {d.get('out_twice', 0)} twice, " + f"{d.get('out_retries', 0)} sends retried |", + f"| Lost so far | in: {len(lost_in)} {lost_in[:10] if lost_in else ''}, out: {len(lost_out)} {lost_out[:10] if lost_out else ''} " + f"(not there {LOST_AFTER // 60} min after sending) |", + f"| IMAP IDLE | {d.get('idle_notices', 0)} new-mail notices, {d.get('idle_drops', 0)} drops; " + f"longest quiet so far {L.get('idle', {}).get('longest_quiet_s', 0)} s |", + f"| Webmail tabs | {d.get('web_requests', 0)} requests, {d.get('web_failed', 0)} failed, " + f"{d.get('web_streams', 0)} streams ({d.get('web_cut_off', 0)} cut off), {d.get('web_signins', 0)} sign-ins |", + f"| Restarts | {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos |", + f"| Load generator | started {d.get('load_starts', 0)} more time(s) |", + "", + "## Chaos", + "", + ] + lines += [f"- {w['scenario']}: {w.get('result', 'still running')} ({fmt(w['start'])})" for w in windows] or ["- none today"] + if unplanned: + lines += ["", "## Unplanned", ""] + [f"- {fmt(e['at'])} {e['kind']}: {e['detail']}" for e in unplanned[:30]] + kinds = {} + for e in evs: + kinds.setdefault(e["kind"], []).append(e) + if kinds: + lines += ["", "## Events", ""] + for k, es in sorted(kinds.items()): + lines.append(f"- {k}: {len(es)}; first: {es[0]['detail']}") + text = "\n".join(lines) + "\n" + open(os.path.join(STATE_DIR, f"day-{day}.md"), "w").write(text) + print(text) + return ok + + +def final(): + started = float(open(os.path.join(STATE_DIR, "started")).read()) + days = sorted(int(m.group(1)) for f in os.listdir(STATE_DIR) if (m := re.fullmatch(r"day-(\d+)\.md", f))) + c, L, W = counters() + lost_in, lost_out = missing(L, W, time.time()) + windows = chaos_windows() + restarts = [e for e in events_between(started, time.time()) if e["kind"] in ("restart", "pod-new")] + unplanned = [e for e in restarts if not in_chaos(e["at"], windows)] + twice = c.get("in_twice", 0) + c.get("out_twice", 0) + chaos_ok = all(w.get("result") == "ok" for w in windows) + ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok + lines = [f"# Soak: {'PASSED' if ok else 'FAILED'}", "", + f"{len(days)} day(s) from {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(started))}, context `{KCTX}`.", "", + f"- Mail in: {c.get('in_sent', 0)} sent, {c.get('in_arrived', 0)} arrived, {len(lost_in)} lost, {c.get('in_twice', 0)} twice", + f"- Mail out: {c.get('out_sent', 0)} sent, {c.get('out_received', 0)} reached the sink, {len(lost_out)} lost, {c.get('out_twice', 0)} twice", + f"- IMAP IDLE: {c.get('idle_notices', 0)} notices, {c.get('idle_drops', 0)} drops", + f"- Webmail: {c.get('web_requests', 0)} requests, {c.get('web_failed', 0)} failed, {c.get('web_cut_off', 0)} streams cut off", + f"- Restarts: {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos", + f"- Chaos: " + ", ".join(f"{w['scenario']} {w.get('result', '?')}" for w in windows), "", + "| Day | |", "|---|---|"] + for dnum in days: + first = open(os.path.join(STATE_DIR, f"day-{dnum}.md")).readline().strip("# \n") + lines.append(f"| {dnum} | {first} |") + text = "\n".join(lines) + "\n" + open(os.path.join(STATE_DIR, "report.md"), "w").write(text) + print(text) + return ok + + +if __name__ == "__main__": + what = sys.argv[1] + if what == "collect": + collect() + elif what == "report": + sys.exit(0 if report(int(sys.argv[2])) else 1) + elif what == "final": + sys.exit(0 if final() else 1) + else: + sys.exit(__doc__) diff --git a/lab/soak/soak.sh b/lab/soak/soak.sh new file mode 100755 index 0000000..41d3cc3 --- /dev/null +++ b/lab/soak/soak.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L7: the soak. Steady load for SOAK_DAYS days (7) on the lab's seeded +# install, one chaos scenario a day, a report at the end of each day and +# one for the whole run. +# +# lab/soak/soak.sh start checks the install, then runs in the +# background (a systemd user unit) +# lab/soak/soak.sh status where it's up to, and today's numbers so far +# lab/soak/soak.sh stop +# lab/soak/soak.sh report the reports written so far +# +# The load (soak-load.py) comes from this host through the load balancer: +# mail in, mail out to the sink, IMAP IDLE. The webmail tabs +# (chart/webmail-tabs.py) run in a pod, as there's no ingress. Chaos is +# chart/chaos-test.sh with one scenario a day, CHAOS_AFTER seconds into the +# day (2 h). Reports and state go to SOAK_DIR; a run is resumed, not +# restarted, by `start` after a stop or a reboot, until it's done. +# +# A shorter run, to try the harness: SOAK_DAY=1800 SOAK_DAYS=2 CHAOS_AFTER=300 +# +# The unit stops when its user logs out unless lingering is on +# (`loginctl enable-linger`), which is the machine owner's call. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +. "$HERE/../versions.env" +export KCTX="${KCTX:-kind-$KIND_CLUSTER}" +K=(kubectl --context "$KCTX") +LAB_STATE="${LAB_STATE:-$HOME/.cache/inbuxa-lab}" +export STATE="${SOAK_DIR:-$LAB_STATE/soak}" +export SOAK_DAY="${SOAK_DAY:-86400}" +SOAK_DAYS="${SOAK_DAYS:-7}" +CHAOS_AFTER="${CHAOS_AFTER:-7200}" +SCENARIOS=(pod-kill drain pg-failover nats-restart s3-outage pod-kill drain) +UNIT=inbuxa-soak +fail() { echo "!! $*" >&2; exit 1; } +lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } + +tabs_pod() { + "${K[@]}" create namespace lab-soak --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" label namespace lab-soak pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null + "${K[@]}" -n lab-soak create configmap soak-tabs --from-file="$HERE/../chart/webmail-tabs.py" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" -n lab-soak create secret generic soak-tabs --from-file=password="$LAB_STATE/alice.password" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" -n lab-soak apply -f - >/dev/null </dev/null +} + +start() { + systemctl --user is-active --quiet "$UNIT" && fail "already running: $0 status" + mkdir -p "$STATE" + if [ -f "$STATE/started" ] && [ -f "$STATE/report.md" ]; then + fail "the run in $STATE is finished; move it away to start another" + fi + if [ ! -f "$STATE/started" ]; then + echo "== every client check first (it also routes sink.test to the sink)" + PORTS="${PORTS:-18080}" STATE="$LAB_STATE" "$HERE/../chart/mailcheck.sh" >"$STATE/mailcheck.log" 2>&1 \ + || { tail -20 "$STATE/mailcheck.log"; fail "the install isn't healthy; not starting"; } + date +%s >"$STATE/started" + fi + echo "== the webmail tabs" + tabs_pod + systemd-run --user --unit "$UNIT" --collect --quiet \ + --setenv=KCTX="$KCTX" --setenv=SOAK_DIR="$STATE" --setenv=SOAK_DAY="$SOAK_DAY" \ + --setenv=SOAK_DAYS="$SOAK_DAYS" --setenv=CHAOS_AFTER="$CHAOS_AFTER" --setenv=LAB_STATE="$LAB_STATE" \ + --setenv=PATH="$PATH" --setenv=HOME="$HOME" \ + bash "$HERE/soak.sh" run + echo "== running: $SOAK_DAYS day(s) of ${SOAK_DAY}s; $0 status" +} + +# The unit's body: never exits on a hiccup, only when the run is done +run() { + set +e + mkdir -p "$STATE" + local started load_pid="" day now since + started="$(cat "$STATE/started")" + stop_load() { [ -n "$load_pid" ] && kill "$load_pid" 2>/dev/null; wait "$load_pid" 2>/dev/null; } + trap 'stop_load; exit 0' TERM INT + while :; do + if [ -z "$load_pid" ] || ! kill -0 "$load_pid" 2>/dev/null; then + ACCOUNT=alice@lab.test PASSWORD_FILE="$LAB_STATE/alice.password" \ + python3 "$HERE/soak-load.py" "$(lb)" >>"$STATE/load.log" 2>&1 & + load_pid=$! + fi + python3 "$HERE/soak-watch.py" collect >>"$STATE/watch.log" 2>&1 + + now=$(date +%s) + day=$(( (now - started) / SOAK_DAY + 1 )) + since=$(( (now - started) % SOAK_DAY )) + # Yesterday's report, once + if [ "$day" -gt 1 ] && [ ! -f "$STATE/day-$((day - 1)).md" ]; then + python3 "$HERE/soak-watch.py" report "$((day - 1))" >/dev/null 2>>"$STATE/watch.log" + fi + if [ "$day" -gt "$SOAK_DAYS" ]; then + python3 "$HERE/soak-watch.py" final >/dev/null 2>>"$STATE/watch.log" + stop_load + "${K[@]}" -n lab-soak scale deploy/soak-tabs --replicas=0 >/dev/null 2>&1 + exit 0 + fi + # Today's chaos, once + if [ "$since" -ge "$CHAOS_AFTER" ] && [ ! -f "$STATE/chaos-day-$day.log" ]; then + local sc="${SCENARIOS[$(( (day - 1) % ${#SCENARIOS[@]} ))]}" t0 rc + t0=$(date +%s) + printf '{"day": %d, "scenario": "%s", "start": %d}\n' "$day" "$sc" "$t0" >>"$STATE/chaos.jsonl" + SCENARIOS="$sc" STATE="$LAB_STATE" PORTS=18180 "$HERE/../chart/chaos-test.sh" >"$STATE/chaos-day-$day.log" 2>&1 + rc=$? + python3 - "$STATE/chaos.jsonl" "$day" "$rc" "$(date +%s)" <<'PY' +import json, sys +path, day, rc, end = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]) +rows = [json.loads(l) for l in open(path) if l.strip()] +for r in rows: + if r["day"] == day and "end" not in r: + r["end"], r["result"] = end, "ok" if rc == 0 else f"FAILED (exit {rc}, see chaos-day-{day}.log)" +open(path, "w").write("".join(json.dumps(r) + "\n" for r in rows)) +PY + fi + sleep 60 + done +} + +status() { + if systemctl --user is-active --quiet "$UNIT"; then echo "== running"; else echo "== not running"; fi + [ -f "$STATE/started" ] || { echo " no run in $STATE"; return 0; } + local started now; started="$(cat "$STATE/started")"; now=$(date +%s) + echo " day $(( (now - started) / SOAK_DAY + 1 )) of $SOAK_DAYS, $(( (now - started) % SOAK_DAY / 60 )) min in" + python3 - "$STATE" <<'PY' +import json, os, sys +d = sys.argv[1] +L = json.load(open(os.path.join(d, "load.json"))) +W = json.load(open(os.path.join(d, "watch.json"))) if os.path.exists(os.path.join(d, "watch.json")) else {"out": {}, "tabs": {}} +i, o, idle = L["inbound"], L["outbound"], L["idle"] +print(f" in: {i['sent']} sent, {i['arrived']} arrived, {len(L['pending'])} pending, {i['twice']} twice") +print(f" out: {o['sent']} sent, {len(W['out'])} at the sink") +print(f" idle: {idle['notices']} notices, {idle['drops']} drops") +t = list(W["tabs"].values()) +print(f" webmail: {sum(x.get('requests', 0) for x in t)} requests, {sum(x.get('failed', 0) for x in t)} failed") +PY + ls "$STATE"/day-*.md 2>/dev/null | sed 's/^/ /' || true +} + +case "${1:-}" in + start) start ;; + run) run ;; + status) status ;; + stop) systemctl --user stop "$UNIT" 2>/dev/null || true; echo "== stopped (start resumes it)" ;; + report) cat "$STATE"/day-*.md "$STATE"/report.md 2>/dev/null || echo "no reports yet" ;; + *) sed -n '4,25p' "$0"; exit 2 ;; +esac