Files
jcoffey-dev 315e84e242 The soak: a week of steady load and daily chaos (K-L7)
lab/soak/soak.sh start runs SOAK_DAYS (7) days on an installed chart as a
systemd user unit: soak-load.py from this host through the load balancer
(mail in every 10 s, out to the sink every minute, two IMAP clients in
IDLE), three webmail tabs in a pod (chart/webmail-tabs.py, now with a
summary every minute and signing in again when its session runs out),
chart/chaos-test.sh with one scenario a day, and soak-watch.py: the sink's
log, the tabs' summaries and every pod in the namespaces each minute,
day-N.md at each day's end and report.md for the run.

It passes with no message lost or doubled and no restart outside the
chaos windows. Every count survives the load generator restarting: what
was sent is saved before the next send, and an arrival is saved, with its
IMAP UID, before it's deleted.
2026-11-20 01:07:27 -07:00

154 lines
6.5 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""Open webmail tabs through a restart (K-W5), run inside the cluster so
every connection goes through the webmail Service, as a browser's would
through an ingress.
ACCOUNT=... PASSWORD_FILE=... STOP_FILE=... webmail-tabs.py WEBMAIL SERVER
Signs in once, then until STOP_FILE appears:
- TABS tabs (default 3), each holding /api/events open as EventSource
does: when a stream ends it reconnects after a second
- a JMAP Mailbox/get every 200 ms, as a busy tab's requests
and prints one JSON line: requests made and failed (with the errors), and
per tab how its streams ended (cleanly, or cut off mid-stream) and the
longest wait for a stream to deliver again. A clean end is what the
browser takes in its stride; a failed request is what shows an error.
For the soak (K-L7): INTERVAL sets the time between requests (0.2 s);
SUMMARY_EVERY (seconds) prints that line every so often while running,
with "at" and the errors since the last one; with no STOP_FILE it runs
until stopped.
"""
import http.client, json, os, sys, threading, time, urllib.parse
webmail, server = sys.argv[1:3]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
stop = os.environ.get("STOP_FILE", "/nonexistent/stop")
TABS = int(os.environ.get("TABS", "3"))
INTERVAL = float(os.environ.get("INTERVAL", "0.2"))
SUMMARY_EVERY = float(os.environ.get("SUMMARY_EVERY", "0"))
def request(base, method, path, headers=None, body=None, timeout=30):
u = urllib.parse.urlsplit(base)
conn = http.client.HTTPConnection(u.hostname, u.port, timeout=timeout)
try:
conn.request(method, path, body=body, headers=headers or {})
res = conn.getresponse()
return res, res.read()
finally:
conn.close()
def cookie(res, name):
for header, value in res.getheaders():
if header.lower() == "set-cookie" and value.startswith(name + "="):
return value.split(";", 1)[0].split("=", 1)[1]
return None
def sign_in():
res, _ = request(webmail, "GET", "/api/auth/oauth/start?username=" + urllib.parse.quote(account))
q = dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(res.getheader("location")).query))
bound = cookie(res, "ihm_session_signin")
res, data = request(server, "POST", "/api/auth", {"content-type": "application/json"}, json.dumps({
"type": "authCode", "accountName": account, "accountSecret": password, "clientId": q["client_id"],
"redirectUri": q["redirect_uri"], "scope": q.get("scope"), "codeChallenge": q["code_challenge"],
"codeChallengeMethod": q["code_challenge_method"], "state": q["state"]}))
a = json.loads(data)
code = a.get("client_code") or a.get("clientCode")
res, _ = request(webmail, "GET", "/api/auth/callback?" + urllib.parse.urlencode({"code": code, "state": q["state"]}),
{"cookie": f"ihm_session_signin={bound}"})
session = cookie(res, "ihm_session")
res, data = request(webmail, "GET", "/api/auth/session", {"cookie": f"ihm_session={session}"})
return session, json.loads(data)["primaryAccounts"]["urn:ietf:params:jmap:mail"]
session, mail_account = sign_in()
H = {"cookie": f"ihm_session={session}"}
lock = threading.Lock()
report = {"requests": 0, "failed": 0, "signins": 1, "errors": [], "tabs": []}
def tab(i):
me = {"streams": 0, "clean_ends": 0, "cut_off": 0, "longest_gap_s": 0.0}
with lock:
report["tabs"].append(me)
last_event = time.time()
u = urllib.parse.urlsplit(webmail)
while not os.path.exists(stop):
try:
conn = http.client.HTTPConnection(u.hostname, u.port, timeout=45)
conn.request("GET", "/api/events?types=*&closeafter=no&ping=30", headers=dict(H, accept="text/event-stream"))
res = conn.getresponse()
if res.status == 401: # signed out: the request loop signs in again
conn.close()
time.sleep(1)
continue
if res.status != 200:
raise RuntimeError(f"HTTP {res.status}")
me["streams"] += 1
while not os.path.exists(stop):
line = res.readline()
if not line:
me["clean_ends"] += 1 # the stream ended properly
break
if line.startswith((b"event:", b":")):
now = time.time()
me["longest_gap_s"] = max(me["longest_gap_s"], round(now - last_event, 1))
last_event = now
conn.close()
except Exception as e: # reset, refused, an incomplete chunk: cut off
if os.path.exists(stop):
break
me["cut_off"] += 1
with lock:
report["errors"].append(f"tab {i} stream: {type(e).__name__}: {e}"[:160])
time.sleep(1) # EventSource's first retry, as push.ts does
threads = [threading.Thread(target=tab, args=(i,), daemon=True) for i in range(TABS)]
for t in threads:
t.start()
body = json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
"methodCalls": [["Mailbox/get", {"accountId": mail_account, "properties": ["name"]}, "0"]]})
next_summary = time.time() + SUMMARY_EVERY
while not os.path.exists(stop):
with lock:
report["requests"] += 1
try:
res, data = request(webmail, "POST", "/api/jmap", dict(H, **{"content-type": "application/json",
"x-requested-with": "ihasmail"}), body)
if res.status == 401:
# The session ran out (12 h by default): sign in again, as the
# page sends the browser back to; the tabs pick up the new cookie
session, mail_account = sign_in()
H["cookie"] = f"ihm_session={session}"
with lock:
report["signins"] += 1
report["requests"] -= 1
continue
if res.status != 200 or b"Mailbox/get" not in data:
raise RuntimeError(f"HTTP {res.status} {data[:80]!r}")
except Exception as e:
with lock:
report["failed"] += 1
report["errors"].append(f"request: {type(e).__name__}: {e}"[:160])
time.sleep(INTERVAL)
if SUMMARY_EVERY and time.time() >= next_summary:
next_summary += SUMMARY_EVERY
with lock:
print(json.dumps(dict(report, at=time.time(), errors=report["errors"][:15])), flush=True)
report["errors"] = []
for t in threads:
t.join(timeout=50)
report["errors"] = report["errors"][:15]
print(json.dumps(report))