ci / check (pull_request) Successful in 19s
The trial (two 30-minute days) found both. Outbound mail went to one address, the same one mailcheck.py uses, and ran into the default limit of 25 an hour from one sender domain to one recipient; the drain day's check afterwards was refused for it. Outbound now goes to soak-0 ... [email protected] in turn. And the run read PASSED with that day's chaos FAILED: a day is clean, and a run passes, only when every scenario recovered.
287 lines
12 KiB
Python
Executable File
287 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""What the soak (K-L7) sees, and its reports.
|
|
|
|
soak-watch.py collect once a minute, from soak.sh
|
|
soak-watch.py report DAY that day's report, STATE/day-DAY.md
|
|
soak-watch.py final the whole run, STATE/report.md
|
|
|
|
collect reads, with kubectl (KCTX):
|
|
- the sink's log: every message soak-load.py sent out, counted once per
|
|
delivery (by the sink's own receive time)
|
|
- the webmail tabs' summaries (chart/webmail-tabs.py, one pod or several
|
|
over the run, summed)
|
|
- every pod in the inbuxa, data, lab-sink and lab-soak namespaces: a pod
|
|
that's new, or a container that restarted, is an event, planned when
|
|
it falls in a chaos window (STATE/chaos.jsonl), unplanned otherwise
|
|
|
|
Everything is kept in STATE/watch.json, events in STATE/events.jsonl
|
|
beside soak-load.py's. Reports compare against the snapshot the previous
|
|
day's report took, so each day stands on its own.
|
|
"""
|
|
import json, os, re, subprocess, sys, time
|
|
|
|
STATE_DIR = os.environ["STATE"]
|
|
KCTX = os.environ["KCTX"]
|
|
DAY = int(os.environ.get("SOAK_DAY", "86400"))
|
|
NAMESPACES = ("inbuxa", "data", "lab-sink", "lab-soak")
|
|
WATCH = os.path.join(STATE_DIR, "watch.json")
|
|
EVENTS = os.path.join(STATE_DIR, "events.jsonl")
|
|
LOST_AFTER = int(os.environ.get("LOST_AFTER", "3600")) # not there an hour after it was sent
|
|
|
|
|
|
def kubectl(*args):
|
|
return subprocess.run(["kubectl", "--context", KCTX, *args], capture_output=True, text=True, timeout=120).stdout
|
|
|
|
|
|
def load(path, default):
|
|
try:
|
|
return json.load(open(path))
|
|
except (OSError, ValueError):
|
|
return default
|
|
|
|
|
|
def save(path, data):
|
|
tmp = path + ".tmp"
|
|
with open(tmp, "w") as f:
|
|
json.dump(data, f)
|
|
os.replace(tmp, path)
|
|
|
|
|
|
def event(kind, detail, at=None):
|
|
with open(EVENTS, "a") as f:
|
|
f.write(json.dumps({"at": at or time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
|
|
|
|
|
|
def new_lines(W, ns, selector, key):
|
|
"""Log lines of every pod matching `selector` since we last looked."""
|
|
out = []
|
|
for pod in kubectl("-n", ns, "get", "pods", "-l", selector, "-o", "name").split():
|
|
since = W["since"].get(f"{key}/{pod}")
|
|
args = ["-n", ns, "logs", pod, "--timestamps"] + (["--since-time", since] if since else [])
|
|
last = since
|
|
for line in kubectl(*args).splitlines():
|
|
ts, _, text = line.partition(" ")
|
|
if since and ts <= since:
|
|
continue # --since-time is inclusive, to the second
|
|
out.append((pod, text))
|
|
last = ts
|
|
if last:
|
|
W["since"][f"{key}/{pod}"] = last
|
|
return out
|
|
|
|
|
|
def collect():
|
|
W = load(WATCH, {"since": {}, "out": {}, "tabs": {}, "pods": {}, "first": True})
|
|
load_state = load(os.path.join(STATE_DIR, "load.json"), {})
|
|
run = load_state.get("run")
|
|
|
|
# The sink: each soak message that reached it
|
|
for _, text in new_lines(W, "lab-sink", "app=lab-sink", "sink"):
|
|
try:
|
|
rec = json.loads(text)
|
|
except ValueError:
|
|
continue
|
|
m = re.fullmatch(rf"soak {run} out (\d+)", rec.get("subject", ""))
|
|
if m:
|
|
W["out"][m.group(1)] = W["out"].get(m.group(1), 0) + 1
|
|
if W["out"][m.group(1)] == 2:
|
|
event("out-twice", f"out {m.group(1)} reached the sink again")
|
|
|
|
# The webmail tabs: the latest summary of each pod
|
|
for pod, text in new_lines(W, "lab-soak", "app=soak-tabs", "tabs"):
|
|
try:
|
|
rec = json.loads(text)
|
|
except ValueError:
|
|
continue
|
|
for e in rec.get("errors", []):
|
|
event("webmail", f"{pod.split('/')[-1]}: {e}", rec.get("at"))
|
|
rec.pop("errors", None)
|
|
W["tabs"][pod] = rec
|
|
|
|
# The pods
|
|
now = time.time()
|
|
seen = {}
|
|
for ns in NAMESPACES:
|
|
for p in json.loads(kubectl("-n", ns, "get", "pods", "-o", "json") or '{"items": []}')["items"]:
|
|
uid = p["metadata"]["uid"]
|
|
name = f"{ns}/{p['metadata']['name']}"
|
|
statuses = p.get("status", {}).get("containerStatuses", []) or []
|
|
restarts = sum(c.get("restartCount", 0) for c in statuses)
|
|
seen[uid] = {"name": name, "restarts": restarts}
|
|
before = W["pods"].get(uid)
|
|
if before is None and not W["first"]:
|
|
owner = (p["metadata"].get("ownerReferences") or [{}])[0].get("kind", "none")
|
|
event("pod-new", f"{name} ({owner})", now)
|
|
elif before and restarts > before["restarts"]:
|
|
why = ", ".join(sorted({(c.get("lastState", {}).get("terminated") or {}).get("reason", "?")
|
|
for c in statuses if c.get("restartCount", 0)}))
|
|
event("restart", f"{name}: {restarts - before['restarts']} more ({why})", now)
|
|
for uid, p in W["pods"].items():
|
|
if uid not in seen:
|
|
event("pod-gone", p["name"], now)
|
|
W["pods"], W["first"] = seen, False
|
|
save(WATCH, W)
|
|
|
|
|
|
def counters():
|
|
"""Every running total, flattened, for day-to-day differences."""
|
|
L = load(os.path.join(STATE_DIR, "load.json"), {})
|
|
W = load(WATCH, {"out": {}, "tabs": {}})
|
|
tabs = list(W.get("tabs", {}).values())
|
|
c = {f"in_{k}": v for k, v in L.get("inbound", {}).items() if k != "next"}
|
|
c.update({f"out_{k}": v for k, v in L.get("outbound", {}).items() if k != "next"})
|
|
c.update({f"idle_{k}": v for k, v in L.get("idle", {}).items() if k != "longest_quiet_s"})
|
|
c["in_late"] = L.get("arrived_late", 0)
|
|
c["out_received"] = len(W.get("out", {}))
|
|
c["out_twice"] = sum(1 for v in W.get("out", {}).values() if v > 1)
|
|
c["load_starts"] = L.get("starts", 0)
|
|
for k in ("requests", "failed", "signins"):
|
|
c[f"web_{k}"] = sum(t.get(k, 0) for t in tabs)
|
|
for k in ("streams", "clean_ends", "cut_off"):
|
|
c[f"web_{k}"] = sum(x.get(k, 0) for t in tabs for x in t.get("tabs", []))
|
|
return c, L, W
|
|
|
|
|
|
def chaos_windows():
|
|
out = []
|
|
for line in open(os.path.join(STATE_DIR, "chaos.jsonl")) if os.path.exists(os.path.join(STATE_DIR, "chaos.jsonl")) else []:
|
|
try:
|
|
out.append(json.loads(line))
|
|
except ValueError:
|
|
pass
|
|
return out
|
|
|
|
|
|
def events_between(a, b):
|
|
out = []
|
|
if os.path.exists(EVENTS):
|
|
for line in open(EVENTS):
|
|
try:
|
|
e = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if a <= e["at"] < b:
|
|
out.append(e)
|
|
return out
|
|
|
|
|
|
def in_chaos(at, windows):
|
|
# A minute's grace after, for what the collector notices late
|
|
return any(w["start"] <= at <= w.get("end", time.time()) + 120 for w in windows)
|
|
|
|
|
|
def day_numbers(day, started):
|
|
"""(this day's counters, the day's start and end)."""
|
|
c, L, W = counters()
|
|
snap = os.path.join(STATE_DIR, f"day-{day}.json")
|
|
prev = load(os.path.join(STATE_DIR, f"day-{day - 1}.json"), {}) if day > 1 else {}
|
|
# Taken once, at the day's end; a report written again uses it
|
|
if os.path.exists(snap):
|
|
c = load(snap, c)
|
|
else:
|
|
save(snap, c)
|
|
delta = {k: v - prev.get(k, 0) for k, v in c.items()}
|
|
return delta, c, L, W, started + (day - 1) * DAY, started + day * DAY
|
|
|
|
|
|
def missing(L, W, now):
|
|
lost_in = sorted(int(n) for n, t in L.get("pending", {}).items() if now - t > LOST_AFTER)
|
|
lost_out = sorted(int(n) for n, t in L.get("out_sent", {}).items()
|
|
if now - t > LOST_AFTER and n not in W.get("out", {}))
|
|
return lost_in, lost_out
|
|
|
|
|
|
def report(day):
|
|
started = float(open(os.path.join(STATE_DIR, "started")).read())
|
|
d, c, L, W, a, b = day_numbers(day, started)
|
|
now = time.time()
|
|
lost_in, lost_out = missing(L, W, now)
|
|
windows = [w for w in chaos_windows() if a <= w["start"] < b]
|
|
evs = events_between(a, b)
|
|
restarts = [e for e in evs if e["kind"] in ("restart", "pod-new")]
|
|
unplanned = [e for e in restarts if not in_chaos(e["at"], chaos_windows())]
|
|
twice = d.get("in_twice", 0) + d.get("out_twice", 0)
|
|
chaos_ok = all(w.get("result") == "ok" for w in windows)
|
|
ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok
|
|
fmt = lambda t: time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(t))
|
|
lines = [
|
|
f"# Soak day {day}: {'clean' if ok else 'NOT clean'}",
|
|
"",
|
|
f"{fmt(a)} to {fmt(b)}, run `{L.get('run')}`, context `{KCTX}`.",
|
|
"",
|
|
"| | |",
|
|
"|---|---|",
|
|
f"| Mail in | {d.get('in_sent', 0)} sent, {d.get('in_arrived', 0)} arrived, {d.get('in_late', 0)} after 15 min, "
|
|
f"{d.get('in_twice', 0)} twice, {d.get('in_retries', 0)} sends retried; slowest so far {L.get('slowest_s', 0)} s |",
|
|
f"| Mail out | {d.get('out_sent', 0)} sent, {d.get('out_received', 0)} reached the sink, {d.get('out_twice', 0)} twice, "
|
|
f"{d.get('out_retries', 0)} sends retried |",
|
|
f"| Lost so far | in: {len(lost_in)} {lost_in[:10] if lost_in else ''}, out: {len(lost_out)} {lost_out[:10] if lost_out else ''} "
|
|
f"(not there {LOST_AFTER // 60} min after sending) |",
|
|
f"| IMAP IDLE | {d.get('idle_notices', 0)} new-mail notices, {d.get('idle_drops', 0)} drops; "
|
|
f"longest quiet so far {L.get('idle', {}).get('longest_quiet_s', 0)} s |",
|
|
f"| Webmail tabs | {d.get('web_requests', 0)} requests, {d.get('web_failed', 0)} failed, "
|
|
f"{d.get('web_streams', 0)} streams ({d.get('web_cut_off', 0)} cut off), {d.get('web_signins', 0)} sign-ins |",
|
|
f"| Restarts | {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos |",
|
|
f"| Load generator | started {d.get('load_starts', 0)} more time(s) |",
|
|
"",
|
|
"## Chaos",
|
|
"",
|
|
]
|
|
lines += [f"- {w['scenario']}: {w.get('result', 'still running')} ({fmt(w['start'])})" for w in windows] or ["- none today"]
|
|
if unplanned:
|
|
lines += ["", "## Unplanned", ""] + [f"- {fmt(e['at'])} {e['kind']}: {e['detail']}" for e in unplanned[:30]]
|
|
kinds = {}
|
|
for e in evs:
|
|
kinds.setdefault(e["kind"], []).append(e)
|
|
if kinds:
|
|
lines += ["", "## Events", ""]
|
|
for k, es in sorted(kinds.items()):
|
|
lines.append(f"- {k}: {len(es)}; first: {es[0]['detail']}")
|
|
text = "\n".join(lines) + "\n"
|
|
open(os.path.join(STATE_DIR, f"day-{day}.md"), "w").write(text)
|
|
print(text)
|
|
return ok
|
|
|
|
|
|
def final():
|
|
started = float(open(os.path.join(STATE_DIR, "started")).read())
|
|
days = sorted(int(m.group(1)) for f in os.listdir(STATE_DIR) if (m := re.fullmatch(r"day-(\d+)\.md", f)))
|
|
c, L, W = counters()
|
|
lost_in, lost_out = missing(L, W, time.time())
|
|
windows = chaos_windows()
|
|
restarts = [e for e in events_between(started, time.time()) if e["kind"] in ("restart", "pod-new")]
|
|
unplanned = [e for e in restarts if not in_chaos(e["at"], windows)]
|
|
twice = c.get("in_twice", 0) + c.get("out_twice", 0)
|
|
chaos_ok = all(w.get("result") == "ok" for w in windows)
|
|
ok = not lost_in and not lost_out and not twice and not unplanned and chaos_ok
|
|
lines = [f"# Soak: {'PASSED' if ok else 'FAILED'}", "",
|
|
f"{len(days)} day(s) from {time.strftime('%Y-%m-%d %H:%M UTC', time.gmtime(started))}, context `{KCTX}`.", "",
|
|
f"- Mail in: {c.get('in_sent', 0)} sent, {c.get('in_arrived', 0)} arrived, {len(lost_in)} lost, {c.get('in_twice', 0)} twice",
|
|
f"- Mail out: {c.get('out_sent', 0)} sent, {c.get('out_received', 0)} reached the sink, {len(lost_out)} lost, {c.get('out_twice', 0)} twice",
|
|
f"- IMAP IDLE: {c.get('idle_notices', 0)} notices, {c.get('idle_drops', 0)} drops",
|
|
f"- Webmail: {c.get('web_requests', 0)} requests, {c.get('web_failed', 0)} failed, {c.get('web_cut_off', 0)} streams cut off",
|
|
f"- Restarts: {len(unplanned)} unplanned, {len(restarts) - len(unplanned)} during chaos",
|
|
f"- Chaos: " + ", ".join(f"{w['scenario']} {w.get('result', '?')}" for w in windows), "",
|
|
"| Day | |", "|---|---|"]
|
|
for dnum in days:
|
|
first = open(os.path.join(STATE_DIR, f"day-{dnum}.md")).readline().strip("# \n")
|
|
lines.append(f"| {dnum} | {first} |")
|
|
text = "\n".join(lines) + "\n"
|
|
open(os.path.join(STATE_DIR, "report.md"), "w").write(text)
|
|
print(text)
|
|
return ok
|
|
|
|
|
|
if __name__ == "__main__":
|
|
what = sys.argv[1]
|
|
if what == "collect":
|
|
collect()
|
|
elif what == "report":
|
|
sys.exit(0 if report(int(sys.argv[2])) else 1)
|
|
elif what == "final":
|
|
sys.exit(0 if final() else 1)
|
|
else:
|
|
sys.exit(__doc__)
|