Files
jcoffey-dev 130c1e405f
ci / check (pull_request) Successful in 21s
Mail senders: a 421 to QUIT isn't a failed send
The chaos tests saw a message arrive twice after a pod kill or a drain,
three times in about twenty runs. Each time the send had been retried after
"421 4.3.0 Server shutting down". That 421 was the answer to QUIT, after
the end of DATA had its 250: smtplib's with-statement sends QUIT on the way
out and raises on anything but 221, so the lab took a delivered message for
a failed one and sent it again. The server did nothing wrong.

mailflow.py and soak-load.py now QUIT and close whatever the answer, and
retry only what failed before the 250. The soak running now has the old
sender; a message twice in its report is checked against its retries.
2026-11-20 16:25:47 -07:00

290 lines
11 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""The soak's steady mail load (K-L7), from outside the cluster through its
load balancer, as real clients reach it. Runs until killed, and picks up
where it left off when started again: everything it knows is in STATE.
ACCOUNT=user@domain PASSWORD_FILE=... STATE=dir soak-load.py LB
inbound a message every IN_EVERY s (10) over SMTP 25, each from its own
sender domain, retried like an MTA when refused
verify every 30 s, over IMAPS: which arrived (any mailbox), then
deleted, so the mailbox stays small; a number seen again after
that came twice
outbound a message every OUT_EVERY s (60) over submission 465 with AUTH,
to [email protected] ... [email protected] in turn (the server
allows 25 an hour from one sender domain to one recipient, and
[email protected] is left to chart/mailcheck.py); soak-watch.py
checks the sink received each once
idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're
told of new mail, and how often they drop. They watch the
mailbox the soak's mail lands in (verify finds it: unknown
senders' mail goes to Junk Mail), as a client watches INBOX
STATE/load.json is rewritten every 30 s; STATE/events.jsonl gets a line
for anything that went wrong. The lab's certificate is self-signed, so TLS
isn't verified.
"""
import imaplib, json, os, re, smtplib, socket, ssl, sys, threading, time, uuid
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
lb = sys.argv[1]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
state_dir = os.environ["STATE"]
IN_EVERY = float(os.environ.get("IN_EVERY", "10"))
OUT_EVERY = float(os.environ.get("OUT_EVERY", "60"))
IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2"))
SINK_SPREAD = int(os.environ.get("SINK_SPREAD", "10"))
tls = ssl.create_default_context()
tls.check_hostname = False
tls.verify_mode = ssl.CERT_NONE
lock = threading.RLock() # event() and save() are called with it held
STATE = os.path.join(state_dir, "load.json")
EVENTS = os.path.join(state_dir, "events.jsonl")
try:
S = json.load(open(STATE))
except (OSError, ValueError):
S = {"run": uuid.uuid4().hex[:8], "started": time.time(),
"inbound": {"next": 1, "sent": 0, "retries": 0, "arrived": 0, "twice": 0},
"pending": {}, "arrived_late": 0, "slowest_s": 0,
"outbound": {"next": 1, "sent": 0, "retries": 0}, "out_sent": {},
"idle": {"notices": 0, "drops": 0, "longest_quiet_s": 0},
"seen": {}, "starts": 0}
S["starts"] += 1
run = S["run"]
def event(kind, detail):
with lock:
with open(EVENTS, "a") as f:
f.write(json.dumps({"at": time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
def save():
with lock:
# A duplicate a day late is still caught; older arrivals are dropped
old = time.time() - 2 * 86400
S["seen"] = {n: v for n, v in S.get("seen", {}).items() if v[1] > old}
tmp = STATE + ".tmp"
with open(tmp, "w") as f:
json.dump(S, f)
os.replace(tmp, STATE)
def message(frm, to, subject):
msg = EmailMessage()
msg["From"], msg["To"], msg["Subject"] = frm, to, subject
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain=f"{run}.soak.test")
msg.set_content(subject + "\n")
return msg
def quit_quietly(s):
"""QUIT, whatever the answer: the message is the server's once DATA got
its 250, and a stopping server answers QUIT 421 (see chart/mailflow.py)."""
try:
s.quit()
except (OSError, smtplib.SMTPException):
pass
finally:
s.close()
def every(seconds, fn):
"""fn() every `seconds`, keeping to the schedule; fn handles its errors."""
nxt = time.time()
while True:
fn()
nxt += seconds
time.sleep(max(0, nxt - time.time()))
def inbound():
with lock:
n = S["inbound"]["next"]
# A sender domain per message: the default throttle allows 25 an hour
# from one sender domain to one recipient
msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}")
try:
# As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py
s = smtplib.SMTP(timeout=20)
try:
s.connect(lb, 25)
s.sock.settimeout(300)
s.send_message(msg)
finally:
quit_quietly(s)
except (OSError, smtplib.SMTPException) as e:
with lock:
S["inbound"]["retries"] += 1
event("inbound-retry", f"{n}: {type(e).__name__}: {e}")
return # the same number goes again next time
with lock:
S["inbound"]["next"] = n + 1
S["inbound"]["sent"] += 1
S["pending"][str(n)] = time.time()
save() # at once: a restart must never send a number twice
def outbound():
with lock:
n = S["outbound"]["next"]
msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}")
try:
s = smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20)
try:
s.sock.settimeout(300)
s.login(account, password)
s.send_message(msg)
finally:
quit_quietly(s)
except (OSError, smtplib.SMTPException) as e:
with lock:
S["outbound"]["retries"] += 1
event("outbound-retry", f"{n}: {type(e).__name__}: {e}")
return
with lock:
S["outbound"]["next"] = n + 1
S["outbound"]["sent"] += 1
S["out_sent"][str(n)] = time.time()
save()
SUBJECT = re.compile(rf"soak {run} in (\d+)")
def verify():
try:
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=60)
except OSError as e:
event("verify", f"connect: {e}")
return
try:
m.login(account, password)
_, boxes = m.list()
for line in boxes:
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
if m.select(name)[0] != "OK":
continue
_, data = m.search(None, "SUBJECT", f'"soak {run} in"')
ids = data[0].split()
if not ids:
continue
_, hdrs = m.fetch(b",".join(ids), "(UID BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
now = time.time()
for part in hdrs:
if not isinstance(part, tuple):
continue
mm = SUBJECT.search(part[1].decode(errors="replace"))
uid = re.search(rb"UID (\d+)", part[0])
if not mm or not uid:
continue
n, copy = mm.group(1), f"{name}:{uid.group(1).decode()}"
with lock:
sent = S["pending"].pop(n, None)
if sent is not None:
S["inbound"]["arrived"] += 1
S["seen"][n] = [copy, now]
S["landed"] = name
took = now - sent
S["slowest_s"] = max(S["slowest_s"], round(took))
if took > 900:
S["arrived_late"] += 1
elif S["seen"].get(n, [None])[0] != copy:
S["inbound"]["twice"] += 1
S["seen"][n] = [copy, now]
event("twice", f"in {n} arrived again ({name})")
# else: this very copy, left by a delete that was cut short
# What arrived is on disk before it's deleted, so a restart in
# between neither loses it nor counts it twice
save()
m.store(b",".join(ids), "+FLAGS", "\\Deleted")
m.expunge()
except (OSError, imaplib.IMAP4.error) as e:
event("verify", f"{type(e).__name__}: {e}")
finally:
try:
m.logout()
except Exception:
pass
save()
def idle(i):
"""One client in IDLE, as a desktop mail app holds it; re-issued every
25 minutes (RFC 2177 asks for under 29)."""
while not S.get("landed"):
time.sleep(5)
last_notice = time.time()
while True:
sock = None
try:
raw = socket.create_connection((lb, 993), timeout=30)
sock = tls.wrap_socket(raw, server_hostname="mail.lab.test")
f = sock.makefile("rb")
def cmd(tag, text):
sock.sendall(f"{tag} {text}\r\n".encode())
while True:
line = f.readline()
if not line:
raise ConnectionError("closed")
if line.startswith(tag.encode() + b" "):
if b" OK" not in line[:12]:
raise ConnectionError(line.decode(errors="replace").strip())
return
f.readline() # greeting
cmd("a1", f'LOGIN "{account}" "{password}"')
cmd("a2", f"SELECT {S['landed']}")
n = 3
while True:
tag = f"a{n}"
n += 1
sock.sendall(f"{tag} IDLE\r\n".encode())
sock.settimeout(60)
renew = time.time() + 25 * 60
while time.time() < renew:
try:
line = f.readline()
except socket.timeout:
continue
if not line:
raise ConnectionError("closed in IDLE")
if line.endswith(b"EXISTS\r\n"):
now = time.time()
with lock:
S["idle"]["notices"] += 1
S["idle"]["longest_quiet_s"] = max(S["idle"]["longest_quiet_s"], round(now - last_notice))
last_notice = now
sock.sendall(b"DONE\r\n")
sock.settimeout(30)
while not f.readline().startswith(tag.encode() + b" "):
pass
except (OSError, ConnectionError, ssl.SSLError) as e:
with lock:
S["idle"]["drops"] += 1
event("idle-drop", f"client {i}: {type(e).__name__}: {e}")
finally:
if sock:
try:
sock.close()
except OSError:
pass
time.sleep(5)
save()
event("start", f"run {run}, start {S['starts']}")
for target in ([lambda: every(IN_EVERY, inbound), lambda: every(OUT_EVERY, outbound), lambda: every(30, verify)]
+ [(lambda i=i: idle(i)) for i in range(IDLE_CLIENTS)]):
threading.Thread(target=target, daemon=True).start()
while True:
time.sleep(30)
save()