ci / check (pull_request) Successful in 21s
The chaos tests saw a message arrive twice after a pod kill or a drain, three times in about twenty runs. Each time the send had been retried after "421 4.3.0 Server shutting down". That 421 was the answer to QUIT, after the end of DATA had its 250: smtplib's with-statement sends QUIT on the way out and raises on anything but 221, so the lab took a delivered message for a failed one and sent it again. The server did nothing wrong. mailflow.py and soak-load.py now QUIT and close whatever the answer, and retry only what failed before the 250. The soak running now has the old sender; a message twice in its report is checked against its retries.
290 lines
11 KiB
Python
Executable File
290 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""The soak's steady mail load (K-L7), from outside the cluster through its
|
|
load balancer, as real clients reach it. Runs until killed, and picks up
|
|
where it left off when started again: everything it knows is in STATE.
|
|
|
|
ACCOUNT=user@domain PASSWORD_FILE=... STATE=dir soak-load.py LB
|
|
|
|
inbound a message every IN_EVERY s (10) over SMTP 25, each from its own
|
|
sender domain, retried like an MTA when refused
|
|
verify every 30 s, over IMAPS: which arrived (any mailbox), then
|
|
deleted, so the mailbox stays small; a number seen again after
|
|
that came twice
|
|
outbound a message every OUT_EVERY s (60) over submission 465 with AUTH,
|
|
to [email protected] ... [email protected] in turn (the server
|
|
allows 25 an hour from one sender domain to one recipient, and
|
|
[email protected] is left to chart/mailcheck.py); soak-watch.py
|
|
checks the sink received each once
|
|
idle IDLE_CLIENTS (2) IMAP connections in IDLE: how often they're
|
|
told of new mail, and how often they drop. They watch the
|
|
mailbox the soak's mail lands in (verify finds it: unknown
|
|
senders' mail goes to Junk Mail), as a client watches INBOX
|
|
|
|
STATE/load.json is rewritten every 30 s; STATE/events.jsonl gets a line
|
|
for anything that went wrong. The lab's certificate is self-signed, so TLS
|
|
isn't verified.
|
|
"""
|
|
import imaplib, json, os, re, smtplib, socket, ssl, sys, threading, time, uuid
|
|
from email.message import EmailMessage
|
|
from email.utils import formatdate, make_msgid
|
|
|
|
lb = sys.argv[1]
|
|
account = os.environ["ACCOUNT"]
|
|
password = open(os.environ["PASSWORD_FILE"]).read().strip()
|
|
state_dir = os.environ["STATE"]
|
|
IN_EVERY = float(os.environ.get("IN_EVERY", "10"))
|
|
OUT_EVERY = float(os.environ.get("OUT_EVERY", "60"))
|
|
IDLE_CLIENTS = int(os.environ.get("IDLE_CLIENTS", "2"))
|
|
SINK_SPREAD = int(os.environ.get("SINK_SPREAD", "10"))
|
|
tls = ssl.create_default_context()
|
|
tls.check_hostname = False
|
|
tls.verify_mode = ssl.CERT_NONE
|
|
lock = threading.RLock() # event() and save() are called with it held
|
|
STATE = os.path.join(state_dir, "load.json")
|
|
EVENTS = os.path.join(state_dir, "events.jsonl")
|
|
|
|
try:
|
|
S = json.load(open(STATE))
|
|
except (OSError, ValueError):
|
|
S = {"run": uuid.uuid4().hex[:8], "started": time.time(),
|
|
"inbound": {"next": 1, "sent": 0, "retries": 0, "arrived": 0, "twice": 0},
|
|
"pending": {}, "arrived_late": 0, "slowest_s": 0,
|
|
"outbound": {"next": 1, "sent": 0, "retries": 0}, "out_sent": {},
|
|
"idle": {"notices": 0, "drops": 0, "longest_quiet_s": 0},
|
|
"seen": {}, "starts": 0}
|
|
S["starts"] += 1
|
|
run = S["run"]
|
|
|
|
|
|
def event(kind, detail):
|
|
with lock:
|
|
with open(EVENTS, "a") as f:
|
|
f.write(json.dumps({"at": time.time(), "kind": kind, "detail": str(detail)[:200]}) + "\n")
|
|
|
|
|
|
def save():
|
|
with lock:
|
|
# A duplicate a day late is still caught; older arrivals are dropped
|
|
old = time.time() - 2 * 86400
|
|
S["seen"] = {n: v for n, v in S.get("seen", {}).items() if v[1] > old}
|
|
tmp = STATE + ".tmp"
|
|
with open(tmp, "w") as f:
|
|
json.dump(S, f)
|
|
os.replace(tmp, STATE)
|
|
|
|
|
|
def message(frm, to, subject):
|
|
msg = EmailMessage()
|
|
msg["From"], msg["To"], msg["Subject"] = frm, to, subject
|
|
msg["Date"] = formatdate()
|
|
msg["Message-ID"] = make_msgid(domain=f"{run}.soak.test")
|
|
msg.set_content(subject + "\n")
|
|
return msg
|
|
|
|
|
|
def quit_quietly(s):
|
|
"""QUIT, whatever the answer: the message is the server's once DATA got
|
|
its 250, and a stopping server answers QUIT 421 (see chart/mailflow.py)."""
|
|
try:
|
|
s.quit()
|
|
except (OSError, smtplib.SMTPException):
|
|
pass
|
|
finally:
|
|
s.close()
|
|
|
|
|
|
def every(seconds, fn):
|
|
"""fn() every `seconds`, keeping to the schedule; fn handles its errors."""
|
|
nxt = time.time()
|
|
while True:
|
|
fn()
|
|
nxt += seconds
|
|
time.sleep(max(0, nxt - time.time()))
|
|
|
|
|
|
def inbound():
|
|
with lock:
|
|
n = S["inbound"]["next"]
|
|
# A sender domain per message: the default throttle allows 25 an hour
|
|
# from one sender domain to one recipient
|
|
msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}")
|
|
try:
|
|
# As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py
|
|
s = smtplib.SMTP(timeout=20)
|
|
try:
|
|
s.connect(lb, 25)
|
|
s.sock.settimeout(300)
|
|
s.send_message(msg)
|
|
finally:
|
|
quit_quietly(s)
|
|
except (OSError, smtplib.SMTPException) as e:
|
|
with lock:
|
|
S["inbound"]["retries"] += 1
|
|
event("inbound-retry", f"{n}: {type(e).__name__}: {e}")
|
|
return # the same number goes again next time
|
|
with lock:
|
|
S["inbound"]["next"] = n + 1
|
|
S["inbound"]["sent"] += 1
|
|
S["pending"][str(n)] = time.time()
|
|
save() # at once: a restart must never send a number twice
|
|
|
|
|
|
def outbound():
|
|
with lock:
|
|
n = S["outbound"]["next"]
|
|
msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}")
|
|
try:
|
|
s = smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20)
|
|
try:
|
|
s.sock.settimeout(300)
|
|
s.login(account, password)
|
|
s.send_message(msg)
|
|
finally:
|
|
quit_quietly(s)
|
|
except (OSError, smtplib.SMTPException) as e:
|
|
with lock:
|
|
S["outbound"]["retries"] += 1
|
|
event("outbound-retry", f"{n}: {type(e).__name__}: {e}")
|
|
return
|
|
with lock:
|
|
S["outbound"]["next"] = n + 1
|
|
S["outbound"]["sent"] += 1
|
|
S["out_sent"][str(n)] = time.time()
|
|
save()
|
|
|
|
|
|
SUBJECT = re.compile(rf"soak {run} in (\d+)")
|
|
|
|
|
|
def verify():
|
|
try:
|
|
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=60)
|
|
except OSError as e:
|
|
event("verify", f"connect: {e}")
|
|
return
|
|
try:
|
|
m.login(account, password)
|
|
_, boxes = m.list()
|
|
for line in boxes:
|
|
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
|
|
if m.select(name)[0] != "OK":
|
|
continue
|
|
_, data = m.search(None, "SUBJECT", f'"soak {run} in"')
|
|
ids = data[0].split()
|
|
if not ids:
|
|
continue
|
|
_, hdrs = m.fetch(b",".join(ids), "(UID BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
|
|
now = time.time()
|
|
for part in hdrs:
|
|
if not isinstance(part, tuple):
|
|
continue
|
|
mm = SUBJECT.search(part[1].decode(errors="replace"))
|
|
uid = re.search(rb"UID (\d+)", part[0])
|
|
if not mm or not uid:
|
|
continue
|
|
n, copy = mm.group(1), f"{name}:{uid.group(1).decode()}"
|
|
with lock:
|
|
sent = S["pending"].pop(n, None)
|
|
if sent is not None:
|
|
S["inbound"]["arrived"] += 1
|
|
S["seen"][n] = [copy, now]
|
|
S["landed"] = name
|
|
took = now - sent
|
|
S["slowest_s"] = max(S["slowest_s"], round(took))
|
|
if took > 900:
|
|
S["arrived_late"] += 1
|
|
elif S["seen"].get(n, [None])[0] != copy:
|
|
S["inbound"]["twice"] += 1
|
|
S["seen"][n] = [copy, now]
|
|
event("twice", f"in {n} arrived again ({name})")
|
|
# else: this very copy, left by a delete that was cut short
|
|
# What arrived is on disk before it's deleted, so a restart in
|
|
# between neither loses it nor counts it twice
|
|
save()
|
|
m.store(b",".join(ids), "+FLAGS", "\\Deleted")
|
|
m.expunge()
|
|
except (OSError, imaplib.IMAP4.error) as e:
|
|
event("verify", f"{type(e).__name__}: {e}")
|
|
finally:
|
|
try:
|
|
m.logout()
|
|
except Exception:
|
|
pass
|
|
save()
|
|
|
|
|
|
def idle(i):
|
|
"""One client in IDLE, as a desktop mail app holds it; re-issued every
|
|
25 minutes (RFC 2177 asks for under 29)."""
|
|
while not S.get("landed"):
|
|
time.sleep(5)
|
|
last_notice = time.time()
|
|
while True:
|
|
sock = None
|
|
try:
|
|
raw = socket.create_connection((lb, 993), timeout=30)
|
|
sock = tls.wrap_socket(raw, server_hostname="mail.lab.test")
|
|
f = sock.makefile("rb")
|
|
|
|
def cmd(tag, text):
|
|
sock.sendall(f"{tag} {text}\r\n".encode())
|
|
while True:
|
|
line = f.readline()
|
|
if not line:
|
|
raise ConnectionError("closed")
|
|
if line.startswith(tag.encode() + b" "):
|
|
if b" OK" not in line[:12]:
|
|
raise ConnectionError(line.decode(errors="replace").strip())
|
|
return
|
|
|
|
f.readline() # greeting
|
|
cmd("a1", f'LOGIN "{account}" "{password}"')
|
|
cmd("a2", f"SELECT {S['landed']}")
|
|
n = 3
|
|
while True:
|
|
tag = f"a{n}"
|
|
n += 1
|
|
sock.sendall(f"{tag} IDLE\r\n".encode())
|
|
sock.settimeout(60)
|
|
renew = time.time() + 25 * 60
|
|
while time.time() < renew:
|
|
try:
|
|
line = f.readline()
|
|
except socket.timeout:
|
|
continue
|
|
if not line:
|
|
raise ConnectionError("closed in IDLE")
|
|
if line.endswith(b"EXISTS\r\n"):
|
|
now = time.time()
|
|
with lock:
|
|
S["idle"]["notices"] += 1
|
|
S["idle"]["longest_quiet_s"] = max(S["idle"]["longest_quiet_s"], round(now - last_notice))
|
|
last_notice = now
|
|
sock.sendall(b"DONE\r\n")
|
|
sock.settimeout(30)
|
|
while not f.readline().startswith(tag.encode() + b" "):
|
|
pass
|
|
except (OSError, ConnectionError, ssl.SSLError) as e:
|
|
with lock:
|
|
S["idle"]["drops"] += 1
|
|
event("idle-drop", f"client {i}: {type(e).__name__}: {e}")
|
|
finally:
|
|
if sock:
|
|
try:
|
|
sock.close()
|
|
except OSError:
|
|
pass
|
|
time.sleep(5)
|
|
|
|
|
|
save()
|
|
event("start", f"run {run}, start {S['starts']}")
|
|
for target in ([lambda: every(IN_EVERY, inbound), lambda: every(OUT_EVERY, outbound), lambda: every(30, verify)]
|
|
+ [(lambda i=i: idle(i)) for i in range(IDLE_CLIENTS)]):
|
|
threading.Thread(target=target, daemon=True).start()
|
|
while True:
|
|
time.sleep(30)
|
|
save()
|