From 130c1e405f288d6fdcfad5a675edf87d57b25338 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 16:25:47 -0700 Subject: [PATCH] Mail senders: a 421 to QUIT isn't a failed send The chaos tests saw a message arrive twice after a pod kill or a drain, three times in about twenty runs. Each time the send had been retried after "421 4.3.0 Server shutting down". That 421 was the answer to QUIT, after the end of DATA had its 250: smtplib's with-statement sends QUIT on the way out and raises on anything but 221, so the lab took a delivered message for a failed one and sent it again. The server did nothing wrong. mailflow.py and soak-load.py now QUIT and close whatever the answer, and retry only what failed before the 250. The soak running now has the old sender; a message twice in its report is checked against its retries. --- lab/chart/mailflow.py | 18 +++++++++++++++++- lab/soak/soak-load.py | 21 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/lab/chart/mailflow.py b/lab/chart/mailflow.py index d4827bf..eadfe64 100755 --- a/lab/chart/mailflow.py +++ b/lab/chart/mailflow.py @@ -39,10 +39,26 @@ def send(n): # As an MTA waits (RFC 5321 4.5.3.2): a dead address fails fast, but a # slow reply is waited for, up to minutes. Giving up early on the reply # to the end of DATA and sending again is how a sender makes duplicates. - with smtplib.SMTP(timeout=20) as s: + s = smtplib.SMTP(timeout=20) + try: s.connect(lb, 25) s.sock.settimeout(300) s.send_message(msg) + finally: + quit_quietly(s) + + +def quit_quietly(s): + """QUIT, whatever the answer. The message is the server's once the end + of DATA got its 250; a server that's stopping answers QUIT with 421, and + taking that as a failure and sending again makes a duplicate (smtplib's + `with` does just that: its __exit__ raises on anything but 221).""" + try: + s.quit() + except (OSError, smtplib.SMTPException): + pass + finally: + s.close() accepted, retries, started = 0, 0, time.time() diff --git a/lab/soak/soak-load.py b/lab/soak/soak-load.py index e348d24..97afec6 100755 --- a/lab/soak/soak-load.py +++ b/lab/soak/soak-load.py @@ -84,6 +84,17 @@ def message(frm, to, subject): return msg +def quit_quietly(s): + """QUIT, whatever the answer: the message is the server's once DATA got + its 250, and a stopping server answers QUIT 421 (see chart/mailflow.py).""" + try: + s.quit() + except (OSError, smtplib.SMTPException): + pass + finally: + s.close() + + def every(seconds, fn): """fn() every `seconds`, keeping to the schedule; fn handles its errors.""" nxt = time.time() @@ -101,10 +112,13 @@ def inbound(): msg = message(f"soak@m{n}.{run}.sender.test", account, f"soak {run} in {n}") try: # As an MTA waits (RFC 5321 4.5.3.2); see chart/mailflow.py - with smtplib.SMTP(timeout=20) as s: + s = smtplib.SMTP(timeout=20) + try: s.connect(lb, 25) s.sock.settimeout(300) s.send_message(msg) + finally: + quit_quietly(s) except (OSError, smtplib.SMTPException) as e: with lock: S["inbound"]["retries"] += 1 @@ -122,10 +136,13 @@ def outbound(): n = S["outbound"]["next"] msg = message(account, f"soak-{n % SINK_SPREAD}@sink.test", f"soak {run} out {n}") try: - with smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) as s: + s = smtplib.SMTP_SSL(lb, 465, context=tls, timeout=20) + try: s.sock.settimeout(300) s.login(account, password) s.send_message(msg) + finally: + quit_quietly(s) except (OSError, smtplib.SMTPException) as e: with lock: S["outbound"]["retries"] += 1 -- 2.54.0