Files
jcoffey-dev ea991b75fd
ci / check (pull_request) Successful in 40s
Chaos: a missing message's queue id, and what the server's logs say on it
When the chaos test finds an accepted message missing, it couldn't tell a
late delivery from a lost one: a PostgreSQL failover left one message
missing after 7 minutes with no retry. mailflow.py now keeps the queue id
from each 250 ("queued with id ...", hex) and prints it for each message
missing, and chaos-test.sh then shows the server's log lines for that id
(decimal there): delivered late, deferred and why, or never seen.
2026-11-20 16:44:18 -07:00

147 lines
5.4 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""Mail flowing through an upgrade (K-L5): sends one message a second over
SMTP (25), each from its own sender domain, until STOP_FILE appears, as a sending MTA would, so a refused or
dropped connection is retried rather than lost. Then waits for every
message the server accepted to show up over IMAPS (993) and checks each
arrived exactly once.
ACCOUNT=user@domain PASSWORD_FILE=... STOP_FILE=... mailflow.py LB
Prints a summary line and exits 1 if anything accepted was lost or came
twice; for a message missing, the queue id the server gave it. The lab's certificate is self-signed, so TLS isn't verified.
"""
import imaplib, os, re, smtplib, ssl, sys, time, uuid
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
lb = sys.argv[1]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
stop = os.environ["STOP_FILE"]
run = uuid.uuid4().hex[:8]
tls = ssl.create_default_context()
tls.check_hostname = False
tls.verify_mode = ssl.CERT_NONE
def send(n):
msg = EmailMessage()
# A sender domain per message: the server's default throttle allows 25
# an hour from one sender domain to one recipient
msg["From"] = f"lab-sender@m{n}.{run}.sender.test"
msg["To"] = account
msg["Subject"] = f"mailflow {run} {n}"
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test")
msg.set_content(f"mailflow {run} message {n}\n")
# As an MTA waits (RFC 5321 4.5.3.2): a dead address fails fast, but a
# slow reply is waited for, up to minutes. Giving up early on the reply
# to the end of DATA and sending again is how a sender makes duplicates.
s = smtplib.SMTP(timeout=20)
try:
s.connect(lb, 25)
s.sock.settimeout(300)
s.ehlo_or_helo_if_needed()
code, resp = s.mail(msg["From"])
if code != 250:
raise smtplib.SMTPSenderRefused(code, resp, msg["From"])
code, resp = s.rcpt(account)
if code not in (250, 251):
raise smtplib.SMTPRecipientsRefused({account: (code, resp)})
code, resp = s.data(msg.as_bytes())
if code != 250:
raise smtplib.SMTPDataError(code, resp)
finally:
quit_quietly(s)
# "250 2.0.0 Message queued with id 1a2b...": the server's logs name the
# same message by that id, in decimal
m = re.search(rb"queued with id ([0-9a-f]+)", resp)
return int(m.group(1), 16) if m else None
def quit_quietly(s):
"""QUIT, whatever the answer. The message is the server's once the end
of DATA got its 250; a server that's stopping answers QUIT with 421, and
taking that as a failure and sending again makes a duplicate (smtplib's
`with` does just that: its __exit__ raises on anything but 221)."""
try:
s.quit()
except (OSError, smtplib.SMTPException):
pass
finally:
s.close()
accepted, retries, started = 0, 0, time.time()
queue_ids = {}
while not os.path.exists(stop):
n = accepted + 1
try:
queue_ids[n] = send(n)
accepted = n
except (OSError, smtplib.SMTPException) as e:
# Refused, reset or a 4xx: a real MTA tries again later
retries += 1
print(f" retry {n}: {type(e).__name__}: {e}"[:160], flush=True)
time.sleep(2)
continue
time.sleep(1)
sent_for = time.time() - started
def count():
"""How many times each message number of this run is in the account."""
seen = {}
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30)
try:
m.login(account, password)
_, boxes = m.list()
for line in boxes:
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
if m.select(name, readonly=True)[0] != "OK":
continue
_, data = m.search(None, "SUBJECT", f'"mailflow {run}"')
ids = data[0].split()
if not ids:
continue
_, hdrs = m.fetch(b",".join(ids), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
for part in hdrs:
if isinstance(part, tuple):
mm = re.search(rf"mailflow {run} (\d+)", part[1].decode())
if mm:
seen[int(mm.group(1))] = seen.get(int(mm.group(1)), 0) + 1
finally:
try:
m.logout()
except Exception:
pass
return seen
# WAIT: how long to wait for what was accepted (an outage can delay delivery)
deadline = time.time() + int(os.environ.get("WAIT", "180"))
while True:
try:
seen = count()
except (OSError, imaplib.IMAP4.error) as e:
print(f" imap: {e}"[:160], flush=True)
seen = {}
missing = [n for n in range(1, accepted + 1) if n not in seen]
if not missing or time.time() > deadline:
break
time.sleep(5)
twice = sorted(n for n, c in seen.items() if c > 1)
print(f"mailflow {run}: {accepted} accepted over {sent_for:.0f}s, {retries} retried, "
f"{accepted - len(missing)} arrived, {len(missing)} missing, {len(twice)} twice")
if missing:
print(f" missing: {missing[:20]}")
# One line per message for chaos-test.sh to look up in the server's logs
for n in missing[:20]:
print(f" queue id {n}: {queue_ids.get(n)}")
if twice:
print(f" twice: {twice[:20]}")
sys.exit(1 if missing or twice or accepted == 0 else 0)