ci / check (pull_request) Successful in 40s
When the chaos test finds an accepted message missing, it couldn't tell a
late delivery from a lost one: a PostgreSQL failover left one message
missing after 7 minutes with no retry. mailflow.py now keeps the queue id
from each 250 ("queued with id ...", hex) and prints it for each message
missing, and chaos-test.sh then shows the server's log lines for that id
(decimal there): delivered late, deferred and why, or never seen.
147 lines
5.4 KiB
Python
Executable File
147 lines
5.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""Mail flowing through an upgrade (K-L5): sends one message a second over
|
|
SMTP (25), each from its own sender domain, until STOP_FILE appears, as a sending MTA would, so a refused or
|
|
dropped connection is retried rather than lost. Then waits for every
|
|
message the server accepted to show up over IMAPS (993) and checks each
|
|
arrived exactly once.
|
|
|
|
ACCOUNT=user@domain PASSWORD_FILE=... STOP_FILE=... mailflow.py LB
|
|
|
|
Prints a summary line and exits 1 if anything accepted was lost or came
|
|
twice; for a message missing, the queue id the server gave it. The lab's certificate is self-signed, so TLS isn't verified.
|
|
"""
|
|
import imaplib, os, re, smtplib, ssl, sys, time, uuid
|
|
from email.message import EmailMessage
|
|
from email.utils import formatdate, make_msgid
|
|
|
|
lb = sys.argv[1]
|
|
account = os.environ["ACCOUNT"]
|
|
password = open(os.environ["PASSWORD_FILE"]).read().strip()
|
|
stop = os.environ["STOP_FILE"]
|
|
run = uuid.uuid4().hex[:8]
|
|
tls = ssl.create_default_context()
|
|
tls.check_hostname = False
|
|
tls.verify_mode = ssl.CERT_NONE
|
|
|
|
|
|
def send(n):
|
|
msg = EmailMessage()
|
|
# A sender domain per message: the server's default throttle allows 25
|
|
# an hour from one sender domain to one recipient
|
|
msg["From"] = f"lab-sender@m{n}.{run}.sender.test"
|
|
msg["To"] = account
|
|
msg["Subject"] = f"mailflow {run} {n}"
|
|
msg["Date"] = formatdate()
|
|
msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test")
|
|
msg.set_content(f"mailflow {run} message {n}\n")
|
|
# As an MTA waits (RFC 5321 4.5.3.2): a dead address fails fast, but a
|
|
# slow reply is waited for, up to minutes. Giving up early on the reply
|
|
# to the end of DATA and sending again is how a sender makes duplicates.
|
|
s = smtplib.SMTP(timeout=20)
|
|
try:
|
|
s.connect(lb, 25)
|
|
s.sock.settimeout(300)
|
|
s.ehlo_or_helo_if_needed()
|
|
code, resp = s.mail(msg["From"])
|
|
if code != 250:
|
|
raise smtplib.SMTPSenderRefused(code, resp, msg["From"])
|
|
code, resp = s.rcpt(account)
|
|
if code not in (250, 251):
|
|
raise smtplib.SMTPRecipientsRefused({account: (code, resp)})
|
|
code, resp = s.data(msg.as_bytes())
|
|
if code != 250:
|
|
raise smtplib.SMTPDataError(code, resp)
|
|
finally:
|
|
quit_quietly(s)
|
|
# "250 2.0.0 Message queued with id 1a2b...": the server's logs name the
|
|
# same message by that id, in decimal
|
|
m = re.search(rb"queued with id ([0-9a-f]+)", resp)
|
|
return int(m.group(1), 16) if m else None
|
|
|
|
|
|
def quit_quietly(s):
|
|
"""QUIT, whatever the answer. The message is the server's once the end
|
|
of DATA got its 250; a server that's stopping answers QUIT with 421, and
|
|
taking that as a failure and sending again makes a duplicate (smtplib's
|
|
`with` does just that: its __exit__ raises on anything but 221)."""
|
|
try:
|
|
s.quit()
|
|
except (OSError, smtplib.SMTPException):
|
|
pass
|
|
finally:
|
|
s.close()
|
|
|
|
|
|
accepted, retries, started = 0, 0, time.time()
|
|
queue_ids = {}
|
|
while not os.path.exists(stop):
|
|
n = accepted + 1
|
|
try:
|
|
queue_ids[n] = send(n)
|
|
accepted = n
|
|
except (OSError, smtplib.SMTPException) as e:
|
|
# Refused, reset or a 4xx: a real MTA tries again later
|
|
retries += 1
|
|
print(f" retry {n}: {type(e).__name__}: {e}"[:160], flush=True)
|
|
time.sleep(2)
|
|
continue
|
|
time.sleep(1)
|
|
sent_for = time.time() - started
|
|
|
|
|
|
def count():
|
|
"""How many times each message number of this run is in the account."""
|
|
seen = {}
|
|
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30)
|
|
try:
|
|
m.login(account, password)
|
|
_, boxes = m.list()
|
|
for line in boxes:
|
|
name = line.decode().rsplit(' "/" ', 1)[-1].strip()
|
|
if m.select(name, readonly=True)[0] != "OK":
|
|
continue
|
|
_, data = m.search(None, "SUBJECT", f'"mailflow {run}"')
|
|
ids = data[0].split()
|
|
if not ids:
|
|
continue
|
|
_, hdrs = m.fetch(b",".join(ids), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
|
|
for part in hdrs:
|
|
if isinstance(part, tuple):
|
|
mm = re.search(rf"mailflow {run} (\d+)", part[1].decode())
|
|
if mm:
|
|
seen[int(mm.group(1))] = seen.get(int(mm.group(1)), 0) + 1
|
|
finally:
|
|
try:
|
|
m.logout()
|
|
except Exception:
|
|
pass
|
|
return seen
|
|
|
|
|
|
# WAIT: how long to wait for what was accepted (an outage can delay delivery)
|
|
deadline = time.time() + int(os.environ.get("WAIT", "180"))
|
|
while True:
|
|
try:
|
|
seen = count()
|
|
except (OSError, imaplib.IMAP4.error) as e:
|
|
print(f" imap: {e}"[:160], flush=True)
|
|
seen = {}
|
|
missing = [n for n in range(1, accepted + 1) if n not in seen]
|
|
if not missing or time.time() > deadline:
|
|
break
|
|
time.sleep(5)
|
|
|
|
twice = sorted(n for n, c in seen.items() if c > 1)
|
|
print(f"mailflow {run}: {accepted} accepted over {sent_for:.0f}s, {retries} retried, "
|
|
f"{accepted - len(missing)} arrived, {len(missing)} missing, {len(twice)} twice")
|
|
if missing:
|
|
print(f" missing: {missing[:20]}")
|
|
# One line per message for chaos-test.sh to look up in the server's logs
|
|
for n in missing[:20]:
|
|
print(f" queue id {n}: {queue_ids.get(n)}")
|
|
if twice:
|
|
print(f" twice: {twice[:20]}")
|
|
sys.exit(1 if missing or twice or accepted == 0 else 0)
|