Chaos: a missing message's queue id, and what the server's logs say on it #42

Merged
jcoffey-dev merged 1 commits from feat/mailflow-queue-ids into main 2026-11-21 00:02:02 +00:00
2 changed files with 27 additions and 3 deletions

No files matched your search

+7
View File
@@ -137,6 +137,13 @@ for sc in $SCENARIOS; do
# The retries too: why each send was refused says what the fault did
sed 's/^/ /' "$WORK/$sc.log"
TOOK[$sc]=$(( $(date +%s) - start ))
# A message missing: what the server's logs say about it, by its queue id
# (delivered late, deferred and why, or nothing at all)
for id in $(sed -n 's/^ queue id [0-9]*: \([0-9][0-9]*\)$/\1/p' "$WORK/$sc.log"); do
echo " -- the server's logs on queue id $id"
"${K[@]}" -n inbuxa logs -l app.kubernetes.io/component=server --prefix --tail=-1 --since=30m 2>/dev/null \
| sed 's/\x1b\[[0-9;]*m//g' | grep -wF -- "$id" | tail -20 | cut -c1-400 | sed 's/^/ /' || true
done
if [ "$fault" != 0 ]; then RESULT[$sc]="FAIL (the fault or the recovery)"
elif [ "$flow" != 0 ]; then RESULT[$sc]="FAIL (mail lost or doubled)"
else RESULT[$sc]="ok"; fi
+20 -3
View File
@@ -10,7 +10,7 @@ arrived exactly once.
ACCOUNT=user@domain PASSWORD_FILE=... STOP_FILE=... mailflow.py LB
Prints a summary line and exits 1 if anything accepted was lost or came
twice. The lab's certificate is self-signed, so TLS isn't verified.
twice; for a message missing, the queue id the server gave it. The lab's certificate is self-signed, so TLS isn't verified.
"""
import imaplib, os, re, smtplib, ssl, sys, time, uuid
from email.message import EmailMessage
@@ -43,9 +43,22 @@ def send(n):
try:
s.connect(lb, 25)
s.sock.settimeout(300)
s.send_message(msg)
s.ehlo_or_helo_if_needed()
code, resp = s.mail(msg["From"])
if code != 250:
raise smtplib.SMTPSenderRefused(code, resp, msg["From"])
code, resp = s.rcpt(account)
if code not in (250, 251):
raise smtplib.SMTPRecipientsRefused({account: (code, resp)})
code, resp = s.data(msg.as_bytes())
if code != 250:
raise smtplib.SMTPDataError(code, resp)
finally:
quit_quietly(s)
# "250 2.0.0 Message queued with id 1a2b...": the server's logs name the
# same message by that id, in decimal
m = re.search(rb"queued with id ([0-9a-f]+)", resp)
return int(m.group(1), 16) if m else None
def quit_quietly(s):
@@ -62,10 +75,11 @@ def quit_quietly(s):
accepted, retries, started = 0, 0, time.time()
queue_ids = {}
while not os.path.exists(stop):
n = accepted + 1
try:
send(n)
queue_ids[n] = send(n)
accepted = n
except (OSError, smtplib.SMTPException) as e:
# Refused, reset or a 4xx: a real MTA tries again later
@@ -124,6 +138,9 @@ print(f"mailflow {run}: {accepted} accepted over {sent_for:.0f}s, {retries} retr
f"{accepted - len(missing)} arrived, {len(missing)} missing, {len(twice)} twice")
if missing:
print(f" missing: {missing[:20]}")
# One line per message for chaos-test.sh to look up in the server's logs
for n in missing[:20]:
print(f" queue id {n}: {queue_ids.get(n)}")
if twice:
print(f" twice: {twice[:20]}")
sys.exit(1 if missing or twice or accepted == 0 else 0)