From ea991b75fd259fbc85c91df12f3290fd002bd74f Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 16:44:18 -0700 Subject: [PATCH] Chaos: a missing message's queue id, and what the server's logs say on it When the chaos test finds an accepted message missing, it couldn't tell a late delivery from a lost one: a PostgreSQL failover left one message missing after 7 minutes with no retry. mailflow.py now keeps the queue id from each 250 ("queued with id ...", hex) and prints it for each message missing, and chaos-test.sh then shows the server's log lines for that id (decimal there): delivered late, deferred and why, or never seen. --- lab/chart/chaos-test.sh | 7 +++++++ lab/chart/mailflow.py | 23 ++++++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lab/chart/chaos-test.sh b/lab/chart/chaos-test.sh index 52189c1..c0b8ef6 100755 --- a/lab/chart/chaos-test.sh +++ b/lab/chart/chaos-test.sh @@ -137,6 +137,13 @@ for sc in $SCENARIOS; do # The retries too: why each send was refused says what the fault did sed 's/^/ /' "$WORK/$sc.log" TOOK[$sc]=$(( $(date +%s) - start )) + # A message missing: what the server's logs say about it, by its queue id + # (delivered late, deferred and why, or nothing at all) + for id in $(sed -n 's/^ queue id [0-9]*: \([0-9][0-9]*\)$/\1/p' "$WORK/$sc.log"); do + echo " -- the server's logs on queue id $id" + "${K[@]}" -n inbuxa logs -l app.kubernetes.io/component=server --prefix --tail=-1 --since=30m 2>/dev/null \ + | sed 's/\x1b\[[0-9;]*m//g' | grep -wF -- "$id" | tail -20 | cut -c1-400 | sed 's/^/ /' || true + done if [ "$fault" != 0 ]; then RESULT[$sc]="FAIL (the fault or the recovery)" elif [ "$flow" != 0 ]; then RESULT[$sc]="FAIL (mail lost or doubled)" else RESULT[$sc]="ok"; fi diff --git a/lab/chart/mailflow.py b/lab/chart/mailflow.py index eadfe64..469e49c 100755 --- a/lab/chart/mailflow.py +++ b/lab/chart/mailflow.py @@ -10,7 +10,7 @@ arrived exactly once. ACCOUNT=user@domain PASSWORD_FILE=... STOP_FILE=... mailflow.py LB Prints a summary line and exits 1 if anything accepted was lost or came -twice. The lab's certificate is self-signed, so TLS isn't verified. +twice; for a message missing, the queue id the server gave it. The lab's certificate is self-signed, so TLS isn't verified. """ import imaplib, os, re, smtplib, ssl, sys, time, uuid from email.message import EmailMessage @@ -43,9 +43,22 @@ def send(n): try: s.connect(lb, 25) s.sock.settimeout(300) - s.send_message(msg) + s.ehlo_or_helo_if_needed() + code, resp = s.mail(msg["From"]) + if code != 250: + raise smtplib.SMTPSenderRefused(code, resp, msg["From"]) + code, resp = s.rcpt(account) + if code not in (250, 251): + raise smtplib.SMTPRecipientsRefused({account: (code, resp)}) + code, resp = s.data(msg.as_bytes()) + if code != 250: + raise smtplib.SMTPDataError(code, resp) finally: quit_quietly(s) + # "250 2.0.0 Message queued with id 1a2b...": the server's logs name the + # same message by that id, in decimal + m = re.search(rb"queued with id ([0-9a-f]+)", resp) + return int(m.group(1), 16) if m else None def quit_quietly(s): @@ -62,10 +75,11 @@ def quit_quietly(s): accepted, retries, started = 0, 0, time.time() +queue_ids = {} while not os.path.exists(stop): n = accepted + 1 try: - send(n) + queue_ids[n] = send(n) accepted = n except (OSError, smtplib.SMTPException) as e: # Refused, reset or a 4xx: a real MTA tries again later @@ -124,6 +138,9 @@ print(f"mailflow {run}: {accepted} accepted over {sent_for:.0f}s, {retries} retr f"{accepted - len(missing)} arrived, {len(missing)} missing, {len(twice)} twice") if missing: print(f" missing: {missing[:20]}") + # One line per message for chaos-test.sh to look up in the server's logs + for n in missing[:20]: + print(f" queue id {n}: {queue_ids.get(n)}") if twice: print(f" twice: {twice[:20]}") sys.exit(1 if missing or twice or accepted == 0 else 0) -- 2.54.0