diff --git a/lab/chart/crossing-test.sh b/lab/chart/crossing-test.sh new file mode 100755 index 0000000..ce17c84 --- /dev/null +++ b/lab/chart/crossing-test.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L5, crossing: a store made and filled by a published 2026.x server, then +# upgraded by the chart to a 27.1 release, with the layout change the +# crossing brings: +# +# 1. a fresh store set up by OLD (fresh-store.sh), and OLD taking mail +# through the chart's own mail Service, as one pod with the chart's +# labels (OLD predates the chart: it runs as the lab's spike pods do) +# 2. the mailbox digests: every mailbox's count and a hash over its messages +# 3. helm upgrade to TO without acknowledging the layout change: refused, +# the store untouched +# 4. acknowledged: the hook upgrades the store, the Deployments come up on +# TO, and the digests are what OLD left +# +# OLD=2026.11.18.3 TO=27.1-rc.34 lab/chart/crossing-test.sh +# +# WIPES the lab's inbuxa database, and leaves a store fresh-store.sh made. +# seed-store.sh and a helm upgrade put the lab back as the other tests want it. +set -euo pipefail + +cd "$(dirname "$0")" +OLD="${OLD:?the 2026.x release to start from, e.g. 2026.11.18.3}" +TO="${TO:?the 27.1 release to cross to, e.g. 27.1-rc.34}" +MESSAGES="${MESSAGES:-15}" +IMAGE=ghcr.io/inbuxa/inbuxa-server +K=(kubectl --context kind-inbuxa-lab) +H=(helm --kube-context kind-inbuxa-lab -n inbuxa) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +WORK="$(mktemp -d)" +trap '"${K[@]}" -n inbuxa delete pod inbuxa-old --ignore-not-found >/dev/null 2>&1; rm -rf "$WORK"' EXIT +fail() { echo "!! $*" >&2; exit 1; } + +digest_of() { docker buildx imagetools inspect "$IMAGE:$1" --format '{{json .Manifest.Digest}}' | tr -d '"'; } +mark() { "${K[@]}" -n data exec inbuxa-pg-1 -c postgres -- psql -d inbuxa -At \ + -c "select encode(v,'hex') from \"_\" where k = 'Vs'::bytea"; } +lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } +mailbox_digest() { ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$(lb)" "$@"; } +upgrade() { + "${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml --set server.seed.enabled=false \ + --set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \ + --set server.image.digest="$to_d" --wait --timeout 20m "$@" +} +hook_log() { "${K[@]}" -n inbuxa logs job/inbuxa-upgrade 2>/dev/null | grep -v '^20..-' || true; } + +old_d="$(digest_of "$OLD")"; to_d="$(digest_of "$TO")" +[[ "$old_d" =~ ^sha256: && "$to_d" =~ ^sha256: ]] || fail "couldn't read the digests of $OLD and $TO" +echo "$OLD = $old_d" +echo "$TO = $to_d" + +echo "== 1. a store made by $OLD" +IMAGE="$IMAGE@$old_d" PULL=IfNotPresent ./fresh-store.sh +echo " layout mark: $(mark)" +# OLD serving mail, behind the chart's Service and NetworkPolicies. The +# Service targets ports by name, so the pod names them as the chart's do. +PORTS="$("${K[@]}" -n inbuxa get deploy inbuxa-front -o jsonpath='{.spec.template.spec.containers[0].ports}')" +sed -n '/^ template:/,$p' ../spike/core.yaml | sed '1d' | + sed -e "s|^ ||" \ + -e "s|metadata: {labels: {app: inbuxa-server, group: core}}|metadata: {name: inbuxa-old, namespace: inbuxa, labels: {app.kubernetes.io/name: inbuxa, app.kubernetes.io/instance: inbuxa, app.kubernetes.io/component: server, inbuxa.org/serves-mail: \"true\"}}|" \ + -e "s|image: inbuxa-lab/server:rc27|image: $IMAGE@$old_d|" \ + -e "s|imagePullPolicy: Never|imagePullPolicy: IfNotPresent\n ports: $PORTS|" \ + -e '/topologySpreadConstraints:/,+1d' > "$WORK/old.yaml" +"${K[@]}" -n inbuxa delete pod inbuxa-old --ignore-not-found --wait >/dev/null +{ printf 'apiVersion: v1\nkind: Pod\n'; cat "$WORK/old.yaml"; } | "${K[@]}" apply -f - >/dev/null +"${K[@]}" -n inbuxa wait --for=condition=Ready pod/inbuxa-old --timeout=300s >/dev/null || + { "${K[@]}" -n inbuxa logs inbuxa-old | tail -20; fail "$OLD never got ready"; } + +echo "== 2. mail into $OLD, and its digests" +for _ in $(seq 1 30); do timeout 5 bash -c "/dev/null && break; sleep 2; done +mailbox_digest send "$MESSAGES" > "$WORK/before" +sed 's/^/ /' "$WORK/before" +grep -q . "$WORK/before" || fail "no mailboxes" +"${K[@]}" -n inbuxa delete pod inbuxa-old --wait >/dev/null +before_mark="$(mark)" + +echo "== 3. to $TO, not acknowledged" +if upgrade >/dev/null 2>&1; then fail "the crossing went through unacknowledged"; fi +echo " refused; the hook said:"; hook_log | sed 's/^/ /' +[ "$(mark)" = "$before_mark" ] || fail "the refused upgrade changed the store" + +echo "== 4. acknowledged" +start=$(date +%s) +upgrade --set server.upgrade.acknowledgeLayout=3 >/dev/null +echo " crossed in $(( $(date +%s) - start ))s; the hook said:"; hook_log | sed 's/^/ /' +[ "$(mark)" = 00000003 ] || fail "the store isn't at layout 3: $(mark)" +for _ in $(seq 1 60); do + got="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{" "}{.status.containerStatuses[0].ready}{"\n"}{end}')" + [ "$(grep -c . <<<"$got")" -ge 5 ] && ! grep -qv "@$to_d true\$" <<<"$got" && break + sleep 3 +done +grep -qv "@$to_d true\$" <<<"$got" && fail "not every server pod runs $TO" +mailbox_digest > "$WORK/after" +diff "$WORK/before" "$WORK/after" >/dev/null || + { diff "$WORK/before" "$WORK/after" | sed 's/^/ /'; fail "the mail isn't what $OLD left"; } +echo " digests match" +ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 mailtest.py "$(lb)" +echo "== $OLD to $TO: crossed, every message as it was" diff --git a/lab/chart/fresh-store.sh b/lab/chart/fresh-store.sh index 7d26501..d7e8e4a 100755 --- a/lab/chart/fresh-store.sh +++ b/lab/chart/fresh-store.sh @@ -14,6 +14,8 @@ set -euo pipefail cd "$(dirname "$0")" K=(kubectl --context kind-inbuxa-lab) IMAGE="${IMAGE:-inbuxa-lab/server:main}" +# Never for an image loaded into kind; IfNotPresent for a published one +PULL="${PULL:-Never}" PORTS="${PORTS:-pop3s=off}" DOMAIN="${DOMAIN:-lab.test}" STATE="${STATE:-$HOME/.cache/inbuxa-lab}" @@ -21,11 +23,17 @@ mkdir -p "$STATE" && chmod 700 "$STATE" secret() { "${K[@]}" -n inbuxa get secret "$1" -o "jsonpath={.data.$2}" | base64 -d; } -# A one-off server pod from ../spike, on this image and these listener ports +# A one-off server pod from ../spike, on this image and these listener ports. +# It carries the chart's server labels, so the chart's deny-all and server +# NetworkPolicies cover it, as the lab's Kyverno policy requires. +LABELS='app.kubernetes.io/name: inbuxa, app.kubernetes.io/instance: inbuxa, app.kubernetes.io/component: server' + run_pod() { local file="$1" name="$2" "${K[@]}" -n inbuxa delete pod "$name" --ignore-not-found --wait >/dev/null sed -e "s|image: inbuxa-lab/server:rc27|image: $IMAGE|" \ + -e "s|imagePullPolicy: Never|imagePullPolicy: $PULL|" \ + -e "s|^ name: $name\$| name: $name\n labels: {$LABELS}|" -e '/^ labels: {app: inbuxa-setup}$/d' \ -e "s| - {name: INBUXA_HOSTNAME, value: $name}| - {name: INBUXA_HOSTNAME, value: $name}\n - {name: INBUXA_LISTENER_PORTS, value: \"$PORTS\"}|" \ "../spike/$file" | "${K[@]}" apply -f - >/dev/null "${K[@]}" -n inbuxa wait --for=condition=Ready "pod/$name" --timeout=180s >/dev/null diff --git a/lab/chart/mailbox-digest.py b/lab/chart/mailbox-digest.py index eb14926..54c2cab 100755 --- a/lab/chart/mailbox-digest.py +++ b/lab/chart/mailbox-digest.py @@ -51,10 +51,13 @@ if len(sys.argv) > 2 and sys.argv[2] == "send": m = imap() before = total(m) run = uuid.uuid4().hex[:8] - with smtplib.SMTP(lb, 25, timeout=30) as s: - for i in range(count): + # A session per message: servers cap the messages one session may send + for i in range(count): + with smtplib.SMTP(lb, 25, timeout=30) as s: msg = EmailMessage() - msg["From"] = "lab-sender@sender.test" + # Its own sender domain: the default throttle allows 25 an hour + # from one sender domain to one recipient + msg["From"] = f"lab-sender@m{i}.{run}.sender.test" msg["To"] = account msg["Subject"] = f"backup-test {run} {i}" msg["Date"] = formatdate() @@ -63,6 +66,8 @@ if len(sys.argv) > 2 and sys.argv[2] == "send": msg.add_attachment((f"line {i}: " + uuid.uuid4().hex + "\n") * (60 + 17 * i), filename=f"notes-{i}.txt") s.send_message(msg) + # Under the default 5 a second from one address + time.sleep(0.3) for _ in range(60): if total(m) >= before + count: break diff --git a/lab/chart/mailflow.py b/lab/chart/mailflow.py new file mode 100755 index 0000000..4bfd083 --- /dev/null +++ b/lab/chart/mailflow.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""Mail flowing through an upgrade (K-L5): sends one message a second over +SMTP (25), each from its own sender domain, until STOP_FILE appears, as a sending MTA would, so a refused or +dropped connection is retried rather than lost. Then waits for every +message the server accepted to show up over IMAPS (993) and checks each +arrived exactly once. + + ACCOUNT=user@domain PASSWORD_FILE=... STOP_FILE=... mailflow.py LB + +Prints a summary line and exits 1 if anything accepted was lost or came +twice. The lab's certificate is self-signed, so TLS isn't verified. +""" +import imaplib, os, re, smtplib, ssl, sys, time, uuid +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +lb = sys.argv[1] +account = os.environ["ACCOUNT"] +password = open(os.environ["PASSWORD_FILE"]).read().strip() +stop = os.environ["STOP_FILE"] +run = uuid.uuid4().hex[:8] +tls = ssl.create_default_context() +tls.check_hostname = False +tls.verify_mode = ssl.CERT_NONE + + +def send(n): + msg = EmailMessage() + # A sender domain per message: the server's default throttle allows 25 + # an hour from one sender domain to one recipient + msg["From"] = f"lab-sender@m{n}.{run}.sender.test" + msg["To"] = account + msg["Subject"] = f"mailflow {run} {n}" + msg["Date"] = formatdate() + msg["Message-ID"] = make_msgid(domain=f"{run}.sender.test") + msg.set_content(f"mailflow {run} message {n}\n") + with smtplib.SMTP(lb, 25, timeout=20) as s: + s.send_message(msg) + + +accepted, retries, started = 0, 0, time.time() +while not os.path.exists(stop): + n = accepted + 1 + try: + send(n) + accepted = n + except (OSError, smtplib.SMTPException) as e: + # Refused, reset or a 4xx: a real MTA tries again later + retries += 1 + print(f" retry {n}: {type(e).__name__}: {e}"[:160], flush=True) + time.sleep(2) + continue + time.sleep(1) +sent_for = time.time() - started + + +def count(): + """How many times each message number of this run is in the account.""" + seen = {} + m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30) + try: + m.login(account, password) + _, boxes = m.list() + for line in boxes: + name = line.decode().rsplit(' "/" ', 1)[-1].strip() + if m.select(name, readonly=True)[0] != "OK": + continue + _, data = m.search(None, "SUBJECT", f'"mailflow {run}"') + ids = data[0].split() + if not ids: + continue + _, hdrs = m.fetch(b",".join(ids), "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])") + for part in hdrs: + if isinstance(part, tuple): + mm = re.search(rf"mailflow {run} (\d+)", part[1].decode()) + if mm: + seen[int(mm.group(1))] = seen.get(int(mm.group(1)), 0) + 1 + finally: + try: + m.logout() + except Exception: + pass + return seen + + +deadline = time.time() + 180 +while True: + try: + seen = count() + except (OSError, imaplib.IMAP4.error) as e: + print(f" imap: {e}"[:160], flush=True) + seen = {} + missing = [n for n in range(1, accepted + 1) if n not in seen] + if not missing or time.time() > deadline: + break + time.sleep(5) + +twice = sorted(n for n, c in seen.items() if c > 1) +print(f"mailflow {run}: {accepted} accepted over {sent_for:.0f}s, {retries} retried, " + f"{accepted - len(missing)} arrived, {len(missing)} missing, {len(twice)} twice") +if missing: + print(f" missing: {missing[:20]}") +if twice: + print(f" twice: {twice[:20]}") +sys.exit(1 if missing or twice or accepted == 0 else 0) diff --git a/lab/chart/mailtest.py b/lab/chart/mailtest.py index 977365d..171a83e 100755 --- a/lab/chart/mailtest.py +++ b/lab/chart/mailtest.py @@ -21,7 +21,9 @@ tls.verify_mode = ssl.CERT_NONE msg = EmailMessage() # A sender domain with no DNS at all: one that publishes DMARC (example.net # does, p=reject) would send the message to Junk -msg["From"] = "lab-sender@sender.test" +# ...and one of its own each run: the server's default throttle allows 25 +# messages an hour from one sender domain to one recipient +msg["From"] = f"lab-sender@{token[:12]}.sender.test" msg["To"] = account msg["Subject"] = f"chart smoke {token}" msg["Date"] = formatdate() diff --git a/lab/chart/rolling-upgrade-test.sh b/lab/chart/rolling-upgrade-test.sh new file mode 100755 index 0000000..11f077b --- /dev/null +++ b/lab/chart/rolling-upgrade-test.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L5, rolling: the published server images, release N-1 to release N, +# with mail flowing the whole way. On the lab's seeded install: +# +# 1. helm upgrade to FROM (ghcr.io, pinned by digest); mail works +# 2. mail starts flowing (mailflow.py: a message a second, retried like +# an MTA would when a pod goes away) +# 3. helm upgrade to TO: the upgrade hook finds no layout change, so the +# Deployments roll +# 4. every server pod runs TO's digest, mail still works, and every +# message accepted during the roll arrived exactly once +# +# FROM=27.1-rc.33 TO=27.1-rc.34 lab/chart/rolling-upgrade-test.sh +# +# Leaves the lab on TO. Needs docker (to read the digests) and the kind +# nodes to reach ghcr.io. +set -euo pipefail + +cd "$(dirname "$0")" +FROM="${FROM:?the release to start from, e.g. 27.1-rc.33}" +TO="${TO:?the release to upgrade to, e.g. 27.1-rc.34}" +IMAGE=ghcr.io/inbuxa/inbuxa-server +K=(kubectl --context kind-inbuxa-lab) +H=(helm --kube-context kind-inbuxa-lab -n inbuxa) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +WORK="$(mktemp -d)" +FLOW="" +trap '[ -n "$FLOW" ] && kill "$FLOW" 2>/dev/null; rm -rf "$WORK"' EXIT +fail() { echo "!! $*" >&2; exit 1; } + +digest() { docker buildx imagetools inspect "$IMAGE:$1" --format '{{json .Manifest.Digest}}' | tr -d '"'; } +upgrade() { # digest, then helm's own flags + local d="$1"; shift + "${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml \ + --set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \ + --set server.image.digest="$d" --wait --timeout 20m "$@" +} +# Every server pod is ready and runs exactly this digest, once the old ones +# have finished terminating (helm's --wait returns before they have) +on_digest() { + local want="$1" got + for _ in $(seq 1 60); do + got="$("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server \ + -o jsonpath='{range .items[*]}{.status.containerStatuses[0].imageID}{" "}{.status.containerStatuses[0].ready}{"\n"}{end}')" + if [ "$(grep -c . <<<"$got")" -ge 5 ] && ! grep -qv "@$want true\$" <<<"$got"; then return 0; fi + sleep 3 + done + echo "$got"; return 1 +} +hook_log() { "${K[@]}" -n inbuxa logs job/inbuxa-upgrade 2>/dev/null | grep -v '^20..-' || true; } +lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } +mail() { ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 mailtest.py "$(lb)"; } + +from_d="$(digest "$FROM")"; to_d="$(digest "$TO")" +[[ "$from_d" =~ ^sha256: && "$to_d" =~ ^sha256: ]] || fail "couldn't read the digests of $FROM and $TO" +echo "$FROM = $from_d" +echo "$TO = $to_d" + +echo "== 1. on $FROM" +start=$(date +%s) +upgrade "$from_d" >/dev/null +echo " in $(( $(date +%s) - start ))s" +on_digest "$from_d" || fail "not every server pod runs $FROM" +mail + +echo "== 2. mail flowing" +ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" STOP_FILE="$WORK/stop" \ + python3 mailflow.py "$(lb)" > "$WORK/flow.log" 2>&1 & +FLOW=$! +sleep 20 + +echo "== 3. rolling to $TO" +start=$(date +%s) +upgrade "$to_d" >/dev/null +echo " in $(( $(date +%s) - start ))s; the hook said:"; hook_log | sed 's/^/ /' +sleep 20 +touch "$WORK/stop" +flow=0; wait "$FLOW" || flow=$?; FLOW="" +sed 's/^/ /' "$WORK/flow.log" + +echo "== 4. on $TO" +on_digest "$to_d" || fail "not every server pod runs $TO" +mail +[ "$flow" = 0 ] || fail "mail accepted during the roll was lost or doubled" +echo "== $FROM to $TO: rolled with mail flowing, nothing lost" diff --git a/lab/chart/seed-job.yaml b/lab/chart/seed-job.yaml index 2da3279..ff7414c 100644 --- a/lab/chart/seed-job.yaml +++ b/lab/chart/seed-job.yaml @@ -12,7 +12,9 @@ spec: ttlSecondsAfterFinished: 3600 template: metadata: - labels: {app: inbuxa-seed} + # The chart's seed labels: the chart's deny-all and jobs NetworkPolicies + # cover it, as the lab's Kyverno policy requires + labels: {app: inbuxa-seed, app.kubernetes.io/name: inbuxa, app.kubernetes.io/instance: inbuxa, app.kubernetes.io/component: seed} spec: restartPolicy: Never automountServiceAccountToken: false diff --git a/lab/chart/seed-store.sh b/lab/chart/seed-store.sh index 688591b..a8ed739 100755 --- a/lab/chart/seed-store.sh +++ b/lab/chart/seed-store.sh @@ -11,6 +11,8 @@ set -euo pipefail cd "$(dirname "$0")" K=(kubectl --context kind-inbuxa-lab) IMAGE="${IMAGE:-inbuxa-lab/server:main}" +# Never for an image loaded into kind; IfNotPresent for a published one +PULL="${PULL:-Never}" STATE="${STATE:-$HOME/.cache/inbuxa-lab}" mkdir -p "$STATE" && chmod 700 "$STATE" @@ -41,7 +43,7 @@ rm -f "$STATE/s3-access-key" echo "== the seed run" "${K[@]}" -n inbuxa delete job inbuxa-seed --ignore-not-found --wait >/dev/null -sed "s|image: IMAGE|image: $IMAGE|" seed-job.yaml | "${K[@]}" apply -f - >/dev/null +sed -e "s|image: IMAGE|image: $IMAGE|" -e "s|imagePullPolicy: Never|imagePullPolicy: $PULL|" seed-job.yaml | "${K[@]}" apply -f - >/dev/null if ! "${K[@]}" -n inbuxa wait --for=condition=Complete job/inbuxa-seed --timeout=300s >/dev/null 2>&1; then "${K[@]}" -n inbuxa logs job/inbuxa-seed --tail=40 >&2 || true echo "!! the seed run failed" >&2