From d400d710c2cdc92814026aed0747a76a017f099c Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 13:48:49 -0700 Subject: [PATCH] backup-test: backup and restore through the chart's CronJobs (K-L6, K-C11) A run of the backup CronJob exports the store to a Garage bucket; then a fresh cluster's stores (the database dropped and made again, the blob bucket emptied), the import from the newest finished export by a Job made from the CronJob's own pod, and the chart back on the restored store. Every mailbox's count and message hashes (read whole over IMAP) must match, and both store check CronJobs must find nothing wrong. RESTORE_ONLY=1 restores the newest export: the way back from a failed run. A failed import leaves the server pods stopped, never started on the empty store. s3.py: just enough S3, standard library only (SigV4, path-style). mailbox-digest.py: an account's mail as one line per mailbox, and test mail with attachments. --- lab/chart/backup-test.sh | 200 ++++++++++++++++++++++++++++++++++++ lab/chart/mailbox-digest.py | 87 ++++++++++++++++ lab/chart/s3.py | 86 ++++++++++++++++ 3 files changed, 373 insertions(+) create mode 100755 lab/chart/backup-test.sh create mode 100755 lab/chart/mailbox-digest.py create mode 100755 lab/chart/s3.py diff --git a/lab/chart/backup-test.sh b/lab/chart/backup-test.sh new file mode 100755 index 0000000..6e4dab7 --- /dev/null +++ b/lab/chart/backup-test.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C11 and K-L6: backup and restore. With server.backup and both store +# checks on: +# 1. mail for alice@lab.test, with attachments, and a digest of every +# mailbox (count, and a hash over each message read whole over IMAP); +# 2. a run of the chart's backup CronJob, which exports the store to the +# bucket inbuxa-backups in Garage; +# 3. a fresh cluster's stores: the server pods stopped, the database +# dropped and made again, the blob bucket emptied; +# 4. the export imported, by a Job made from the CronJob's own pod (same +# config, Secrets and keys), from s3://inbuxa-backups/lab: the newest +# finished export; +# 5. the chart back up on the restored store: the digest must match, and +# both store check CronJobs, run once each, must find nothing wrong. +# +# THIS WIPES THE LAB'S STORE AND PUTS THE EXPORT BACK: when step 4 fails, +# the store is empty, and fresh-store.sh sets up a new one. Secrets stay out +# of stdout: the backup key goes to a Secret and to files under $STATE. +# +# SERVER_TAG=kc11 WEBMAIL_TAG=kw3 lab/chart/backup-test.sh +# +# RESTORE_ONLY=1 skips the mail and the backup, and restores the newest +# finished export into fresh stores (steps 3 to 5, printing the digest): +# the way back from a failed run. +set -euo pipefail + +cd "$(dirname "$0")" +CTX=kind-inbuxa-lab +K=(kubectl --context "$CTX" -n inbuxa) +D=(kubectl --context "$CTX" -n data) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +SERVER_TAG="${SERVER_TAG:?the server image tag in kind}" +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +EXTRA=() +[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG") +BUCKET=inbuxa-backups +BLOBS=inbuxa-blobs +KEY=inbuxa-lab-backup +URL="s3://$BUCKET/lab" +S3_PORT=13900 +fails=0 +work=$(mktemp -d) +STAMP=$(date +%s) + +check() { # + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi +} +g() { "${D[@]}" exec deploy/garage -- /garage "$@" 2>/dev/null; } +BACKUP=( + --set server.backup.enabled=true --set server.backup.url="$URL" + --set server.backup.endpoint=http://garage.data.svc:3900 + --set server.backup.region=garage --set server.backup.existingSecret="$KEY" + --set server.storeCheck.quick.enabled=true --set server.storeCheck.full.enabled=true +) +install() { # [helm args] + helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \ + -f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \ + "${EXTRA[@]}" "$@" --wait --timeout 15m >/dev/null + for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done +} +FORWARD_PID="" +STORE_EMPTY="" +cleanup() { + [ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null + "${K[@]}" delete job -l lab.inbuxa.org/backup-test=true --ignore-not-found >/dev/null 2>&1 || true + # Never start servers on the emptied store: their defaults would be data + # an import refuses to go over + if [ -n "$STORE_EMPTY" ]; then + echo "!! the server pods are stopped and the store is empty: RESTORE_ONLY=1 $0 imports the newest export, or fresh-store.sh sets up a new store" >&2 + else + install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2 + fi + rm -rf "$work" +} +trap cleanup EXIT + +s3() { # [arg]: lab/chart/s3.py as the backup key + S3_ENDPOINT="http://127.0.0.1:$S3_PORT" S3_REGION=garage \ + S3_ACCESS_KEY_FILE="$STATE/backup-s3.access" S3_SECRET_KEY_FILE="$STATE/backup-s3.secret" \ + python3 s3.py "$@" +} +run_job() { # [--import URL]: a Job from the CronJob's template; waits; log in $work/.log + local cron="$1" job="$2"; shift 2 + "${K[@]}" get cronjob "$cron" -o json | python3 -c ' +import json, sys +cron = json.load(sys.stdin); job, extra = sys.argv[1], sys.argv[2:] +spec = cron["spec"]["jobTemplate"]["spec"] +spec["backoffLimit"] = 0 +if extra: + spec["template"]["spec"]["containers"][0]["args"] = extra + ["--config", "/etc/inbuxa/config.json"] +labels = dict(cron["spec"]["jobTemplate"]["metadata"].get("labels", {}), **{"lab.inbuxa.org/backup-test": "true"}) +print(json.dumps({"apiVersion": "batch/v1", "kind": "Job", + "metadata": {"name": job, "labels": labels}, "spec": spec})) +' "$job" "$@" | "${K[@]}" apply -f - >/dev/null + local status="" + for _ in $(seq 400); do + status=$("${K[@]}" get job "$job" -o jsonpath='{range .status.conditions[?(@.status=="True")]}{.type}{end}') + case "$status" in *Complete*|*Failed*) break ;; esac + sleep 3 + done + "${K[@]}" logs "job/$job" --all-containers >"$work/$job.log" 2>&1 || true + case "$status" in *Complete*) return 0 ;; *) return 1 ;; esac +} +digest() { ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$LB" "$@"; } + +echo "== the backup bucket and its key" +g bucket info "$BUCKET" >/dev/null || g bucket create "$BUCKET" >/dev/null +g key info "$KEY" >/dev/null || g key create "$KEY" >/dev/null +g bucket allow --read --write --owner "$BUCKET" --key "$KEY" >/dev/null +# The test empties the blob bucket for the fresh store, as this key +g bucket allow --read --write "$BLOBS" --key "$KEY" >/dev/null +umask 077 +g key info --show-secret "$KEY" | awk -F': *' '/^Key ID/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.access" +g key info --show-secret "$KEY" | awk -F': *' '/^Secret key/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.secret" +umask 022 +[ -s "$STATE/backup-s3.access" ] && [ -s "$STATE/backup-s3.secret" ] || { echo "!! couldn't read the key from Garage" >&2; exit 1; } +"${K[@]}" create secret generic "$KEY" --from-file=access-key="$STATE/backup-s3.access" \ + --from-file=secret-key="$STATE/backup-s3.secret" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${D[@]}" port-forward svc/garage "$S3_PORT:3900" >/dev/null 2>&1 & +FORWARD_PID=$! +for _ in $(seq 30); do s3 ls "$BUCKET" x >/dev/null 2>&1 && break; sleep 1; done + +echo "== the chart, with the backup and store checks on" +install "${BACKUP[@]}" +LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +check "CronJobs" "$("${K[@]}" get cronjob -l app.kubernetes.io/instance=inbuxa -o name | sort | tr '\n' ' ')" \ + "cronjob.batch/inbuxa-backup cronjob.batch/inbuxa-store-check-full cronjob.batch/inbuxa-store-check-quick " + +if [ -z "${RESTORE_ONLY:-}" ]; then +echo "== 1. mail, and its digest" +digest send 5 >"$work/before" +cat "$work/before" | sed 's/^/ /' +held=$(awk -F'\t' '{n += $2} END {print n + 0}' "$work/before") +[ "$held" -ge 5 ] && echo "ok alice's mailboxes hold $held messages" || { echo "FAIL alice's mailboxes hold $held"; fails=$((fails + 1)); } + +echo "== 2. the backup CronJob, once" +if run_job inbuxa-backup "lab-backup-$STAMP"; then echo "ok the export finished"; else echo "FAIL the export: $(tail -3 "$work/lab-backup-$STAMP.log")"; exit 1; fi +where=$(sed -nE 's/^Exported .* to (s3:\/\/[^ ]+)\.$/\1/p' "$work/lab-backup-$STAMP.log" | tail -1) +name=${where##*/} +echo " $(grep '^Exported' "$work/lab-backup-$STAMP.log")" +s3 get "$BUCKET" "lab/$name/export.json" >"$work/manifest.json" +check "export.json lists the files in the bucket" \ + "$(python3 -c 'import json,sys; print(sorted(f["name"] for f in json.load(open(sys.argv[1]))["files"]))' "$work/manifest.json")" \ + "$(s3 ls "$BUCKET" "lab/$name/" | sed "s|^lab/$name/||" | grep -v '^export.json$' | python3 -c 'import sys; print(sorted(l.strip() for l in sys.stdin))')" +# The blobs' subspace is t (116) +grep -q '"subspace": 116' "$work/manifest.json" && echo "ok the export holds the blobs" || { echo "FAIL no blobs in the export"; fails=$((fails + 1)); } +fi + +echo "== 3. a fresh cluster's stores" +for d in front mta core; do "${K[@]}" scale deploy/inbuxa-$d --replicas=0 >/dev/null; done +"${K[@]}" wait --for=delete pod -l app.kubernetes.io/component=server --timeout=300s >/dev/null 2>&1 || true +STORE_EMPTY=1 +primary=$("${D[@]}" get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}') +"${D[@]}" exec "$primary" -c postgres -- psql -q -v ON_ERROR_STOP=1 \ + -c 'DROP DATABASE IF EXISTS inbuxa WITH (FORCE)' -c 'CREATE DATABASE inbuxa OWNER inbuxa' >/dev/null +echo " database dropped and made again; $(s3 empty "$BLOBS") blobs deleted from $BLOBS" +check "blobs left in $BLOBS" "$(s3 ls "$BLOBS" | wc -l)" 0 + +echo "== 4. the import, from the newest finished export" +if run_job inbuxa-backup "lab-restore-$STAMP" --import "$URL"; then + echo "ok the import finished" +else + echo "FAIL the import: $(tail -5 "$work/lab-restore-$STAMP.log")" + echo "!! the lab's store is empty: fresh-store.sh sets up a new one" >&2 + exit 1 +fi +STORE_EMPTY="" +if [ -z "${RESTORE_ONLY:-}" ]; then +grep -q "Importing the newest finished export, s3://$BUCKET/lab/$name" "$work/lab-restore-$STAMP.log" && + echo "ok it chose $name" || { echo "FAIL it didn't choose $name"; fails=$((fails + 1)); } +else + echo " $(grep 'Importing the newest' "$work/lab-restore-$STAMP.log")" +fi +grep -q "into the blob store the export's settings name" "$work/lab-restore-$STAMP.log" && + echo "ok blobs went to the blob store the settings name" || { echo "FAIL blobs went elsewhere"; fails=$((fails + 1)); } +echo " $BLOBS holds $(s3 ls "$BLOBS" | wc -l) blobs again" + +echo "== 5. the chart on the restored store" +install "${BACKUP[@]}" +for _ in $(seq 30); do digest >"$work/after" 2>/dev/null && break; sleep 5; done +if [ -n "${RESTORE_ONLY:-}" ]; then + sed 's/^/ /' "$work/after" +elif diff -q "$work/before" "$work/after" >/dev/null; then + echo "ok every mailbox's count and digest match" +else + echo "FAIL the mail differs:"; diff "$work/before" "$work/after" | sed 's/^/ /'; fails=$((fails + 1)) +fi +for mode in quick full; do + if run_job "inbuxa-store-check-$mode" "lab-check-$mode-$STAMP"; then + echo "ok store check $mode: $(tail -1 "$work/lab-check-$mode-$STAMP.log")" + else + echo "FAIL store check $mode: $(tail -2 "$work/lab-check-$mode-$STAMP.log")"; fails=$((fails + 1)) + fi +done + +echo +if [ "$fails" -eq 0 ]; then echo "backup-test: all passed"; else echo "backup-test: $fails failed"; exit 1; fi diff --git a/lab/chart/mailbox-digest.py b/lab/chart/mailbox-digest.py new file mode 100755 index 0000000..eb14926 --- /dev/null +++ b/lab/chart/mailbox-digest.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""An account's mail, as something two stores can be compared by: one line +per mailbox with its message count and a SHA-256 over the sorted SHA-256s +of its messages, read whole over IMAPS (993). Or, with `send N`, N messages +to the account over SMTP (25) first, each with an attachment, and waits +until IMAP shows them (in any mailbox). + + ACCOUNT=user@domain PASSWORD_FILE=... mailbox-digest.py LB [send N] + +The lab's certificate is self-signed, so TLS isn't verified. +""" +import hashlib, imaplib, os, re, smtplib, ssl, sys, time, uuid +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +lb = sys.argv[1] +account = os.environ["ACCOUNT"] +password = open(os.environ["PASSWORD_FILE"]).read().strip() +tls = ssl.create_default_context() +tls.check_hostname = False +tls.verify_mode = ssl.CERT_NONE + + +def imap(): + m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30) + m.login(account, password) + return m + + +def names(m): + typ, boxes = m.list() + for line in boxes: + name = re.search(rb'"([^"]*)"$|(\S+)$', line.strip()) + yield (name.group(1) or name.group(2)).decode() + + +def total(m): + # Every mailbox: the spam filter may put a test message in Junk + count = 0 + for name in names(m): + typ, data = m.select(f'"{name}"', readonly=True) + if typ == "OK": + count += int(data[0]) + return count + + +if len(sys.argv) > 2 and sys.argv[2] == "send": + count = int(sys.argv[3]) + m = imap() + before = total(m) + run = uuid.uuid4().hex[:8] + with smtplib.SMTP(lb, 25, timeout=30) as s: + for i in range(count): + msg = EmailMessage() + msg["From"] = "lab-sender@sender.test" + msg["To"] = account + msg["Subject"] = f"backup-test {run} {i}" + msg["Date"] = formatdate() + msg["Message-ID"] = make_msgid(domain="sender.test") + msg.set_content(f"Sent by the inbuxa-k8s-lab backup test, {i} of {count}.\n") + msg.add_attachment((f"line {i}: " + uuid.uuid4().hex + "\n") * (60 + 17 * i), + filename=f"notes-{i}.txt") + s.send_message(msg) + for _ in range(60): + if total(m) >= before + count: + break + time.sleep(2) + else: + sys.exit(f"only {total(m) - before} of {count} arrived") + m.logout() + +m = imap() +for name in list(names(m)): + typ, data = m.select(f'"{name}"', readonly=True) + if typ != "OK": + continue + count = int(data[0]) + hashes = [] + if count: + typ, data = m.fetch(f"1:{count}", "(BODY.PEEK[])") + hashes = sorted(hashlib.sha256(part[1]).hexdigest() + for part in data if isinstance(part, tuple)) + digest = hashlib.sha256("\n".join(hashes).encode()).hexdigest() + print(f"{name}\t{count}\t{digest}") +m.logout() diff --git a/lab/chart/s3.py b/lab/chart/s3.py new file mode 100755 index 0000000..fcf96aa --- /dev/null +++ b/lab/chart/s3.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""Just enough S3 for the lab's tests, with the standard library only: +path-style requests signed with SigV4, to the lab's Garage through a +port-forward. + + S3_ENDPOINT=http://127.0.0.1:13900 S3_REGION=garage \\ + S3_ACCESS_KEY_FILE=... S3_SECRET_KEY_FILE=... s3.py [arg] + + ls [prefix] keys under prefix, one per line + get the object, to stdout + empty [prefix] deletes every object under prefix; prints how many +""" +import datetime, hashlib, hmac, os, sys, urllib.parse, urllib.request +import xml.etree.ElementTree as ET + +ENDPOINT = os.environ["S3_ENDPOINT"].rstrip("/") +REGION = os.environ.get("S3_REGION", "us-east-1") +ACCESS = open(os.environ["S3_ACCESS_KEY_FILE"]).read().strip() +SECRET = open(os.environ["S3_SECRET_KEY_FILE"]).read().strip() +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def request(method, bucket, key="", query=None): + query = query or {} + host = urllib.parse.urlsplit(ENDPOINT).netloc + path = "/" + bucket + ("/" + urllib.parse.quote(key, safe="/~") if key else "") + canonical_query = "&".join( + f"{urllib.parse.quote(k, safe='~')}={urllib.parse.quote(str(v), safe='~')}" + for k, v in sorted(query.items()) + ) + now = datetime.datetime.now(datetime.timezone.utc) + amz_date, date = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d") + payload = hashlib.sha256(b"").hexdigest() + headers = {"host": host, "x-amz-content-sha256": payload, "x-amz-date": amz_date} + signed = ";".join(sorted(headers)) + canonical = "\n".join([ + method, path, canonical_query, + "".join(f"{k}:{headers[k]}\n" for k in sorted(headers)), signed, payload, + ]) + scope = f"{date}/{REGION}/s3/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", amz_date, scope, + hashlib.sha256(canonical.encode()).hexdigest()]) + k = ("AWS4" + SECRET).encode() + for part in (date, REGION, "s3", "aws4_request"): + k = hmac.new(k, part.encode(), hashlib.sha256).digest() + signature = hmac.new(k, to_sign.encode(), hashlib.sha256).hexdigest() + headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, " + f"SignedHeaders={signed}, Signature={signature}") + url = ENDPOINT + path + ("?" + canonical_query if canonical_query else "") + req = urllib.request.Request(url, method=method, headers=headers) + with urllib.request.urlopen(req, timeout=60) as r: + return r.read() + + +def keys(bucket, prefix=""): + token = None + while True: + query = {"list-type": "2", "prefix": prefix} + if token: + query["continuation-token"] = token + root = ET.fromstring(request("GET", bucket, query=query)) + for c in root.iter(NS + "Contents"): + yield c.find(NS + "Key").text + token_el = root.find(NS + "NextContinuationToken") + if root.findtext(NS + "IsTruncated") != "true" or token_el is None: + return + token = token_el.text + + +command, bucket = sys.argv[1], sys.argv[2] +arg = sys.argv[3] if len(sys.argv) > 3 else "" +if command == "ls": + for key in keys(bucket, arg): + print(key) +elif command == "get": + sys.stdout.buffer.write(request("GET", bucket, arg)) +elif command == "empty": + gone = 0 + for key in list(keys(bucket, arg)): + request("DELETE", bucket, key) + gone += 1 + print(gone) +else: + sys.exit(f"unknown command {command}") -- 2.54.0