diff --git a/lab/chart/backup-test.sh b/lab/chart/backup-test.sh new file mode 100755 index 0000000..6e4dab7 --- /dev/null +++ b/lab/chart/backup-test.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C11 and K-L6: backup and restore. With server.backup and both store +# checks on: +# 1. mail for alice@lab.test, with attachments, and a digest of every +# mailbox (count, and a hash over each message read whole over IMAP); +# 2. a run of the chart's backup CronJob, which exports the store to the +# bucket inbuxa-backups in Garage; +# 3. a fresh cluster's stores: the server pods stopped, the database +# dropped and made again, the blob bucket emptied; +# 4. the export imported, by a Job made from the CronJob's own pod (same +# config, Secrets and keys), from s3://inbuxa-backups/lab: the newest +# finished export; +# 5. the chart back up on the restored store: the digest must match, and +# both store check CronJobs, run once each, must find nothing wrong. +# +# THIS WIPES THE LAB'S STORE AND PUTS THE EXPORT BACK: when step 4 fails, +# the store is empty, and fresh-store.sh sets up a new one. Secrets stay out +# of stdout: the backup key goes to a Secret and to files under $STATE. +# +# SERVER_TAG=kc11 WEBMAIL_TAG=kw3 lab/chart/backup-test.sh +# +# RESTORE_ONLY=1 skips the mail and the backup, and restores the newest +# finished export into fresh stores (steps 3 to 5, printing the digest): +# the way back from a failed run. +set -euo pipefail + +cd "$(dirname "$0")" +CTX=kind-inbuxa-lab +K=(kubectl --context "$CTX" -n inbuxa) +D=(kubectl --context "$CTX" -n data) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +SERVER_TAG="${SERVER_TAG:?the server image tag in kind}" +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +EXTRA=() +[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG") +BUCKET=inbuxa-backups +BLOBS=inbuxa-blobs +KEY=inbuxa-lab-backup +URL="s3://$BUCKET/lab" +S3_PORT=13900 +fails=0 +work=$(mktemp -d) +STAMP=$(date +%s) + +check() { # + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi +} +g() { "${D[@]}" exec deploy/garage -- /garage "$@" 2>/dev/null; } +BACKUP=( + --set server.backup.enabled=true --set server.backup.url="$URL" + --set server.backup.endpoint=http://garage.data.svc:3900 + --set server.backup.region=garage --set server.backup.existingSecret="$KEY" + --set server.storeCheck.quick.enabled=true --set server.storeCheck.full.enabled=true +) +install() { # [helm args] + helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \ + -f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \ + "${EXTRA[@]}" "$@" --wait --timeout 15m >/dev/null + for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done +} +FORWARD_PID="" +STORE_EMPTY="" +cleanup() { + [ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null + "${K[@]}" delete job -l lab.inbuxa.org/backup-test=true --ignore-not-found >/dev/null 2>&1 || true + # Never start servers on the emptied store: their defaults would be data + # an import refuses to go over + if [ -n "$STORE_EMPTY" ]; then + echo "!! the server pods are stopped and the store is empty: RESTORE_ONLY=1 $0 imports the newest export, or fresh-store.sh sets up a new store" >&2 + else + install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2 + fi + rm -rf "$work" +} +trap cleanup EXIT + +s3() { # [arg]: lab/chart/s3.py as the backup key + S3_ENDPOINT="http://127.0.0.1:$S3_PORT" S3_REGION=garage \ + S3_ACCESS_KEY_FILE="$STATE/backup-s3.access" S3_SECRET_KEY_FILE="$STATE/backup-s3.secret" \ + python3 s3.py "$@" +} +run_job() { # [--import URL]: a Job from the CronJob's template; waits; log in $work/.log + local cron="$1" job="$2"; shift 2 + "${K[@]}" get cronjob "$cron" -o json | python3 -c ' +import json, sys +cron = json.load(sys.stdin); job, extra = sys.argv[1], sys.argv[2:] +spec = cron["spec"]["jobTemplate"]["spec"] +spec["backoffLimit"] = 0 +if extra: + spec["template"]["spec"]["containers"][0]["args"] = extra + ["--config", "/etc/inbuxa/config.json"] +labels = dict(cron["spec"]["jobTemplate"]["metadata"].get("labels", {}), **{"lab.inbuxa.org/backup-test": "true"}) +print(json.dumps({"apiVersion": "batch/v1", "kind": "Job", + "metadata": {"name": job, "labels": labels}, "spec": spec})) +' "$job" "$@" | "${K[@]}" apply -f - >/dev/null + local status="" + for _ in $(seq 400); do + status=$("${K[@]}" get job "$job" -o jsonpath='{range .status.conditions[?(@.status=="True")]}{.type}{end}') + case "$status" in *Complete*|*Failed*) break ;; esac + sleep 3 + done + "${K[@]}" logs "job/$job" --all-containers >"$work/$job.log" 2>&1 || true + case "$status" in *Complete*) return 0 ;; *) return 1 ;; esac +} +digest() { ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$LB" "$@"; } + +echo "== the backup bucket and its key" +g bucket info "$BUCKET" >/dev/null || g bucket create "$BUCKET" >/dev/null +g key info "$KEY" >/dev/null || g key create "$KEY" >/dev/null +g bucket allow --read --write --owner "$BUCKET" --key "$KEY" >/dev/null +# The test empties the blob bucket for the fresh store, as this key +g bucket allow --read --write "$BLOBS" --key "$KEY" >/dev/null +umask 077 +g key info --show-secret "$KEY" | awk -F': *' '/^Key ID/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.access" +g key info --show-secret "$KEY" | awk -F': *' '/^Secret key/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.secret" +umask 022 +[ -s "$STATE/backup-s3.access" ] && [ -s "$STATE/backup-s3.secret" ] || { echo "!! couldn't read the key from Garage" >&2; exit 1; } +"${K[@]}" create secret generic "$KEY" --from-file=access-key="$STATE/backup-s3.access" \ + --from-file=secret-key="$STATE/backup-s3.secret" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${D[@]}" port-forward svc/garage "$S3_PORT:3900" >/dev/null 2>&1 & +FORWARD_PID=$! +for _ in $(seq 30); do s3 ls "$BUCKET" x >/dev/null 2>&1 && break; sleep 1; done + +echo "== the chart, with the backup and store checks on" +install "${BACKUP[@]}" +LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}') +check "CronJobs" "$("${K[@]}" get cronjob -l app.kubernetes.io/instance=inbuxa -o name | sort | tr '\n' ' ')" \ + "cronjob.batch/inbuxa-backup cronjob.batch/inbuxa-store-check-full cronjob.batch/inbuxa-store-check-quick " + +if [ -z "${RESTORE_ONLY:-}" ]; then +echo "== 1. mail, and its digest" +digest send 5 >"$work/before" +cat "$work/before" | sed 's/^/ /' +held=$(awk -F'\t' '{n += $2} END {print n + 0}' "$work/before") +[ "$held" -ge 5 ] && echo "ok alice's mailboxes hold $held messages" || { echo "FAIL alice's mailboxes hold $held"; fails=$((fails + 1)); } + +echo "== 2. the backup CronJob, once" +if run_job inbuxa-backup "lab-backup-$STAMP"; then echo "ok the export finished"; else echo "FAIL the export: $(tail -3 "$work/lab-backup-$STAMP.log")"; exit 1; fi +where=$(sed -nE 's/^Exported .* to (s3:\/\/[^ ]+)\.$/\1/p' "$work/lab-backup-$STAMP.log" | tail -1) +name=${where##*/} +echo " $(grep '^Exported' "$work/lab-backup-$STAMP.log")" +s3 get "$BUCKET" "lab/$name/export.json" >"$work/manifest.json" +check "export.json lists the files in the bucket" \ + "$(python3 -c 'import json,sys; print(sorted(f["name"] for f in json.load(open(sys.argv[1]))["files"]))' "$work/manifest.json")" \ + "$(s3 ls "$BUCKET" "lab/$name/" | sed "s|^lab/$name/||" | grep -v '^export.json$' | python3 -c 'import sys; print(sorted(l.strip() for l in sys.stdin))')" +# The blobs' subspace is t (116) +grep -q '"subspace": 116' "$work/manifest.json" && echo "ok the export holds the blobs" || { echo "FAIL no blobs in the export"; fails=$((fails + 1)); } +fi + +echo "== 3. a fresh cluster's stores" +for d in front mta core; do "${K[@]}" scale deploy/inbuxa-$d --replicas=0 >/dev/null; done +"${K[@]}" wait --for=delete pod -l app.kubernetes.io/component=server --timeout=300s >/dev/null 2>&1 || true +STORE_EMPTY=1 +primary=$("${D[@]}" get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}') +"${D[@]}" exec "$primary" -c postgres -- psql -q -v ON_ERROR_STOP=1 \ + -c 'DROP DATABASE IF EXISTS inbuxa WITH (FORCE)' -c 'CREATE DATABASE inbuxa OWNER inbuxa' >/dev/null +echo " database dropped and made again; $(s3 empty "$BLOBS") blobs deleted from $BLOBS" +check "blobs left in $BLOBS" "$(s3 ls "$BLOBS" | wc -l)" 0 + +echo "== 4. the import, from the newest finished export" +if run_job inbuxa-backup "lab-restore-$STAMP" --import "$URL"; then + echo "ok the import finished" +else + echo "FAIL the import: $(tail -5 "$work/lab-restore-$STAMP.log")" + echo "!! the lab's store is empty: fresh-store.sh sets up a new one" >&2 + exit 1 +fi +STORE_EMPTY="" +if [ -z "${RESTORE_ONLY:-}" ]; then +grep -q "Importing the newest finished export, s3://$BUCKET/lab/$name" "$work/lab-restore-$STAMP.log" && + echo "ok it chose $name" || { echo "FAIL it didn't choose $name"; fails=$((fails + 1)); } +else + echo " $(grep 'Importing the newest' "$work/lab-restore-$STAMP.log")" +fi +grep -q "into the blob store the export's settings name" "$work/lab-restore-$STAMP.log" && + echo "ok blobs went to the blob store the settings name" || { echo "FAIL blobs went elsewhere"; fails=$((fails + 1)); } +echo " $BLOBS holds $(s3 ls "$BLOBS" | wc -l) blobs again" + +echo "== 5. the chart on the restored store" +install "${BACKUP[@]}" +for _ in $(seq 30); do digest >"$work/after" 2>/dev/null && break; sleep 5; done +if [ -n "${RESTORE_ONLY:-}" ]; then + sed 's/^/ /' "$work/after" +elif diff -q "$work/before" "$work/after" >/dev/null; then + echo "ok every mailbox's count and digest match" +else + echo "FAIL the mail differs:"; diff "$work/before" "$work/after" | sed 's/^/ /'; fails=$((fails + 1)) +fi +for mode in quick full; do + if run_job "inbuxa-store-check-$mode" "lab-check-$mode-$STAMP"; then + echo "ok store check $mode: $(tail -1 "$work/lab-check-$mode-$STAMP.log")" + else + echo "FAIL store check $mode: $(tail -2 "$work/lab-check-$mode-$STAMP.log")"; fails=$((fails + 1)) + fi +done + +echo +if [ "$fails" -eq 0 ]; then echo "backup-test: all passed"; else echo "backup-test: $fails failed"; exit 1; fi diff --git a/lab/chart/mailbox-digest.py b/lab/chart/mailbox-digest.py new file mode 100755 index 0000000..eb14926 --- /dev/null +++ b/lab/chart/mailbox-digest.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""An account's mail, as something two stores can be compared by: one line +per mailbox with its message count and a SHA-256 over the sorted SHA-256s +of its messages, read whole over IMAPS (993). Or, with `send N`, N messages +to the account over SMTP (25) first, each with an attachment, and waits +until IMAP shows them (in any mailbox). + + ACCOUNT=user@domain PASSWORD_FILE=... mailbox-digest.py LB [send N] + +The lab's certificate is self-signed, so TLS isn't verified. +""" +import hashlib, imaplib, os, re, smtplib, ssl, sys, time, uuid +from email.message import EmailMessage +from email.utils import formatdate, make_msgid + +lb = sys.argv[1] +account = os.environ["ACCOUNT"] +password = open(os.environ["PASSWORD_FILE"]).read().strip() +tls = ssl.create_default_context() +tls.check_hostname = False +tls.verify_mode = ssl.CERT_NONE + + +def imap(): + m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30) + m.login(account, password) + return m + + +def names(m): + typ, boxes = m.list() + for line in boxes: + name = re.search(rb'"([^"]*)"$|(\S+)$', line.strip()) + yield (name.group(1) or name.group(2)).decode() + + +def total(m): + # Every mailbox: the spam filter may put a test message in Junk + count = 0 + for name in names(m): + typ, data = m.select(f'"{name}"', readonly=True) + if typ == "OK": + count += int(data[0]) + return count + + +if len(sys.argv) > 2 and sys.argv[2] == "send": + count = int(sys.argv[3]) + m = imap() + before = total(m) + run = uuid.uuid4().hex[:8] + with smtplib.SMTP(lb, 25, timeout=30) as s: + for i in range(count): + msg = EmailMessage() + msg["From"] = "lab-sender@sender.test" + msg["To"] = account + msg["Subject"] = f"backup-test {run} {i}" + msg["Date"] = formatdate() + msg["Message-ID"] = make_msgid(domain="sender.test") + msg.set_content(f"Sent by the inbuxa-k8s-lab backup test, {i} of {count}.\n") + msg.add_attachment((f"line {i}: " + uuid.uuid4().hex + "\n") * (60 + 17 * i), + filename=f"notes-{i}.txt") + s.send_message(msg) + for _ in range(60): + if total(m) >= before + count: + break + time.sleep(2) + else: + sys.exit(f"only {total(m) - before} of {count} arrived") + m.logout() + +m = imap() +for name in list(names(m)): + typ, data = m.select(f'"{name}"', readonly=True) + if typ != "OK": + continue + count = int(data[0]) + hashes = [] + if count: + typ, data = m.fetch(f"1:{count}", "(BODY.PEEK[])") + hashes = sorted(hashlib.sha256(part[1]).hexdigest() + for part in data if isinstance(part, tuple)) + digest = hashlib.sha256("\n".join(hashes).encode()).hexdigest() + print(f"{name}\t{count}\t{digest}") +m.logout() diff --git a/lab/chart/s3.py b/lab/chart/s3.py new file mode 100755 index 0000000..fcf96aa --- /dev/null +++ b/lab/chart/s3.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +"""Just enough S3 for the lab's tests, with the standard library only: +path-style requests signed with SigV4, to the lab's Garage through a +port-forward. + + S3_ENDPOINT=http://127.0.0.1:13900 S3_REGION=garage \\ + S3_ACCESS_KEY_FILE=... S3_SECRET_KEY_FILE=... s3.py [arg] + + ls [prefix] keys under prefix, one per line + get the object, to stdout + empty [prefix] deletes every object under prefix; prints how many +""" +import datetime, hashlib, hmac, os, sys, urllib.parse, urllib.request +import xml.etree.ElementTree as ET + +ENDPOINT = os.environ["S3_ENDPOINT"].rstrip("/") +REGION = os.environ.get("S3_REGION", "us-east-1") +ACCESS = open(os.environ["S3_ACCESS_KEY_FILE"]).read().strip() +SECRET = open(os.environ["S3_SECRET_KEY_FILE"]).read().strip() +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def request(method, bucket, key="", query=None): + query = query or {} + host = urllib.parse.urlsplit(ENDPOINT).netloc + path = "/" + bucket + ("/" + urllib.parse.quote(key, safe="/~") if key else "") + canonical_query = "&".join( + f"{urllib.parse.quote(k, safe='~')}={urllib.parse.quote(str(v), safe='~')}" + for k, v in sorted(query.items()) + ) + now = datetime.datetime.now(datetime.timezone.utc) + amz_date, date = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d") + payload = hashlib.sha256(b"").hexdigest() + headers = {"host": host, "x-amz-content-sha256": payload, "x-amz-date": amz_date} + signed = ";".join(sorted(headers)) + canonical = "\n".join([ + method, path, canonical_query, + "".join(f"{k}:{headers[k]}\n" for k in sorted(headers)), signed, payload, + ]) + scope = f"{date}/{REGION}/s3/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", amz_date, scope, + hashlib.sha256(canonical.encode()).hexdigest()]) + k = ("AWS4" + SECRET).encode() + for part in (date, REGION, "s3", "aws4_request"): + k = hmac.new(k, part.encode(), hashlib.sha256).digest() + signature = hmac.new(k, to_sign.encode(), hashlib.sha256).hexdigest() + headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, " + f"SignedHeaders={signed}, Signature={signature}") + url = ENDPOINT + path + ("?" + canonical_query if canonical_query else "") + req = urllib.request.Request(url, method=method, headers=headers) + with urllib.request.urlopen(req, timeout=60) as r: + return r.read() + + +def keys(bucket, prefix=""): + token = None + while True: + query = {"list-type": "2", "prefix": prefix} + if token: + query["continuation-token"] = token + root = ET.fromstring(request("GET", bucket, query=query)) + for c in root.iter(NS + "Contents"): + yield c.find(NS + "Key").text + token_el = root.find(NS + "NextContinuationToken") + if root.findtext(NS + "IsTruncated") != "true" or token_el is None: + return + token = token_el.text + + +command, bucket = sys.argv[1], sys.argv[2] +arg = sys.argv[3] if len(sys.argv) > 3 else "" +if command == "ls": + for key in keys(bucket, arg): + print(key) +elif command == "get": + sys.stdout.buffer.write(request("GET", bucket, arg)) +elif command == "empty": + gone = 0 + for key in list(keys(bucket, arg)): + request("DELETE", bucket, key) + gone += 1 + print(gone) +else: + sys.exit(f"unknown command {command}")