backup-test: backup and restore through the chart's CronJobs (K-L6) #29

Merged
jcoffey-dev merged 1 commits from feat/backup-test into main 2026-11-19 21:04:08 +00:00
3 changed files with 373 additions and 0 deletions

No files matched your search

+200
View File
@@ -0,0 +1,200 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-C11 and K-L6: backup and restore. With server.backup and both store
# checks on:
# 1. mail for [email protected], with attachments, and a digest of every
# mailbox (count, and a hash over each message read whole over IMAP);
# 2. a run of the chart's backup CronJob, which exports the store to the
# bucket inbuxa-backups in Garage;
# 3. a fresh cluster's stores: the server pods stopped, the database
# dropped and made again, the blob bucket emptied;
# 4. the export imported, by a Job made from the CronJob's own pod (same
# config, Secrets and keys), from s3://inbuxa-backups/lab: the newest
# finished export;
# 5. the chart back up on the restored store: the digest must match, and
# both store check CronJobs, run once each, must find nothing wrong.
#
# THIS WIPES THE LAB'S STORE AND PUTS THE EXPORT BACK: when step 4 fails,
# the store is empty, and fresh-store.sh sets up a new one. Secrets stay out
# of stdout: the backup key goes to a Secret and to files under $STATE.
#
# SERVER_TAG=kc11 WEBMAIL_TAG=kw3 lab/chart/backup-test.sh
#
# RESTORE_ONLY=1 skips the mail and the backup, and restores the newest
# finished export into fresh stores (steps 3 to 5, printing the digest):
# the way back from a failed run.
set -euo pipefail
cd "$(dirname "$0")"
CTX=kind-inbuxa-lab
K=(kubectl --context "$CTX" -n inbuxa)
D=(kubectl --context "$CTX" -n data)
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
SERVER_TAG="${SERVER_TAG:?the server image tag in kind}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
EXTRA=()
[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG")
BUCKET=inbuxa-backups
BLOBS=inbuxa-blobs
KEY=inbuxa-lab-backup
URL="s3://$BUCKET/lab"
S3_PORT=13900
fails=0
work=$(mktemp -d)
STAMP=$(date +%s)
check() { # <name> <got> <want>
if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi
}
g() { "${D[@]}" exec deploy/garage -- /garage "$@" 2>/dev/null; }
BACKUP=(
--set server.backup.enabled=true --set server.backup.url="$URL"
--set server.backup.endpoint=http://garage.data.svc:3900
--set server.backup.region=garage --set server.backup.existingSecret="$KEY"
--set server.storeCheck.quick.enabled=true --set server.storeCheck.full.enabled=true
)
install() { # [helm args]
helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \
-f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \
"${EXTRA[@]}" "$@" --wait --timeout 15m >/dev/null
for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done
}
FORWARD_PID=""
STORE_EMPTY=""
cleanup() {
[ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null
"${K[@]}" delete job -l lab.inbuxa.org/backup-test=true --ignore-not-found >/dev/null 2>&1 || true
# Never start servers on the emptied store: their defaults would be data
# an import refuses to go over
if [ -n "$STORE_EMPTY" ]; then
echo "!! the server pods are stopped and the store is empty: RESTORE_ONLY=1 $0 imports the newest export, or fresh-store.sh sets up a new store" >&2
else
install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2
fi
rm -rf "$work"
}
trap cleanup EXIT
s3() { # <command> <bucket> [arg]: lab/chart/s3.py as the backup key
S3_ENDPOINT="http://127.0.0.1:$S3_PORT" S3_REGION=garage \
S3_ACCESS_KEY_FILE="$STATE/backup-s3.access" S3_SECRET_KEY_FILE="$STATE/backup-s3.secret" \
python3 s3.py "$@"
}
run_job() { # <cronjob> <job> [--import URL]: a Job from the CronJob's template; waits; log in $work/<job>.log
local cron="$1" job="$2"; shift 2
"${K[@]}" get cronjob "$cron" -o json | python3 -c '
import json, sys
cron = json.load(sys.stdin); job, extra = sys.argv[1], sys.argv[2:]
spec = cron["spec"]["jobTemplate"]["spec"]
spec["backoffLimit"] = 0
if extra:
spec["template"]["spec"]["containers"][0]["args"] = extra + ["--config", "/etc/inbuxa/config.json"]
labels = dict(cron["spec"]["jobTemplate"]["metadata"].get("labels", {}), **{"lab.inbuxa.org/backup-test": "true"})
print(json.dumps({"apiVersion": "batch/v1", "kind": "Job",
"metadata": {"name": job, "labels": labels}, "spec": spec}))
' "$job" "$@" | "${K[@]}" apply -f - >/dev/null
local status=""
for _ in $(seq 400); do
status=$("${K[@]}" get job "$job" -o jsonpath='{range .status.conditions[?(@.status=="True")]}{.type}{end}')
case "$status" in *Complete*|*Failed*) break ;; esac
sleep 3
done
"${K[@]}" logs "job/$job" --all-containers >"$work/$job.log" 2>&1 || true
case "$status" in *Complete*) return 0 ;; *) return 1 ;; esac
}
digest() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$LB" "$@"; }
echo "== the backup bucket and its key"
g bucket info "$BUCKET" >/dev/null || g bucket create "$BUCKET" >/dev/null
g key info "$KEY" >/dev/null || g key create "$KEY" >/dev/null
g bucket allow --read --write --owner "$BUCKET" --key "$KEY" >/dev/null
# The test empties the blob bucket for the fresh store, as this key
g bucket allow --read --write "$BLOBS" --key "$KEY" >/dev/null
umask 077
g key info --show-secret "$KEY" | awk -F': *' '/^Key ID/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.access"
g key info --show-secret "$KEY" | awk -F': *' '/^Secret key/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.secret"
umask 022
[ -s "$STATE/backup-s3.access" ] && [ -s "$STATE/backup-s3.secret" ] || { echo "!! couldn't read the key from Garage" >&2; exit 1; }
"${K[@]}" create secret generic "$KEY" --from-file=access-key="$STATE/backup-s3.access" \
--from-file=secret-key="$STATE/backup-s3.secret" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${D[@]}" port-forward svc/garage "$S3_PORT:3900" >/dev/null 2>&1 &
FORWARD_PID=$!
for _ in $(seq 30); do s3 ls "$BUCKET" x >/dev/null 2>&1 && break; sleep 1; done
echo "== the chart, with the backup and store checks on"
install "${BACKUP[@]}"
LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
check "CronJobs" "$("${K[@]}" get cronjob -l app.kubernetes.io/instance=inbuxa -o name | sort | tr '\n' ' ')" \
"cronjob.batch/inbuxa-backup cronjob.batch/inbuxa-store-check-full cronjob.batch/inbuxa-store-check-quick "
if [ -z "${RESTORE_ONLY:-}" ]; then
echo "== 1. mail, and its digest"
digest send 5 >"$work/before"
cat "$work/before" | sed 's/^/ /'
held=$(awk -F'\t' '{n += $2} END {print n + 0}' "$work/before")
[ "$held" -ge 5 ] && echo "ok alice's mailboxes hold $held messages" || { echo "FAIL alice's mailboxes hold $held"; fails=$((fails + 1)); }
echo "== 2. the backup CronJob, once"
if run_job inbuxa-backup "lab-backup-$STAMP"; then echo "ok the export finished"; else echo "FAIL the export: $(tail -3 "$work/lab-backup-$STAMP.log")"; exit 1; fi
where=$(sed -nE 's/^Exported .* to (s3:\/\/[^ ]+)\.$/\1/p' "$work/lab-backup-$STAMP.log" | tail -1)
name=${where##*/}
echo " $(grep '^Exported' "$work/lab-backup-$STAMP.log")"
s3 get "$BUCKET" "lab/$name/export.json" >"$work/manifest.json"
check "export.json lists the files in the bucket" \
"$(python3 -c 'import json,sys; print(sorted(f["name"] for f in json.load(open(sys.argv[1]))["files"]))' "$work/manifest.json")" \
"$(s3 ls "$BUCKET" "lab/$name/" | sed "s|^lab/$name/||" | grep -v '^export.json$' | python3 -c 'import sys; print(sorted(l.strip() for l in sys.stdin))')"
# The blobs' subspace is t (116)
grep -q '"subspace": 116' "$work/manifest.json" && echo "ok the export holds the blobs" || { echo "FAIL no blobs in the export"; fails=$((fails + 1)); }
fi
echo "== 3. a fresh cluster's stores"
for d in front mta core; do "${K[@]}" scale deploy/inbuxa-$d --replicas=0 >/dev/null; done
"${K[@]}" wait --for=delete pod -l app.kubernetes.io/component=server --timeout=300s >/dev/null 2>&1 || true
STORE_EMPTY=1
primary=$("${D[@]}" get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}')
"${D[@]}" exec "$primary" -c postgres -- psql -q -v ON_ERROR_STOP=1 \
-c 'DROP DATABASE IF EXISTS inbuxa WITH (FORCE)' -c 'CREATE DATABASE inbuxa OWNER inbuxa' >/dev/null
echo " database dropped and made again; $(s3 empty "$BLOBS") blobs deleted from $BLOBS"
check "blobs left in $BLOBS" "$(s3 ls "$BLOBS" | wc -l)" 0
echo "== 4. the import, from the newest finished export"
if run_job inbuxa-backup "lab-restore-$STAMP" --import "$URL"; then
echo "ok the import finished"
else
echo "FAIL the import: $(tail -5 "$work/lab-restore-$STAMP.log")"
echo "!! the lab's store is empty: fresh-store.sh sets up a new one" >&2
exit 1
fi
STORE_EMPTY=""
if [ -z "${RESTORE_ONLY:-}" ]; then
grep -q "Importing the newest finished export, s3://$BUCKET/lab/$name" "$work/lab-restore-$STAMP.log" &&
echo "ok it chose $name" || { echo "FAIL it didn't choose $name"; fails=$((fails + 1)); }
else
echo " $(grep 'Importing the newest' "$work/lab-restore-$STAMP.log")"
fi
grep -q "into the blob store the export's settings name" "$work/lab-restore-$STAMP.log" &&
echo "ok blobs went to the blob store the settings name" || { echo "FAIL blobs went elsewhere"; fails=$((fails + 1)); }
echo " $BLOBS holds $(s3 ls "$BLOBS" | wc -l) blobs again"
echo "== 5. the chart on the restored store"
install "${BACKUP[@]}"
for _ in $(seq 30); do digest >"$work/after" 2>/dev/null && break; sleep 5; done
if [ -n "${RESTORE_ONLY:-}" ]; then
sed 's/^/ /' "$work/after"
elif diff -q "$work/before" "$work/after" >/dev/null; then
echo "ok every mailbox's count and digest match"
else
echo "FAIL the mail differs:"; diff "$work/before" "$work/after" | sed 's/^/ /'; fails=$((fails + 1))
fi
for mode in quick full; do
if run_job "inbuxa-store-check-$mode" "lab-check-$mode-$STAMP"; then
echo "ok store check $mode: $(tail -1 "$work/lab-check-$mode-$STAMP.log")"
else
echo "FAIL store check $mode: $(tail -2 "$work/lab-check-$mode-$STAMP.log")"; fails=$((fails + 1))
fi
done
echo
if [ "$fails" -eq 0 ]; then echo "backup-test: all passed"; else echo "backup-test: $fails failed"; exit 1; fi
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""An account's mail, as something two stores can be compared by: one line
per mailbox with its message count and a SHA-256 over the sorted SHA-256s
of its messages, read whole over IMAPS (993). Or, with `send N`, N messages
to the account over SMTP (25) first, each with an attachment, and waits
until IMAP shows them (in any mailbox).
ACCOUNT=user@domain PASSWORD_FILE=... mailbox-digest.py LB [send N]
The lab's certificate is self-signed, so TLS isn't verified.
"""
import hashlib, imaplib, os, re, smtplib, ssl, sys, time, uuid
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
lb = sys.argv[1]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
tls = ssl.create_default_context()
tls.check_hostname = False
tls.verify_mode = ssl.CERT_NONE
def imap():
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30)
m.login(account, password)
return m
def names(m):
typ, boxes = m.list()
for line in boxes:
name = re.search(rb'"([^"]*)"$|(\S+)$', line.strip())
yield (name.group(1) or name.group(2)).decode()
def total(m):
# Every mailbox: the spam filter may put a test message in Junk
count = 0
for name in names(m):
typ, data = m.select(f'"{name}"', readonly=True)
if typ == "OK":
count += int(data[0])
return count
if len(sys.argv) > 2 and sys.argv[2] == "send":
count = int(sys.argv[3])
m = imap()
before = total(m)
run = uuid.uuid4().hex[:8]
with smtplib.SMTP(lb, 25, timeout=30) as s:
for i in range(count):
msg = EmailMessage()
msg["From"] = "[email protected]"
msg["To"] = account
msg["Subject"] = f"backup-test {run} {i}"
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain="sender.test")
msg.set_content(f"Sent by the inbuxa-k8s-lab backup test, {i} of {count}.\n")
msg.add_attachment((f"line {i}: " + uuid.uuid4().hex + "\n") * (60 + 17 * i),
filename=f"notes-{i}.txt")
s.send_message(msg)
for _ in range(60):
if total(m) >= before + count:
break
time.sleep(2)
else:
sys.exit(f"only {total(m) - before} of {count} arrived")
m.logout()
m = imap()
for name in list(names(m)):
typ, data = m.select(f'"{name}"', readonly=True)
if typ != "OK":
continue
count = int(data[0])
hashes = []
if count:
typ, data = m.fetch(f"1:{count}", "(BODY.PEEK[])")
hashes = sorted(hashlib.sha256(part[1]).hexdigest()
for part in data if isinstance(part, tuple))
digest = hashlib.sha256("\n".join(hashes).encode()).hexdigest()
print(f"{name}\t{count}\t{digest}")
m.logout()
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""Just enough S3 for the lab's tests, with the standard library only:
path-style requests signed with SigV4, to the lab's Garage through a
port-forward.
S3_ENDPOINT=http://127.0.0.1:13900 S3_REGION=garage \\
S3_ACCESS_KEY_FILE=... S3_SECRET_KEY_FILE=... s3.py <command> <bucket> [arg]
ls <bucket> [prefix] keys under prefix, one per line
get <bucket> <key> the object, to stdout
empty <bucket> [prefix] deletes every object under prefix; prints how many
"""
import datetime, hashlib, hmac, os, sys, urllib.parse, urllib.request
import xml.etree.ElementTree as ET
ENDPOINT = os.environ["S3_ENDPOINT"].rstrip("/")
REGION = os.environ.get("S3_REGION", "us-east-1")
ACCESS = open(os.environ["S3_ACCESS_KEY_FILE"]).read().strip()
SECRET = open(os.environ["S3_SECRET_KEY_FILE"]).read().strip()
NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
def request(method, bucket, key="", query=None):
query = query or {}
host = urllib.parse.urlsplit(ENDPOINT).netloc
path = "/" + bucket + ("/" + urllib.parse.quote(key, safe="/~") if key else "")
canonical_query = "&".join(
f"{urllib.parse.quote(k, safe='~')}={urllib.parse.quote(str(v), safe='~')}"
for k, v in sorted(query.items())
)
now = datetime.datetime.now(datetime.timezone.utc)
amz_date, date = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
payload = hashlib.sha256(b"").hexdigest()
headers = {"host": host, "x-amz-content-sha256": payload, "x-amz-date": amz_date}
signed = ";".join(sorted(headers))
canonical = "\n".join([
method, path, canonical_query,
"".join(f"{k}:{headers[k]}\n" for k in sorted(headers)), signed, payload,
])
scope = f"{date}/{REGION}/s3/aws4_request"
to_sign = "\n".join(["AWS4-HMAC-SHA256", amz_date, scope,
hashlib.sha256(canonical.encode()).hexdigest()])
k = ("AWS4" + SECRET).encode()
for part in (date, REGION, "s3", "aws4_request"):
k = hmac.new(k, part.encode(), hashlib.sha256).digest()
signature = hmac.new(k, to_sign.encode(), hashlib.sha256).hexdigest()
headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, "
f"SignedHeaders={signed}, Signature={signature}")
url = ENDPOINT + path + ("?" + canonical_query if canonical_query else "")
req = urllib.request.Request(url, method=method, headers=headers)
with urllib.request.urlopen(req, timeout=60) as r:
return r.read()
def keys(bucket, prefix=""):
token = None
while True:
query = {"list-type": "2", "prefix": prefix}
if token:
query["continuation-token"] = token
root = ET.fromstring(request("GET", bucket, query=query))
for c in root.iter(NS + "Contents"):
yield c.find(NS + "Key").text
token_el = root.find(NS + "NextContinuationToken")
if root.findtext(NS + "IsTruncated") != "true" or token_el is None:
return
token = token_el.text
command, bucket = sys.argv[1], sys.argv[2]
arg = sys.argv[3] if len(sys.argv) > 3 else ""
if command == "ls":
for key in keys(bucket, arg):
print(key)
elif command == "get":
sys.stdout.buffer.write(request("GET", bucket, arg))
elif command == "empty":
gone = 0
for key in list(keys(bucket, arg)):
request("DELETE", bucket, key)
gone += 1
print(gone)
else:
sys.exit(f"unknown command {command}")