backup-test: backup and restore through the chart's CronJobs (K-L6) #29
No files matched your search
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-C11 and K-L6: backup and restore. With server.backup and both store
|
||||
# checks on:
|
||||
# 1. mail for [email protected], with attachments, and a digest of every
|
||||
# mailbox (count, and a hash over each message read whole over IMAP);
|
||||
# 2. a run of the chart's backup CronJob, which exports the store to the
|
||||
# bucket inbuxa-backups in Garage;
|
||||
# 3. a fresh cluster's stores: the server pods stopped, the database
|
||||
# dropped and made again, the blob bucket emptied;
|
||||
# 4. the export imported, by a Job made from the CronJob's own pod (same
|
||||
# config, Secrets and keys), from s3://inbuxa-backups/lab: the newest
|
||||
# finished export;
|
||||
# 5. the chart back up on the restored store: the digest must match, and
|
||||
# both store check CronJobs, run once each, must find nothing wrong.
|
||||
#
|
||||
# THIS WIPES THE LAB'S STORE AND PUTS THE EXPORT BACK: when step 4 fails,
|
||||
# the store is empty, and fresh-store.sh sets up a new one. Secrets stay out
|
||||
# of stdout: the backup key goes to a Secret and to files under $STATE.
|
||||
#
|
||||
# SERVER_TAG=kc11 WEBMAIL_TAG=kw3 lab/chart/backup-test.sh
|
||||
#
|
||||
# RESTORE_ONLY=1 skips the mail and the backup, and restores the newest
|
||||
# finished export into fresh stores (steps 3 to 5, printing the digest):
|
||||
# the way back from a failed run.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
CTX=kind-inbuxa-lab
|
||||
K=(kubectl --context "$CTX" -n inbuxa)
|
||||
D=(kubectl --context "$CTX" -n data)
|
||||
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
|
||||
SERVER_TAG="${SERVER_TAG:?the server image tag in kind}"
|
||||
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
|
||||
EXTRA=()
|
||||
[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG")
|
||||
BUCKET=inbuxa-backups
|
||||
BLOBS=inbuxa-blobs
|
||||
KEY=inbuxa-lab-backup
|
||||
URL="s3://$BUCKET/lab"
|
||||
S3_PORT=13900
|
||||
fails=0
|
||||
work=$(mktemp -d)
|
||||
STAMP=$(date +%s)
|
||||
|
||||
check() { # <name> <got> <want>
|
||||
if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi
|
||||
}
|
||||
g() { "${D[@]}" exec deploy/garage -- /garage "$@" 2>/dev/null; }
|
||||
BACKUP=(
|
||||
--set server.backup.enabled=true --set server.backup.url="$URL"
|
||||
--set server.backup.endpoint=http://garage.data.svc:3900
|
||||
--set server.backup.region=garage --set server.backup.existingSecret="$KEY"
|
||||
--set server.storeCheck.quick.enabled=true --set server.storeCheck.full.enabled=true
|
||||
)
|
||||
install() { # [helm args]
|
||||
helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \
|
||||
-f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \
|
||||
"${EXTRA[@]}" "$@" --wait --timeout 15m >/dev/null
|
||||
for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done
|
||||
}
|
||||
FORWARD_PID=""
|
||||
STORE_EMPTY=""
|
||||
cleanup() {
|
||||
[ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null
|
||||
"${K[@]}" delete job -l lab.inbuxa.org/backup-test=true --ignore-not-found >/dev/null 2>&1 || true
|
||||
# Never start servers on the emptied store: their defaults would be data
|
||||
# an import refuses to go over
|
||||
if [ -n "$STORE_EMPTY" ]; then
|
||||
echo "!! the server pods are stopped and the store is empty: RESTORE_ONLY=1 $0 imports the newest export, or fresh-store.sh sets up a new store" >&2
|
||||
else
|
||||
install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2
|
||||
fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
s3() { # <command> <bucket> [arg]: lab/chart/s3.py as the backup key
|
||||
S3_ENDPOINT="http://127.0.0.1:$S3_PORT" S3_REGION=garage \
|
||||
S3_ACCESS_KEY_FILE="$STATE/backup-s3.access" S3_SECRET_KEY_FILE="$STATE/backup-s3.secret" \
|
||||
python3 s3.py "$@"
|
||||
}
|
||||
run_job() { # <cronjob> <job> [--import URL]: a Job from the CronJob's template; waits; log in $work/<job>.log
|
||||
local cron="$1" job="$2"; shift 2
|
||||
"${K[@]}" get cronjob "$cron" -o json | python3 -c '
|
||||
import json, sys
|
||||
cron = json.load(sys.stdin); job, extra = sys.argv[1], sys.argv[2:]
|
||||
spec = cron["spec"]["jobTemplate"]["spec"]
|
||||
spec["backoffLimit"] = 0
|
||||
if extra:
|
||||
spec["template"]["spec"]["containers"][0]["args"] = extra + ["--config", "/etc/inbuxa/config.json"]
|
||||
labels = dict(cron["spec"]["jobTemplate"]["metadata"].get("labels", {}), **{"lab.inbuxa.org/backup-test": "true"})
|
||||
print(json.dumps({"apiVersion": "batch/v1", "kind": "Job",
|
||||
"metadata": {"name": job, "labels": labels}, "spec": spec}))
|
||||
' "$job" "$@" | "${K[@]}" apply -f - >/dev/null
|
||||
local status=""
|
||||
for _ in $(seq 400); do
|
||||
status=$("${K[@]}" get job "$job" -o jsonpath='{range .status.conditions[?(@.status=="True")]}{.type}{end}')
|
||||
case "$status" in *Complete*|*Failed*) break ;; esac
|
||||
sleep 3
|
||||
done
|
||||
"${K[@]}" logs "job/$job" --all-containers >"$work/$job.log" 2>&1 || true
|
||||
case "$status" in *Complete*) return 0 ;; *) return 1 ;; esac
|
||||
}
|
||||
digest() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$LB" "$@"; }
|
||||
|
||||
echo "== the backup bucket and its key"
|
||||
g bucket info "$BUCKET" >/dev/null || g bucket create "$BUCKET" >/dev/null
|
||||
g key info "$KEY" >/dev/null || g key create "$KEY" >/dev/null
|
||||
g bucket allow --read --write --owner "$BUCKET" --key "$KEY" >/dev/null
|
||||
# The test empties the blob bucket for the fresh store, as this key
|
||||
g bucket allow --read --write "$BLOBS" --key "$KEY" >/dev/null
|
||||
umask 077
|
||||
g key info --show-secret "$KEY" | awk -F': *' '/^Key ID/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.access"
|
||||
g key info --show-secret "$KEY" | awk -F': *' '/^Secret key/{print $2}' | tr -d '\r\n' >"$STATE/backup-s3.secret"
|
||||
umask 022
|
||||
[ -s "$STATE/backup-s3.access" ] && [ -s "$STATE/backup-s3.secret" ] || { echo "!! couldn't read the key from Garage" >&2; exit 1; }
|
||||
"${K[@]}" create secret generic "$KEY" --from-file=access-key="$STATE/backup-s3.access" \
|
||||
--from-file=secret-key="$STATE/backup-s3.secret" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${D[@]}" port-forward svc/garage "$S3_PORT:3900" >/dev/null 2>&1 &
|
||||
FORWARD_PID=$!
|
||||
for _ in $(seq 30); do s3 ls "$BUCKET" x >/dev/null 2>&1 && break; sleep 1; done
|
||||
|
||||
echo "== the chart, with the backup and store checks on"
|
||||
install "${BACKUP[@]}"
|
||||
LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
|
||||
check "CronJobs" "$("${K[@]}" get cronjob -l app.kubernetes.io/instance=inbuxa -o name | sort | tr '\n' ' ')" \
|
||||
"cronjob.batch/inbuxa-backup cronjob.batch/inbuxa-store-check-full cronjob.batch/inbuxa-store-check-quick "
|
||||
|
||||
if [ -z "${RESTORE_ONLY:-}" ]; then
|
||||
echo "== 1. mail, and its digest"
|
||||
digest send 5 >"$work/before"
|
||||
cat "$work/before" | sed 's/^/ /'
|
||||
held=$(awk -F'\t' '{n += $2} END {print n + 0}' "$work/before")
|
||||
[ "$held" -ge 5 ] && echo "ok alice's mailboxes hold $held messages" || { echo "FAIL alice's mailboxes hold $held"; fails=$((fails + 1)); }
|
||||
|
||||
echo "== 2. the backup CronJob, once"
|
||||
if run_job inbuxa-backup "lab-backup-$STAMP"; then echo "ok the export finished"; else echo "FAIL the export: $(tail -3 "$work/lab-backup-$STAMP.log")"; exit 1; fi
|
||||
where=$(sed -nE 's/^Exported .* to (s3:\/\/[^ ]+)\.$/\1/p' "$work/lab-backup-$STAMP.log" | tail -1)
|
||||
name=${where##*/}
|
||||
echo " $(grep '^Exported' "$work/lab-backup-$STAMP.log")"
|
||||
s3 get "$BUCKET" "lab/$name/export.json" >"$work/manifest.json"
|
||||
check "export.json lists the files in the bucket" \
|
||||
"$(python3 -c 'import json,sys; print(sorted(f["name"] for f in json.load(open(sys.argv[1]))["files"]))' "$work/manifest.json")" \
|
||||
"$(s3 ls "$BUCKET" "lab/$name/" | sed "s|^lab/$name/||" | grep -v '^export.json$' | python3 -c 'import sys; print(sorted(l.strip() for l in sys.stdin))')"
|
||||
# The blobs' subspace is t (116)
|
||||
grep -q '"subspace": 116' "$work/manifest.json" && echo "ok the export holds the blobs" || { echo "FAIL no blobs in the export"; fails=$((fails + 1)); }
|
||||
fi
|
||||
|
||||
echo "== 3. a fresh cluster's stores"
|
||||
for d in front mta core; do "${K[@]}" scale deploy/inbuxa-$d --replicas=0 >/dev/null; done
|
||||
"${K[@]}" wait --for=delete pod -l app.kubernetes.io/component=server --timeout=300s >/dev/null 2>&1 || true
|
||||
STORE_EMPTY=1
|
||||
primary=$("${D[@]}" get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}')
|
||||
"${D[@]}" exec "$primary" -c postgres -- psql -q -v ON_ERROR_STOP=1 \
|
||||
-c 'DROP DATABASE IF EXISTS inbuxa WITH (FORCE)' -c 'CREATE DATABASE inbuxa OWNER inbuxa' >/dev/null
|
||||
echo " database dropped and made again; $(s3 empty "$BLOBS") blobs deleted from $BLOBS"
|
||||
check "blobs left in $BLOBS" "$(s3 ls "$BLOBS" | wc -l)" 0
|
||||
|
||||
echo "== 4. the import, from the newest finished export"
|
||||
if run_job inbuxa-backup "lab-restore-$STAMP" --import "$URL"; then
|
||||
echo "ok the import finished"
|
||||
else
|
||||
echo "FAIL the import: $(tail -5 "$work/lab-restore-$STAMP.log")"
|
||||
echo "!! the lab's store is empty: fresh-store.sh sets up a new one" >&2
|
||||
exit 1
|
||||
fi
|
||||
STORE_EMPTY=""
|
||||
if [ -z "${RESTORE_ONLY:-}" ]; then
|
||||
grep -q "Importing the newest finished export, s3://$BUCKET/lab/$name" "$work/lab-restore-$STAMP.log" &&
|
||||
echo "ok it chose $name" || { echo "FAIL it didn't choose $name"; fails=$((fails + 1)); }
|
||||
else
|
||||
echo " $(grep 'Importing the newest' "$work/lab-restore-$STAMP.log")"
|
||||
fi
|
||||
grep -q "into the blob store the export's settings name" "$work/lab-restore-$STAMP.log" &&
|
||||
echo "ok blobs went to the blob store the settings name" || { echo "FAIL blobs went elsewhere"; fails=$((fails + 1)); }
|
||||
echo " $BLOBS holds $(s3 ls "$BLOBS" | wc -l) blobs again"
|
||||
|
||||
echo "== 5. the chart on the restored store"
|
||||
install "${BACKUP[@]}"
|
||||
for _ in $(seq 30); do digest >"$work/after" 2>/dev/null && break; sleep 5; done
|
||||
if [ -n "${RESTORE_ONLY:-}" ]; then
|
||||
sed 's/^/ /' "$work/after"
|
||||
elif diff -q "$work/before" "$work/after" >/dev/null; then
|
||||
echo "ok every mailbox's count and digest match"
|
||||
else
|
||||
echo "FAIL the mail differs:"; diff "$work/before" "$work/after" | sed 's/^/ /'; fails=$((fails + 1))
|
||||
fi
|
||||
for mode in quick full; do
|
||||
if run_job "inbuxa-store-check-$mode" "lab-check-$mode-$STAMP"; then
|
||||
echo "ok store check $mode: $(tail -1 "$work/lab-check-$mode-$STAMP.log")"
|
||||
else
|
||||
echo "FAIL store check $mode: $(tail -2 "$work/lab-check-$mode-$STAMP.log")"; fails=$((fails + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
if [ "$fails" -eq 0 ]; then echo "backup-test: all passed"; else echo "backup-test: $fails failed"; exit 1; fi
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""An account's mail, as something two stores can be compared by: one line
|
||||
per mailbox with its message count and a SHA-256 over the sorted SHA-256s
|
||||
of its messages, read whole over IMAPS (993). Or, with `send N`, N messages
|
||||
to the account over SMTP (25) first, each with an attachment, and waits
|
||||
until IMAP shows them (in any mailbox).
|
||||
|
||||
ACCOUNT=user@domain PASSWORD_FILE=... mailbox-digest.py LB [send N]
|
||||
|
||||
The lab's certificate is self-signed, so TLS isn't verified.
|
||||
"""
|
||||
import hashlib, imaplib, os, re, smtplib, ssl, sys, time, uuid
|
||||
from email.message import EmailMessage
|
||||
from email.utils import formatdate, make_msgid
|
||||
|
||||
lb = sys.argv[1]
|
||||
account = os.environ["ACCOUNT"]
|
||||
password = open(os.environ["PASSWORD_FILE"]).read().strip()
|
||||
tls = ssl.create_default_context()
|
||||
tls.check_hostname = False
|
||||
tls.verify_mode = ssl.CERT_NONE
|
||||
|
||||
|
||||
def imap():
|
||||
m = imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30)
|
||||
m.login(account, password)
|
||||
return m
|
||||
|
||||
|
||||
def names(m):
|
||||
typ, boxes = m.list()
|
||||
for line in boxes:
|
||||
name = re.search(rb'"([^"]*)"$|(\S+)$', line.strip())
|
||||
yield (name.group(1) or name.group(2)).decode()
|
||||
|
||||
|
||||
def total(m):
|
||||
# Every mailbox: the spam filter may put a test message in Junk
|
||||
count = 0
|
||||
for name in names(m):
|
||||
typ, data = m.select(f'"{name}"', readonly=True)
|
||||
if typ == "OK":
|
||||
count += int(data[0])
|
||||
return count
|
||||
|
||||
|
||||
if len(sys.argv) > 2 and sys.argv[2] == "send":
|
||||
count = int(sys.argv[3])
|
||||
m = imap()
|
||||
before = total(m)
|
||||
run = uuid.uuid4().hex[:8]
|
||||
with smtplib.SMTP(lb, 25, timeout=30) as s:
|
||||
for i in range(count):
|
||||
msg = EmailMessage()
|
||||
msg["From"] = "[email protected]"
|
||||
msg["To"] = account
|
||||
msg["Subject"] = f"backup-test {run} {i}"
|
||||
msg["Date"] = formatdate()
|
||||
msg["Message-ID"] = make_msgid(domain="sender.test")
|
||||
msg.set_content(f"Sent by the inbuxa-k8s-lab backup test, {i} of {count}.\n")
|
||||
msg.add_attachment((f"line {i}: " + uuid.uuid4().hex + "\n") * (60 + 17 * i),
|
||||
filename=f"notes-{i}.txt")
|
||||
s.send_message(msg)
|
||||
for _ in range(60):
|
||||
if total(m) >= before + count:
|
||||
break
|
||||
time.sleep(2)
|
||||
else:
|
||||
sys.exit(f"only {total(m) - before} of {count} arrived")
|
||||
m.logout()
|
||||
|
||||
m = imap()
|
||||
for name in list(names(m)):
|
||||
typ, data = m.select(f'"{name}"', readonly=True)
|
||||
if typ != "OK":
|
||||
continue
|
||||
count = int(data[0])
|
||||
hashes = []
|
||||
if count:
|
||||
typ, data = m.fetch(f"1:{count}", "(BODY.PEEK[])")
|
||||
hashes = sorted(hashlib.sha256(part[1]).hexdigest()
|
||||
for part in data if isinstance(part, tuple))
|
||||
digest = hashlib.sha256("\n".join(hashes).encode()).hexdigest()
|
||||
print(f"{name}\t{count}\t{digest}")
|
||||
m.logout()
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Just enough S3 for the lab's tests, with the standard library only:
|
||||
path-style requests signed with SigV4, to the lab's Garage through a
|
||||
port-forward.
|
||||
|
||||
S3_ENDPOINT=http://127.0.0.1:13900 S3_REGION=garage \\
|
||||
S3_ACCESS_KEY_FILE=... S3_SECRET_KEY_FILE=... s3.py <command> <bucket> [arg]
|
||||
|
||||
ls <bucket> [prefix] keys under prefix, one per line
|
||||
get <bucket> <key> the object, to stdout
|
||||
empty <bucket> [prefix] deletes every object under prefix; prints how many
|
||||
"""
|
||||
import datetime, hashlib, hmac, os, sys, urllib.parse, urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
ENDPOINT = os.environ["S3_ENDPOINT"].rstrip("/")
|
||||
REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
ACCESS = open(os.environ["S3_ACCESS_KEY_FILE"]).read().strip()
|
||||
SECRET = open(os.environ["S3_SECRET_KEY_FILE"]).read().strip()
|
||||
NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
|
||||
|
||||
|
||||
def request(method, bucket, key="", query=None):
|
||||
query = query or {}
|
||||
host = urllib.parse.urlsplit(ENDPOINT).netloc
|
||||
path = "/" + bucket + ("/" + urllib.parse.quote(key, safe="/~") if key else "")
|
||||
canonical_query = "&".join(
|
||||
f"{urllib.parse.quote(k, safe='~')}={urllib.parse.quote(str(v), safe='~')}"
|
||||
for k, v in sorted(query.items())
|
||||
)
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
amz_date, date = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
|
||||
payload = hashlib.sha256(b"").hexdigest()
|
||||
headers = {"host": host, "x-amz-content-sha256": payload, "x-amz-date": amz_date}
|
||||
signed = ";".join(sorted(headers))
|
||||
canonical = "\n".join([
|
||||
method, path, canonical_query,
|
||||
"".join(f"{k}:{headers[k]}\n" for k in sorted(headers)), signed, payload,
|
||||
])
|
||||
scope = f"{date}/{REGION}/s3/aws4_request"
|
||||
to_sign = "\n".join(["AWS4-HMAC-SHA256", amz_date, scope,
|
||||
hashlib.sha256(canonical.encode()).hexdigest()])
|
||||
k = ("AWS4" + SECRET).encode()
|
||||
for part in (date, REGION, "s3", "aws4_request"):
|
||||
k = hmac.new(k, part.encode(), hashlib.sha256).digest()
|
||||
signature = hmac.new(k, to_sign.encode(), hashlib.sha256).hexdigest()
|
||||
headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, "
|
||||
f"SignedHeaders={signed}, Signature={signature}")
|
||||
url = ENDPOINT + path + ("?" + canonical_query if canonical_query else "")
|
||||
req = urllib.request.Request(url, method=method, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def keys(bucket, prefix=""):
|
||||
token = None
|
||||
while True:
|
||||
query = {"list-type": "2", "prefix": prefix}
|
||||
if token:
|
||||
query["continuation-token"] = token
|
||||
root = ET.fromstring(request("GET", bucket, query=query))
|
||||
for c in root.iter(NS + "Contents"):
|
||||
yield c.find(NS + "Key").text
|
||||
token_el = root.find(NS + "NextContinuationToken")
|
||||
if root.findtext(NS + "IsTruncated") != "true" or token_el is None:
|
||||
return
|
||||
token = token_el.text
|
||||
|
||||
|
||||
command, bucket = sys.argv[1], sys.argv[2]
|
||||
arg = sys.argv[3] if len(sys.argv) > 3 else ""
|
||||
if command == "ls":
|
||||
for key in keys(bucket, arg):
|
||||
print(key)
|
||||
elif command == "get":
|
||||
sys.stdout.buffer.write(request("GET", bucket, arg))
|
||||
elif command == "empty":
|
||||
gone = 0
|
||||
for key in list(keys(bucket, arg)):
|
||||
request("DELETE", bucket, key)
|
||||
gone += 1
|
||||
print(gone)
|
||||
else:
|
||||
sys.exit(f"unknown command {command}")
|
||||
Reference in new issue
Block a user