Files
jcoffey-dev d400d710c2 backup-test: backup and restore through the chart's CronJobs (K-L6, K-C11)
A run of the backup CronJob exports the store to a Garage bucket; then
a fresh cluster's stores (the database dropped and made again, the blob
bucket emptied), the import from the newest finished export by a Job
made from the CronJob's own pod, and the chart back on the restored
store. Every mailbox's count and message hashes (read whole over IMAP)
must match, and both store check CronJobs must find nothing wrong.
RESTORE_ONLY=1 restores the newest export: the way back from a failed
run. A failed import leaves the server pods stopped, never started on
the empty store.

s3.py: just enough S3, standard library only (SigV4, path-style).
mailbox-digest.py: an account's mail as one line per mailbox, and test
mail with attachments.
2026-11-19 13:48:49 -07:00

87 lines
3.5 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
"""Just enough S3 for the lab's tests, with the standard library only:
path-style requests signed with SigV4, to the lab's Garage through a
port-forward.
S3_ENDPOINT=http://127.0.0.1:13900 S3_REGION=garage \\
S3_ACCESS_KEY_FILE=... S3_SECRET_KEY_FILE=... s3.py <command> <bucket> [arg]
ls <bucket> [prefix] keys under prefix, one per line
get <bucket> <key> the object, to stdout
empty <bucket> [prefix] deletes every object under prefix; prints how many
"""
import datetime, hashlib, hmac, os, sys, urllib.parse, urllib.request
import xml.etree.ElementTree as ET
ENDPOINT = os.environ["S3_ENDPOINT"].rstrip("/")
REGION = os.environ.get("S3_REGION", "us-east-1")
ACCESS = open(os.environ["S3_ACCESS_KEY_FILE"]).read().strip()
SECRET = open(os.environ["S3_SECRET_KEY_FILE"]).read().strip()
NS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
def request(method, bucket, key="", query=None):
query = query or {}
host = urllib.parse.urlsplit(ENDPOINT).netloc
path = "/" + bucket + ("/" + urllib.parse.quote(key, safe="/~") if key else "")
canonical_query = "&".join(
f"{urllib.parse.quote(k, safe='~')}={urllib.parse.quote(str(v), safe='~')}"
for k, v in sorted(query.items())
)
now = datetime.datetime.now(datetime.timezone.utc)
amz_date, date = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d")
payload = hashlib.sha256(b"").hexdigest()
headers = {"host": host, "x-amz-content-sha256": payload, "x-amz-date": amz_date}
signed = ";".join(sorted(headers))
canonical = "\n".join([
method, path, canonical_query,
"".join(f"{k}:{headers[k]}\n" for k in sorted(headers)), signed, payload,
])
scope = f"{date}/{REGION}/s3/aws4_request"
to_sign = "\n".join(["AWS4-HMAC-SHA256", amz_date, scope,
hashlib.sha256(canonical.encode()).hexdigest()])
k = ("AWS4" + SECRET).encode()
for part in (date, REGION, "s3", "aws4_request"):
k = hmac.new(k, part.encode(), hashlib.sha256).digest()
signature = hmac.new(k, to_sign.encode(), hashlib.sha256).hexdigest()
headers["authorization"] = (f"AWS4-HMAC-SHA256 Credential={ACCESS}/{scope}, "
f"SignedHeaders={signed}, Signature={signature}")
url = ENDPOINT + path + ("?" + canonical_query if canonical_query else "")
req = urllib.request.Request(url, method=method, headers=headers)
with urllib.request.urlopen(req, timeout=60) as r:
return r.read()
def keys(bucket, prefix=""):
token = None
while True:
query = {"list-type": "2", "prefix": prefix}
if token:
query["continuation-token"] = token
root = ET.fromstring(request("GET", bucket, query=query))
for c in root.iter(NS + "Contents"):
yield c.find(NS + "Key").text
token_el = root.find(NS + "NextContinuationToken")
if root.findtext(NS + "IsTruncated") != "true" or token_el is None:
return
token = token_el.text
command, bucket = sys.argv[1], sys.argv[2]
arg = sys.argv[3] if len(sys.argv) > 3 else ""
if command == "ls":
for key in keys(bucket, arg):
print(key)
elif command == "get":
sys.stdout.buffer.write(request("GET", bucket, arg))
elif command == "empty":
gone = 0
for key in list(keys(bucket, arg)):
request("DELETE", bucket, key)
gone += 1
print(gone)
else:
sys.exit(f"unknown command {command}")