Lab: cluster, first boot, mail Services and the chart's smoke test #17

Merged
jcoffey-dev merged 6 commits from feat/chart-smoke into main 2026-11-18 19:39:00 +00:00
24 changed files with 831 additions and 0 deletions

No files matched your search

+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Creates the smoke test's mail account, through a recovery-mode node.
RECOVERY_ADMIN=admin:pw PASSWORD_FILE=... accounts.py URL DOMAIN USER
The password is generated here and written to PASSWORD_FILE (0600); it
never goes to stdout. An account that exists already gets the new password.
"""
import base64, json, os, secrets, sys, urllib.request
url, domain, user = sys.argv[1].rstrip("/"), sys.argv[2], sys.argv[3]
auth = "Basic " + base64.b64encode(os.environ["RECOVERY_ADMIN"].encode()).decode()
USING = ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"]
def call(calls):
req = urllib.request.Request(url + "/jmap", data=json.dumps({"using": USING, "methodCalls": calls}).encode(),
headers={"Content-Type": "application/json", "Authorization": auth})
return json.load(urllib.request.urlopen(req, timeout=60))["methodResponses"]
r = call([["x:Domain/query", {"filter": {"name": domain}}, "q"],
["x:Domain/get", {"#ids": {"resultOf": "q", "name": "x:Domain/query", "path": "/ids"}, "properties": ["name"]}, "g"]])
domains = [d for d in r[1][1]["list"] if d["name"] == domain]
if not domains:
sys.exit(f"no domain {domain}: {json.dumps(r)[:400]}")
domain_id = domains[0]["id"]
password = secrets.token_urlsafe(24)
r = call([["x:Account/set", {"create": {"a": {
"@type": "User", "name": user, "domainId": domain_id,
"credentials": {"0": {"@type": "Password", "secret": password}}, "roles": {"@type": "User"}}}}, "c"]])
created = r[0][1].get("created", {})
if not created:
sys.exit("account not created: " + json.dumps(r[0][1].get("notCreated"))[:400])
fd = os.open(os.environ["PASSWORD_FILE"], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
os.write(fd, password.encode()); os.close(fd)
print(f"account {user}@{domain} created")
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env bash
# A fresh, set-up store for the chart smoke test, in place of whatever the
# lab's database held: first boot (bootstrap-mode pod), then the blob store,
# coordinator and role groups, and one mail account (recovery-mode pod).
#
# IMAGE=inbuxa-lab/server:main lab/chart/fresh-store.sh
#
# This wipes the lab's inbuxa database. Secrets stay out of stdout: the
# administrator and the test account's password go to files under $STATE
# (0600). The steps the chart can't do yet (first boot and the registry
# setup) are here, by hand, until it can.
set -euo pipefail
cd "$(dirname "$0")"
K=(kubectl --context kind-inbuxa-lab)
IMAGE="${IMAGE:-inbuxa-lab/server:main}"
PORTS="${PORTS:-pop3s=off}"
DOMAIN="${DOMAIN:-lab.test}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
mkdir -p "$STATE" && chmod 700 "$STATE"
secret() { "${K[@]}" -n inbuxa get secret "$1" -o "jsonpath={.data.$2}" | base64 -d; }
# A one-off server pod from ../spike, on this image and these listener ports
run_pod() {
local file="$1" name="$2"
"${K[@]}" -n inbuxa delete pod "$name" --ignore-not-found --wait >/dev/null
sed -e "s|image: inbuxa-lab/server:rc27|image: $IMAGE|" \
-e "s| - {name: INBUXA_HOSTNAME, value: $name}| - {name: INBUXA_HOSTNAME, value: $name}\n - {name: INBUXA_LISTENER_PORTS, value: \"$PORTS\"}|" \
"../spike/$file" | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n inbuxa wait --for=condition=Ready "pod/$name" --timeout=180s >/dev/null
}
# Port-forward 18080 to a pod's 8080 until stop_forward; waits for it to answer
FORWARD_PID=""
forward() {
"${K[@]}" -n inbuxa port-forward "pod/$1" 18080:8080 >/dev/null 2>&1 &
FORWARD_PID=$!
for _ in $(seq 1 60); do
curl -fsS -o /dev/null http://127.0.0.1:18080/healthz/live 2>/dev/null && return 0
sleep 1
done
echo "!! $1 never answered on 8080" >&2
return 1
}
stop_forward() { [ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null; FORWARD_PID=""; }
trap stop_forward EXIT
echo "== stopping everything on the old store"
"${K[@]}" -n inbuxa scale deploy --all --replicas=0 >/dev/null 2>&1 || true
"${K[@]}" -n inbuxa wait --for=delete pod -l app.kubernetes.io/component=server --timeout=180s >/dev/null 2>&1 || true
"${K[@]}" -n inbuxa wait --for=delete pod -l app=inbuxa-server --timeout=180s >/dev/null 2>&1 || true
echo "== a fresh database"
primary="$("${K[@]}" -n data get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}')"
"${K[@]}" -n data exec "$primary" -c postgres -- psql -q -v ON_ERROR_STOP=1 \
-c 'DROP DATABASE IF EXISTS inbuxa WITH (FORCE)' -c 'CREATE DATABASE inbuxa OWNER inbuxa'
echo "== first boot"
run_pod bootstrap-pod.yaml inbuxa-bootstrap
forward inbuxa-bootstrap
BOOTSTRAP_ADMIN="$(secret inbuxa-recovery admin)" ADMIN_OUT="$STATE/admin.json" \
python3 ../spike/bootstrap.py http://127.0.0.1:18080 "mail.$DOMAIN" "$DOMAIN"
stop_forward
"${K[@]}" -n inbuxa delete pod inbuxa-bootstrap --wait >/dev/null
echo "== blob store, coordinator, role groups, a mail account"
run_pod setup-pod.yaml inbuxa-setup
forward inbuxa-setup
RECOVERY_ADMIN="$(secret inbuxa-recovery admin)" S3_ACCESS_KEY="$(secret inbuxa-s3 access-key)" \
python3 ../spike/setup-registry.py http://127.0.0.1:18080 all >"$STATE/setup.log" \
|| { cat "$STATE/setup.log" >&2; exit 1; }
grep -E " OK$" "$STATE/setup.log"
RECOVERY_ADMIN="$(secret inbuxa-recovery admin)" PASSWORD_FILE="$STATE/alice.password" \
python3 accounts.py http://127.0.0.1:18080 "$DOMAIN" alice
stop_forward
"${K[@]}" -n inbuxa delete pod inbuxa-setup --wait >/dev/null
echo "== done: alice@$DOMAIN, password in $STATE/alice.password"
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""The chart smoke test's mail check: a message in over SMTP on port 25,
then found over IMAPS on 993, both through the mail Service's address.
ACCOUNT=user@domain PASSWORD_FILE=... mailtest.py LB_ADDRESS
The server's certificate is self-signed in the lab, so TLS isn't verified.
"""
import imaplib, os, smtplib, ssl, sys, time, uuid
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
lb = sys.argv[1]
account = os.environ["ACCOUNT"]
password = open(os.environ["PASSWORD_FILE"]).read().strip()
token = uuid.uuid4().hex
tls = ssl.create_default_context()
tls.check_hostname = False
tls.verify_mode = ssl.CERT_NONE
msg = EmailMessage()
# A sender domain with no DNS at all: one that publishes DMARC (example.net
# does, p=reject) would send the message to Junk
msg["From"] = "[email protected]"
msg["To"] = account
msg["Subject"] = f"chart smoke {token}"
msg["Date"] = formatdate()
msg["Message-ID"] = make_msgid(domain="sender.test")
msg.set_content("Sent by the inbuxa-k8s-lab chart smoke test.\n")
started = time.time()
with smtplib.SMTP(lb, 25, timeout=30) as smtp:
smtp.ehlo("lab-client.sender.test")
if smtp.has_extn("starttls"):
smtp.starttls(context=tls)
smtp.ehlo("lab-client.sender.test")
smtp.send_message(msg)
print(f"SMTP: accepted for {account}")
deadline = started + 90
while True:
with imaplib.IMAP4_SSL(lb, 993, ssl_context=tls, timeout=30) as imap:
imap.login(account, password)
# The lab has no public DNS, so the sender can't pass SPF, DKIM or
# reverse DNS and the spam filter files it in Junk, rightly. The test
# is the path through the chart, so either folder will do.
where = None
for folder in ("INBOX", "Junk Mail", "Junk"):
if imap.select(f'"{folder}"', readonly=True)[0] != "OK":
continue
_, found = imap.search(None, "SUBJECT", f'"{token}"')
if found[0].split():
where = folder
break
if where:
print(f"IMAPS: delivered to {where}, found after {time.time() - started:.1f}s")
break
if time.time() > deadline:
sys.exit("IMAPS: the message never arrived")
time.sleep(3)
+47
View File
@@ -0,0 +1,47 @@
# The seed run, as the chart will make it: `inbuxa --seed` sets up the new
# store from seed.json (ConfigMap inbuxa-lab-seed) and exits. Its secrets come
# from Secret inbuxa-seed, mounted at /etc/inbuxa/seed-secrets, which only
# this Job sees. The server's own secrets are where every node reads them.
apiVersion: batch/v1
kind: Job
metadata:
name: inbuxa-seed
namespace: inbuxa
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
metadata:
labels: {app: inbuxa-seed}
spec:
restartPolicy: Never
automountServiceAccountToken: false
enableServiceLinks: false
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: seed
image: IMAGE
imagePullPolicy: Never
args: [--seed, /etc/inbuxa/seed/seed.json, --config, /etc/inbuxa/config.json]
env:
- {name: INBUXA_HOSTNAME, value: seed}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: config, mountPath: /etc/inbuxa/config.json, subPath: config.json}
- {name: seed, mountPath: /etc/inbuxa/seed}
- {name: seed-secrets, mountPath: /etc/inbuxa/seed-secrets}
- {name: data-secrets, mountPath: /etc/inbuxa/secrets}
- {name: var, mountPath: /var/lib/inbuxa}
- {name: tmp, mountPath: /tmp}
volumes:
- {name: config, configMap: {name: inbuxa-config}}
- {name: seed, configMap: {name: inbuxa-lab-seed}}
- {name: seed-secrets, secret: {secretName: inbuxa-seed, defaultMode: 0440}}
- name: data-secrets
projected:
defaultMode: 0440
sources:
- secret: {name: inbuxa-data, items: [{key: pg-password, path: pg-password}, {key: nats-password, path: nats-password}]}
- secret: {name: inbuxa-s3, items: [{key: secret-key, path: s3-secret-key}]}
- {name: var, emptyDir: {}}
- {name: tmp, emptyDir: {}}
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env bash
# A new store set up by the server's --seed, the way the chart will do it:
# wipes the lab's inbuxa database, then runs seed-job.yaml with seed.json.
#
# IMAGE=inbuxa-lab/server:main lab/chart/seed-store.sh
#
# The administrator's and alice's passwords are generated here, put in
# Secret inbuxa-seed, and kept under $STATE (0600); never printed.
set -euo pipefail
cd "$(dirname "$0")"
K=(kubectl --context kind-inbuxa-lab)
IMAGE="${IMAGE:-inbuxa-lab/server:main}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
mkdir -p "$STATE" && chmod 700 "$STATE"
new_password() { python3 -c 'import secrets; print(secrets.token_urlsafe(24))'; }
umask 077
echo "== stopping everything on the old store"
"${K[@]}" -n inbuxa scale deploy --all --replicas=0 >/dev/null 2>&1 || true
"${K[@]}" -n inbuxa wait --for=delete pod -l app.kubernetes.io/component=server --timeout=180s >/dev/null 2>&1 || true
echo "== a fresh database"
primary="$("${K[@]}" -n data get cluster inbuxa-pg -o jsonpath='{.status.currentPrimary}')"
"${K[@]}" -n data exec "$primary" -c postgres -- psql -q -v ON_ERROR_STOP=1 \
-c 'DROP DATABASE IF EXISTS inbuxa WITH (FORCE)' -c 'CREATE DATABASE inbuxa OWNER inbuxa'
echo "== the seed and its secrets"
new_password > "$STATE/admin.password"
new_password > "$STATE/alice.password"
"${K[@]}" -n inbuxa get secret inbuxa-s3 -o jsonpath='{.data.access-key}' | base64 -d > "$STATE/s3-access-key"
"${K[@]}" -n inbuxa delete secret inbuxa-seed --ignore-not-found >/dev/null
"${K[@]}" -n inbuxa create secret generic inbuxa-seed \
--from-file=admin-password="$STATE/admin.password" \
--from-file=alice-password="$STATE/alice.password" \
--from-file=s3-access-key="$STATE/s3-access-key" >/dev/null
rm -f "$STATE/s3-access-key"
"${K[@]}" -n inbuxa create configmap inbuxa-lab-seed --from-file=seed.json --dry-run=client -o yaml \
| "${K[@]}" apply -f - >/dev/null
echo "== the seed run"
"${K[@]}" -n inbuxa delete job inbuxa-seed --ignore-not-found --wait >/dev/null
sed "s|image: IMAGE|image: $IMAGE|" seed-job.yaml | "${K[@]}" apply -f - >/dev/null
if ! "${K[@]}" -n inbuxa wait --for=condition=Complete job/inbuxa-seed --timeout=300s >/dev/null 2>&1; then
"${K[@]}" -n inbuxa logs job/inbuxa-seed --tail=40 >&2 || true
echo "!! the seed run failed" >&2
exit 1
fi
"${K[@]}" -n inbuxa logs job/inbuxa-seed | grep -E "seed|store layout" | grep -v "^20..-" || true
echo "== done: [email protected] and [email protected], passwords in $STATE"
+60
View File
@@ -0,0 +1,60 @@
{
"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"],
"methodCalls": [
["x:Domain/set", {"create": {"d": {
"name": "lab.test",
"dkimManagement": {"@type": "Automatic"}
}}}, "domain"],
["x:SystemSettings/set", {"update": {"singleton": {
"defaultHostname": "mail.lab.test",
"defaultDomainId": "#d"
}}}, "settings"],
["x:BlobStore/set", {"update": {"singleton": {
"@type": "S3", "bucket": "inbuxa-blobs",
"region": {"@type": "Custom", "customEndpoint": "http://garage.data.svc:3900", "customRegion": "garage"},
"accessKey": {"@type": "Value", "value": {"$file": "/etc/inbuxa/seed-secrets/s3-access-key"}},
"secretKey": {"@type": "File", "filePath": "/etc/inbuxa/secrets/s3-secret-key"},
"timeout": 30000, "maxRetries": 3, "verifyAfterWrite": false
}}}, "blob"],
["x:Coordinator/set", {"update": {"singleton": {
"@type": "Nats", "addresses": {"nats.data.svc:4222": true}, "authUsername": "inbuxa",
"authSecret": {"@type": "File", "filePath": "/etc/inbuxa/secrets/nats-password"}
}}}, "coordinator"],
["x:ClusterRole/set", {"create": {
"peer": {
"name": "peer",
"description": "front: protocols, task queue, indexing, maintenance; no scheduler, no outbound",
"tasks": {"@type": "DisableSome", "taskTypes": {
"taskScheduler": true, "outboundMta": true, "spamClassifierTraining": true,
"metricsCalculate": true, "metricsPush": true, "pushNotifications": true}},
"listeners": {"@type": "EnableAll"}
},
"peer-mta": {
"name": "peer-mta",
"description": "mta: peer plus outbound mail",
"tasks": {"@type": "DisableSome", "taskTypes": {
"taskScheduler": true, "spamClassifierTraining": true,
"metricsCalculate": true, "metricsPush": true, "pushNotifications": true}},
"listeners": {"@type": "EnableAll"}
}
}}, "roles"],
["x:Account/set", {"create": {
"admin": {
"@type": "User", "name": "admin", "domainId": "#d",
"description": "System administrator",
"credentials": {"0": {"@type": "Password", "secret": {"$file": "/etc/inbuxa/seed-secrets/admin-password"}}},
"roles": {"@type": "Admin"}
},
"alice": {
"@type": "User", "name": "alice", "domainId": "#d",
"credentials": {"0": {"@type": "Password", "secret": {"$file": "/etc/inbuxa/seed-secrets/alice-password"}}},
"roles": {"@type": "User"}
}
}}, "accounts"]
]
}
+39
View File
@@ -0,0 +1,39 @@
# The chart's values for the lab: the data layer from lab/data-layer, the
# image loaded into kind, and the secrets lab/up.sh created.
server:
image:
repository: inbuxa-lab/server
tag: main
pullPolicy: Never
dataStore:
"@type": PostgreSql
host: inbuxa-pg-rw.data.svc
port: 5432
database: inbuxa
authUsername: inbuxa
authSecret: {"@type": File, filePath: /etc/inbuxa/secrets/pg-password}
useTls: false
poolMaxConnections: 20
secretMounts:
- secretName: inbuxa-data
items: [{key: pg-password, path: pg-password}, {key: nats-password, path: nats-password}]
- secretName: inbuxa-s3
items: [{key: secret-key, path: s3-secret-key}]
groups:
front: {enabled: true, replicas: 2, role: peer}
mta: {enabled: true, replicas: 2, role: peer-mta}
core: {enabled: true, replicas: 1, role: ""}
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {cpu: "1", memory: 1Gi}
# The lab's setup (seed.json), from ConfigMap inbuxa-lab-seed and Secret inbuxa-seed,
# which seed-store.sh makes
seed:
enabled: true
existingConfigMap: inbuxa-lab-seed
secretMounts:
- secretName: inbuxa-seed
items:
- {key: admin-password, path: admin-password}
- {key: alice-password, path: alice-password}
- {key: s3-access-key, path: s3-access-key}
+55
View File
@@ -0,0 +1,55 @@
# A single Garage node (replication factor 1) is enough for the lab: the blob
# store is the operator's to make highly available, not the chart's.
apiVersion: v1
kind: ConfigMap
metadata:
name: garage
namespace: data
data:
garage.toml: |
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"
db_engine = "sqlite"
replication_factor = 1
rpc_bind_addr = "[::]:3901"
rpc_public_addr = "127.0.0.1:3901"
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
[admin]
api_bind_addr = "[::]:3903"
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: garage
namespace: data
spec:
replicas: 1
selector: {matchLabels: {app: garage}}
template:
metadata: {labels: {app: garage}}
spec:
securityContext: {runAsNonRoot: true, runAsUser: 1000, fsGroup: 1000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: garage
image: GARAGE_IMAGE
env:
- {name: GARAGE_RPC_SECRET, valueFrom: {secretKeyRef: {name: garage-rpc, key: secret}}}
ports: [{containerPort: 3900, name: s3}]
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: config, mountPath: /etc/garage.toml, subPath: garage.toml}
- {name: data, mountPath: /var/lib/garage}
volumes:
- {name: config, configMap: {name: garage}}
- {name: data, emptyDir: {}}
---
apiVersion: v1
kind: Service
metadata:
name: garage
namespace: data
spec:
selector: {app: garage}
ports: [{name: s3, port: 3900, targetPort: 3900}]
+20
View File
@@ -0,0 +1,20 @@
# Three NATS servers with a single user; the password comes from the Secret
# nats-inbuxa (key: password) through the environment.
config:
cluster:
enabled: true
replicas: 3
merge:
authorization:
user: inbuxa
password: << $NATS_INBUXA_PASSWORD >>
container:
env:
NATS_INBUXA_PASSWORD:
valueFrom:
secretKeyRef: {name: nats-inbuxa, key: password}
podTemplate:
topologySpreadConstraints:
kubernetes.io/hostname:
maxSkew: 1
whenUnsatisfiable: DoNotSchedule
+19
View File
@@ -0,0 +1,19 @@
# Two instances so a primary failover can be tested. CloudNativePG creates the
# Secret inbuxa-pg-app (username, password) and the Service inbuxa-pg-rw.
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: inbuxa-pg
namespace: data
spec:
instances: 2
imageName: ghcr.io/cloudnative-pg/postgresql:17
storage:
size: 2Gi
bootstrap:
initdb:
database: inbuxa
owner: inbuxa
affinity:
enablePodAntiAffinity: true
topologyKey: kubernetes.io/hostname
Executable
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
# Remove the lab cluster and everything in it.
set -euo pipefail
. "$(cd "$(dirname "$0")" && pwd)/versions.env"
kind delete cluster --name "$KIND_CLUSTER"
+10
View File
@@ -0,0 +1,10 @@
# One control plane and three workers, so pods can spread and a node can be
# drained or removed. Ports aren't mapped to the host: mail reaches the cluster
# through MetalLB addresses on the kind Docker network.
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
+11
View File
@@ -0,0 +1,11 @@
# Lab stand-in for a cloud or hardware load balancer that speaks PROXY v2.
defaults
mode tcp
timeout connect 5s
timeout client 60s
timeout server 60s
frontend smtp
bind :2525
default_backend inbuxa
backend inbuxa
server lb 172.19.255.201:10025 send-proxy-v2
+13
View File
@@ -0,0 +1,13 @@
# Mode A: the load balancer sends traffic only to nodes running a server pod
# and keeps the client's address (no SNAT). No PROXY protocol involved.
apiVersion: v1
kind: Service
metadata: {name: inbuxa-mail-local, namespace: inbuxa}
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector: {app: inbuxa-server}
ports:
- {name: smtp, port: 25, targetPort: 25}
- {name: submissions, port: 465, targetPort: 465}
- {name: imaps, port: 993, targetPort: 993}
+14
View File
@@ -0,0 +1,14 @@
# Mode B: an external load balancer (HAProxy in the lab) sends PROXY v2 to a
# listener that trusts PROXY headers from the load balancer's own address
# only. externalTrafficPolicy Local keeps that address visible: with the
# default Cluster policy, SNAT makes every peer look like a node, so trusting
# nodes lets anyone who reaches the port forge a client address (spike S6).
apiVersion: v1
kind: Service
metadata: {name: inbuxa-mail-proxy, namespace: inbuxa}
spec:
type: LoadBalancer
externalTrafficPolicy: Local
selector: {app: inbuxa-server}
ports:
- {name: smtp-proxy, port: 10025, targetPort: 10025}
+13
View File
@@ -0,0 +1,13 @@
# Addresses MetalLB hands to LoadBalancer Services: the top of the kind
# Docker network, which the host and other containers on it can reach.
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata: {name: lab, namespace: metallb-system}
spec:
addresses: ["172.19.255.200-172.19.255.250"]
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata: {name: lab, namespace: metallb-system}
spec:
ipAddressPools: [lab]
+35
View File
@@ -0,0 +1,35 @@
# First boot: a server with no config.json starts in bootstrap mode, where
# x:Bootstrap/set names the data store and creates the administrator, the
# default listeners and the signing keys. /etc/inbuxa is writable here
# because bootstrap writes config.json; normal nodes mount it read-only.
# INBUXA_RECOVERY_ADMIN pins the temporary administrator; without it the
# server prints a one-time password to the log, which would reach whatever
# collects pod logs.
apiVersion: v1
kind: Pod
metadata:
name: inbuxa-bootstrap
namespace: inbuxa
spec:
automountServiceAccountToken: false
restartPolicy: Never
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: server
image: inbuxa-lab/server:rc27
imagePullPolicy: Never
env:
- {name: INBUXA_HOSTNAME, value: bootstrap}
- {name: INBUXA_RECOVERY_ADMIN, valueFrom: {secretKeyRef: {name: inbuxa-recovery, key: admin}}}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}}
volumeMounts:
- {name: etc, mountPath: /etc/inbuxa}
- {name: data-secrets, mountPath: /etc/inbuxa/secrets}
- {name: var, mountPath: /var/lib/inbuxa}
- {name: tmp, mountPath: /tmp}
volumes:
- {name: etc, emptyDir: {}}
- name: data-secrets
secret: {secretName: inbuxa-data, items: [{key: pg-password, path: pg-password}]}
- {name: var, emptyDir: {}}
- {name: tmp, emptyDir: {}}
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""First boot of a fresh store through a bootstrap-mode node (bootstrap-pod.yaml).
BOOTSTRAP_ADMIN=admin:pw bootstrap.py http://127.0.0.1:18080 MAIL_HOST DOMAIN
Names PostgreSQL as the data store (password as a File secret), creates the
administrator, listeners and keys. Prints the administrator's username; the
password goes to the file named by ADMIN_OUT (0600), never to stdout.
"""
import base64, json, os, sys, urllib.request
url, host, domain = sys.argv[1].rstrip("/"), sys.argv[2], sys.argv[3]
auth = "Basic " + base64.b64encode(os.environ["BOOTSTRAP_ADMIN"].encode()).decode()
update = {
"serverHostname": host, "defaultDomain": domain, "requestTlsCertificate": False, "generateDkimKeys": True,
"tracer": {"@type": "Stdout", "enable": True, "level": "info", "ansi": False, "multiline": False, "lossy": False,
"events": {}, "eventsPolicy": "exclude"},
"dataStore": {"@type": "PostgreSql", "host": "inbuxa-pg-rw.data.svc", "port": 5432, "database": "inbuxa",
"authUsername": "inbuxa", "authSecret": {"@type": "File", "filePath": "/etc/inbuxa/secrets/pg-password"},
"useTls": False, "poolMaxConnections": 20},
}
body = {"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"],
"methodCalls": [["x:Bootstrap/set", {"update": {"singleton": update}}, "0"]]}
req = urllib.request.Request(url + "/jmap", data=json.dumps(body).encode(), headers={"Content-Type": "application/json", "Authorization": auth})
resp = json.load(urllib.request.urlopen(req, timeout=120))["methodResponses"][0]
if resp[0] == "error" or resp[1].get("notUpdated"):
sys.exit("bootstrap failed: " + json.dumps(resp)[:800])
admin = resp[1]["updated"]["singleton"]
out = os.environ.get("ADMIN_OUT", "admin.json")
fd = os.open(out, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
os.write(fd, json.dumps(admin).encode()); os.close(fd)
print("administrator:", admin.get("username"), "(password in", out + ")")
+41
View File
@@ -0,0 +1,41 @@
# Spike S4: the "core" role group (no INBUXA_ROLE: every task, including the
# scheduler) at two replicas, to see whether background work is ever done
# twice. Pod Security "restricted" settings throughout.
apiVersion: apps/v1
kind: Deployment
metadata:
name: inbuxa-core
namespace: inbuxa
spec:
replicas: 2
selector: {matchLabels: {app: inbuxa-server, group: core}}
template:
metadata: {labels: {app: inbuxa-server, group: core}}
spec:
automountServiceAccountToken: false
terminationGracePeriodSeconds: 60
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
topologySpreadConstraints:
- {maxSkew: 1, topologyKey: kubernetes.io/hostname, whenUnsatisfiable: DoNotSchedule, labelSelector: {matchLabels: {app: inbuxa-server}}}
containers:
- name: server
image: inbuxa-lab/server:rc27
imagePullPolicy: Never
env:
- name: INBUXA_HOSTNAME
valueFrom: {fieldRef: {fieldPath: metadata.name}}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}}
volumeMounts:
- {name: config, mountPath: /etc/inbuxa/config.json, subPath: config.json, readOnly: true}
- {name: data-secrets, mountPath: /etc/inbuxa/secrets, readOnly: true}
- {name: var, mountPath: /var/lib/inbuxa}
- {name: tmp, mountPath: /tmp}
volumes:
- {name: config, configMap: {name: inbuxa-config}}
- name: data-secrets
projected:
sources:
- secret: {name: inbuxa-data, items: [{key: pg-password, path: pg-password}, {key: nats-password, path: nats-password}]}
- secret: {name: inbuxa-s3, items: [{key: secret-key, path: s3-secret-key}]}
- {name: var, emptyDir: {}}
- {name: tmp, emptyDir: {}}
+12
View File
@@ -0,0 +1,12 @@
# Server pieces shared by the spike manifests: config.json (the data store,
# with its password as a File secret) and nothing else. Everything else lives
# in the store.
apiVersion: v1
kind: ConfigMap
metadata:
name: inbuxa-config
namespace: inbuxa
data:
config.json: |
{"@type":"PostgreSql","host":"inbuxa-pg-rw.data.svc","port":5432,"database":"inbuxa","authUsername":"inbuxa",
"authSecret":{"@type":"File","filePath":"/etc/inbuxa/secrets/pg-password"},"useTls":false,"poolMaxConnections":20}
+35
View File
@@ -0,0 +1,35 @@
# A one-off server in recovery mode: port 8080 only, the recovery admin from
# Secret inbuxa-recovery. Used to point a fresh store at NATS and S3 and to
# create the role groups before any normal node starts.
apiVersion: v1
kind: Pod
metadata:
name: inbuxa-setup
namespace: inbuxa
labels: {app: inbuxa-setup}
spec:
automountServiceAccountToken: false
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: server
image: inbuxa-lab/server:rc27
imagePullPolicy: Never
env:
- {name: INBUXA_HOSTNAME, value: setup}
- {name: INBUXA_RECOVERY_MODE, value: "1"}
- {name: INBUXA_RECOVERY_ADMIN, valueFrom: {secretKeyRef: {name: inbuxa-recovery, key: admin}}}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL], add: [NET_BIND_SERVICE]}}
volumeMounts:
- {name: config, mountPath: /etc/inbuxa/config.json, subPath: config.json}
- {name: data-secrets, mountPath: /etc/inbuxa/secrets}
- {name: var, mountPath: /var/lib/inbuxa}
- {name: tmp, mountPath: /tmp}
volumes:
- {name: config, configMap: {name: inbuxa-config}}
- name: data-secrets
projected:
sources:
- secret: {name: inbuxa-data, items: [{key: pg-password, path: pg-password}, {key: nats-password, path: nats-password}]}
- secret: {name: inbuxa-s3, items: [{key: secret-key, path: s3-secret-key}]}
- {name: var, emptyDir: {}}
- {name: tmp, emptyDir: {}}
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Point a fresh store at the lab's NATS and S3 and create the role groups,
through a recovery-mode node (setup-pod.yaml) reached on a local port.
RECOVERY_ADMIN=admin:pw S3_ACCESS_KEY=... setup-registry.py http://127.0.0.1:18080 [all|show]
Secrets stay out of the registry: NATS and S3 secrets are File secrets every
node reads from /etc/inbuxa/secrets. Only the S3 access key id is a value.
Role groups (the chart's front / mta / core):
core no INBUXA_ROLE: everything, including the task scheduler
peer protocols, task queue, indexing, maintenance; no scheduler, no
outbound, no spam training, no metrics, no push notifications
peer-mta peer plus outbound mail
"""
import base64, json, os, sys, urllib.error, urllib.request
URL = sys.argv[1].rstrip("/")
STEP = sys.argv[2] if len(sys.argv) > 2 else "all"
AUTH = "Basic " + base64.b64encode(os.environ["RECOVERY_ADMIN"].encode()).decode()
USING = ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"]
SECRETS = "/etc/inbuxa/secrets"
PEER_OFF = ["taskScheduler", "outboundMta", "spamClassifierTraining", "metricsCalculate", "metricsPush", "pushNotifications"]
def call(calls):
req = urllib.request.Request(URL + "/jmap", data=json.dumps({"using": USING, "methodCalls": calls}).encode(),
headers={"Content-Type": "application/json", "Authorization": AUTH})
try:
with urllib.request.urlopen(req, timeout=60) as r:
return json.load(r)["methodResponses"]
except urllib.error.HTTPError as e:
sys.exit(f"HTTP {e.code}: {e.read().decode()[:500]}")
def redact(o):
if isinstance(o, dict):
return {k: "<redacted>" if any(x in k.lower() for x in ("secret", "password", "token")) else redact(v) for k, v in o.items()}
return [redact(x) for x in o] if isinstance(o, list) else o
def ok(resps):
bad = False
for name, args, tag in resps:
problems = {k: args[k] for k in ("notCreated", "notUpdated", "notDestroyed") if args.get(k)}
if name == "error" or problems:
bad = True
print(tag, "PROBLEM", json.dumps(redact(problems or args))[:600])
else:
print(tag, "OK")
if bad:
sys.exit(1)
if STEP == "all":
ok(call([["x:BlobStore/set", {"update": {"singleton": {
"@type": "S3", "bucket": "inbuxa-blobs",
"region": {"@type": "Custom", "customEndpoint": "http://garage.data.svc:3900", "customRegion": "garage"},
"accessKey": {"@type": "Value", "value": os.environ["S3_ACCESS_KEY"]},
"secretKey": {"@type": "File", "filePath": f"{SECRETS}/s3-secret-key"},
"timeout": 30000, "maxRetries": 3, "verifyAfterWrite": False}}}, "blob"]]))
ok(call([["x:Coordinator/set", {"update": {"singleton": {
"@type": "Nats", "addresses": {"nats.data.svc:4222": True}, "authUsername": "inbuxa",
"authSecret": {"@type": "File", "filePath": f"{SECRETS}/nats-password"}}}}, "coord"]]))
have = {r["name"]: r["id"] for r in call([["x:ClusterRole/get", {"ids": None}, "r"]])[0][1]["list"]}
want = {
"peer": {"description": "front: protocols, task queue, indexing, maintenance; no scheduler, no outbound",
"tasks": {"@type": "DisableSome", "taskTypes": {t: True for t in PEER_OFF}}, "listeners": {"@type": "EnableAll"}},
"peer-mta": {"description": "mta: peer plus outbound mail",
"tasks": {"@type": "DisableSome", "taskTypes": {t: True for t in PEER_OFF if t != "outboundMta"}},
"listeners": {"@type": "EnableAll"}},
}
ok(call([["x:ClusterRole/set", {"create": {n: dict(v, name=n) for n, v in want.items() if n not in have},
"update": {have[n]: v for n, v in want.items() if n in have}}, "roles"]]))
r = call([["x:BlobStore/get", {"ids": ["singleton"]}, "b"], ["x:Coordinator/get", {"ids": ["singleton"]}, "c"],
["x:ClusterRole/get", {"ids": None, "properties": ["name"]}, "r"], ["x:NetworkListener/get", {"ids": None}, "l"]])
print(json.dumps(redact(r), indent=1)[:5000])
Executable
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env bash
# Bring up the lab: a kind cluster with the data layer inbuxa needs
# (PostgreSQL via CloudNativePG, NATS, Garage for S3), plus the Secrets the
# server reads, in namespace "inbuxa". Prints how long each part took.
set -euo pipefail
here=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=versions.env
. "$here/versions.env"
t0=$SECONDS
step() { printf '\n== %s (%ss)\n' "$1" "$((SECONDS - t0))"; }
rand() { head -c 24 /dev/urandom | base64 | tr -d '/+=' ; }
step "kind cluster"
kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" || kind create cluster --name "$KIND_CLUSTER" --config "$here/kind.yaml" --wait 120s
kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null
for ns in data inbuxa; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; done
step "CloudNativePG operator"
helm upgrade --install cnpg cnpg/cloudnative-pg --version "$CNPG_CHART" -n cnpg-system --create-namespace --wait >/dev/null
kubectl -n data apply -f "$here/data-layer/postgres.yaml" >/dev/null
step "NATS"
kubectl -n data get secret nats-inbuxa >/dev/null 2>&1 || kubectl -n data create secret generic nats-inbuxa --from-literal=password="$(rand)" >/dev/null
helm upgrade --install nats nats/nats --version "$NATS_CHART" -n data -f "$here/data-layer/nats-values.yaml" >/dev/null
step "Garage"
kubectl -n data get secret garage-rpc >/dev/null 2>&1 || kubectl -n data create secret generic garage-rpc --from-literal=secret="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')" >/dev/null
sed "s|GARAGE_IMAGE|$GARAGE_IMAGE|" "$here/data-layer/garage.yaml" | kubectl -n data apply -f - >/dev/null
kubectl -n data rollout status deploy/garage --timeout=180s >/dev/null
g() { kubectl -n data exec deploy/garage -- /garage "$@"; }
if ! g bucket info inbuxa-blobs >/dev/null 2>&1; then
node=$(g status 2>/dev/null | awk '/^[0-9a-f]{16}/{print $1; exit}')
g layout assign -z lab -c 2G "$node" >/dev/null
g layout apply --version 1 >/dev/null
g bucket create inbuxa-blobs >/dev/null
keyout=$(g key create inbuxa)
g bucket allow --read --write --owner inbuxa-blobs --key inbuxa >/dev/null
kid=$(awk -F': *' '/Key ID/{print $2; exit}' <<<"$keyout")
ksec=$(awk -F': *' '/Secret key/{print $2; exit}' <<<"$keyout")
kubectl -n inbuxa create secret generic inbuxa-s3 --from-literal=access-key="$kid" --from-literal=secret-key="$ksec" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
fi
step "waiting for PostgreSQL and NATS"
kubectl -n data wait cluster.postgresql.cnpg.io/inbuxa-pg --for=condition=Ready --timeout=600s >/dev/null
kubectl -n data rollout status statefulset/nats --timeout=300s >/dev/null
step "Secrets for the server (namespace inbuxa)"
pgpw=$(kubectl -n data get secret inbuxa-pg-app -o jsonpath='{.data.password}' | base64 -d)
natspw=$(kubectl -n data get secret nats-inbuxa -o jsonpath='{.data.password}' | base64 -d)
kubectl -n inbuxa create secret generic inbuxa-data --from-literal=pg-password="$pgpw" --from-literal=nats-password="$natspw" \
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
kubectl -n inbuxa get secret inbuxa-recovery >/dev/null 2>&1 || kubectl -n inbuxa create secret generic inbuxa-recovery --from-literal=admin="admin:$(rand)" >/dev/null
step "done"
kubectl get nodes -o wide | awk '{print $1, $2, $5}' | column -t
kubectl -n data get pods -o wide | awk '{print $1, $3, $7}' | column -t
+8
View File
@@ -0,0 +1,8 @@
# Pinned versions for the lab. Bump deliberately, in their own PR.
KIND_CLUSTER=inbuxa-lab
CNPG_CHART=0.29.2 # CloudNativePG operator 1.30.2
NATS_CHART=2.15.2
METALLB_CHART=0.16.2
GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0
PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17
SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27}