lab/e2e.sh: up.sh, then the chart with the images release/suite.yaml names, seeded, then mail, the webmail across replicas and the console, timing each part. lab/tools.sh installs kind, kubectl and Helm at pinned versions against their published checksums. .github/workflows/e2e.yml runs both on ubuntu-24.04 and ubuntu-24.04-arm (GitHub replica only). up.sh now does what a fresh machine needs: adds the Helm repositories, installs MetalLB with a pool computed from kind's network, takes KIND_NODE_IMAGE, and waits out two cold-start races (Kyverno's webhook, Garage's status) that a reused bench cluster never showed. The README gets the suite's header, badges and notes.
98 lines
5.6 KiB
Bash
Executable File
98 lines
5.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Bring up the lab: a kind cluster with the data layer inbuxa needs
|
|
# (PostgreSQL via CloudNativePG, NATS, Garage for S3), plus the Secrets the
|
|
# server reads, in namespace "inbuxa". Prints how long each part took.
|
|
set -euo pipefail
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
# shellcheck source=versions.env
|
|
. "$here/versions.env"
|
|
t0=$SECONDS
|
|
step() { printf '\n== %s (%ss)\n' "$1" "$((SECONDS - t0))"; }
|
|
rand() { head -c 24 /dev/urandom | base64 | tr -d '/+=' ; }
|
|
|
|
step "Helm repositories"
|
|
helm repo add cnpg https://cloudnative-pg.github.io/charts --force-update >/dev/null
|
|
helm repo add nats https://nats-io.github.io/k8s/helm/charts/ --force-update >/dev/null
|
|
helm repo add metallb https://metallb.github.io/metallb --force-update >/dev/null
|
|
helm repo add kyverno https://kyverno.github.io/kyverno/ --force-update >/dev/null
|
|
helm repo update >/dev/null
|
|
|
|
step "kind cluster"
|
|
# KIND_NODE_IMAGE picks the Kubernetes version (CI's matrix); unset, kind's own
|
|
kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" ||
|
|
kind create cluster --name "$KIND_CLUSTER" --config "$here/kind.yaml" --wait 120s \
|
|
${KIND_NODE_IMAGE:+--image "$KIND_NODE_IMAGE"}
|
|
kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null
|
|
for ns in data inbuxa; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; done
|
|
# inbuxa's pods run under Pod Security "restricted", enforced (K-C4), and
|
|
# the lab's own policies (policy/defaults.yaml, K-L3)
|
|
kubectl label --overwrite namespace inbuxa inbuxa.org/policy=enforce \
|
|
pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null
|
|
|
|
step "MetalLB"
|
|
# LoadBalancer addresses from the top of kind's Docker network, which the
|
|
# host reaches: computed, since that network's subnet differs by machine
|
|
helm upgrade --install metallb metallb/metallb --version "$METALLB_CHART" -n metallb-system --create-namespace --wait >/dev/null
|
|
subnet=$(docker network inspect kind -f '{{range .IPAM.Config}}{{.Subnet}} {{end}}' | tr ' ' '\n' | grep -v ':' | head -1)
|
|
pool=$(python3 -c 'import ipaddress,sys; n=ipaddress.ip_network(sys.argv[1]); print(f"{n.broadcast_address-55}-{n.broadcast_address-5}")' "$subnet")
|
|
sed "s|172.19.255.200-172.19.255.250|$pool|" "$here/network/metallb-pool.yaml" | kubectl apply -f - >/dev/null
|
|
echo " LoadBalancer pool $pool ($subnet)"
|
|
|
|
step "Kyverno"
|
|
helm upgrade --install kyverno kyverno/kyverno --version "$KYVERNO_CHART" -n kyverno --create-namespace \
|
|
--set cleanupController.enabled=false --set reportsController.enabled=false --wait >/dev/null
|
|
# Helm's --wait returns before the webhook answers on a new cluster
|
|
for i in $(seq 1 30); do
|
|
kubectl apply -f "$here/policy/defaults.yaml" >/dev/null 2>"$here/.policy-apply.err" && break
|
|
[ "$i" = 30 ] && { cat "$here/.policy-apply.err" >&2; rm -f "$here/.policy-apply.err"; exit 1; }
|
|
sleep 3
|
|
done
|
|
rm -f "$here/.policy-apply.err"
|
|
|
|
step "CloudNativePG operator"
|
|
helm upgrade --install cnpg cnpg/cloudnative-pg --version "$CNPG_CHART" -n cnpg-system --create-namespace --wait >/dev/null
|
|
kubectl -n data apply -f "$here/data-layer/postgres.yaml" >/dev/null
|
|
|
|
step "NATS"
|
|
kubectl -n data get secret nats-inbuxa >/dev/null 2>&1 || kubectl -n data create secret generic nats-inbuxa --from-literal=password="$(rand)" >/dev/null
|
|
helm upgrade --install nats nats/nats --version "$NATS_CHART" -n data -f "$here/data-layer/nats-values.yaml" >/dev/null
|
|
|
|
step "Garage"
|
|
kubectl -n data get secret garage-rpc >/dev/null 2>&1 || kubectl -n data create secret generic garage-rpc --from-literal=secret="$(head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n')" >/dev/null
|
|
sed "s|GARAGE_IMAGE|$GARAGE_IMAGE|" "$here/data-layer/garage.yaml" | kubectl -n data apply -f - >/dev/null
|
|
kubectl -n data rollout status deploy/garage --timeout=180s >/dev/null
|
|
g() { kubectl -n data exec deploy/garage -- /garage "$@"; }
|
|
if ! g bucket info inbuxa-blobs >/dev/null 2>&1; then
|
|
# Ready to Kubernetes isn't ready to answer: wait for Garage to list itself
|
|
node=""
|
|
for _ in $(seq 1 30); do
|
|
node=$( (g status 2>/dev/null || true) | awk '/^[0-9a-f]{16}/{print $1; exit}')
|
|
[ -n "$node" ] && break
|
|
sleep 2
|
|
done
|
|
[ -n "$node" ] || { echo "!! Garage never listed its node" >&2; exit 1; }
|
|
g layout assign -z lab -c 2G "$node" >/dev/null
|
|
g layout apply --version 1 >/dev/null
|
|
g bucket create inbuxa-blobs >/dev/null
|
|
keyout=$(g key create inbuxa)
|
|
g bucket allow --read --write --owner inbuxa-blobs --key inbuxa >/dev/null
|
|
kid=$(awk -F': *' '/Key ID/{print $2; exit}' <<<"$keyout")
|
|
ksec=$(awk -F': *' '/Secret key/{print $2; exit}' <<<"$keyout")
|
|
kubectl -n inbuxa create secret generic inbuxa-s3 --from-literal=access-key="$kid" --from-literal=secret-key="$ksec" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
|
fi
|
|
|
|
step "waiting for PostgreSQL and NATS"
|
|
kubectl -n data wait cluster.postgresql.cnpg.io/inbuxa-pg --for=condition=Ready --timeout=600s >/dev/null
|
|
kubectl -n data rollout status statefulset/nats --timeout=300s >/dev/null
|
|
|
|
step "Secrets for the server (namespace inbuxa)"
|
|
pgpw=$(kubectl -n data get secret inbuxa-pg-app -o jsonpath='{.data.password}' | base64 -d)
|
|
natspw=$(kubectl -n data get secret nats-inbuxa -o jsonpath='{.data.password}' | base64 -d)
|
|
kubectl -n inbuxa create secret generic inbuxa-data --from-literal=pg-password="$pgpw" --from-literal=nats-password="$natspw" \
|
|
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
|
kubectl -n inbuxa get secret inbuxa-recovery >/dev/null 2>&1 || kubectl -n inbuxa create secret generic inbuxa-recovery --from-literal=admin="admin:$(rand)" >/dev/null
|
|
|
|
step "done"
|
|
kubectl get nodes -o wide | awk '{print $1, $2, $5}' | column -t
|
|
kubectl -n data get pods -o wide | awk '{print $1, $3, $7}' | column -t
|