From b34fce62de1693014c9088d22709da892a31bdb6 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 22:09:28 -0700 Subject: [PATCH] check-test: the pre-flight check names what's broken (K-C5) Runs pods shaped like the chart's server pod with one part of the data layer broken on purpose: wrong PostgreSQL, S3 or NATS credentials, and NATS or S3 refusing the namespace. Each must stop in its init container with the right exit code and the part named in its termination message. frontends-test: PORTS picks the local ports, and a port something else already listens on stops the test: before, the port-forward failed quietly and the test talked to whatever was there. --- lab/chart/check-test.sh | 154 ++++++++++++++++++++++++++++++++++++ lab/chart/frontends-test.sh | 19 +++-- 2 files changed, 167 insertions(+), 6 deletions(-) create mode 100755 lab/chart/check-test.sh diff --git a/lab/chart/check-test.sh b/lab/chart/check-test.sh new file mode 100755 index 0000000..56ae3b1 --- /dev/null +++ b/lab/chart/check-test.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C5: the server pods' pre-flight check (inbuxa --check, the chart's +# init container) names the part of the data layer it can't use, and why, in +# the pod's status. Each case runs a pod shaped like the chart's server pod, +# in a scratch namespace under Pod Security "restricted", with the lab's +# config.json and secrets, one of them broken on purpose: +# +# nothing broken the pod starts +# wrong PostgreSQL password data store: credentials refused (77) +# wrong S3 secret key blob store: credentials refused (77) +# wrong NATS password coordinator: credentials refused (77) +# NATS not reachable coordinator: unreachable (69) +# S3 not reachable blob store: unreachable (69) +# +# Needs a server image with --check (27.1-rc.32): SERVER_TAG, or what the +# lab's server pods run. +# +# lab/chart/check-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +K=(kubectl --context kind-inbuxa-lab) +NS=check-test +IMAGE="${SERVER_TAG:+inbuxa-lab/server:$SERVER_TAG}" +IMAGE="${IMAGE:-$("${K[@]}" -n inbuxa get deploy inbuxa-front -o jsonpath='{.spec.template.spec.containers[0].image}')}" +fails=0 + +"${K[@]}" create namespace "$NS" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${K[@]}" label --overwrite namespace "$NS" pod-security.kubernetes.io/enforce=restricted >/dev/null +"${K[@]}" -n inbuxa get configmap inbuxa-server -o json | python3 -c ' +import json, sys +d = json.load(sys.stdin) +print(json.dumps({"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "server"}, "data": d["data"]}))' | + "${K[@]}" -n "$NS" apply -f - >/dev/null + +# secrets [key=wrong]...: the lab's secrets, as the pods see them, with any +# key named replaced by a wrong value. Nothing is printed. +secrets() { + { "${K[@]}" -n inbuxa get secret inbuxa-data -o json; "${K[@]}" -n inbuxa get secret inbuxa-s3 -o json; } | + python3 -c ' +import base64, json, sys +docs, text = [], sys.stdin.read() +dec = json.JSONDecoder(); i = 0 +while i < len(text): + if text[i].isspace(): i += 1; continue + d, i = dec.raw_decode(text, i); docs.append(d) +files = {"pg-password": docs[0]["data"]["pg-password"], "nats-password": docs[0]["data"]["nats-password"], + "s3-secret-key": docs[1]["data"]["secret-key"]} +for name in sys.argv[1:]: + files[name] = base64.b64encode(b"wrong-on-purpose").decode() +print(json.dumps({"apiVersion": "v1", "kind": "Secret", "metadata": {"name": "secrets"}, "data": files}))' "$@" | + "${K[@]}" -n "$NS" apply -f - >/dev/null +} + +# block nats|s3: that service refuses this namespace and no other. It's +# done on the service's side, with a policy in "data": the lab's CNI +# (kindnet) enforces ingress rules, not egress ones. +block() { + local selector + case $1 in + nats) selector='{"matchLabels":{"app.kubernetes.io/name":"nats","app.kubernetes.io/component":"nats"}}' ;; + s3) selector='{"matchLabels":{"app":"garage"}}' ;; + esac + echo "{\"apiVersion\":\"networking.k8s.io/v1\",\"kind\":\"NetworkPolicy\", + \"metadata\":{\"name\":\"check-test-block\",\"namespace\":\"data\"}, + \"spec\":{\"podSelector\":$selector,\"policyTypes\":[\"Ingress\"],\"ingress\":[{\"from\":[{\"namespaceSelector\": + {\"matchExpressions\":[{\"key\":\"kubernetes.io/metadata.name\",\"operator\":\"NotIn\",\"values\":[\"$NS\"]}]}}]}]}}" | + "${K[@]}" apply -f - >/dev/null + # The CNI takes a moment to enforce a new policy + sleep 10 +} +unblock() { "${K[@]}" -n data delete networkpolicy check-test-block --ignore-not-found >/dev/null; } +trap 'unblock; "${K[@]}" delete namespace "$NS" --wait=false >/dev/null 2>&1 || true' EXIT + +pod() { # : the chart's server pod, cut down to the check and a stand-in + cat </dev/null +apiVersion: v1 +kind: Pod +metadata: {name: $1} +spec: + automountServiceAccountToken: false + restartPolicy: Never + securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}} + initContainers: + - name: check + image: $IMAGE + imagePullPolicy: Never + args: ["--config", "/etc/inbuxa/config.json", "--check"] + terminationMessagePolicy: FallbackToLogsOnError + env: [{name: INBUXA_HOSTNAME, valueFrom: {fieldRef: {fieldPath: metadata.name}}}] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumeMounts: + - {name: config, mountPath: /etc/inbuxa/config.json, subPath: config.json, readOnly: true} + - {name: secrets, mountPath: /etc/inbuxa/secrets, readOnly: true} + - {name: tmp, mountPath: /tmp} + containers: + - name: server + image: $IMAGE + imagePullPolicy: Never + command: ["sh", "-c", "exit 0"] + securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}} + volumes: + - {name: config, configMap: {name: server}} + - {name: secrets, secret: {secretName: secrets, defaultMode: 0440}} + - {name: tmp, emptyDir: {}} +EOF +} + +# case_ [secrets args] [-- nats|s3] +case_() { + local name=$1 want=$2 words=$3; shift 3 + local keys=() blocked="" + while [ $# -gt 0 ] && [ "$1" != -- ]; do keys+=("$1"); shift; done + [ "${1:-}" = -- ] && blocked=$2 + secrets "${keys[@]}" + unblock + [ -z "$blocked" ] || block "$blocked" + local p="check-$RANDOM" + pod "$p" + local code="" status="" + for _ in $(seq 120); do + code=$("${K[@]}" -n "$NS" get pod "$p" -o jsonpath='{.status.initContainerStatuses[0].state.terminated.exitCode}') + [ -n "$code" ] && break + sleep 2 + done + sleep 1 + status=$("${K[@]}" -n "$NS" get pod "$p" --no-headers -o custom-columns=S:.status.phase,R:.status.initContainerStatuses[0].state.terminated.reason) + local message + message=$("${K[@]}" -n "$NS" get pod "$p" -o jsonpath='{.status.initContainerStatuses[0].state.terminated.message}') + # A check that passed leaves no message, only its log + [ "$code" = 0 ] && message=$("${K[@]}" -n "$NS" logs "$p" -c check) + if [ "$code" = "$want" ] && grep -q "$words" <<<"$message"; then + echo "ok $name: exit $code, $(grep -m1 "$words" <<<"$message" | cut -c1-110)" + else + echo "FAIL $name: exit ${code:-none} (want $want), status $status" + sed 's/^/ /' <<<"$message" + fails=$((fails + 1)) + fi + "${K[@]}" -n "$NS" delete pod "$p" --wait=false >/dev/null +} + +echo "== $IMAGE" +case_ "nothing broken" 0 "ok coordinator: NATS" +case_ "wrong PostgreSQL password" 77 "data store: credentials refused" pg-password +case_ "wrong S3 secret key" 77 "blob store: credentials refused" s3-secret-key +case_ "wrong NATS password" 77 "coordinator: credentials refused" nats-password +case_ "NATS refuses this namespace" 69 "coordinator: unreachable" -- nats +case_ "S3 refuses this namespace" 69 "blob store: unreachable" -- s3 + +echo +[ "$fails" = 0 ] && echo "== all pass" || { echo "== $fails failed"; exit 1; } diff --git a/lab/chart/frontends-test.sh b/lab/chart/frontends-test.sh index cd9e902..1741c14 100755 --- a/lab/chart/frontends-test.sh +++ b/lab/chart/frontends-test.sh @@ -24,9 +24,16 @@ PIDS=() cleanup() { for p in "${PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; } trap cleanup EXIT forward() { # + # Something else on the port would answer in the forward's place + if ss -ltn "sport = :$1" | grep -q LISTEN; then + echo "!! port $1 is in use by something else: PORTS= $0" >&2 + exit 1 + fi "${K[@]}" port-forward "$2" "$1:8080" >/dev/null 2>&1 & PIDS+=($!) } +# Four local ports in a row, for the server, the two webmail pods and the console +P="${PORTS:-18080}" echo "== install the webmail and the console" helm --kube-context kind-inbuxa-lab -n inbuxa upgrade inbuxa "$CHART" \ @@ -40,18 +47,18 @@ sleep 5 mapfile -t webmail < <("${K[@]}" get pods -l app.kubernetes.io/component=webmail \ --field-selector status.phase=Running -o name | sort) [ "${#webmail[@]}" -ge 2 ] || { echo "!! fewer than two webmail pods"; exit 1; } -forward 18081 "${webmail[0]}" -forward 18082 "${webmail[1]}" -forward 18080 svc/inbuxa-http -forward 18083 svc/inbuxa-console +forward $((P + 1)) "${webmail[0]}" +forward $((P + 2)) "${webmail[1]}" +forward $P svc/inbuxa-http +forward $((P + 3)) svc/inbuxa-console sleep 3 echo "== webmail: start on ${webmail[0]#pod/}, finish on ${webmail[1]#pod/}" -python3 signin.py http://127.0.0.1:18081 http://127.0.0.1:18082 http://127.0.0.1:18080 \ +python3 signin.py http://127.0.0.1:$((P + 1)) http://127.0.0.1:$((P + 2)) http://127.0.0.1:$P \ alice@lab.test "$STATE/alice.password" echo "== console" -python3 console-signin.py http://127.0.0.1:18083 http://127.0.0.1:18080 \ +python3 console-signin.py http://127.0.0.1:$((P + 3)) http://127.0.0.1:$P \ https://admin.lab.test https://mail.lab.test admin@lab.test "$STATE/admin.password" echo "== both pass" -- 2.54.0