diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/README.md b/README.md index f2bcb3a..91e19e0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ # inbuxa-k8s-lab -Test lab for inbuxa on Kubernetes: end-to-end, chaos, upgrade and soak tests \ No newline at end of file +Test lab for inbuxa on Kubernetes: end-to-end, chaos, upgrade and soak tests. + +`lab/up.sh` brings up a kind cluster with the data layer, Kyverno and the +policies. It needs the Helm repositories `cnpg`, `nats`, `metallb` and +`kyverno` (https://kyverno.github.io/kyverno/); the versions are pinned in +`lab/versions.env`. + +The `inbuxa` namespace enforces Pod Security "restricted" and the policies in +`lab/policy/defaults.yaml`: a read-only root, no service-account token (except +the upgrade Job's), secrets never in the environment, and a deny-all +NetworkPolicy over every pod. `lab/policy/policy-test.sh` checks that the +chart passes them and that a bad value is refused. diff --git a/lab/policy/defaults.yaml b/lab/policy/defaults.yaml new file mode 100644 index 0000000..8de557b --- /dev/null +++ b/lab/policy/defaults.yaml @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# The chart's security defaults that Pod Security "restricted" doesn't check +# (K-L3). They apply in namespaces labelled inbuxa.org/policy=enforce, and to +# Deployments and Jobs as well as Pods, so a bad value is refused when it's +# applied, not later when a ReplicaSet can't make its pods. +apiVersion: policies.kyverno.io/v1 +kind: ValidatingPolicy +metadata: + name: inbuxa-read-only-root +spec: + validationActions: [Deny] + matchConstraints: + namespaceSelector: + matchLabels: {inbuxa.org/policy: enforce} + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE, UPDATE] + resources: [pods] + validations: + - expression: >- + object.spec.containers.all(c, has(c.securityContext) && has(c.securityContext.readOnlyRootFilesystem) + && c.securityContext.readOnlyRootFilesystem == true) + && (!has(object.spec.initContainers) || object.spec.initContainers.all(c, has(c.securityContext) + && has(c.securityContext.readOnlyRootFilesystem) && c.securityContext.readOnlyRootFilesystem == true)) + message: every container runs with a read-only root filesystem +--- +apiVersion: policies.kyverno.io/v1 +kind: ValidatingPolicy +metadata: + name: inbuxa-no-service-account-token +spec: + validationActions: [Deny] + matchConstraints: + namespaceSelector: + matchLabels: {inbuxa.org/policy: enforce} + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE, UPDATE] + resources: [pods] + validations: + # The upgrade Job scales the server down through a Role only it holds + - expression: >- + (has(object.spec.automountServiceAccountToken) && object.spec.automountServiceAccountToken == false) + || (has(object.metadata.labels) && object.metadata.labels[?"app.kubernetes.io/component"].orValue("") == "upgrade") + message: no pod mounts a service-account token, except the upgrade Job's +--- +apiVersion: policies.kyverno.io/v1 +kind: ValidatingPolicy +metadata: + name: inbuxa-secrets-as-files +spec: + # Warn until the chart's server image is 27.1-rc.32 (inbuxa-charts#19): + # before it, the server takes the webmail's client secret from its + # environment. Then Deny. + validationActions: [Warn] + matchConstraints: + namespaceSelector: + matchLabels: {inbuxa.org/policy: enforce} + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE, UPDATE] + resources: [pods] + validations: + - expression: >- + object.spec.containers.all(c, + (!has(c.env) || c.env.all(e, !has(e.valueFrom) || !has(e.valueFrom.secretKeyRef))) + && (!has(c.envFrom) || c.envFrom.all(e, !has(e.secretRef)))) + message: secrets are mounted as files, never set in the environment +--- +apiVersion: policies.kyverno.io/v1 +kind: ValidatingPolicy +metadata: + name: inbuxa-network-policy +spec: + validationActions: [Deny] + matchConstraints: + namespaceSelector: + matchLabels: {inbuxa.org/policy: enforce} + resourceRules: + - apiGroups: [""] + apiVersions: [v1] + operations: [CREATE, UPDATE] + resources: [pods] + variables: + - name: labels + expression: 'has(object.metadata.labels) ? object.metadata.labels : {}' + - name: policies + expression: resource.List("networking.k8s.io/v1", "networkpolicies", namespaceObject.metadata.name).items + validations: + # A deny-all: both directions, no rules, and a selector this pod matches + - expression: >- + dyn(variables.policies).exists(np, + np.spec.?policyTypes.orValue([]).exists(t, t == "Ingress") + && np.spec.?policyTypes.orValue([]).exists(t, t == "Egress") + && size(np.spec.?ingress.orValue([])) == 0 + && size(np.spec.?egress.orValue([])) == 0 + && !has(np.spec.podSelector.matchExpressions) + && np.spec.podSelector.?matchLabels.orValue({}).all(k, + k in variables.labels && variables.labels[k] == np.spec.podSelector.matchLabels[k])) + message: every pod is under a deny-all NetworkPolicy (networkPolicy.enabled) diff --git a/lab/policy/policy-test.sh b/lab/policy/policy-test.sh new file mode 100755 index 0000000..d5a3b5b --- /dev/null +++ b/lab/policy/policy-test.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L3: the chart as the lab installs it passes Pod Security "restricted" and +# the policies in defaults.yaml, and what breaks a default is refused. Runs +# against a scratch namespace with server-side dry runs, so it changes +# nothing but that namespace's NetworkPolicies, and removes it after. +# +# lab/policy/policy-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +NS=policy-test +K=(kubectl --context kind-inbuxa-lab) +H=(helm --kube-context kind-inbuxa-lab) +fails=0 +pass() { echo "ok $*"; } +fail() { echo "FAIL $*"; fails=$((fails + 1)); } + +"${K[@]}" apply -f defaults.yaml 2>&1 | grep -v '^Warning: ' >/dev/null +"${K[@]}" create namespace "$NS" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${K[@]}" label --overwrite namespace "$NS" inbuxa.org/policy=enforce \ + pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null +trap '"${K[@]}" delete namespace "$NS" --wait=false >/dev/null 2>&1 || true' EXIT + +# render [helm args]: the chart as the lab installs it +render() { + local release=$1; shift + "${H[@]}" template "$release" "$CHART" -n "$NS" -f ../chart/values-lab.yaml -f ../chart/values-frontends.yaml \ + --set server.upgrade.acknowledgeLayout=3 "$@" +} +# pick [patch]: those documents only, optionally changed by a Python +# expression on each one (d) +pick() { + python3 -c ' +import sys, yaml +kinds, patch = sys.argv[1].split(","), (sys.argv[2] if len(sys.argv) > 2 else "") +docs = [d for d in yaml.safe_load_all(sys.stdin) if d and d["kind"] in kinds] +for d in docs: + if patch: exec(patch) +print(yaml.safe_dump_all(docs))' "$@" +} +dry_run() { "${K[@]}" apply -n "$NS" --dry-run=server -f - 2>&1 || true; } + +echo "== the chart as the lab installs it" +render pt | pick NetworkPolicy | "${K[@]}" apply -n "$NS" -f - >/dev/null +out=$(render pt | pick Deployment,Job | dry_run) +if grep -q 'denied\|violate PodSecurity' <<<"$out"; then fail "refused:"; echo "$out"; else pass "every Deployment and Job admitted"; fi +# Until the chart's server image is 27.1-rc.32 (inbuxa-charts#19) +grep -q 'secrets are mounted as files' <<<"$out" && echo " (warned: a secret in the server's environment, until rc.32)" + +echo "== bad values" +out=$(render pt2 --set networkPolicy.enabled=false | pick Deployment | dry_run) +grep -q 'deny-all NetworkPolicy' <<<"$out" && pass "networkPolicy.enabled=false is refused" || { fail "networkPolicy.enabled=false got through"; echo "$out"; } + +out=$(render pt --set-json 'webmail.extraEnv=[{"name":"LEAK","valueFrom":{"secretKeyRef":{"name":"inbuxa-data","key":"pg-password"}}}]' \ + --show-only templates/webmail.yaml | pick Deployment | dry_run) +grep -q 'secrets are mounted as files' <<<"$out" && pass "a secret in webmail.extraEnv is flagged (warn until rc.32, then refused)" \ + || { fail "a secret in webmail.extraEnv got through"; echo "$out"; } + +if render pt --set-json 'webmail.securityContext={"readOnlyRootFilesystem":false}' >/dev/null 2>&1; then + fail "the schema took webmail.securityContext" +else + pass "no value turns the security context off: the schema refuses webmail.securityContext" +fi + +echo "== what the chart can't render, if someone edits a manifest" +out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["containers"][0]["securityContext"]["readOnlyRootFilesystem"] = False' | dry_run) +grep -q 'read-only root filesystem' <<<"$out" && pass "a writable root is refused" || { fail "a writable root got through"; echo "$out"; } + +out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["automountServiceAccountToken"] = True' | dry_run) +grep -q 'service-account token' <<<"$out" && pass "a service-account token is refused" || { fail "a service-account token got through"; echo "$out"; } + +out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["containers"][0]["securityContext"]["capabilities"] = {"add": ["NET_ADMIN"]}' | dry_run) +grep -q 'violate PodSecurity' <<<"$out" && pass "an added capability is refused (Pod Security restricted)" || { fail "an added capability got through"; echo "$out"; } + +echo +[ "$fails" = 0 ] && echo "== all pass" || { echo "== $fails failed"; exit 1; } diff --git a/lab/up.sh b/lab/up.sh index 69d5d23..b07f8ed 100755 --- a/lab/up.sh +++ b/lab/up.sh @@ -14,10 +14,16 @@ step "kind cluster" kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" || kind create cluster --name "$KIND_CLUSTER" --config "$here/kind.yaml" --wait 120s kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null for ns in data inbuxa; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; done -# inbuxa's pods run under Pod Security "restricted", enforced (K-C4) -kubectl label --overwrite namespace inbuxa \ +# inbuxa's pods run under Pod Security "restricted", enforced (K-C4), and +# the lab's own policies (policy/defaults.yaml, K-L3) +kubectl label --overwrite namespace inbuxa inbuxa.org/policy=enforce \ pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null +step "Kyverno" +helm upgrade --install kyverno kyverno/kyverno --version "$KYVERNO_CHART" -n kyverno --create-namespace \ + --set cleanupController.enabled=false --set reportsController.enabled=false --wait >/dev/null +kubectl apply -f "$here/policy/defaults.yaml" >/dev/null + step "CloudNativePG operator" helm upgrade --install cnpg cnpg/cloudnative-pg --version "$CNPG_CHART" -n cnpg-system --create-namespace --wait >/dev/null kubectl -n data apply -f "$here/data-layer/postgres.yaml" >/dev/null diff --git a/lab/versions.env b/lab/versions.env index 35a0cc3..37711a5 100644 --- a/lab/versions.env +++ b/lab/versions.env @@ -3,6 +3,7 @@ KIND_CLUSTER=inbuxa-lab CNPG_CHART=0.29.2 # CloudNativePG operator 1.30.2 NATS_CHART=2.15.2 METALLB_CHART=0.16.2 +KYVERNO_CHART=3.9.2 # Kyverno 1.19.2 GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0 PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17 SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27}