Kyverno policies for the chart's defaults, and a test that bad values fail #22

Merged
jcoffey-dev merged 1 commits from feat/policy-checks into main 2026-11-19 01:37:57 +00:00
6 changed files with 207 additions and 3 deletions

No files matched your search

+1
View File
@@ -0,0 +1 @@
__pycache__/
+12 -1
View File
@@ -1,3 +1,14 @@
# inbuxa-k8s-lab
Test lab for inbuxa on Kubernetes: end-to-end, chaos, upgrade and soak tests
Test lab for inbuxa on Kubernetes: end-to-end, chaos, upgrade and soak tests.
`lab/up.sh` brings up a kind cluster with the data layer, Kyverno and the
policies. It needs the Helm repositories `cnpg`, `nats`, `metallb` and
`kyverno` (https://kyverno.github.io/kyverno/); the versions are pinned in
`lab/versions.env`.
The `inbuxa` namespace enforces Pod Security "restricted" and the policies in
`lab/policy/defaults.yaml`: a read-only root, no service-account token (except
the upgrade Job's), secrets never in the environment, and a deny-all
NetworkPolicy over every pod. `lab/policy/policy-test.sh` checks that the
chart passes them and that a bad value is refused.
+105
View File
@@ -0,0 +1,105 @@
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# The chart's security defaults that Pod Security "restricted" doesn't check
# (K-L3). They apply in namespaces labelled inbuxa.org/policy=enforce, and to
# Deployments and Jobs as well as Pods, so a bad value is refused when it's
# applied, not later when a ReplicaSet can't make its pods.
apiVersion: policies.kyverno.io/v1
kind: ValidatingPolicy
metadata:
name: inbuxa-read-only-root
spec:
validationActions: [Deny]
matchConstraints:
namespaceSelector:
matchLabels: {inbuxa.org/policy: enforce}
resourceRules:
- apiGroups: [""]
apiVersions: [v1]
operations: [CREATE, UPDATE]
resources: [pods]
validations:
- expression: >-
object.spec.containers.all(c, has(c.securityContext) && has(c.securityContext.readOnlyRootFilesystem)
&& c.securityContext.readOnlyRootFilesystem == true)
&& (!has(object.spec.initContainers) || object.spec.initContainers.all(c, has(c.securityContext)
&& has(c.securityContext.readOnlyRootFilesystem) && c.securityContext.readOnlyRootFilesystem == true))
message: every container runs with a read-only root filesystem
---
apiVersion: policies.kyverno.io/v1
kind: ValidatingPolicy
metadata:
name: inbuxa-no-service-account-token
spec:
validationActions: [Deny]
matchConstraints:
namespaceSelector:
matchLabels: {inbuxa.org/policy: enforce}
resourceRules:
- apiGroups: [""]
apiVersions: [v1]
operations: [CREATE, UPDATE]
resources: [pods]
validations:
# The upgrade Job scales the server down through a Role only it holds
- expression: >-
(has(object.spec.automountServiceAccountToken) && object.spec.automountServiceAccountToken == false)
|| (has(object.metadata.labels) && object.metadata.labels[?"app.kubernetes.io/component"].orValue("") == "upgrade")
message: no pod mounts a service-account token, except the upgrade Job's
---
apiVersion: policies.kyverno.io/v1
kind: ValidatingPolicy
metadata:
name: inbuxa-secrets-as-files
spec:
# Warn until the chart's server image is 27.1-rc.32 (inbuxa-charts#19):
# before it, the server takes the webmail's client secret from its
# environment. Then Deny.
validationActions: [Warn]
matchConstraints:
namespaceSelector:
matchLabels: {inbuxa.org/policy: enforce}
resourceRules:
- apiGroups: [""]
apiVersions: [v1]
operations: [CREATE, UPDATE]
resources: [pods]
validations:
- expression: >-
object.spec.containers.all(c,
(!has(c.env) || c.env.all(e, !has(e.valueFrom) || !has(e.valueFrom.secretKeyRef)))
&& (!has(c.envFrom) || c.envFrom.all(e, !has(e.secretRef))))
message: secrets are mounted as files, never set in the environment
---
apiVersion: policies.kyverno.io/v1
kind: ValidatingPolicy
metadata:
name: inbuxa-network-policy
spec:
validationActions: [Deny]
matchConstraints:
namespaceSelector:
matchLabels: {inbuxa.org/policy: enforce}
resourceRules:
- apiGroups: [""]
apiVersions: [v1]
operations: [CREATE, UPDATE]
resources: [pods]
variables:
- name: labels
expression: 'has(object.metadata.labels) ? object.metadata.labels : {}'
- name: policies
expression: resource.List("networking.k8s.io/v1", "networkpolicies", namespaceObject.metadata.name).items
validations:
# A deny-all: both directions, no rules, and a selector this pod matches
- expression: >-
dyn(variables.policies).exists(np,
np.spec.?policyTypes.orValue([]).exists(t, t == "Ingress")
&& np.spec.?policyTypes.orValue([]).exists(t, t == "Egress")
&& size(np.spec.?ingress.orValue([])) == 0
&& size(np.spec.?egress.orValue([])) == 0
&& !has(np.spec.podSelector.matchExpressions)
&& np.spec.podSelector.?matchLabels.orValue({}).all(k,
k in variables.labels && variables.labels[k] == np.spec.podSelector.matchLabels[k]))
message: every pod is under a deny-all NetworkPolicy (networkPolicy.enabled)
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L3: the chart as the lab installs it passes Pod Security "restricted" and
# the policies in defaults.yaml, and what breaks a default is refused. Runs
# against a scratch namespace with server-side dry runs, so it changes
# nothing but that namespace's NetworkPolicies, and removes it after.
#
# lab/policy/policy-test.sh
set -euo pipefail
cd "$(dirname "$0")"
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
NS=policy-test
K=(kubectl --context kind-inbuxa-lab)
H=(helm --kube-context kind-inbuxa-lab)
fails=0
pass() { echo "ok $*"; }
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
"${K[@]}" apply -f defaults.yaml 2>&1 | grep -v '^Warning: <nil>' >/dev/null
"${K[@]}" create namespace "$NS" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" label --overwrite namespace "$NS" inbuxa.org/policy=enforce \
pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null
trap '"${K[@]}" delete namespace "$NS" --wait=false >/dev/null 2>&1 || true' EXIT
# render <release> [helm args]: the chart as the lab installs it
render() {
local release=$1; shift
"${H[@]}" template "$release" "$CHART" -n "$NS" -f ../chart/values-lab.yaml -f ../chart/values-frontends.yaml \
--set server.upgrade.acknowledgeLayout=3 "$@"
}
# pick <kinds> [patch]: those documents only, optionally changed by a Python
# expression on each one (d)
pick() {
python3 -c '
import sys, yaml
kinds, patch = sys.argv[1].split(","), (sys.argv[2] if len(sys.argv) > 2 else "")
docs = [d for d in yaml.safe_load_all(sys.stdin) if d and d["kind"] in kinds]
for d in docs:
if patch: exec(patch)
print(yaml.safe_dump_all(docs))' "$@"
}
dry_run() { "${K[@]}" apply -n "$NS" --dry-run=server -f - 2>&1 || true; }
echo "== the chart as the lab installs it"
render pt | pick NetworkPolicy | "${K[@]}" apply -n "$NS" -f - >/dev/null
out=$(render pt | pick Deployment,Job | dry_run)
if grep -q 'denied\|violate PodSecurity' <<<"$out"; then fail "refused:"; echo "$out"; else pass "every Deployment and Job admitted"; fi
# Until the chart's server image is 27.1-rc.32 (inbuxa-charts#19)
grep -q 'secrets are mounted as files' <<<"$out" && echo " (warned: a secret in the server's environment, until rc.32)"
echo "== bad values"
out=$(render pt2 --set networkPolicy.enabled=false | pick Deployment | dry_run)
grep -q 'deny-all NetworkPolicy' <<<"$out" && pass "networkPolicy.enabled=false is refused" || { fail "networkPolicy.enabled=false got through"; echo "$out"; }
out=$(render pt --set-json 'webmail.extraEnv=[{"name":"LEAK","valueFrom":{"secretKeyRef":{"name":"inbuxa-data","key":"pg-password"}}}]' \
--show-only templates/webmail.yaml | pick Deployment | dry_run)
grep -q 'secrets are mounted as files' <<<"$out" && pass "a secret in webmail.extraEnv is flagged (warn until rc.32, then refused)" \
|| { fail "a secret in webmail.extraEnv got through"; echo "$out"; }
if render pt --set-json 'webmail.securityContext={"readOnlyRootFilesystem":false}' >/dev/null 2>&1; then
fail "the schema took webmail.securityContext"
else
pass "no value turns the security context off: the schema refuses webmail.securityContext"
fi
echo "== what the chart can't render, if someone edits a manifest"
out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["containers"][0]["securityContext"]["readOnlyRootFilesystem"] = False' | dry_run)
grep -q 'read-only root filesystem' <<<"$out" && pass "a writable root is refused" || { fail "a writable root got through"; echo "$out"; }
out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["automountServiceAccountToken"] = True' | dry_run)
grep -q 'service-account token' <<<"$out" && pass "a service-account token is refused" || { fail "a service-account token got through"; echo "$out"; }
out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["containers"][0]["securityContext"]["capabilities"] = {"add": ["NET_ADMIN"]}' | dry_run)
grep -q 'violate PodSecurity' <<<"$out" && pass "an added capability is refused (Pod Security restricted)" || { fail "an added capability got through"; echo "$out"; }
echo
[ "$fails" = 0 ] && echo "== all pass" || { echo "== $fails failed"; exit 1; }
+8 -2
View File
@@ -14,10 +14,16 @@ step "kind cluster"
kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" || kind create cluster --name "$KIND_CLUSTER" --config "$here/kind.yaml" --wait 120s
kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null
for ns in data inbuxa; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; done
# inbuxa's pods run under Pod Security "restricted", enforced (K-C4)
kubectl label --overwrite namespace inbuxa \
# inbuxa's pods run under Pod Security "restricted", enforced (K-C4), and
# the lab's own policies (policy/defaults.yaml, K-L3)
kubectl label --overwrite namespace inbuxa inbuxa.org/policy=enforce \
pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null
step "Kyverno"
helm upgrade --install kyverno kyverno/kyverno --version "$KYVERNO_CHART" -n kyverno --create-namespace \
--set cleanupController.enabled=false --set reportsController.enabled=false --wait >/dev/null
kubectl apply -f "$here/policy/defaults.yaml" >/dev/null
step "CloudNativePG operator"
helm upgrade --install cnpg cnpg/cloudnative-pg --version "$CNPG_CHART" -n cnpg-system --create-namespace --wait >/dev/null
kubectl -n data apply -f "$here/data-layer/postgres.yaml" >/dev/null
+1
View File
@@ -3,6 +3,7 @@ KIND_CLUSTER=inbuxa-lab
CNPG_CHART=0.29.2 # CloudNativePG operator 1.30.2
NATS_CHART=2.15.2
METALLB_CHART=0.16.2
KYVERNO_CHART=3.9.2 # Kyverno 1.19.2
GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0
PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17
SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27}