Kyverno policies for the chart's defaults, and a test that bad values fail #22
No files matched your search
@@ -0,0 +1 @@
|
||||
__pycache__/
|
||||
@@ -1,3 +1,14 @@
|
||||
# inbuxa-k8s-lab
|
||||
|
||||
Test lab for inbuxa on Kubernetes: end-to-end, chaos, upgrade and soak tests
|
||||
Test lab for inbuxa on Kubernetes: end-to-end, chaos, upgrade and soak tests.
|
||||
|
||||
`lab/up.sh` brings up a kind cluster with the data layer, Kyverno and the
|
||||
policies. It needs the Helm repositories `cnpg`, `nats`, `metallb` and
|
||||
`kyverno` (https://kyverno.github.io/kyverno/); the versions are pinned in
|
||||
`lab/versions.env`.
|
||||
|
||||
The `inbuxa` namespace enforces Pod Security "restricted" and the policies in
|
||||
`lab/policy/defaults.yaml`: a read-only root, no service-account token (except
|
||||
the upgrade Job's), secrets never in the environment, and a deny-all
|
||||
NetworkPolicy over every pod. `lab/policy/policy-test.sh` checks that the
|
||||
chart passes them and that a bad value is refused.
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# The chart's security defaults that Pod Security "restricted" doesn't check
|
||||
# (K-L3). They apply in namespaces labelled inbuxa.org/policy=enforce, and to
|
||||
# Deployments and Jobs as well as Pods, so a bad value is refused when it's
|
||||
# applied, not later when a ReplicaSet can't make its pods.
|
||||
apiVersion: policies.kyverno.io/v1
|
||||
kind: ValidatingPolicy
|
||||
metadata:
|
||||
name: inbuxa-read-only-root
|
||||
spec:
|
||||
validationActions: [Deny]
|
||||
matchConstraints:
|
||||
namespaceSelector:
|
||||
matchLabels: {inbuxa.org/policy: enforce}
|
||||
resourceRules:
|
||||
- apiGroups: [""]
|
||||
apiVersions: [v1]
|
||||
operations: [CREATE, UPDATE]
|
||||
resources: [pods]
|
||||
validations:
|
||||
- expression: >-
|
||||
object.spec.containers.all(c, has(c.securityContext) && has(c.securityContext.readOnlyRootFilesystem)
|
||||
&& c.securityContext.readOnlyRootFilesystem == true)
|
||||
&& (!has(object.spec.initContainers) || object.spec.initContainers.all(c, has(c.securityContext)
|
||||
&& has(c.securityContext.readOnlyRootFilesystem) && c.securityContext.readOnlyRootFilesystem == true))
|
||||
message: every container runs with a read-only root filesystem
|
||||
---
|
||||
apiVersion: policies.kyverno.io/v1
|
||||
kind: ValidatingPolicy
|
||||
metadata:
|
||||
name: inbuxa-no-service-account-token
|
||||
spec:
|
||||
validationActions: [Deny]
|
||||
matchConstraints:
|
||||
namespaceSelector:
|
||||
matchLabels: {inbuxa.org/policy: enforce}
|
||||
resourceRules:
|
||||
- apiGroups: [""]
|
||||
apiVersions: [v1]
|
||||
operations: [CREATE, UPDATE]
|
||||
resources: [pods]
|
||||
validations:
|
||||
# The upgrade Job scales the server down through a Role only it holds
|
||||
- expression: >-
|
||||
(has(object.spec.automountServiceAccountToken) && object.spec.automountServiceAccountToken == false)
|
||||
|| (has(object.metadata.labels) && object.metadata.labels[?"app.kubernetes.io/component"].orValue("") == "upgrade")
|
||||
message: no pod mounts a service-account token, except the upgrade Job's
|
||||
---
|
||||
apiVersion: policies.kyverno.io/v1
|
||||
kind: ValidatingPolicy
|
||||
metadata:
|
||||
name: inbuxa-secrets-as-files
|
||||
spec:
|
||||
# Warn until the chart's server image is 27.1-rc.32 (inbuxa-charts#19):
|
||||
# before it, the server takes the webmail's client secret from its
|
||||
# environment. Then Deny.
|
||||
validationActions: [Warn]
|
||||
matchConstraints:
|
||||
namespaceSelector:
|
||||
matchLabels: {inbuxa.org/policy: enforce}
|
||||
resourceRules:
|
||||
- apiGroups: [""]
|
||||
apiVersions: [v1]
|
||||
operations: [CREATE, UPDATE]
|
||||
resources: [pods]
|
||||
validations:
|
||||
- expression: >-
|
||||
object.spec.containers.all(c,
|
||||
(!has(c.env) || c.env.all(e, !has(e.valueFrom) || !has(e.valueFrom.secretKeyRef)))
|
||||
&& (!has(c.envFrom) || c.envFrom.all(e, !has(e.secretRef))))
|
||||
message: secrets are mounted as files, never set in the environment
|
||||
---
|
||||
apiVersion: policies.kyverno.io/v1
|
||||
kind: ValidatingPolicy
|
||||
metadata:
|
||||
name: inbuxa-network-policy
|
||||
spec:
|
||||
validationActions: [Deny]
|
||||
matchConstraints:
|
||||
namespaceSelector:
|
||||
matchLabels: {inbuxa.org/policy: enforce}
|
||||
resourceRules:
|
||||
- apiGroups: [""]
|
||||
apiVersions: [v1]
|
||||
operations: [CREATE, UPDATE]
|
||||
resources: [pods]
|
||||
variables:
|
||||
- name: labels
|
||||
expression: 'has(object.metadata.labels) ? object.metadata.labels : {}'
|
||||
- name: policies
|
||||
expression: resource.List("networking.k8s.io/v1", "networkpolicies", namespaceObject.metadata.name).items
|
||||
validations:
|
||||
# A deny-all: both directions, no rules, and a selector this pod matches
|
||||
- expression: >-
|
||||
dyn(variables.policies).exists(np,
|
||||
np.spec.?policyTypes.orValue([]).exists(t, t == "Ingress")
|
||||
&& np.spec.?policyTypes.orValue([]).exists(t, t == "Egress")
|
||||
&& size(np.spec.?ingress.orValue([])) == 0
|
||||
&& size(np.spec.?egress.orValue([])) == 0
|
||||
&& !has(np.spec.podSelector.matchExpressions)
|
||||
&& np.spec.podSelector.?matchLabels.orValue({}).all(k,
|
||||
k in variables.labels && variables.labels[k] == np.spec.podSelector.matchLabels[k]))
|
||||
message: every pod is under a deny-all NetworkPolicy (networkPolicy.enabled)
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-L3: the chart as the lab installs it passes Pod Security "restricted" and
|
||||
# the policies in defaults.yaml, and what breaks a default is refused. Runs
|
||||
# against a scratch namespace with server-side dry runs, so it changes
|
||||
# nothing but that namespace's NetworkPolicies, and removes it after.
|
||||
#
|
||||
# lab/policy/policy-test.sh
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
|
||||
NS=policy-test
|
||||
K=(kubectl --context kind-inbuxa-lab)
|
||||
H=(helm --kube-context kind-inbuxa-lab)
|
||||
fails=0
|
||||
pass() { echo "ok $*"; }
|
||||
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
||||
|
||||
"${K[@]}" apply -f defaults.yaml 2>&1 | grep -v '^Warning: <nil>' >/dev/null
|
||||
"${K[@]}" create namespace "$NS" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" label --overwrite namespace "$NS" inbuxa.org/policy=enforce \
|
||||
pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null
|
||||
trap '"${K[@]}" delete namespace "$NS" --wait=false >/dev/null 2>&1 || true' EXIT
|
||||
|
||||
# render <release> [helm args]: the chart as the lab installs it
|
||||
render() {
|
||||
local release=$1; shift
|
||||
"${H[@]}" template "$release" "$CHART" -n "$NS" -f ../chart/values-lab.yaml -f ../chart/values-frontends.yaml \
|
||||
--set server.upgrade.acknowledgeLayout=3 "$@"
|
||||
}
|
||||
# pick <kinds> [patch]: those documents only, optionally changed by a Python
|
||||
# expression on each one (d)
|
||||
pick() {
|
||||
python3 -c '
|
||||
import sys, yaml
|
||||
kinds, patch = sys.argv[1].split(","), (sys.argv[2] if len(sys.argv) > 2 else "")
|
||||
docs = [d for d in yaml.safe_load_all(sys.stdin) if d and d["kind"] in kinds]
|
||||
for d in docs:
|
||||
if patch: exec(patch)
|
||||
print(yaml.safe_dump_all(docs))' "$@"
|
||||
}
|
||||
dry_run() { "${K[@]}" apply -n "$NS" --dry-run=server -f - 2>&1 || true; }
|
||||
|
||||
echo "== the chart as the lab installs it"
|
||||
render pt | pick NetworkPolicy | "${K[@]}" apply -n "$NS" -f - >/dev/null
|
||||
out=$(render pt | pick Deployment,Job | dry_run)
|
||||
if grep -q 'denied\|violate PodSecurity' <<<"$out"; then fail "refused:"; echo "$out"; else pass "every Deployment and Job admitted"; fi
|
||||
# Until the chart's server image is 27.1-rc.32 (inbuxa-charts#19)
|
||||
grep -q 'secrets are mounted as files' <<<"$out" && echo " (warned: a secret in the server's environment, until rc.32)"
|
||||
|
||||
echo "== bad values"
|
||||
out=$(render pt2 --set networkPolicy.enabled=false | pick Deployment | dry_run)
|
||||
grep -q 'deny-all NetworkPolicy' <<<"$out" && pass "networkPolicy.enabled=false is refused" || { fail "networkPolicy.enabled=false got through"; echo "$out"; }
|
||||
|
||||
out=$(render pt --set-json 'webmail.extraEnv=[{"name":"LEAK","valueFrom":{"secretKeyRef":{"name":"inbuxa-data","key":"pg-password"}}}]' \
|
||||
--show-only templates/webmail.yaml | pick Deployment | dry_run)
|
||||
grep -q 'secrets are mounted as files' <<<"$out" && pass "a secret in webmail.extraEnv is flagged (warn until rc.32, then refused)" \
|
||||
|| { fail "a secret in webmail.extraEnv got through"; echo "$out"; }
|
||||
|
||||
if render pt --set-json 'webmail.securityContext={"readOnlyRootFilesystem":false}' >/dev/null 2>&1; then
|
||||
fail "the schema took webmail.securityContext"
|
||||
else
|
||||
pass "no value turns the security context off: the schema refuses webmail.securityContext"
|
||||
fi
|
||||
|
||||
echo "== what the chart can't render, if someone edits a manifest"
|
||||
out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["containers"][0]["securityContext"]["readOnlyRootFilesystem"] = False' | dry_run)
|
||||
grep -q 'read-only root filesystem' <<<"$out" && pass "a writable root is refused" || { fail "a writable root got through"; echo "$out"; }
|
||||
|
||||
out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["automountServiceAccountToken"] = True' | dry_run)
|
||||
grep -q 'service-account token' <<<"$out" && pass "a service-account token is refused" || { fail "a service-account token got through"; echo "$out"; }
|
||||
|
||||
out=$(render pt | pick Deployment 'd["spec"]["template"]["spec"]["containers"][0]["securityContext"]["capabilities"] = {"add": ["NET_ADMIN"]}' | dry_run)
|
||||
grep -q 'violate PodSecurity' <<<"$out" && pass "an added capability is refused (Pod Security restricted)" || { fail "an added capability got through"; echo "$out"; }
|
||||
|
||||
echo
|
||||
[ "$fails" = 0 ] && echo "== all pass" || { echo "== $fails failed"; exit 1; }
|
||||
@@ -14,10 +14,16 @@ step "kind cluster"
|
||||
kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER" || kind create cluster --name "$KIND_CLUSTER" --config "$here/kind.yaml" --wait 120s
|
||||
kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null
|
||||
for ns in data inbuxa; do kubectl create namespace "$ns" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; done
|
||||
# inbuxa's pods run under Pod Security "restricted", enforced (K-C4)
|
||||
kubectl label --overwrite namespace inbuxa \
|
||||
# inbuxa's pods run under Pod Security "restricted", enforced (K-C4), and
|
||||
# the lab's own policies (policy/defaults.yaml, K-L3)
|
||||
kubectl label --overwrite namespace inbuxa inbuxa.org/policy=enforce \
|
||||
pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/warn=restricted >/dev/null
|
||||
|
||||
step "Kyverno"
|
||||
helm upgrade --install kyverno kyverno/kyverno --version "$KYVERNO_CHART" -n kyverno --create-namespace \
|
||||
--set cleanupController.enabled=false --set reportsController.enabled=false --wait >/dev/null
|
||||
kubectl apply -f "$here/policy/defaults.yaml" >/dev/null
|
||||
|
||||
step "CloudNativePG operator"
|
||||
helm upgrade --install cnpg cnpg/cloudnative-pg --version "$CNPG_CHART" -n cnpg-system --create-namespace --wait >/dev/null
|
||||
kubectl -n data apply -f "$here/data-layer/postgres.yaml" >/dev/null
|
||||
|
||||
@@ -3,6 +3,7 @@ KIND_CLUSTER=inbuxa-lab
|
||||
CNPG_CHART=0.29.2 # CloudNativePG operator 1.30.2
|
||||
NATS_CHART=2.15.2
|
||||
METALLB_CHART=0.16.2
|
||||
KYVERNO_CHART=3.9.2 # Kyverno 1.19.2
|
||||
GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0
|
||||
PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17
|
||||
SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27}
|
||||
Reference in new issue
Block a user