From 6ef0f1acfbb1f221846aedbf9668cd97787fd147 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 23:16:50 -0700 Subject: [PATCH] Mail out: podLabels per group, and a check of the egress address (K-C7) server.egress.expectedAddresses and checkUrl: after each install and upgrade a Job placed like the mta pods (node selector, tolerations, affinity, pod labels) asks which address it leaves from, and records the answer on the mta Deployment as an event, a Warning when it isn't one of the expected addresses or can't be learned. It never fails the install. Its token may create events and nothing else. podLabels on each server group, for an egress gateway that selects the mta pods. --- CHANGELOG.md | 7 + charts/inbuxa/ci/everything-values.yaml | 4 + charts/inbuxa/files/egress-check.sh | 42 +++++ charts/inbuxa/templates/networkpolicy.yaml | 8 +- .../inbuxa/templates/server/deployment.yaml | 3 + .../inbuxa/templates/server/egress-check.yaml | 170 ++++++++++++++++++ charts/inbuxa/tests/egress_test.yaml | 83 +++++++++ charts/inbuxa/values.schema.json | 28 +++ charts/inbuxa/values.yaml | 24 ++- 9 files changed, 365 insertions(+), 4 deletions(-) create mode 100644 charts/inbuxa/files/egress-check.sh create mode 100644 charts/inbuxa/templates/server/egress-check.yaml create mode 100644 charts/inbuxa/tests/egress_test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8048071..f1d103a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,13 @@ runs, pinned by digest. passwords between all users would ban the ingress for everyone. `server.trustedProxies` lists the proxies when the pod network isn't a private range. Needs server 27.1-rc.32; older servers ignore it. +- Mail out from a known address (`server.egress`): with + `expectedAddresses` and `checkUrl` set, a Job placed like the mta pods + asks which address it leaves from after each install and upgrade, and + records the answer on the mta Deployment as an event, a Warning when it + isn't one of the expected addresses. It never fails an install. Its token + may create events and nothing else. `podLabels` on each server group, for + an egress gateway that selects the mta pods. - `networkPolicy`, on by default: every pod starts from deny all, in and out, and gets only what it needs. The server takes mail on its listeners from anywhere and connects anywhere; `networkPolicy.httpFrom` and diff --git a/charts/inbuxa/ci/everything-values.yaml b/charts/inbuxa/ci/everything-values.yaml index ca71e68..b2f8cd5 100644 --- a/charts/inbuxa/ci/everything-values.yaml +++ b/charts/inbuxa/ci/everything-values.yaml @@ -22,6 +22,9 @@ server: trustedProxies: [10.0.0.0/8, 100.64.0.0/10] listeners: pop3s: 995 + egress: + expectedAddresses: [198.51.100.7, "2001:db8::7"] + checkUrl: https://ip.example.org/ groups: front: replicas: 3 @@ -30,6 +33,7 @@ server: mta: nodeSelector: {example.org/egress: mail} tolerations: [{key: example.org/egress, operator: Exists, effect: NoSchedule}] + podLabels: {example.org/egress-gateway: mail} resources: requests: {cpu: 500m, memory: 1Gi} limits: {cpu: "4", memory: 4Gi} diff --git a/charts/inbuxa/files/egress-check.sh b/charts/inbuxa/files/egress-check.sh new file mode 100644 index 0000000..bb0d72a --- /dev/null +++ b/charts/inbuxa/files/egress-check.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C7: the address outbound mail leaves from, from a pod placed like the mta +# pods, against the addresses expected (the ones with PTR records and in +# SPF). The answer goes on the mta Deployment as an event: Normal when it +# matches, Warning when it doesn't or can't be learned. It never fails, so it +# never holds up an install or an upgrade. Only sh, sed, tr and curl: the +# image has nothing more. +# +# CHECK_URL answers a GET with the caller's address as plain text. +set -u +sa=/var/run/secrets/kubernetes.io/serviceaccount + +event() { # + now=$(date -u +%Y-%m-%dT%H:%M:%SZ) + message=$(printf '%s' "$3" | sed 's/\\/\\\\/g; s/"/\\"/g') + body=$(printf '{"apiVersion":"v1","kind":"Event","metadata":{"generateName":"%s.","namespace":"%s"},"involvedObject":{"apiVersion":"apps/v1","kind":"Deployment","name":"%s","namespace":"%s"},"type":"%s","reason":"%s","message":"%s","firstTimestamp":"%s","lastTimestamp":"%s","count":1,"source":{"component":"inbuxa-egress-check"}}' \ + "$DEPLOYMENT" "$NAMESPACE" "$DEPLOYMENT" "$NAMESPACE" "$1" "$2" "$message" "$now" "$now") + echo "$1 $2: $3" + curl -fsS -o /dev/null --cacert "$sa/ca.crt" -H "Authorization: Bearer $(cat "$sa/token")" \ + -H 'Content-Type: application/json' -X POST -d "$body" \ + "https://kubernetes.default.svc/api/v1/namespaces/$NAMESPACE/events" || + echo "couldn't record the event on deployment/$DEPLOYMENT" +} + +seen=$(curl -fsS -m 20 "$CHECK_URL" 2>/dev/null | tr -d ' \r\n') +case "$seen" in + "" | *[!0-9a-fA-F.:]*) + event Warning EgressUnknown "Couldn't learn the address outbound mail leaves from: $CHECK_URL gave no address. Expected one of: $EXPECTED." + exit 0 + ;; +esac +for address in $EXPECTED; do + if [ "$address" = "$seen" ]; then + event Normal EgressAddress "Outbound mail leaves from $seen, as expected." + exit 0 + fi +done +event Warning EgressMismatch "Outbound mail leaves from $seen, which is not one of the expected addresses ($EXPECTED). Receivers check its reverse DNS and SPF: mail may be refused or marked as spam." +exit 0 diff --git a/charts/inbuxa/templates/networkpolicy.yaml b/charts/inbuxa/templates/networkpolicy.yaml index d2a91cc..d910af9 100644 --- a/charts/inbuxa/templates/networkpolicy.yaml +++ b/charts/inbuxa/templates/networkpolicy.yaml @@ -9,7 +9,8 @@ all, in and out; the policies after it open only what each one needs: server its mail listeners and https from anywhere (networkPolicy.serverEgress anywhere; http from the webmail narrows it) and from networkPolicy.httpFrom - seed, upgrade nothing as the server + seed, upgrade, nothing as the server + egress-check webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80 and 443 anywhere, for remote images and calendars (networkPolicy.webmailInternet) @@ -82,7 +83,8 @@ spec: {{- include "inbuxa.networkPolicy.serverEgress" $root | nindent 4 }} --- # The seed and upgrade Jobs reach the data store as the server does; the -# upgrade Job reaches the Kubernetes API too. Nothing connects to them. +# upgrade Job reaches the Kubernetes API too, and the egress check the API +# and server.egress.checkUrl. Nothing connects to them. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -95,7 +97,7 @@ spec: matchLabels: {{- include "inbuxa.selectorLabels" $root | nindent 6 }} matchExpressions: - - {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade]} + - {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check]} policyTypes: [Ingress, Egress] egress: {{- include "inbuxa.networkPolicy.serverEgress" $root | nindent 4 }} diff --git a/charts/inbuxa/templates/server/deployment.yaml b/charts/inbuxa/templates/server/deployment.yaml index 43ed7e7..ec6db37 100644 --- a/charts/inbuxa/templates/server/deployment.yaml +++ b/charts/inbuxa/templates/server/deployment.yaml @@ -51,6 +51,9 @@ spec: template: metadata: labels: + {{- with $g.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- include "inbuxa.labels" $root | nindent 8 }} app.kubernetes.io/component: server inbuxa.org/group: {{ $group }} diff --git a/charts/inbuxa/templates/server/egress-check.yaml b/charts/inbuxa/templates/server/egress-check.yaml new file mode 100644 index 0000000..2218bac --- /dev/null +++ b/charts/inbuxa/templates/server/egress-check.yaml @@ -0,0 +1,170 @@ +{{/* +SPDX-FileCopyrightText: 2026 Coffey Labs LLC +SPDX-License-Identifier: Apache-2.0 + +After each install and upgrade, when server.egress.expectedAddresses is set: +a Job placed like the mta pods (their node selector, tolerations, affinity +and labels, so an egress gateway that picks them picks it too) asks +server.egress.checkUrl which address it came from, and records the answer +on the mta Deployment as an event, a Warning when it isn't one of the +expected addresses. The script is files/egress-check.sh. It never fails. + +Its token may create events in this namespace and nothing else. +*/}} +{{- $root := . }} +{{- $server := .Values.server }} +{{- $egress := $server.egress }} +{{- $mta := $server.groups.mta }} +{{- if and $egress.expectedAddresses $mta.enabled }} +{{- if not $egress.checkUrl }} +{{- fail "server.egress.checkUrl is required with server.egress.expectedAddresses: a URL that answers with the caller's address as plain text" }} +{{- end }} +{{- $name := printf "%s-egress-check" (include "inbuxa.fullname" $root) }} +{{- $hook := dict "helm.sh/hook" "post-install,post-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: egress-check + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }} +automountServiceAccountToken: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: egress-check + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: egress-check + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: egress-check + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $hook) | nindent 4 }} +data: + egress-check.sh: | + {{- $root.Files.Get "files/egress-check.sh" | nindent 4 }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ $root.Release.Namespace }} + labels: + {{- include "inbuxa.labels" $root | nindent 4 }} + app.kubernetes.io/component: egress-check + annotations: + {{- toYaml (merge (dict "helm.sh/hook-weight" "0") $hook) | nindent 4 }} + ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it" + ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing" + ignore-check.kube-linter.io/restart-policy: "Never on purpose: one answer is enough, and the script never fails" +spec: + backoffLimit: 0 + activeDeadlineSeconds: 120 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + {{- with $mta.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "inbuxa.labels" $root | nindent 8 }} + app.kubernetes.io/component: egress-check + spec: + serviceAccountName: {{ $name }} + automountServiceAccountToken: true + enableServiceLinks: false + restartPolicy: Never + {{- with $root.Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + seccompProfile: + type: RuntimeDefault + {{- with $mta.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $mta.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $mta.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: egress-check + image: {{ include "inbuxa.server.image" $root }} + imagePullPolicy: {{ $server.image.pullPolicy }} + command: ["/bin/sh", "/etc/inbuxa/egress-check.sh"] + env: + - name: CHECK_URL + value: {{ $egress.checkUrl | quote }} + - name: EXPECTED + value: {{ join " " $egress.expectedAddresses | quote }} + - name: NAMESPACE + value: {{ $root.Release.Namespace | quote }} + - name: DEPLOYMENT + value: {{ printf "%s-mta" (include "inbuxa.fullname" $root) | quote }} + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: script + mountPath: /etc/inbuxa/egress-check.sh + subPath: egress-check.sh + readOnly: true + volumes: + - name: script + configMap: + name: {{ $name }} +{{- end }} diff --git a/charts/inbuxa/tests/egress_test.yaml b/charts/inbuxa/tests/egress_test.yaml new file mode 100644 index 0000000..7ad3e20 --- /dev/null +++ b/charts/inbuxa/tests/egress_test.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +suite: egress +templates: + - server/egress-check.yaml + - server/deployment.yaml +release: + name: mail + namespace: inbuxa +values: + - ../ci/defaults-values.yaml +tests: + - it: checks nothing unless the expected addresses are given + template: server/egress-check.yaml + asserts: + - hasDocuments: + count: 0 + + - it: needs a URL to ask + template: server/egress-check.yaml + set: + server.egress.expectedAddresses: [198.51.100.7] + asserts: + - failedTemplate: + errorPattern: checkUrl is required + + - it: runs where the mta pods run, with a token that may only create events + template: server/egress-check.yaml + set: + server.egress.expectedAddresses: [198.51.100.7, "2001:db8::7"] + server.egress.checkUrl: https://echo.example.test/ip + server.groups.mta.nodeSelector: {pool: mail-out} + server.groups.mta.tolerations: [{key: mail-out, operator: Exists, effect: NoSchedule}] + server.groups.mta.podLabels: {egress: mail} + asserts: + - equal: + path: rules + value: [{apiGroups: [""], resources: [events], verbs: [create]}] + documentSelector: {path: kind, value: Role} + - equal: + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + documentSelector: {path: kind, value: Job} + - equal: + path: spec.template.spec.nodeSelector + value: {pool: mail-out} + documentSelector: {path: kind, value: Job} + - equal: + path: spec.template.spec.tolerations[0].key + value: mail-out + documentSelector: {path: kind, value: Job} + - equal: + path: spec.template.metadata.labels.egress + value: mail + documentSelector: {path: kind, value: Job} + - contains: + path: spec.template.spec.containers[0].env + content: {name: EXPECTED, value: "198.51.100.7 2001:db8::7"} + documentSelector: {path: kind, value: Job} + - contains: + path: spec.template.spec.containers[0].env + content: {name: DEPLOYMENT, value: mail-inbuxa-mta} + documentSelector: {path: kind, value: Job} + + - it: labels a group's pods for an egress gateway + template: server/deployment.yaml + set: + server.groups.mta.podLabels: {egress: mail} + documentIndex: 1 + asserts: + - equal: + path: spec.template.metadata.labels.egress + value: mail + - equal: + path: spec.template.metadata.labels["inbuxa.org/group"] + value: mta + + - it: refuses labels the chart sets itself + template: server/deployment.yaml + set: + server.groups.mta.podLabels: {"inbuxa.org/group": front} + asserts: + - failedTemplate: {} diff --git a/charts/inbuxa/values.schema.json b/charts/inbuxa/values.schema.json index 9db79c9..748855b 100644 --- a/charts/inbuxa/values.schema.json +++ b/charts/inbuxa/values.schema.json @@ -63,6 +63,17 @@ "type": "string" } }, + "podLabels": { + "type": "object", + "propertyNames": { + "not": { + "pattern": "^(app\\.kubernetes\\.io|inbuxa\\.org|helm\\.sh)/" + } + }, + "additionalProperties": { + "type": "string" + } + }, "extraEnv": { "type": "array", "items": { @@ -235,6 +246,23 @@ } } }, + "egress": { + "type": "object", + "additionalProperties": false, + "properties": { + "expectedAddresses": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[0-9a-fA-F.:]+$" + } + }, + "checkUrl": { + "type": "string", + "pattern": "^(https?://\\S+)?$" + } + } + }, "upgrade": { "type": "object", "additionalProperties": false, diff --git a/charts/inbuxa/values.yaml b/charts/inbuxa/values.yaml index f07f731..d407c2f 100644 --- a/charts/inbuxa/values.yaml +++ b/charts/inbuxa/values.yaml @@ -119,9 +119,14 @@ server: tolerations: [] affinity: {} podAnnotations: {} + # Extra labels on the pods, for what selects them by label: an egress + # gateway policy, say. Not app.kubernetes.io/ or inbuxa.org/ ones. + podLabels: {} extraEnv: [] # As front, plus outbound delivery. Place it where its egress address - # is the one your SPF and PTR records name. + # is the one your SPF and PTR records name: on nodes with a fixed + # address (nodeSelector, tolerations), or behind an egress gateway that + # selects it (podLabels). server.egress checks it. mta: enabled: true replicas: 2 @@ -131,6 +136,9 @@ server: tolerations: [] affinity: {} podAnnotations: {} + # Extra labels on the pods, for what selects them by label: an egress + # gateway policy, say. Not app.kubernetes.io/ or inbuxa.org/ ones. + podLabels: {} extraEnv: [] # Everything, including the task scheduler. Exactly one replica: a # second would queue every maintenance task twice. @@ -143,8 +151,22 @@ server: tolerations: [] affinity: {} podAnnotations: {} + # Extra labels on the pods, for what selects them by label: an egress + # gateway policy, say. Not app.kubernetes.io/ or inbuxa.org/ ones. + podLabels: {} extraEnv: [] + # The addresses outbound mail should leave from: the ones with reverse + # DNS (PTR) records and in your SPF. When set, a Job placed like the mta + # pods asks checkUrl which address it came from after each install and + # upgrade, and records the answer on the mta Deployment as an event: a + # Warning when it isn't one of these (kubectl get events). checkUrl answers + # a GET with the caller's address as plain text; pick one you trust, as it + # sees your cluster's address. + egress: + expectedAddresses: [] + checkUrl: "" + # Used for every group that sets no resources of its own resources: requests: -- 2.54.0