Mail out: podLabels per group, and a check of the egress address (K-C7) #25

Merged
jcoffey-dev merged 1 commits from feat/mta-egress into main 2026-11-19 13:41:14 +00:00
9 changed files with 365 additions and 4 deletions

No files matched your search

+7
View File
@@ -51,6 +51,13 @@ runs, pinned by digest.
passwords between all users would ban the ingress for everyone.
`server.trustedProxies` lists the proxies when the pod network isn't a
private range. Needs server 27.1-rc.32; older servers ignore it.
- Mail out from a known address (`server.egress`): with
`expectedAddresses` and `checkUrl` set, a Job placed like the mta pods
asks which address it leaves from after each install and upgrade, and
records the answer on the mta Deployment as an event, a Warning when it
isn't one of the expected addresses. It never fails an install. Its token
may create events and nothing else. `podLabels` on each server group, for
an egress gateway that selects the mta pods.
- `networkPolicy`, on by default: every pod starts from deny all, in and
out, and gets only what it needs. The server takes mail on its listeners
from anywhere and connects anywhere; `networkPolicy.httpFrom` and
+4
View File
@@ -22,6 +22,9 @@ server:
trustedProxies: [10.0.0.0/8, 100.64.0.0/10]
listeners:
pop3s: 995
egress:
expectedAddresses: [198.51.100.7, "2001:db8::7"]
checkUrl: https://ip.example.org/
groups:
front:
replicas: 3
@@ -30,6 +33,7 @@ server:
mta:
nodeSelector: {example.org/egress: mail}
tolerations: [{key: example.org/egress, operator: Exists, effect: NoSchedule}]
podLabels: {example.org/egress-gateway: mail}
resources:
requests: {cpu: 500m, memory: 1Gi}
limits: {cpu: "4", memory: 4Gi}
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-C7: the address outbound mail leaves from, from a pod placed like the mta
# pods, against the addresses expected (the ones with PTR records and in
# SPF). The answer goes on the mta Deployment as an event: Normal when it
# matches, Warning when it doesn't or can't be learned. It never fails, so it
# never holds up an install or an upgrade. Only sh, sed, tr and curl: the
# image has nothing more.
#
# CHECK_URL answers a GET with the caller's address as plain text.
set -u
sa=/var/run/secrets/kubernetes.io/serviceaccount
event() { # <Normal|Warning> <reason> <message>
now=$(date -u +%Y-%m-%dT%H:%M:%SZ)
message=$(printf '%s' "$3" | sed 's/\\/\\\\/g; s/"/\\"/g')
body=$(printf '{"apiVersion":"v1","kind":"Event","metadata":{"generateName":"%s.","namespace":"%s"},"involvedObject":{"apiVersion":"apps/v1","kind":"Deployment","name":"%s","namespace":"%s"},"type":"%s","reason":"%s","message":"%s","firstTimestamp":"%s","lastTimestamp":"%s","count":1,"source":{"component":"inbuxa-egress-check"}}' \
"$DEPLOYMENT" "$NAMESPACE" "$DEPLOYMENT" "$NAMESPACE" "$1" "$2" "$message" "$now" "$now")
echo "$1 $2: $3"
curl -fsS -o /dev/null --cacert "$sa/ca.crt" -H "Authorization: Bearer $(cat "$sa/token")" \
-H 'Content-Type: application/json' -X POST -d "$body" \
"https://kubernetes.default.svc/api/v1/namespaces/$NAMESPACE/events" ||
echo "couldn't record the event on deployment/$DEPLOYMENT"
}
seen=$(curl -fsS -m 20 "$CHECK_URL" 2>/dev/null | tr -d ' \r\n')
case "$seen" in
"" | *[!0-9a-fA-F.:]*)
event Warning EgressUnknown "Couldn't learn the address outbound mail leaves from: $CHECK_URL gave no address. Expected one of: $EXPECTED."
exit 0
;;
esac
for address in $EXPECTED; do
if [ "$address" = "$seen" ]; then
event Normal EgressAddress "Outbound mail leaves from $seen, as expected."
exit 0
fi
done
event Warning EgressMismatch "Outbound mail leaves from $seen, which is not one of the expected addresses ($EXPECTED). Receivers check its reverse DNS and SPF: mail may be refused or marked as spam."
exit 0
+5 -3
View File
@@ -9,7 +9,8 @@ all, in and out; the policies after it open only what each one needs:
server its mail listeners and https from anywhere (networkPolicy.serverEgress
anywhere; http from the webmail narrows it)
and from networkPolicy.httpFrom
seed, upgrade nothing as the server
seed, upgrade, nothing as the server
egress-check
webmail 8080 from networkPolicy.httpFrom DNS; the server's http port; TCP 80
and 443 anywhere, for remote images
and calendars (networkPolicy.webmailInternet)
@@ -82,7 +83,8 @@ spec:
{{- include "inbuxa.networkPolicy.serverEgress" $root | nindent 4 }}
---
# The seed and upgrade Jobs reach the data store as the server does; the
# upgrade Job reaches the Kubernetes API too. Nothing connects to them.
# upgrade Job reaches the Kubernetes API too, and the egress check the API
# and server.egress.checkUrl. Nothing connects to them.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
@@ -95,7 +97,7 @@ spec:
matchLabels:
{{- include "inbuxa.selectorLabels" $root | nindent 6 }}
matchExpressions:
- {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade]}
- {key: app.kubernetes.io/component, operator: In, values: [seed, upgrade, egress-check]}
policyTypes: [Ingress, Egress]
egress:
{{- include "inbuxa.networkPolicy.serverEgress" $root | nindent 4 }}
@@ -51,6 +51,9 @@ spec:
template:
metadata:
labels:
{{- with $g.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- include "inbuxa.labels" $root | nindent 8 }}
app.kubernetes.io/component: server
inbuxa.org/group: {{ $group }}
@@ -0,0 +1,170 @@
{{/*
SPDX-FileCopyrightText: 2026 Coffey Labs LLC
SPDX-License-Identifier: Apache-2.0
After each install and upgrade, when server.egress.expectedAddresses is set:
a Job placed like the mta pods (their node selector, tolerations, affinity
and labels, so an egress gateway that picks them picks it too) asks
server.egress.checkUrl which address it came from, and records the answer
on the mta Deployment as an event, a Warning when it isn't one of the
expected addresses. The script is files/egress-check.sh. It never fails.
Its token may create events in this namespace and nothing else.
*/}}
{{- $root := . }}
{{- $server := .Values.server }}
{{- $egress := $server.egress }}
{{- $mta := $server.groups.mta }}
{{- if and $egress.expectedAddresses $mta.enabled }}
{{- if not $egress.checkUrl }}
{{- fail "server.egress.checkUrl is required with server.egress.expectedAddresses: a URL that answers with the caller's address as plain text" }}
{{- end }}
{{- $name := printf "%s-egress-check" (include "inbuxa.fullname" $root) }}
{{- $hook := dict "helm.sh/hook" "post-install,post-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation" }}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: egress-check
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
automountServiceAccountToken: false
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: egress-check
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: egress-check
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-10") $hook) | nindent 4 }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: {{ $name }}
subjects:
- kind: ServiceAccount
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: egress-check
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $hook) | nindent 4 }}
data:
egress-check.sh: |
{{- $root.Files.Get "files/egress-check.sh" | nindent 4 }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: {{ $name }}
namespace: {{ $root.Release.Namespace }}
labels:
{{- include "inbuxa.labels" $root | nindent 4 }}
app.kubernetes.io/component: egress-check
annotations:
{{- toYaml (merge (dict "helm.sh/hook-weight" "0") $hook) | nindent 4 }}
ignore-check.kube-linter.io/no-liveness-probe: "A one-shot that serves nothing; activeDeadlineSeconds bounds it"
ignore-check.kube-linter.io/no-readiness-probe: "A one-shot that serves nothing"
ignore-check.kube-linter.io/restart-policy: "Never on purpose: one answer is enough, and the script never fails"
spec:
backoffLimit: 0
activeDeadlineSeconds: 120
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
{{- with $mta.podLabels }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- include "inbuxa.labels" $root | nindent 8 }}
app.kubernetes.io/component: egress-check
spec:
serviceAccountName: {{ $name }}
automountServiceAccountToken: true
enableServiceLinks: false
restartPolicy: Never
{{- with $root.Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
securityContext:
runAsNonRoot: true
runAsUser: 2000
runAsGroup: 2000
seccompProfile:
type: RuntimeDefault
{{- with $mta.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with $mta.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with $mta.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
- name: egress-check
image: {{ include "inbuxa.server.image" $root }}
imagePullPolicy: {{ $server.image.pullPolicy }}
command: ["/bin/sh", "/etc/inbuxa/egress-check.sh"]
env:
- name: CHECK_URL
value: {{ $egress.checkUrl | quote }}
- name: EXPECTED
value: {{ join " " $egress.expectedAddresses | quote }}
- name: NAMESPACE
value: {{ $root.Release.Namespace | quote }}
- name: DEPLOYMENT
value: {{ printf "%s-mta" (include "inbuxa.fullname" $root) | quote }}
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
memory: 64Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
capabilities:
drop: [ALL]
volumeMounts:
- name: script
mountPath: /etc/inbuxa/egress-check.sh
subPath: egress-check.sh
readOnly: true
volumes:
- name: script
configMap:
name: {{ $name }}
{{- end }}
+83
View File
@@ -0,0 +1,83 @@
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
suite: egress
templates:
- server/egress-check.yaml
- server/deployment.yaml
release:
name: mail
namespace: inbuxa
values:
- ../ci/defaults-values.yaml
tests:
- it: checks nothing unless the expected addresses are given
template: server/egress-check.yaml
asserts:
- hasDocuments:
count: 0
- it: needs a URL to ask
template: server/egress-check.yaml
set:
server.egress.expectedAddresses: [198.51.100.7]
asserts:
- failedTemplate:
errorPattern: checkUrl is required
- it: runs where the mta pods run, with a token that may only create events
template: server/egress-check.yaml
set:
server.egress.expectedAddresses: [198.51.100.7, "2001:db8::7"]
server.egress.checkUrl: https://echo.example.test/ip
server.groups.mta.nodeSelector: {pool: mail-out}
server.groups.mta.tolerations: [{key: mail-out, operator: Exists, effect: NoSchedule}]
server.groups.mta.podLabels: {egress: mail}
asserts:
- equal:
path: rules
value: [{apiGroups: [""], resources: [events], verbs: [create]}]
documentSelector: {path: kind, value: Role}
- equal:
path: metadata.annotations["helm.sh/hook"]
value: post-install,post-upgrade
documentSelector: {path: kind, value: Job}
- equal:
path: spec.template.spec.nodeSelector
value: {pool: mail-out}
documentSelector: {path: kind, value: Job}
- equal:
path: spec.template.spec.tolerations[0].key
value: mail-out
documentSelector: {path: kind, value: Job}
- equal:
path: spec.template.metadata.labels.egress
value: mail
documentSelector: {path: kind, value: Job}
- contains:
path: spec.template.spec.containers[0].env
content: {name: EXPECTED, value: "198.51.100.7 2001:db8::7"}
documentSelector: {path: kind, value: Job}
- contains:
path: spec.template.spec.containers[0].env
content: {name: DEPLOYMENT, value: mail-inbuxa-mta}
documentSelector: {path: kind, value: Job}
- it: labels a group's pods for an egress gateway
template: server/deployment.yaml
set:
server.groups.mta.podLabels: {egress: mail}
documentIndex: 1
asserts:
- equal:
path: spec.template.metadata.labels.egress
value: mail
- equal:
path: spec.template.metadata.labels["inbuxa.org/group"]
value: mta
- it: refuses labels the chart sets itself
template: server/deployment.yaml
set:
server.groups.mta.podLabels: {"inbuxa.org/group": front}
asserts:
- failedTemplate: {}
+28
View File
@@ -63,6 +63,17 @@
"type": "string"
}
},
"podLabels": {
"type": "object",
"propertyNames": {
"not": {
"pattern": "^(app\\.kubernetes\\.io|inbuxa\\.org|helm\\.sh)/"
}
},
"additionalProperties": {
"type": "string"
}
},
"extraEnv": {
"type": "array",
"items": {
@@ -235,6 +246,23 @@
}
}
},
"egress": {
"type": "object",
"additionalProperties": false,
"properties": {
"expectedAddresses": {
"type": "array",
"items": {
"type": "string",
"pattern": "^[0-9a-fA-F.:]+$"
}
},
"checkUrl": {
"type": "string",
"pattern": "^(https?://\\S+)?$"
}
}
},
"upgrade": {
"type": "object",
"additionalProperties": false,
+23 -1
View File
@@ -119,9 +119,14 @@ server:
tolerations: []
affinity: {}
podAnnotations: {}
# Extra labels on the pods, for what selects them by label: an egress
# gateway policy, say. Not app.kubernetes.io/ or inbuxa.org/ ones.
podLabels: {}
extraEnv: []
# As front, plus outbound delivery. Place it where its egress address
# is the one your SPF and PTR records name.
# is the one your SPF and PTR records name: on nodes with a fixed
# address (nodeSelector, tolerations), or behind an egress gateway that
# selects it (podLabels). server.egress checks it.
mta:
enabled: true
replicas: 2
@@ -131,6 +136,9 @@ server:
tolerations: []
affinity: {}
podAnnotations: {}
# Extra labels on the pods, for what selects them by label: an egress
# gateway policy, say. Not app.kubernetes.io/ or inbuxa.org/ ones.
podLabels: {}
extraEnv: []
# Everything, including the task scheduler. Exactly one replica: a
# second would queue every maintenance task twice.
@@ -143,8 +151,22 @@ server:
tolerations: []
affinity: {}
podAnnotations: {}
# Extra labels on the pods, for what selects them by label: an egress
# gateway policy, say. Not app.kubernetes.io/ or inbuxa.org/ ones.
podLabels: {}
extraEnv: []
# The addresses outbound mail should leave from: the ones with reverse
# DNS (PTR) records and in your SPF. When set, a Job placed like the mta
# pods asks checkUrl which address it came from after each install and
# upgrade, and records the answer on the mta Deployment as an event: a
# Warning when it isn't one of these (kubectl get events). checkUrl answers
# a GET with the caller's address as plain text; pick one you trust, as it
# sees your cluster's address.
egress:
expectedAddresses: []
checkUrl: ""
# Used for every group that sets no resources of its own
resources:
requests: