Argo CD and Flux: install, upgrade and the crossing through GitOps (K-L10) #38

Merged
jcoffey-dev merged 3 commits from feat/gitops-lab into main 2026-11-20 17:55:54 +00:00
8 changed files with 661 additions and 13 deletions

No files matched your search

+52
View File
@@ -147,3 +147,55 @@ jobs:
E2E_LABEL: chaos, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}
CHAOS: "1"
run: inbuxa-k8s-lab/lab/e2e.sh
# GitOps (K-L10): the chart installed, upgraded and crossed from 2026.x by
# Argo CD and by Flux instead of Helm. The lab's own runs, newest minor,
# both architectures.
gitops:
needs: matrix
if: ${{ inputs.chart_ref == '' }}
name: gitops (${{ matrix.tool }}, ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }})
strategy:
fail-fast: false
matrix:
runner: ${{ fromJSON(needs.matrix.outputs.runners) }}
tool: [argocd, flux]
runs-on: ${{ matrix.runner }}
timeout-minutes: 75
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: inbuxa/inbuxa-k8s-lab
ref: ${{ github.sha }}
path: inbuxa-k8s-lab
persist-credentials: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: inbuxa/inbuxa-charts
ref: ${{ vars.CHART_REF || 'main' }}
path: inbuxa-charts
persist-credentials: false
- name: Tools (pinned, checksummed)
run: |
inbuxa-k8s-lab/lab/tools.sh "$HOME/.local/bin"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Lab end to end through ${{ matrix.tool }}, then its tests and the crossing
env:
KIND_NODE_IMAGE: ${{ fromJSON(needs.matrix.outputs.newest).image }}
E2E_LABEL: ${{ matrix.tool }}, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}
INSTALL_WITH: ${{ matrix.tool }}
run: inbuxa-k8s-lab/lab/e2e.sh
- name: What the controller said
if: failure()
env:
TOOL: ${{ matrix.tool }}
run: |
if [ "$TOOL" = argocd ]; then
kubectl -n argocd get application inbuxa -o yaml | sed -n '/^status:/,$p' | head -150
kubectl -n argocd logs statefulset/argocd-application-controller --tail=80 || true
else
kubectl -n flux-system get helmrelease inbuxa -o yaml | sed -n '/^status:/,$p' | head -150
kubectl -n flux-system logs deploy/helm-controller --tail=80 || true
fi
kubectl -n inbuxa get jobs,pods -o wide || true
kubectl -n inbuxa logs job/inbuxa-upgrade --tail=40 || true
+12
View File
@@ -47,6 +47,18 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed
over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest
Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`).
`INSTALL_WITH=argocd lab/e2e.sh` (or `flux`) installs the chart through
Argo CD or Flux instead of Helm, from a Helm repository in the cluster
(`lab/gitops/gitops.sh`), then checks what GitOps can get wrong
(`lab/gitops/gitops-test.sh`). Nothing is applied again unchanged and no
Secret changes across refreshes. The controller puts a hand edit back. An
upgrade with every image pinned by digest rolls everything without signing
out webmail tabs left open across it. Then the 27.1 crossing from a 2026.x
store, as changes to the controller's spec: refused, then acknowledged.
Under Argo CD the chart's generated secrets are off and the Secrets are
made first; Flux runs Helm, so the chart keeps its own. CI runs both
nightly, on the newest minor and both architectures.
`lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs
and checks no request failed and every live-update stream came back.
+25
View File
@@ -17,6 +17,12 @@
#
# OLD=2026.11.18.3 TO=27.1-rc.34 lab/chart/crossing-test.sh
#
# GITOPS=argocd or flux (K-L10): the chart is the controller's
# (lab/gitops), and steps 3 and 4 are changes to its spec, as commits would
# be: the controller is suspended while OLD sets the store up, then given
# TO without the acknowledgement (it must give up), then with it. Takes
# CHART_VERSION, WEBMAIL and CONSOLE for the rest of the spec.
#
# WIPES the lab's inbuxa database, and leaves a store fresh-store.sh made.
# seed-store.sh and a helm upgrade put the lab back as the other tests want it.
set -euo pipefail
@@ -40,6 +46,20 @@ mark() { "${K[@]}" -n data exec inbuxa-pg-1 -c postgres -- psql -d inbuxa -At \
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
mailbox_digest() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$(lb)" "$@"; }
upgrade() {
if [ -n "${GITOPS:-}" ]; then
local since ack=""
[[ "$*" == *acknowledgeLayout=3* ]] && ack=3
since="$(../gitops/gitops.sh op "$GITOPS")"
SERVER="$to_d" SEED=false ACK="$ack" ../gitops/gitops.sh apply "$GITOPS" >/dev/null
if [ -n "$ack" ]; then
../gitops/gitops.sh wait "$GITOPS" "$since" 1800 >/dev/null
else
# Given up on is what's wanted; anything else counts as went through
if ../gitops/gitops.sh refused "$GITOPS" "$since" 900 >/dev/null; then return 1; fi
return 0
fi
return
fi
"${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml --set server.seed.enabled=false \
--set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \
--set server.image.digest="$to_d" --wait --timeout 20m "$@"
@@ -52,6 +72,11 @@ echo "$OLD = $old_d"
echo "$TO = $to_d"
echo "== 1. a store made by $OLD"
if [ -n "${GITOPS:-}" ]; then
: "${WEBMAIL:?}" "${CONSOLE:?}"
../gitops/gitops.sh suspend "$GITOPS"
echo " $GITOPS suspended"
fi
IMAGE="$IMAGE@$old_d" PULL=IfNotPresent ./fresh-store.sh
echo " layout mark: $(mark)"
# OLD serving mail, behind the chart's Service and NetworkPolicies. The
+3
View File
@@ -54,6 +54,9 @@ forward() {
stop_forward() { [ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null; FORWARD_PID=""; }
trap stop_forward EXIT
# The one-off pods' config.json (the chart's own pods have theirs)
"${K[@]}" apply -f ../spike/server-common.yaml >/dev/null
echo "== stopping everything on the old store"
"${K[@]}" -n inbuxa scale deploy --all --replicas=0 >/dev/null 2>&1 || true
"${K[@]}" -n inbuxa wait --for=delete pod -l app.kubernetes.io/component=server --timeout=180s >/dev/null 2>&1 || true
+47 -13
View File
@@ -10,7 +10,13 @@
# JMAP, SMTP out to an outside server, CalDAV, CardDAV, the webmail across
# both replicas and the console
#
# and how long each part took; CHAOS=1 adds the chaos tests after it. CI
# and how long each part took; CHAOS=1 adds the chaos tests after it.
#
# INSTALL_WITH=argocd or flux (K-L10) installs the chart through that
# controller instead of Helm (lab/gitops/gitops.sh), then after the client
# checks runs lab/gitops/gitops-test.sh (steady, self-heal, an upgrade under
# open webmail tabs) and the 27.1 crossing from CROSSING_FROM (versions.env)
# as changes to the controller's spec. CI
# runs this on a fresh runner; on the bench
# it builds the lab from scratch too, so take the lab down first
# (lab/down.sh) unless you mean to reuse it.
@@ -79,18 +85,31 @@ rm -f "$STATE/s3-access-key"
| "${K[@]}" apply -f - >/dev/null
umask 022
echo "== the chart: seeded, two replicas of everything"
s=$SECONDS
helm --kube-context "kind-$KIND_CLUSTER" -n inbuxa upgrade --install inbuxa "$CHART" \
-f "$here/chart/values-lab.yaml" -f "$here/chart/values-frontends.yaml" \
--set server.image.repository=ghcr.io/inbuxa/inbuxa-server --set server.image.tag="$SERVER" \
--set server.image.pullPolicy=IfNotPresent \
--set webmail.image.repository=ghcr.io/inbuxa/inbuxa-webmail --set webmail.image.tag="$WEBMAIL" \
--set webmail.image.pullPolicy=IfNotPresent \
--set console.image.repository=ghcr.io/inbuxa/inbuxa-admin --set console.image.tag="$CONSOLE" \
--set console.image.pullPolicy=IfNotPresent \
--wait --timeout 15m >/dev/null
"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}'
if [ -n "${INSTALL_WITH:-}" ]; then
echo "== $INSTALL_WITH, and the chart as a Helm repository in the cluster"
s=$SECONDS
"$here/gitops/gitops.sh" controller "$INSTALL_WITH"
CHART_REPO="$CHART_REPO" "$here/gitops/gitops.sh" repo
lap "$INSTALL_WITH and the chart repository" "$s"
echo "== the chart through $INSTALL_WITH: seeded, two replicas of everything"
s=$SECONDS
SERVER="$SERVER" WEBMAIL="$WEBMAIL" CONSOLE="$CONSOLE" "$here/gitops/gitops.sh" apply "$INSTALL_WITH" >/dev/null
"$here/gitops/gitops.sh" wait "$INSTALL_WITH" "" 1200 | sed 's/^/ /'
"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}'
else
echo "== the chart: seeded, two replicas of everything"
s=$SECONDS
helm --kube-context "kind-$KIND_CLUSTER" -n inbuxa upgrade --install inbuxa "$CHART" \
-f "$here/chart/values-lab.yaml" -f "$here/chart/values-frontends.yaml" \
--set server.image.repository=ghcr.io/inbuxa/inbuxa-server --set server.image.tag="$SERVER" \
--set server.image.pullPolicy=IfNotPresent \
--set webmail.image.repository=ghcr.io/inbuxa/inbuxa-webmail --set webmail.image.tag="$WEBMAIL" \
--set webmail.image.pullPolicy=IfNotPresent \
--set console.image.repository=ghcr.io/inbuxa/inbuxa-admin --set console.image.tag="$CONSOLE" \
--set console.image.pullPolicy=IfNotPresent \
--wait --timeout 15m >/dev/null
"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}'
fi
lap "chart install (seed, server, webmail, console)" "$s"
echo "== the client checks (chart/mailcheck.sh)"
@@ -106,5 +125,20 @@ if [ -n "${CHAOS:-}" ]; then
lap "chaos: pod kill, drain, PostgreSQL failover, NATS restart, S3 outage" "$s"
fi
# INSTALL_WITH: then the controller's own tests, and the crossing
if [ -n "${INSTALL_WITH:-}" ]; then
echo "== $INSTALL_WITH (gitops/gitops-test.sh)"
s=$SECONDS
SERVER="$SERVER" WEBMAIL="$WEBMAIL" CONSOLE="$CONSOLE" STATE="$STATE" \
"$here/gitops/gitops-test.sh" "$INSTALL_WITH" 3>"$STATE/gitops.env"
lap "$INSTALL_WITH: steady, self-heal, upgrade under open tabs" "$s"
echo "== the 27.1 crossing from $CROSSING_FROM through $INSTALL_WITH (chart/crossing-test.sh)"
s=$SECONDS
# shellcheck disable=SC2046
env $(cat "$STATE/gitops.env") GITOPS="$INSTALL_WITH" OLD="$CROSSING_FROM" TO="$SERVER" STATE="$STATE" \
CHART="$CHART" "$here/chart/crossing-test.sh"
lap "$INSTALL_WITH: the crossing from $CROSSING_FROM, refused, then acknowledged" "$s"
fi
lap "**total**"
echo "== passed in $(( SECONDS - t0 ))s"
+170
View File
@@ -0,0 +1,170 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L10: the chart under a GitOps controller, after it installed it
# (INSTALL_WITH=TOOL lab/e2e.sh does both). TOOL is argocd or flux.
#
# 1. steady: the app stays in sync across refreshes; nothing is re-applied
# and no Secret changes (a chart that renders differently each time
# never settles under Argo CD)
# 2. self-heal: the webmail scaled down by hand comes back to two
# 3. an upgrade, 27.1.0-lab.1 to -lab.2 with every image pinned by digest
# as a chart release pins them: everything rolls, the upgrade hook
# runs, and webmail tabs open across it are never signed out
# 4. after it: the same Secrets, mail works, and steady again
#
# lab/gitops/gitops-test.sh TOOL
#
# Wants SERVER, WEBMAIL and CONSOLE, the tags the install used. Leaves the
# app on -lab.2 with the digests, and prints them as KEY=value lines on fd 3
# when it's open (e2e.sh passes them to the crossing).
set -euo pipefail
here=$(cd "$(dirname "$0")" && pwd)
. "$here/../versions.env"
TOOL="${1:?argocd or flux}"
K=(kubectl --context "kind-$KIND_CLUSTER")
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
G=("$here/gitops.sh")
: "${SERVER:?}" "${WEBMAIL:?}" "${CONSOLE:?}"
WORK="$(mktemp -d)"
trap '"${K[@]}" -n lab-sink delete pod gitops-tabs --ignore-not-found --wait=false >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT
fail() { echo "!! $*" >&2; exit 1; }
# Every Secret in the namespace by name, uid and a hash of its data; Helm's
# own release records change on every upgrade by design, so they're left out
secrets() {
"${K[@]}" -n inbuxa get secrets -o json | python3 -c '
import hashlib, json, sys
for s in sorted(json.load(sys.stdin)["items"], key=lambda s: s["metadata"]["name"]):
if s.get("type") == "helm.sh/release.v1":
continue
h = hashlib.sha256(json.dumps(s.get("data", {}), sort_keys=True).encode()).hexdigest()[:16]
print(s["metadata"]["name"], s["metadata"]["uid"], h)'
}
refresh() {
if [ "$TOOL" = argocd ]; then
"${K[@]}" -n argocd annotate application inbuxa argocd.argoproj.io/refresh=hard --overwrite >/dev/null
else
"${K[@]}" -n flux-system annotate helmrelease inbuxa reconcile.fluxcd.io/requestedAt="$(date +%s%N)" --overwrite >/dev/null
fi
}
steady() {
local before after
before="$("${G[@]}" deployed "$TOOL")"
for i in 1 2 3; do
refresh
sleep 30
"${G[@]}" state "$TOOL" > "$WORK/state"
python3 -c 'import json, sys; sys.exit(0 if json.load(open(sys.argv[1]))["ok"] else 1)' "$WORK/state" \
|| { cat "$WORK/state" >&2; fail "out of sync or unhealthy on refresh $i"; }
done
after="$("${G[@]}" deployed "$TOOL")"
[ "$before" = "$after" ] || fail "$TOOL applied the chart again with nothing changed ($before, then $after)"
secrets > "$WORK/secrets.now"
diff "$WORK/secrets" "$WORK/secrets.now" >/dev/null ||
{ diff "$WORK/secrets" "$WORK/secrets.now" | sed 's/^/ /'; fail "a Secret changed"; }
}
digest_of() { docker buildx imagetools inspect "ghcr.io/inbuxa/$1:$2" --format '{{json .Manifest.Digest}}' | tr -d '"'; }
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
"${G[@]}" wait "$TOOL" "" 600 >/dev/null
secrets > "$WORK/secrets"
grep -q '^inbuxa-webmail' "$WORK/secrets" || fail "no webmail Secret"
echo " $(wc -l < "$WORK/secrets") Secrets: $(awk '{print $1}' "$WORK/secrets" | tr '\n' ' ')"
echo "== 1. steady: three refreshes"
steady
echo " in sync each time; nothing re-applied; every Secret as it was"
echo "== 2. self-heal: the webmail scaled to 1 by hand"
"${K[@]}" -n inbuxa scale deploy/inbuxa-webmail --replicas=1 >/dev/null
for _ in $(seq 1 60); do
[ "$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.replicas}')" = 2 ] && break
sleep 5
done
[ "$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.replicas}')" = 2 ] || fail "$TOOL didn't put the webmail back"
"${K[@]}" -n inbuxa rollout status deploy/inbuxa-webmail --timeout=180s >/dev/null
"${G[@]}" wait "$TOOL" "" 300 >/dev/null
echo " back to 2"
echo "== 3. upgrade to 27.1.0-lab.2, every image by digest, under open webmail tabs"
S_D="$(digest_of inbuxa-server "$SERVER")"; W_D="$(digest_of inbuxa-webmail "$WEBMAIL")"; C_D="$(digest_of inbuxa-admin "$CONSOLE")"
for d in "$S_D" "$W_D" "$C_D"; do [[ "$d" =~ ^sha256: ]] || fail "couldn't read a digest: $d"; done
"${K[@]}" create namespace lab-sink --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-sink create configmap webmail-tabs --from-file="$here/../chart/webmail-tabs.py" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-sink create secret generic webmail-tabs --from-file=password="$STATE/alice.password" \
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" -n lab-sink delete pod gitops-tabs --ignore-not-found --wait >/dev/null
"${K[@]}" -n lab-sink apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: gitops-tabs}
spec:
restartPolicy: Never
automountServiceAccountToken: false
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: tabs
image: $PG_IMAGE
imagePullPolicy: IfNotPresent
command: [python3, -u, /tabs/webmail-tabs.py, "http://inbuxa-webmail.inbuxa.svc:8080", "http://inbuxa-http.inbuxa.svc:8080"]
env:
- {name: ACCOUNT, value: [email protected]}
- {name: PASSWORD_FILE, value: /secret/password}
- {name: STOP_FILE, value: /tmp/stop}
- {name: INTERVAL, value: "1"}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: tabs, mountPath: /tabs}
- {name: secret, mountPath: /secret}
- {name: tmp, mountPath: /tmp}
volumes:
- {name: tabs, configMap: {name: webmail-tabs}}
- {name: secret, secret: {secretName: webmail-tabs, defaultMode: 0440}}
- {name: tmp, emptyDir: {}}
EOF
"${K[@]}" -n lab-sink wait --for=condition=Ready pod/gitops-tabs --timeout=120s >/dev/null
sleep 10
[ "$("${K[@]}" -n lab-sink get pod gitops-tabs -o jsonpath='{.status.phase}')" = Running ] ||
{ "${K[@]}" -n lab-sink logs gitops-tabs | tail -20 >&2; fail "the tabs stopped before the upgrade"; }
since="$("${G[@]}" op "$TOOL")"
start=$SECONDS
CHART_VERSION=27.1.0-lab.2 SERVER="$S_D" WEBMAIL="$W_D" CONSOLE="$C_D" "${G[@]}" apply "$TOOL" | sed 's/^/ /'
"${G[@]}" wait "$TOOL" "$since" 1500 > "$WORK/state"
for d in front mta core webmail console; do
"${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=600s >/dev/null
done
echo " in after $(( SECONDS - start ))s: $(cat "$WORK/state")"
images="$("${K[@]}" -n inbuxa get deploy -o jsonpath='{range .items[*]}{.spec.template.spec.containers[0].image}{"\n"}{end}')"
grep -qv '@sha256:' <<<"$images" && { echo "$images" >&2; fail "a Deployment isn't on a digest"; }
sleep 20
"${K[@]}" -n lab-sink exec gitops-tabs -- touch /tmp/stop
"${K[@]}" -n lab-sink wait --for=jsonpath='{.status.phase}'=Succeeded pod/gitops-tabs --timeout=120s >/dev/null \
|| fail "the tabs didn't finish"
python3 - "$("${K[@]}" -n lab-sink logs gitops-tabs | tail -1)" <<'PY'
import json, sys
r = json.loads(sys.argv[1])
print(f" tabs: {r['requests']} requests, {r['failed']} failed, signed in {r['signins']} time(s)")
for e in r["errors"][:10]:
print(f" {e}")
if r["signins"] != 1:
print("!! the upgrade signed the tabs out", file=sys.stderr)
sys.exit(1)
PY
echo "== 4. after the upgrade"
secrets > "$WORK/secrets.now"
diff "$WORK/secrets" "$WORK/secrets.now" >/dev/null ||
{ diff "$WORK/secrets" "$WORK/secrets.now" | sed 's/^/ /'; fail "the upgrade changed a Secret"; }
echo " every Secret as it was"
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 "$here/../chart/mailtest.py" "$(lb)"
steady
echo " steady again"
if { true >&3; } 2>/dev/null; then
printf 'CHART_VERSION=27.1.0-lab.2\nWEBMAIL=%s\nCONSOLE=%s\n' "$W_D" "$C_D" >&3
fi
echo "== $TOOL: installed, steady, self-healed and upgraded; no Secret changed, nobody signed out"
+342
View File
@@ -0,0 +1,342 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-L10: the chart installed and upgraded by a GitOps controller instead of
# by Helm. TOOL is argocd or flux.
#
# lab/gitops/gitops.sh controller TOOL Argo CD (core) or Flux, from the
# pinned manifest (versions.env)
# lab/gitops/gitops.sh repo the chart as a Helm repository in
# the cluster: versions 27.1.0-lab.1
# and -lab.2, the same chart
# lab/gitops/gitops.sh apply TOOL the Application or HelmRelease, as
# the variables below say; prints
# what it changed
# lab/gitops/gitops.sh wait TOOL [SINCE] until the controller says the
# current spec is in and healthy
# lab/gitops/gitops.sh refused TOOL [SINCE]
# until it says the current spec failed
# lab/gitops/gitops.sh suspend TOOL no syncing until the next apply
# lab/gitops/gitops.sh state TOOL one line: what the controller says
#
# apply reads CHART_VERSION (27.1.0-lab.1), SERVER, WEBMAIL and CONSOLE
# (each a tag or sha256:...), SEED (true) and ACK (the layout to
# acknowledge, if any). Self-heal is on in both: Argo CD's selfHeal, Flux's
# drift detection.
#
# Secrets: under Argo CD the chart renders without the cluster, so
# generateSecrets is off and the webmail's and the metrics Secrets are made
# here first, as the docs will tell people to. Flux runs Helm itself, so the
# chart makes and keeps its own, as under Helm.
#
# The chart is the checkout's (CHART_REPO, as e2e.sh); the lab's values files
# go inside the package (lab/), as values files in a chart repository do.
set -euo pipefail
here=$(cd "$(dirname "$0")" && pwd)
. "$here/../versions.env"
CHART_REPO="${CHART_REPO:-$here/../../../inbuxa-charts}"
K=(kubectl --context "kind-$KIND_CLUSTER")
REPO_URL=http://charts.lab-charts.svc:8080
fail() { echo "!! $*" >&2; exit 1; }
controller() {
local tool="$1" url sum work
work="$(mktemp -d)"
case "$tool" in
argocd)
url="https://raw.githubusercontent.com/argoproj/argo-cd/$ARGOCD_VERSION/manifests/core-install.yaml"
sum="$ARGOCD_CORE_SHA256" ;;
flux)
url="https://github.com/fluxcd/flux2/releases/download/$FLUX_VERSION/install.yaml"
sum="$FLUX_INSTALL_SHA256" ;;
*) fail "argocd or flux" ;;
esac
curl -fsSL --retry 3 --proto '=https' -o "$work/install.yaml" "$url"
echo "$sum $work/install.yaml" | sha256sum -c --quiet - || fail "$url isn't the pinned manifest"
if [ "$tool" = argocd ]; then
"${K[@]}" create namespace argocd --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
# Server-side: Argo CD's CRDs are too big for the last-applied annotation
"${K[@]}" -n argocd apply --server-side --force-conflicts -f "$work/install.yaml" >/dev/null
"${K[@]}" -n argocd rollout status deploy/argocd-repo-server --timeout=300s >/dev/null
"${K[@]}" -n argocd rollout status statefulset/argocd-application-controller --timeout=300s >/dev/null
# The core install has no API server, which is what makes the default
# project elsewhere
"${K[@]}" -n argocd apply -f - >/dev/null <<'EOF'
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata: {name: default, namespace: argocd}
spec:
sourceRepos: ["*"]
destinations: [{server: "*", namespace: "*"}]
clusterResourceWhitelist: [{group: "*", kind: "*"}]
EOF
echo " Argo CD $ARGOCD_VERSION (core)"
else
"${K[@]}" apply --server-side --force-conflicts -f "$work/install.yaml" >/dev/null
for d in source-controller helm-controller; do
"${K[@]}" -n flux-system rollout status "deploy/$d" --timeout=300s >/dev/null
done
echo " Flux $FLUX_VERSION"
fi
rm -rf "$work"
}
repo() {
local work v
work="$(mktemp -d)"
mkdir -p "$work/charts"
for v in 27.1.0-lab.1 27.1.0-lab.2; do
rm -rf "$work/inbuxa" && cp -r "$CHART_REPO/charts/inbuxa" "$work/inbuxa"
mkdir -p "$work/inbuxa/lab"
cp "$here/../chart/values-lab.yaml" "$here/../chart/values-frontends.yaml" "$work/inbuxa/lab/"
helm package "$work/inbuxa" --version "$v" -d "$work/charts" >/dev/null
done
helm repo index "$work/charts" --url "$REPO_URL"
"${K[@]}" create namespace lab-charts --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
"${K[@]}" label namespace lab-charts pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null
"${K[@]}" -n lab-charts create configmap charts --from-file="$work/charts" --dry-run=client -o yaml \
| "${K[@]}" apply --server-side --force-conflicts -f - >/dev/null
"${K[@]}" -n lab-charts apply -f - >/dev/null <<EOF
apiVersion: apps/v1
kind: Deployment
metadata: {name: charts}
spec:
replicas: 1
selector: {matchLabels: {app: charts}}
template:
metadata: {labels: {app: charts}}
spec:
automountServiceAccountToken: false
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
containers:
- name: http
image: $PG_IMAGE
imagePullPolicy: IfNotPresent
command: [python3, -m, http.server, "8080", --directory, /charts]
ports: [{name: http, containerPort: 8080}]
readinessProbe: {httpGet: {path: /index.yaml, port: http}}
resources: {requests: {cpu: 10m, memory: 32Mi}, limits: {memory: 64Mi}}
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
volumeMounts: [{name: charts, mountPath: /charts}]
volumes: [{name: charts, configMap: {name: charts}}]
---
apiVersion: v1
kind: Service
metadata: {name: charts}
spec:
selector: {app: charts}
ports: [{name: http, port: 8080, targetPort: http}]
EOF
"${K[@]}" -n lab-charts rollout restart deploy/charts >/dev/null
"${K[@]}" -n lab-charts rollout status deploy/charts --timeout=180s >/dev/null
echo " $REPO_URL: $(cd "$work/charts" && ls *.tgz | tr '\n' ' ')"
rm -rf "$work"
}
# Under Argo CD: the Secrets the chart would otherwise make, made once
secrets() {
local r
r() { python3 -c 'import secrets, string, sys; a = string.ascii_letters + string.digits; print("".join(secrets.choice(a) for _ in range(int(sys.argv[1]))), end="")' "$1"; }
if ! "${K[@]}" -n inbuxa get secret inbuxa-webmail-keys >/dev/null 2>&1; then
"${K[@]}" -n inbuxa create secret generic inbuxa-webmail-keys \
--from-literal=app-secret="$(r 48)" --from-literal=oauth-client-secret="$(r 48)" >/dev/null
fi
if ! "${K[@]}" -n inbuxa get secret inbuxa-metrics-auth >/dev/null 2>&1; then
"${K[@]}" -n inbuxa create secret generic inbuxa-metrics-auth \
--from-literal=username=prometheus --from-literal=password="$(r 40)" >/dev/null
fi
}
# The values on top of the chart's and the lab's files, as JSON (which is YAML)
overrides() {
python3 - "$1" <<'PY'
import json, os, sys
tool = sys.argv[1]
def image(repo, ref):
i = {"repository": repo, "pullPolicy": "IfNotPresent"}
if ref.startswith("sha256:"):
i.update(digest=ref, tag="")
else:
i.update(tag=ref, digest="")
return i
v = {
"server": {
"image": image("ghcr.io/inbuxa/inbuxa-server", os.environ["SERVER"]),
"seed": {"enabled": os.environ.get("SEED", "true") == "true"},
"metrics": {"prometheus": {"enabled": True}},
},
"webmail": {"image": image("ghcr.io/inbuxa/inbuxa-webmail", os.environ["WEBMAIL"])},
"console": {"image": image("ghcr.io/inbuxa/inbuxa-admin", os.environ["CONSOLE"])},
}
if os.environ.get("ACK"):
v["server"]["upgrade"] = {"acknowledgeLayout": int(os.environ["ACK"])}
if tool == "argocd":
v["generateSecrets"] = False
v["webmail"]["existingSecret"] = "inbuxa-webmail-keys"
v["server"]["metrics"]["prometheus"]["existingSecret"] = "inbuxa-metrics-auth"
print(json.dumps(v))
PY
}
apply() {
local tool="$1" version="${CHART_VERSION:-27.1.0-lab.1}" values
: "${SERVER:?}" "${WEBMAIL:?}" "${CONSOLE:?}"
values="$(overrides "$tool")"
if [ "$tool" = argocd ]; then
secrets
# The finalizer: deleting the Application deletes what it made
"${K[@]}" apply -f - <<EOF
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: inbuxa
namespace: argocd
finalizers: [resources-finalizer.argocd.argoproj.io]
spec:
project: default
destination: {server: "https://kubernetes.default.svc", namespace: inbuxa}
source:
repoURL: $REPO_URL
chart: inbuxa
targetRevision: $version
helm:
releaseName: inbuxa
valueFiles: [lab/values-lab.yaml, lab/values-frontends.yaml]
valuesObject: $values
syncPolicy:
automated: {prune: true, selfHeal: true}
retry: {limit: 2, backoff: {duration: 10s, factor: 2, maxDuration: 1m}}
EOF
else
# A failed upgrade isn't rolled back: a refused layout change has
# nothing to undo, and a rollback would run the same check (pre-rollback)
"${K[@]}" apply -f - <<EOF
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata: {name: lab-charts, namespace: flux-system}
spec: {url: "$REPO_URL", interval: 1m}
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata: {name: inbuxa, namespace: flux-system}
spec:
interval: 1m
suspend: false
releaseName: inbuxa
targetNamespace: inbuxa
storageNamespace: inbuxa
chart:
spec:
chart: inbuxa
version: "$version"
sourceRef: {kind: HelmRepository, name: lab-charts}
valuesFiles: [values.yaml, lab/values-lab.yaml, lab/values-frontends.yaml]
install: {timeout: 15m, remediation: {retries: 2}}
upgrade: {timeout: 20m, remediation: {retries: 0}}
driftDetection: {mode: enabled}
values: $values
EOF
fi
}
suspend() {
if [ "$1" = argocd ]; then
"${K[@]}" -n argocd patch application inbuxa --type json \
-p '[{"op": "remove", "path": "/spec/syncPolicy/automated"}]' >/dev/null
else
"${K[@]}" -n flux-system patch helmrelease inbuxa --type merge -p '{"spec": {"suspend": true}}' >/dev/null
fi
}
# One JSON object: what the controller says about the current spec.
# ok: in and healthy; failed: it gave up on it; op: the last sync
# (Argo CD) or the spec generation looked at (Flux), to tell an answer
# about a new spec from the last one; deployed: when the last sync or
# Helm release was, which a steady app doesn't change
state() {
local tool="$1" ns=argocd kind=application
[ "$tool" = flux ] && ns=flux-system kind=helmrelease
"${K[@]}" -n "$ns" get "$kind" inbuxa -o json | python3 -c '
import json, sys
tool, a = sys.argv[1], json.load(sys.stdin)
s, spec = a.get("status", {}), a["spec"]
if tool == "argocd":
op = s.get("operationState", {})
phase = op.get("phase", "")
want = spec["source"]["targetRevision"]
synced = s.get("sync", {}).get("status") == "Synced" and s.get("sync", {}).get("revision") == want
healthy = s.get("health", {}).get("status") == "Healthy"
print(json.dumps({"ok": synced and healthy and phase == "Succeeded", "failed": phase in ("Failed", "Error"),
"op": op.get("startedAt", ""), "deployed": op.get("startedAt", ""), "sync": s.get("sync", {}).get("status"),
"health": s.get("health", {}).get("status"), "phase": phase,
"revision": s.get("sync", {}).get("revision"), "message": op.get("message", "")[:300]}))
else:
cond = {c["type"]: c for c in s.get("conditions", [])}
ready = cond.get("Ready", {})
current = s.get("observedGeneration") == a["metadata"]["generation"]
hist = s.get("history", [])
want = spec["chart"]["spec"]["version"]
# Flux adds +N to the version of a chart it rebuilt with valuesFiles
got = (hist[0].get("chartVersion") or "").split("+")[0] if hist else None
ok = current and ready.get("status") == "True" and got == want
failed = current and ready.get("status") == "False" and (
cond.get("Stalled", {}).get("status") == "True" or ready.get("reason", "").endswith("Failed"))
print(json.dumps({"ok": ok, "failed": failed, "op": "gen:" + str(s.get("observedGeneration")),
"deployed": hist[0].get("lastDeployed", "") if hist else "",
"ready": ready.get("status"), "reason": ready.get("reason"),
"version": got,
"actions": len(hist), "message": ready.get("message", "")[:300]}))
' "$tool"
}
# wait|refused TOOL [SINCE] [TIMEOUT]: SINCE is the op the state had before
# the apply, so the answer is about this spec, not the last one
await() {
local want="$1" tool="$2" since="${3:-}" timeout="${4:-1500}" end s
end=$(( SECONDS + timeout ))
while [ "$SECONDS" -lt "$end" ]; do
s="$(state "$tool" 2>/dev/null || echo '{}')"
if python3 -c '
import json, sys
s, want, since = json.loads(sys.argv[1]), sys.argv[2], sys.argv[3]
new = s.get("op", "") != since or not since
sys.exit(0 if new and s.get(want) else 1)' "$s" "$want" "$since"; then
echo "$s"
return 0
fi
if [ "$want" = failed ] && python3 -c '
import json, sys
s, since = json.loads(sys.argv[1]), sys.argv[2]
sys.exit(0 if s.get("ok") and (s.get("op", "") != since or not since) else 1)' "$s" "$since"; then
echo "$s" >&2
fail "$tool took the spec"
fi
if [ "$want" = ok ] && python3 -c '
import json, sys
s, since = json.loads(sys.argv[1]), sys.argv[2]
sys.exit(0 if s.get("failed") and (s.get("op", "") != since or not since) else 1)' "$s" "$since"; then
echo "$s" >&2
fail "$tool gave up on the spec"
fi
sleep 5
done
echo "$s" >&2
fail "$tool: not $want after ${timeout}s"
}
field() { state "$1" | python3 -c 'import json, sys; print(json.load(sys.stdin)[sys.argv[1]])' "$2"; }
case "${1:-}" in
controller) controller "$2" ;;
repo) repo ;;
apply) apply "$2" ;;
wait) await ok "$2" "${3:-}" "${4:-1500}" ;;
refused) await failed "$2" "${3:-}" "${4:-900}" ;;
suspend) suspend "$2" ;;
state) state "$2" ;;
op) field "$2" op ;;
deployed) field "$2" deployed ;;
*) sed -n '4,33p' "$0"; exit 2 ;;
esac
+10
View File
@@ -7,3 +7,13 @@ KYVERNO_CHART=3.9.2 # Kyverno 1.19.2
GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0
PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17
SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27}
# K-L10: the GitOps controllers, each as its project's install manifest,
# checked by SHA-256 before it's applied. Flux publishes the sum
# (flux_<v>_checksums.txt); Argo CD doesn't, so its sum is the one taken
# when the pin was set.
ARGOCD_VERSION=v3.6.2
ARGOCD_CORE_SHA256=7d39e823fa236666a512bdd294f307360a59f7865e6936fed49fd7b57d348a3f
FLUX_VERSION=v2.10.1
FLUX_INSTALL_SHA256=4db2fb9ce179622327d3a9a36a765bbfc0329fd50690e8d37da7dd0918b09392
# The 2026.x release the GitOps runs cross from (chart/crossing-test.sh)
CROSSING_FROM=2026.11.18.3