Argo CD and Flux: install, upgrade and the crossing through GitOps (K-L10) #38
No files matched your search
@@ -147,3 +147,55 @@ jobs:
|
||||
E2E_LABEL: chaos, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}
|
||||
CHAOS: "1"
|
||||
run: inbuxa-k8s-lab/lab/e2e.sh
|
||||
|
||||
# GitOps (K-L10): the chart installed, upgraded and crossed from 2026.x by
|
||||
# Argo CD and by Flux instead of Helm. The lab's own runs, newest minor,
|
||||
# both architectures.
|
||||
gitops:
|
||||
needs: matrix
|
||||
if: ${{ inputs.chart_ref == '' }}
|
||||
name: gitops (${{ matrix.tool }}, ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
runner: ${{ fromJSON(needs.matrix.outputs.runners) }}
|
||||
tool: [argocd, flux]
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 75
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: inbuxa/inbuxa-k8s-lab
|
||||
ref: ${{ github.sha }}
|
||||
path: inbuxa-k8s-lab
|
||||
persist-credentials: false
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
repository: inbuxa/inbuxa-charts
|
||||
ref: ${{ vars.CHART_REF || 'main' }}
|
||||
path: inbuxa-charts
|
||||
persist-credentials: false
|
||||
- name: Tools (pinned, checksummed)
|
||||
run: |
|
||||
inbuxa-k8s-lab/lab/tools.sh "$HOME/.local/bin"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
- name: Lab end to end through ${{ matrix.tool }}, then its tests and the crossing
|
||||
env:
|
||||
KIND_NODE_IMAGE: ${{ fromJSON(needs.matrix.outputs.newest).image }}
|
||||
E2E_LABEL: ${{ matrix.tool }}, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}
|
||||
INSTALL_WITH: ${{ matrix.tool }}
|
||||
run: inbuxa-k8s-lab/lab/e2e.sh
|
||||
- name: What the controller said
|
||||
if: failure()
|
||||
env:
|
||||
TOOL: ${{ matrix.tool }}
|
||||
run: |
|
||||
if [ "$TOOL" = argocd ]; then
|
||||
kubectl -n argocd get application inbuxa -o yaml | sed -n '/^status:/,$p' | head -150
|
||||
kubectl -n argocd logs statefulset/argocd-application-controller --tail=80 || true
|
||||
else
|
||||
kubectl -n flux-system get helmrelease inbuxa -o yaml | sed -n '/^status:/,$p' | head -150
|
||||
kubectl -n flux-system logs deploy/helm-controller --tail=80 || true
|
||||
fi
|
||||
kubectl -n inbuxa get jobs,pods -o wide || true
|
||||
kubectl -n inbuxa logs job/inbuxa-upgrade --tail=40 || true
|
||||
@@ -47,6 +47,18 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed
|
||||
over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest
|
||||
Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`).
|
||||
|
||||
`INSTALL_WITH=argocd lab/e2e.sh` (or `flux`) installs the chart through
|
||||
Argo CD or Flux instead of Helm, from a Helm repository in the cluster
|
||||
(`lab/gitops/gitops.sh`), then checks what GitOps can get wrong
|
||||
(`lab/gitops/gitops-test.sh`). Nothing is applied again unchanged and no
|
||||
Secret changes across refreshes. The controller puts a hand edit back. An
|
||||
upgrade with every image pinned by digest rolls everything without signing
|
||||
out webmail tabs left open across it. Then the 27.1 crossing from a 2026.x
|
||||
store, as changes to the controller's spec: refused, then acknowledged.
|
||||
Under Argo CD the chart's generated secrets are off and the Secrets are
|
||||
made first; Flux runs Helm, so the chart keeps its own. CI runs both
|
||||
nightly, on the newest minor and both architectures.
|
||||
|
||||
`lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs
|
||||
and checks no request failed and every live-update stream came back.
|
||||
|
||||
|
||||
@@ -17,6 +17,12 @@
|
||||
#
|
||||
# OLD=2026.11.18.3 TO=27.1-rc.34 lab/chart/crossing-test.sh
|
||||
#
|
||||
# GITOPS=argocd or flux (K-L10): the chart is the controller's
|
||||
# (lab/gitops), and steps 3 and 4 are changes to its spec, as commits would
|
||||
# be: the controller is suspended while OLD sets the store up, then given
|
||||
# TO without the acknowledgement (it must give up), then with it. Takes
|
||||
# CHART_VERSION, WEBMAIL and CONSOLE for the rest of the spec.
|
||||
#
|
||||
# WIPES the lab's inbuxa database, and leaves a store fresh-store.sh made.
|
||||
# seed-store.sh and a helm upgrade put the lab back as the other tests want it.
|
||||
set -euo pipefail
|
||||
@@ -40,6 +46,20 @@ mark() { "${K[@]}" -n data exec inbuxa-pg-1 -c postgres -- psql -d inbuxa -At \
|
||||
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
|
||||
mailbox_digest() { ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$(lb)" "$@"; }
|
||||
upgrade() {
|
||||
if [ -n "${GITOPS:-}" ]; then
|
||||
local since ack=""
|
||||
[[ "$*" == *acknowledgeLayout=3* ]] && ack=3
|
||||
since="$(../gitops/gitops.sh op "$GITOPS")"
|
||||
SERVER="$to_d" SEED=false ACK="$ack" ../gitops/gitops.sh apply "$GITOPS" >/dev/null
|
||||
if [ -n "$ack" ]; then
|
||||
../gitops/gitops.sh wait "$GITOPS" "$since" 1800 >/dev/null
|
||||
else
|
||||
# Given up on is what's wanted; anything else counts as went through
|
||||
if ../gitops/gitops.sh refused "$GITOPS" "$since" 900 >/dev/null; then return 1; fi
|
||||
return 0
|
||||
fi
|
||||
return
|
||||
fi
|
||||
"${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml --set server.seed.enabled=false \
|
||||
--set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \
|
||||
--set server.image.digest="$to_d" --wait --timeout 20m "$@"
|
||||
@@ -52,6 +72,11 @@ echo "$OLD = $old_d"
|
||||
echo "$TO = $to_d"
|
||||
|
||||
echo "== 1. a store made by $OLD"
|
||||
if [ -n "${GITOPS:-}" ]; then
|
||||
: "${WEBMAIL:?}" "${CONSOLE:?}"
|
||||
../gitops/gitops.sh suspend "$GITOPS"
|
||||
echo " $GITOPS suspended"
|
||||
fi
|
||||
IMAGE="$IMAGE@$old_d" PULL=IfNotPresent ./fresh-store.sh
|
||||
echo " layout mark: $(mark)"
|
||||
# OLD serving mail, behind the chart's Service and NetworkPolicies. The
|
||||
|
||||
@@ -54,6 +54,9 @@ forward() {
|
||||
stop_forward() { [ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null; FORWARD_PID=""; }
|
||||
trap stop_forward EXIT
|
||||
|
||||
# The one-off pods' config.json (the chart's own pods have theirs)
|
||||
"${K[@]}" apply -f ../spike/server-common.yaml >/dev/null
|
||||
|
||||
echo "== stopping everything on the old store"
|
||||
"${K[@]}" -n inbuxa scale deploy --all --replicas=0 >/dev/null 2>&1 || true
|
||||
"${K[@]}" -n inbuxa wait --for=delete pod -l app.kubernetes.io/component=server --timeout=180s >/dev/null 2>&1 || true
|
||||
|
||||
+47
-13
@@ -10,7 +10,13 @@
|
||||
# JMAP, SMTP out to an outside server, CalDAV, CardDAV, the webmail across
|
||||
# both replicas and the console
|
||||
#
|
||||
# and how long each part took; CHAOS=1 adds the chaos tests after it. CI
|
||||
# and how long each part took; CHAOS=1 adds the chaos tests after it.
|
||||
#
|
||||
# INSTALL_WITH=argocd or flux (K-L10) installs the chart through that
|
||||
# controller instead of Helm (lab/gitops/gitops.sh), then after the client
|
||||
# checks runs lab/gitops/gitops-test.sh (steady, self-heal, an upgrade under
|
||||
# open webmail tabs) and the 27.1 crossing from CROSSING_FROM (versions.env)
|
||||
# as changes to the controller's spec. CI
|
||||
# runs this on a fresh runner; on the bench
|
||||
# it builds the lab from scratch too, so take the lab down first
|
||||
# (lab/down.sh) unless you mean to reuse it.
|
||||
@@ -79,18 +85,31 @@ rm -f "$STATE/s3-access-key"
|
||||
| "${K[@]}" apply -f - >/dev/null
|
||||
umask 022
|
||||
|
||||
echo "== the chart: seeded, two replicas of everything"
|
||||
s=$SECONDS
|
||||
helm --kube-context "kind-$KIND_CLUSTER" -n inbuxa upgrade --install inbuxa "$CHART" \
|
||||
-f "$here/chart/values-lab.yaml" -f "$here/chart/values-frontends.yaml" \
|
||||
--set server.image.repository=ghcr.io/inbuxa/inbuxa-server --set server.image.tag="$SERVER" \
|
||||
--set server.image.pullPolicy=IfNotPresent \
|
||||
--set webmail.image.repository=ghcr.io/inbuxa/inbuxa-webmail --set webmail.image.tag="$WEBMAIL" \
|
||||
--set webmail.image.pullPolicy=IfNotPresent \
|
||||
--set console.image.repository=ghcr.io/inbuxa/inbuxa-admin --set console.image.tag="$CONSOLE" \
|
||||
--set console.image.pullPolicy=IfNotPresent \
|
||||
--wait --timeout 15m >/dev/null
|
||||
"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}'
|
||||
if [ -n "${INSTALL_WITH:-}" ]; then
|
||||
echo "== $INSTALL_WITH, and the chart as a Helm repository in the cluster"
|
||||
s=$SECONDS
|
||||
"$here/gitops/gitops.sh" controller "$INSTALL_WITH"
|
||||
CHART_REPO="$CHART_REPO" "$here/gitops/gitops.sh" repo
|
||||
lap "$INSTALL_WITH and the chart repository" "$s"
|
||||
echo "== the chart through $INSTALL_WITH: seeded, two replicas of everything"
|
||||
s=$SECONDS
|
||||
SERVER="$SERVER" WEBMAIL="$WEBMAIL" CONSOLE="$CONSOLE" "$here/gitops/gitops.sh" apply "$INSTALL_WITH" >/dev/null
|
||||
"$here/gitops/gitops.sh" wait "$INSTALL_WITH" "" 1200 | sed 's/^/ /'
|
||||
"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}'
|
||||
else
|
||||
echo "== the chart: seeded, two replicas of everything"
|
||||
s=$SECONDS
|
||||
helm --kube-context "kind-$KIND_CLUSTER" -n inbuxa upgrade --install inbuxa "$CHART" \
|
||||
-f "$here/chart/values-lab.yaml" -f "$here/chart/values-frontends.yaml" \
|
||||
--set server.image.repository=ghcr.io/inbuxa/inbuxa-server --set server.image.tag="$SERVER" \
|
||||
--set server.image.pullPolicy=IfNotPresent \
|
||||
--set webmail.image.repository=ghcr.io/inbuxa/inbuxa-webmail --set webmail.image.tag="$WEBMAIL" \
|
||||
--set webmail.image.pullPolicy=IfNotPresent \
|
||||
--set console.image.repository=ghcr.io/inbuxa/inbuxa-admin --set console.image.tag="$CONSOLE" \
|
||||
--set console.image.pullPolicy=IfNotPresent \
|
||||
--wait --timeout 15m >/dev/null
|
||||
"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}'
|
||||
fi
|
||||
lap "chart install (seed, server, webmail, console)" "$s"
|
||||
|
||||
echo "== the client checks (chart/mailcheck.sh)"
|
||||
@@ -106,5 +125,20 @@ if [ -n "${CHAOS:-}" ]; then
|
||||
lap "chaos: pod kill, drain, PostgreSQL failover, NATS restart, S3 outage" "$s"
|
||||
fi
|
||||
|
||||
# INSTALL_WITH: then the controller's own tests, and the crossing
|
||||
if [ -n "${INSTALL_WITH:-}" ]; then
|
||||
echo "== $INSTALL_WITH (gitops/gitops-test.sh)"
|
||||
s=$SECONDS
|
||||
SERVER="$SERVER" WEBMAIL="$WEBMAIL" CONSOLE="$CONSOLE" STATE="$STATE" \
|
||||
"$here/gitops/gitops-test.sh" "$INSTALL_WITH" 3>"$STATE/gitops.env"
|
||||
lap "$INSTALL_WITH: steady, self-heal, upgrade under open tabs" "$s"
|
||||
echo "== the 27.1 crossing from $CROSSING_FROM through $INSTALL_WITH (chart/crossing-test.sh)"
|
||||
s=$SECONDS
|
||||
# shellcheck disable=SC2046
|
||||
env $(cat "$STATE/gitops.env") GITOPS="$INSTALL_WITH" OLD="$CROSSING_FROM" TO="$SERVER" STATE="$STATE" \
|
||||
CHART="$CHART" "$here/chart/crossing-test.sh"
|
||||
lap "$INSTALL_WITH: the crossing from $CROSSING_FROM, refused, then acknowledged" "$s"
|
||||
fi
|
||||
|
||||
lap "**total**"
|
||||
echo "== passed in $(( SECONDS - t0 ))s"
|
||||
Executable
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-L10: the chart under a GitOps controller, after it installed it
|
||||
# (INSTALL_WITH=TOOL lab/e2e.sh does both). TOOL is argocd or flux.
|
||||
#
|
||||
# 1. steady: the app stays in sync across refreshes; nothing is re-applied
|
||||
# and no Secret changes (a chart that renders differently each time
|
||||
# never settles under Argo CD)
|
||||
# 2. self-heal: the webmail scaled down by hand comes back to two
|
||||
# 3. an upgrade, 27.1.0-lab.1 to -lab.2 with every image pinned by digest
|
||||
# as a chart release pins them: everything rolls, the upgrade hook
|
||||
# runs, and webmail tabs open across it are never signed out
|
||||
# 4. after it: the same Secrets, mail works, and steady again
|
||||
#
|
||||
# lab/gitops/gitops-test.sh TOOL
|
||||
#
|
||||
# Wants SERVER, WEBMAIL and CONSOLE, the tags the install used. Leaves the
|
||||
# app on -lab.2 with the digests, and prints them as KEY=value lines on fd 3
|
||||
# when it's open (e2e.sh passes them to the crossing).
|
||||
set -euo pipefail
|
||||
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
. "$here/../versions.env"
|
||||
TOOL="${1:?argocd or flux}"
|
||||
K=(kubectl --context "kind-$KIND_CLUSTER")
|
||||
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
|
||||
G=("$here/gitops.sh")
|
||||
: "${SERVER:?}" "${WEBMAIL:?}" "${CONSOLE:?}"
|
||||
WORK="$(mktemp -d)"
|
||||
trap '"${K[@]}" -n lab-sink delete pod gitops-tabs --ignore-not-found --wait=false >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT
|
||||
fail() { echo "!! $*" >&2; exit 1; }
|
||||
|
||||
# Every Secret in the namespace by name, uid and a hash of its data; Helm's
|
||||
# own release records change on every upgrade by design, so they're left out
|
||||
secrets() {
|
||||
"${K[@]}" -n inbuxa get secrets -o json | python3 -c '
|
||||
import hashlib, json, sys
|
||||
for s in sorted(json.load(sys.stdin)["items"], key=lambda s: s["metadata"]["name"]):
|
||||
if s.get("type") == "helm.sh/release.v1":
|
||||
continue
|
||||
h = hashlib.sha256(json.dumps(s.get("data", {}), sort_keys=True).encode()).hexdigest()[:16]
|
||||
print(s["metadata"]["name"], s["metadata"]["uid"], h)'
|
||||
}
|
||||
refresh() {
|
||||
if [ "$TOOL" = argocd ]; then
|
||||
"${K[@]}" -n argocd annotate application inbuxa argocd.argoproj.io/refresh=hard --overwrite >/dev/null
|
||||
else
|
||||
"${K[@]}" -n flux-system annotate helmrelease inbuxa reconcile.fluxcd.io/requestedAt="$(date +%s%N)" --overwrite >/dev/null
|
||||
fi
|
||||
}
|
||||
steady() {
|
||||
local before after
|
||||
before="$("${G[@]}" deployed "$TOOL")"
|
||||
for i in 1 2 3; do
|
||||
refresh
|
||||
sleep 30
|
||||
"${G[@]}" state "$TOOL" > "$WORK/state"
|
||||
python3 -c 'import json, sys; sys.exit(0 if json.load(open(sys.argv[1]))["ok"] else 1)' "$WORK/state" \
|
||||
|| { cat "$WORK/state" >&2; fail "out of sync or unhealthy on refresh $i"; }
|
||||
done
|
||||
after="$("${G[@]}" deployed "$TOOL")"
|
||||
[ "$before" = "$after" ] || fail "$TOOL applied the chart again with nothing changed ($before, then $after)"
|
||||
secrets > "$WORK/secrets.now"
|
||||
diff "$WORK/secrets" "$WORK/secrets.now" >/dev/null ||
|
||||
{ diff "$WORK/secrets" "$WORK/secrets.now" | sed 's/^/ /'; fail "a Secret changed"; }
|
||||
}
|
||||
digest_of() { docker buildx imagetools inspect "ghcr.io/inbuxa/$1:$2" --format '{{json .Manifest.Digest}}' | tr -d '"'; }
|
||||
lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; }
|
||||
|
||||
"${G[@]}" wait "$TOOL" "" 600 >/dev/null
|
||||
secrets > "$WORK/secrets"
|
||||
grep -q '^inbuxa-webmail' "$WORK/secrets" || fail "no webmail Secret"
|
||||
echo " $(wc -l < "$WORK/secrets") Secrets: $(awk '{print $1}' "$WORK/secrets" | tr '\n' ' ')"
|
||||
|
||||
echo "== 1. steady: three refreshes"
|
||||
steady
|
||||
echo " in sync each time; nothing re-applied; every Secret as it was"
|
||||
|
||||
echo "== 2. self-heal: the webmail scaled to 1 by hand"
|
||||
"${K[@]}" -n inbuxa scale deploy/inbuxa-webmail --replicas=1 >/dev/null
|
||||
for _ in $(seq 1 60); do
|
||||
[ "$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.replicas}')" = 2 ] && break
|
||||
sleep 5
|
||||
done
|
||||
[ "$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.replicas}')" = 2 ] || fail "$TOOL didn't put the webmail back"
|
||||
"${K[@]}" -n inbuxa rollout status deploy/inbuxa-webmail --timeout=180s >/dev/null
|
||||
"${G[@]}" wait "$TOOL" "" 300 >/dev/null
|
||||
echo " back to 2"
|
||||
|
||||
echo "== 3. upgrade to 27.1.0-lab.2, every image by digest, under open webmail tabs"
|
||||
S_D="$(digest_of inbuxa-server "$SERVER")"; W_D="$(digest_of inbuxa-webmail "$WEBMAIL")"; C_D="$(digest_of inbuxa-admin "$CONSOLE")"
|
||||
for d in "$S_D" "$W_D" "$C_D"; do [[ "$d" =~ ^sha256: ]] || fail "couldn't read a digest: $d"; done
|
||||
"${K[@]}" create namespace lab-sink --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" -n lab-sink create configmap webmail-tabs --from-file="$here/../chart/webmail-tabs.py" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" -n lab-sink create secret generic webmail-tabs --from-file=password="$STATE/alice.password" \
|
||||
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" -n lab-sink delete pod gitops-tabs --ignore-not-found --wait >/dev/null
|
||||
"${K[@]}" -n lab-sink apply -f - >/dev/null <<EOF
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: {name: gitops-tabs}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, fsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
|
||||
containers:
|
||||
- name: tabs
|
||||
image: $PG_IMAGE
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [python3, -u, /tabs/webmail-tabs.py, "http://inbuxa-webmail.inbuxa.svc:8080", "http://inbuxa-http.inbuxa.svc:8080"]
|
||||
env:
|
||||
- {name: ACCOUNT, value: [email protected]}
|
||||
- {name: PASSWORD_FILE, value: /secret/password}
|
||||
- {name: STOP_FILE, value: /tmp/stop}
|
||||
- {name: INTERVAL, value: "1"}
|
||||
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
|
||||
volumeMounts:
|
||||
- {name: tabs, mountPath: /tabs}
|
||||
- {name: secret, mountPath: /secret}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- {name: tabs, configMap: {name: webmail-tabs}}
|
||||
- {name: secret, secret: {secretName: webmail-tabs, defaultMode: 0440}}
|
||||
- {name: tmp, emptyDir: {}}
|
||||
EOF
|
||||
"${K[@]}" -n lab-sink wait --for=condition=Ready pod/gitops-tabs --timeout=120s >/dev/null
|
||||
sleep 10
|
||||
[ "$("${K[@]}" -n lab-sink get pod gitops-tabs -o jsonpath='{.status.phase}')" = Running ] ||
|
||||
{ "${K[@]}" -n lab-sink logs gitops-tabs | tail -20 >&2; fail "the tabs stopped before the upgrade"; }
|
||||
|
||||
since="$("${G[@]}" op "$TOOL")"
|
||||
start=$SECONDS
|
||||
CHART_VERSION=27.1.0-lab.2 SERVER="$S_D" WEBMAIL="$W_D" CONSOLE="$C_D" "${G[@]}" apply "$TOOL" | sed 's/^/ /'
|
||||
"${G[@]}" wait "$TOOL" "$since" 1500 > "$WORK/state"
|
||||
for d in front mta core webmail console; do
|
||||
"${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=600s >/dev/null
|
||||
done
|
||||
echo " in after $(( SECONDS - start ))s: $(cat "$WORK/state")"
|
||||
images="$("${K[@]}" -n inbuxa get deploy -o jsonpath='{range .items[*]}{.spec.template.spec.containers[0].image}{"\n"}{end}')"
|
||||
grep -qv '@sha256:' <<<"$images" && { echo "$images" >&2; fail "a Deployment isn't on a digest"; }
|
||||
sleep 20
|
||||
|
||||
"${K[@]}" -n lab-sink exec gitops-tabs -- touch /tmp/stop
|
||||
"${K[@]}" -n lab-sink wait --for=jsonpath='{.status.phase}'=Succeeded pod/gitops-tabs --timeout=120s >/dev/null \
|
||||
|| fail "the tabs didn't finish"
|
||||
python3 - "$("${K[@]}" -n lab-sink logs gitops-tabs | tail -1)" <<'PY'
|
||||
import json, sys
|
||||
r = json.loads(sys.argv[1])
|
||||
print(f" tabs: {r['requests']} requests, {r['failed']} failed, signed in {r['signins']} time(s)")
|
||||
for e in r["errors"][:10]:
|
||||
print(f" {e}")
|
||||
if r["signins"] != 1:
|
||||
print("!! the upgrade signed the tabs out", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
|
||||
echo "== 4. after the upgrade"
|
||||
secrets > "$WORK/secrets.now"
|
||||
diff "$WORK/secrets" "$WORK/secrets.now" >/dev/null ||
|
||||
{ diff "$WORK/secrets" "$WORK/secrets.now" | sed 's/^/ /'; fail "the upgrade changed a Secret"; }
|
||||
echo " every Secret as it was"
|
||||
ACCOUNT=[email protected] PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 "$here/../chart/mailtest.py" "$(lb)"
|
||||
steady
|
||||
echo " steady again"
|
||||
if { true >&3; } 2>/dev/null; then
|
||||
printf 'CHART_VERSION=27.1.0-lab.2\nWEBMAIL=%s\nCONSOLE=%s\n' "$W_D" "$C_D" >&3
|
||||
fi
|
||||
echo "== $TOOL: installed, steady, self-healed and upgraded; no Secret changed, nobody signed out"
|
||||
Executable
+342
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# K-L10: the chart installed and upgraded by a GitOps controller instead of
|
||||
# by Helm. TOOL is argocd or flux.
|
||||
#
|
||||
# lab/gitops/gitops.sh controller TOOL Argo CD (core) or Flux, from the
|
||||
# pinned manifest (versions.env)
|
||||
# lab/gitops/gitops.sh repo the chart as a Helm repository in
|
||||
# the cluster: versions 27.1.0-lab.1
|
||||
# and -lab.2, the same chart
|
||||
# lab/gitops/gitops.sh apply TOOL the Application or HelmRelease, as
|
||||
# the variables below say; prints
|
||||
# what it changed
|
||||
# lab/gitops/gitops.sh wait TOOL [SINCE] until the controller says the
|
||||
# current spec is in and healthy
|
||||
# lab/gitops/gitops.sh refused TOOL [SINCE]
|
||||
# until it says the current spec failed
|
||||
# lab/gitops/gitops.sh suspend TOOL no syncing until the next apply
|
||||
# lab/gitops/gitops.sh state TOOL one line: what the controller says
|
||||
#
|
||||
# apply reads CHART_VERSION (27.1.0-lab.1), SERVER, WEBMAIL and CONSOLE
|
||||
# (each a tag or sha256:...), SEED (true) and ACK (the layout to
|
||||
# acknowledge, if any). Self-heal is on in both: Argo CD's selfHeal, Flux's
|
||||
# drift detection.
|
||||
#
|
||||
# Secrets: under Argo CD the chart renders without the cluster, so
|
||||
# generateSecrets is off and the webmail's and the metrics Secrets are made
|
||||
# here first, as the docs will tell people to. Flux runs Helm itself, so the
|
||||
# chart makes and keeps its own, as under Helm.
|
||||
#
|
||||
# The chart is the checkout's (CHART_REPO, as e2e.sh); the lab's values files
|
||||
# go inside the package (lab/), as values files in a chart repository do.
|
||||
set -euo pipefail
|
||||
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
. "$here/../versions.env"
|
||||
CHART_REPO="${CHART_REPO:-$here/../../../inbuxa-charts}"
|
||||
K=(kubectl --context "kind-$KIND_CLUSTER")
|
||||
REPO_URL=http://charts.lab-charts.svc:8080
|
||||
fail() { echo "!! $*" >&2; exit 1; }
|
||||
|
||||
controller() {
|
||||
local tool="$1" url sum work
|
||||
work="$(mktemp -d)"
|
||||
case "$tool" in
|
||||
argocd)
|
||||
url="https://raw.githubusercontent.com/argoproj/argo-cd/$ARGOCD_VERSION/manifests/core-install.yaml"
|
||||
sum="$ARGOCD_CORE_SHA256" ;;
|
||||
flux)
|
||||
url="https://github.com/fluxcd/flux2/releases/download/$FLUX_VERSION/install.yaml"
|
||||
sum="$FLUX_INSTALL_SHA256" ;;
|
||||
*) fail "argocd or flux" ;;
|
||||
esac
|
||||
curl -fsSL --retry 3 --proto '=https' -o "$work/install.yaml" "$url"
|
||||
echo "$sum $work/install.yaml" | sha256sum -c --quiet - || fail "$url isn't the pinned manifest"
|
||||
if [ "$tool" = argocd ]; then
|
||||
"${K[@]}" create namespace argocd --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
# Server-side: Argo CD's CRDs are too big for the last-applied annotation
|
||||
"${K[@]}" -n argocd apply --server-side --force-conflicts -f "$work/install.yaml" >/dev/null
|
||||
"${K[@]}" -n argocd rollout status deploy/argocd-repo-server --timeout=300s >/dev/null
|
||||
"${K[@]}" -n argocd rollout status statefulset/argocd-application-controller --timeout=300s >/dev/null
|
||||
# The core install has no API server, which is what makes the default
|
||||
# project elsewhere
|
||||
"${K[@]}" -n argocd apply -f - >/dev/null <<'EOF'
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata: {name: default, namespace: argocd}
|
||||
spec:
|
||||
sourceRepos: ["*"]
|
||||
destinations: [{server: "*", namespace: "*"}]
|
||||
clusterResourceWhitelist: [{group: "*", kind: "*"}]
|
||||
EOF
|
||||
echo " Argo CD $ARGOCD_VERSION (core)"
|
||||
else
|
||||
"${K[@]}" apply --server-side --force-conflicts -f "$work/install.yaml" >/dev/null
|
||||
for d in source-controller helm-controller; do
|
||||
"${K[@]}" -n flux-system rollout status "deploy/$d" --timeout=300s >/dev/null
|
||||
done
|
||||
echo " Flux $FLUX_VERSION"
|
||||
fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
|
||||
repo() {
|
||||
local work v
|
||||
work="$(mktemp -d)"
|
||||
mkdir -p "$work/charts"
|
||||
for v in 27.1.0-lab.1 27.1.0-lab.2; do
|
||||
rm -rf "$work/inbuxa" && cp -r "$CHART_REPO/charts/inbuxa" "$work/inbuxa"
|
||||
mkdir -p "$work/inbuxa/lab"
|
||||
cp "$here/../chart/values-lab.yaml" "$here/../chart/values-frontends.yaml" "$work/inbuxa/lab/"
|
||||
helm package "$work/inbuxa" --version "$v" -d "$work/charts" >/dev/null
|
||||
done
|
||||
helm repo index "$work/charts" --url "$REPO_URL"
|
||||
"${K[@]}" create namespace lab-charts --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
|
||||
"${K[@]}" label namespace lab-charts pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null
|
||||
"${K[@]}" -n lab-charts create configmap charts --from-file="$work/charts" --dry-run=client -o yaml \
|
||||
| "${K[@]}" apply --server-side --force-conflicts -f - >/dev/null
|
||||
"${K[@]}" -n lab-charts apply -f - >/dev/null <<EOF
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata: {name: charts}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector: {matchLabels: {app: charts}}
|
||||
template:
|
||||
metadata: {labels: {app: charts}}
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext: {runAsNonRoot: true, runAsUser: 2000, runAsGroup: 2000, seccompProfile: {type: RuntimeDefault}}
|
||||
containers:
|
||||
- name: http
|
||||
image: $PG_IMAGE
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: [python3, -m, http.server, "8080", --directory, /charts]
|
||||
ports: [{name: http, containerPort: 8080}]
|
||||
readinessProbe: {httpGet: {path: /index.yaml, port: http}}
|
||||
resources: {requests: {cpu: 10m, memory: 32Mi}, limits: {memory: 64Mi}}
|
||||
securityContext: {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: [ALL]}}
|
||||
volumeMounts: [{name: charts, mountPath: /charts}]
|
||||
volumes: [{name: charts, configMap: {name: charts}}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata: {name: charts}
|
||||
spec:
|
||||
selector: {app: charts}
|
||||
ports: [{name: http, port: 8080, targetPort: http}]
|
||||
EOF
|
||||
"${K[@]}" -n lab-charts rollout restart deploy/charts >/dev/null
|
||||
"${K[@]}" -n lab-charts rollout status deploy/charts --timeout=180s >/dev/null
|
||||
echo " $REPO_URL: $(cd "$work/charts" && ls *.tgz | tr '\n' ' ')"
|
||||
rm -rf "$work"
|
||||
}
|
||||
|
||||
# Under Argo CD: the Secrets the chart would otherwise make, made once
|
||||
secrets() {
|
||||
local r
|
||||
r() { python3 -c 'import secrets, string, sys; a = string.ascii_letters + string.digits; print("".join(secrets.choice(a) for _ in range(int(sys.argv[1]))), end="")' "$1"; }
|
||||
if ! "${K[@]}" -n inbuxa get secret inbuxa-webmail-keys >/dev/null 2>&1; then
|
||||
"${K[@]}" -n inbuxa create secret generic inbuxa-webmail-keys \
|
||||
--from-literal=app-secret="$(r 48)" --from-literal=oauth-client-secret="$(r 48)" >/dev/null
|
||||
fi
|
||||
if ! "${K[@]}" -n inbuxa get secret inbuxa-metrics-auth >/dev/null 2>&1; then
|
||||
"${K[@]}" -n inbuxa create secret generic inbuxa-metrics-auth \
|
||||
--from-literal=username=prometheus --from-literal=password="$(r 40)" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# The values on top of the chart's and the lab's files, as JSON (which is YAML)
|
||||
overrides() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json, os, sys
|
||||
tool = sys.argv[1]
|
||||
def image(repo, ref):
|
||||
i = {"repository": repo, "pullPolicy": "IfNotPresent"}
|
||||
if ref.startswith("sha256:"):
|
||||
i.update(digest=ref, tag="")
|
||||
else:
|
||||
i.update(tag=ref, digest="")
|
||||
return i
|
||||
v = {
|
||||
"server": {
|
||||
"image": image("ghcr.io/inbuxa/inbuxa-server", os.environ["SERVER"]),
|
||||
"seed": {"enabled": os.environ.get("SEED", "true") == "true"},
|
||||
"metrics": {"prometheus": {"enabled": True}},
|
||||
},
|
||||
"webmail": {"image": image("ghcr.io/inbuxa/inbuxa-webmail", os.environ["WEBMAIL"])},
|
||||
"console": {"image": image("ghcr.io/inbuxa/inbuxa-admin", os.environ["CONSOLE"])},
|
||||
}
|
||||
if os.environ.get("ACK"):
|
||||
v["server"]["upgrade"] = {"acknowledgeLayout": int(os.environ["ACK"])}
|
||||
if tool == "argocd":
|
||||
v["generateSecrets"] = False
|
||||
v["webmail"]["existingSecret"] = "inbuxa-webmail-keys"
|
||||
v["server"]["metrics"]["prometheus"]["existingSecret"] = "inbuxa-metrics-auth"
|
||||
print(json.dumps(v))
|
||||
PY
|
||||
}
|
||||
|
||||
apply() {
|
||||
local tool="$1" version="${CHART_VERSION:-27.1.0-lab.1}" values
|
||||
: "${SERVER:?}" "${WEBMAIL:?}" "${CONSOLE:?}"
|
||||
values="$(overrides "$tool")"
|
||||
if [ "$tool" = argocd ]; then
|
||||
secrets
|
||||
# The finalizer: deleting the Application deletes what it made
|
||||
"${K[@]}" apply -f - <<EOF
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: inbuxa
|
||||
namespace: argocd
|
||||
finalizers: [resources-finalizer.argocd.argoproj.io]
|
||||
spec:
|
||||
project: default
|
||||
destination: {server: "https://kubernetes.default.svc", namespace: inbuxa}
|
||||
source:
|
||||
repoURL: $REPO_URL
|
||||
chart: inbuxa
|
||||
targetRevision: $version
|
||||
helm:
|
||||
releaseName: inbuxa
|
||||
valueFiles: [lab/values-lab.yaml, lab/values-frontends.yaml]
|
||||
valuesObject: $values
|
||||
syncPolicy:
|
||||
automated: {prune: true, selfHeal: true}
|
||||
retry: {limit: 2, backoff: {duration: 10s, factor: 2, maxDuration: 1m}}
|
||||
EOF
|
||||
else
|
||||
# A failed upgrade isn't rolled back: a refused layout change has
|
||||
# nothing to undo, and a rollback would run the same check (pre-rollback)
|
||||
"${K[@]}" apply -f - <<EOF
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: HelmRepository
|
||||
metadata: {name: lab-charts, namespace: flux-system}
|
||||
spec: {url: "$REPO_URL", interval: 1m}
|
||||
---
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata: {name: inbuxa, namespace: flux-system}
|
||||
spec:
|
||||
interval: 1m
|
||||
suspend: false
|
||||
releaseName: inbuxa
|
||||
targetNamespace: inbuxa
|
||||
storageNamespace: inbuxa
|
||||
chart:
|
||||
spec:
|
||||
chart: inbuxa
|
||||
version: "$version"
|
||||
sourceRef: {kind: HelmRepository, name: lab-charts}
|
||||
valuesFiles: [values.yaml, lab/values-lab.yaml, lab/values-frontends.yaml]
|
||||
install: {timeout: 15m, remediation: {retries: 2}}
|
||||
upgrade: {timeout: 20m, remediation: {retries: 0}}
|
||||
driftDetection: {mode: enabled}
|
||||
values: $values
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
suspend() {
|
||||
if [ "$1" = argocd ]; then
|
||||
"${K[@]}" -n argocd patch application inbuxa --type json \
|
||||
-p '[{"op": "remove", "path": "/spec/syncPolicy/automated"}]' >/dev/null
|
||||
else
|
||||
"${K[@]}" -n flux-system patch helmrelease inbuxa --type merge -p '{"spec": {"suspend": true}}' >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# One JSON object: what the controller says about the current spec.
|
||||
# ok: in and healthy; failed: it gave up on it; op: the last sync
|
||||
# (Argo CD) or the spec generation looked at (Flux), to tell an answer
|
||||
# about a new spec from the last one; deployed: when the last sync or
|
||||
# Helm release was, which a steady app doesn't change
|
||||
state() {
|
||||
local tool="$1" ns=argocd kind=application
|
||||
[ "$tool" = flux ] && ns=flux-system kind=helmrelease
|
||||
"${K[@]}" -n "$ns" get "$kind" inbuxa -o json | python3 -c '
|
||||
import json, sys
|
||||
tool, a = sys.argv[1], json.load(sys.stdin)
|
||||
s, spec = a.get("status", {}), a["spec"]
|
||||
if tool == "argocd":
|
||||
op = s.get("operationState", {})
|
||||
phase = op.get("phase", "")
|
||||
want = spec["source"]["targetRevision"]
|
||||
synced = s.get("sync", {}).get("status") == "Synced" and s.get("sync", {}).get("revision") == want
|
||||
healthy = s.get("health", {}).get("status") == "Healthy"
|
||||
print(json.dumps({"ok": synced and healthy and phase == "Succeeded", "failed": phase in ("Failed", "Error"),
|
||||
"op": op.get("startedAt", ""), "deployed": op.get("startedAt", ""), "sync": s.get("sync", {}).get("status"),
|
||||
"health": s.get("health", {}).get("status"), "phase": phase,
|
||||
"revision": s.get("sync", {}).get("revision"), "message": op.get("message", "")[:300]}))
|
||||
else:
|
||||
cond = {c["type"]: c for c in s.get("conditions", [])}
|
||||
ready = cond.get("Ready", {})
|
||||
current = s.get("observedGeneration") == a["metadata"]["generation"]
|
||||
hist = s.get("history", [])
|
||||
want = spec["chart"]["spec"]["version"]
|
||||
# Flux adds +N to the version of a chart it rebuilt with valuesFiles
|
||||
got = (hist[0].get("chartVersion") or "").split("+")[0] if hist else None
|
||||
ok = current and ready.get("status") == "True" and got == want
|
||||
failed = current and ready.get("status") == "False" and (
|
||||
cond.get("Stalled", {}).get("status") == "True" or ready.get("reason", "").endswith("Failed"))
|
||||
print(json.dumps({"ok": ok, "failed": failed, "op": "gen:" + str(s.get("observedGeneration")),
|
||||
"deployed": hist[0].get("lastDeployed", "") if hist else "",
|
||||
"ready": ready.get("status"), "reason": ready.get("reason"),
|
||||
"version": got,
|
||||
"actions": len(hist), "message": ready.get("message", "")[:300]}))
|
||||
' "$tool"
|
||||
}
|
||||
|
||||
# wait|refused TOOL [SINCE] [TIMEOUT]: SINCE is the op the state had before
|
||||
# the apply, so the answer is about this spec, not the last one
|
||||
await() {
|
||||
local want="$1" tool="$2" since="${3:-}" timeout="${4:-1500}" end s
|
||||
end=$(( SECONDS + timeout ))
|
||||
while [ "$SECONDS" -lt "$end" ]; do
|
||||
s="$(state "$tool" 2>/dev/null || echo '{}')"
|
||||
if python3 -c '
|
||||
import json, sys
|
||||
s, want, since = json.loads(sys.argv[1]), sys.argv[2], sys.argv[3]
|
||||
new = s.get("op", "") != since or not since
|
||||
sys.exit(0 if new and s.get(want) else 1)' "$s" "$want" "$since"; then
|
||||
echo "$s"
|
||||
return 0
|
||||
fi
|
||||
if [ "$want" = failed ] && python3 -c '
|
||||
import json, sys
|
||||
s, since = json.loads(sys.argv[1]), sys.argv[2]
|
||||
sys.exit(0 if s.get("ok") and (s.get("op", "") != since or not since) else 1)' "$s" "$since"; then
|
||||
echo "$s" >&2
|
||||
fail "$tool took the spec"
|
||||
fi
|
||||
if [ "$want" = ok ] && python3 -c '
|
||||
import json, sys
|
||||
s, since = json.loads(sys.argv[1]), sys.argv[2]
|
||||
sys.exit(0 if s.get("failed") and (s.get("op", "") != since or not since) else 1)' "$s" "$since"; then
|
||||
echo "$s" >&2
|
||||
fail "$tool gave up on the spec"
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "$s" >&2
|
||||
fail "$tool: not $want after ${timeout}s"
|
||||
}
|
||||
|
||||
field() { state "$1" | python3 -c 'import json, sys; print(json.load(sys.stdin)[sys.argv[1]])' "$2"; }
|
||||
|
||||
case "${1:-}" in
|
||||
controller) controller "$2" ;;
|
||||
repo) repo ;;
|
||||
apply) apply "$2" ;;
|
||||
wait) await ok "$2" "${3:-}" "${4:-1500}" ;;
|
||||
refused) await failed "$2" "${3:-}" "${4:-900}" ;;
|
||||
suspend) suspend "$2" ;;
|
||||
state) state "$2" ;;
|
||||
op) field "$2" op ;;
|
||||
deployed) field "$2" deployed ;;
|
||||
*) sed -n '4,33p' "$0"; exit 2 ;;
|
||||
esac
|
||||
@@ -7,3 +7,13 @@ KYVERNO_CHART=3.9.2 # Kyverno 1.19.2
|
||||
GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0
|
||||
PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17
|
||||
SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27}
|
||||
# K-L10: the GitOps controllers, each as its project's install manifest,
|
||||
# checked by SHA-256 before it's applied. Flux publishes the sum
|
||||
# (flux_<v>_checksums.txt); Argo CD doesn't, so its sum is the one taken
|
||||
# when the pin was set.
|
||||
ARGOCD_VERSION=v3.6.2
|
||||
ARGOCD_CORE_SHA256=7d39e823fa236666a512bdd294f307360a59f7865e6936fed49fd7b57d348a3f
|
||||
FLUX_VERSION=v2.10.1
|
||||
FLUX_INSTALL_SHA256=4db2fb9ce179622327d3a9a36a765bbfc0329fd50690e8d37da7dd0918b09392
|
||||
# The 2026.x release the GitOps runs cross from (chart/crossing-test.sh)
|
||||
CROSSING_FROM=2026.11.18.3
|
||||
Reference in new issue
Block a user