diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 42ef104..eea11d4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -147,3 +147,55 @@ jobs: E2E_LABEL: chaos, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }} CHAOS: "1" run: inbuxa-k8s-lab/lab/e2e.sh + + # GitOps (K-L10): the chart installed, upgraded and crossed from 2026.x by + # Argo CD and by Flux instead of Helm. The lab's own runs, newest minor, + # both architectures. + gitops: + needs: matrix + if: ${{ inputs.chart_ref == '' }} + name: gitops (${{ matrix.tool }}, ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }}) + strategy: + fail-fast: false + matrix: + runner: ${{ fromJSON(needs.matrix.outputs.runners) }} + tool: [argocd, flux] + runs-on: ${{ matrix.runner }} + timeout-minutes: 75 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: inbuxa/inbuxa-k8s-lab + ref: ${{ github.sha }} + path: inbuxa-k8s-lab + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: inbuxa/inbuxa-charts + ref: ${{ vars.CHART_REF || 'main' }} + path: inbuxa-charts + persist-credentials: false + - name: Tools (pinned, checksummed) + run: | + inbuxa-k8s-lab/lab/tools.sh "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + - name: Lab end to end through ${{ matrix.tool }}, then its tests and the crossing + env: + KIND_NODE_IMAGE: ${{ fromJSON(needs.matrix.outputs.newest).image }} + E2E_LABEL: ${{ matrix.tool }}, Kubernetes ${{ fromJSON(needs.matrix.outputs.newest).minor }}, ${{ matrix.runner }} + INSTALL_WITH: ${{ matrix.tool }} + run: inbuxa-k8s-lab/lab/e2e.sh + - name: What the controller said + if: failure() + env: + TOOL: ${{ matrix.tool }} + run: | + if [ "$TOOL" = argocd ]; then + kubectl -n argocd get application inbuxa -o yaml | sed -n '/^status:/,$p' | head -150 + kubectl -n argocd logs statefulset/argocd-application-controller --tail=80 || true + else + kubectl -n flux-system get helmrelease inbuxa -o yaml | sed -n '/^status:/,$p' | head -150 + kubectl -n flux-system logs deploy/helm-controller --tail=80 || true + fi + kubectl -n inbuxa get jobs,pods -o wide || true + kubectl -n inbuxa logs job/inbuxa-upgrade --tail=40 || true diff --git a/README.md b/README.md index e8e72a8..ac9935c 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,18 @@ once: a server pod killed, a node drained, the PostgreSQL primary failed over, NATS restarted, S3 frozen for 60 s. CI runs it nightly on the newest Kubernetes minor, both architectures (`CHAOS=1 lab/e2e.sh`). +`INSTALL_WITH=argocd lab/e2e.sh` (or `flux`) installs the chart through +Argo CD or Flux instead of Helm, from a Helm repository in the cluster +(`lab/gitops/gitops.sh`), then checks what GitOps can get wrong +(`lab/gitops/gitops-test.sh`). Nothing is applied again unchanged and no +Secret changes across refreshes. The controller puts a hand edit back. An +upgrade with every image pinned by digest rolls everything without signing +out webmail tabs left open across it. Then the 27.1 crossing from a 2026.x +store, as changes to the controller's spec: refused, then acknowledged. +Under Argo CD the chart's generated secrets are off and the Secrets are +made first; Flux runs Helm, so the chart keeps its own. CI runs both +nightly, on the newest minor and both architectures. + `lab/chart/webmail-restart-test.sh` restarts the webmail under open tabs and checks no request failed and every live-update stream came back. diff --git a/lab/chart/crossing-test.sh b/lab/chart/crossing-test.sh index ce17c84..aeb7c51 100755 --- a/lab/chart/crossing-test.sh +++ b/lab/chart/crossing-test.sh @@ -17,6 +17,12 @@ # # OLD=2026.11.18.3 TO=27.1-rc.34 lab/chart/crossing-test.sh # +# GITOPS=argocd or flux (K-L10): the chart is the controller's +# (lab/gitops), and steps 3 and 4 are changes to its spec, as commits would +# be: the controller is suspended while OLD sets the store up, then given +# TO without the acknowledgement (it must give up), then with it. Takes +# CHART_VERSION, WEBMAIL and CONSOLE for the rest of the spec. +# # WIPES the lab's inbuxa database, and leaves a store fresh-store.sh made. # seed-store.sh and a helm upgrade put the lab back as the other tests want it. set -euo pipefail @@ -40,6 +46,20 @@ mark() { "${K[@]}" -n data exec inbuxa-pg-1 -c postgres -- psql -d inbuxa -At \ lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } mailbox_digest() { ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" python3 mailbox-digest.py "$(lb)" "$@"; } upgrade() { + if [ -n "${GITOPS:-}" ]; then + local since ack="" + [[ "$*" == *acknowledgeLayout=3* ]] && ack=3 + since="$(../gitops/gitops.sh op "$GITOPS")" + SERVER="$to_d" SEED=false ACK="$ack" ../gitops/gitops.sh apply "$GITOPS" >/dev/null + if [ -n "$ack" ]; then + ../gitops/gitops.sh wait "$GITOPS" "$since" 1800 >/dev/null + else + # Given up on is what's wanted; anything else counts as went through + if ../gitops/gitops.sh refused "$GITOPS" "$since" 900 >/dev/null; then return 1; fi + return 0 + fi + return + fi "${H[@]}" upgrade inbuxa "$CHART" -f values-lab.yaml --set server.seed.enabled=false \ --set server.image.repository="$IMAGE" --set server.image.pullPolicy=IfNotPresent \ --set server.image.digest="$to_d" --wait --timeout 20m "$@" @@ -52,6 +72,11 @@ echo "$OLD = $old_d" echo "$TO = $to_d" echo "== 1. a store made by $OLD" +if [ -n "${GITOPS:-}" ]; then + : "${WEBMAIL:?}" "${CONSOLE:?}" + ../gitops/gitops.sh suspend "$GITOPS" + echo " $GITOPS suspended" +fi IMAGE="$IMAGE@$old_d" PULL=IfNotPresent ./fresh-store.sh echo " layout mark: $(mark)" # OLD serving mail, behind the chart's Service and NetworkPolicies. The diff --git a/lab/chart/fresh-store.sh b/lab/chart/fresh-store.sh index d7e8e4a..e6d3732 100755 --- a/lab/chart/fresh-store.sh +++ b/lab/chart/fresh-store.sh @@ -54,6 +54,9 @@ forward() { stop_forward() { [ -n "$FORWARD_PID" ] && kill "$FORWARD_PID" 2>/dev/null; FORWARD_PID=""; } trap stop_forward EXIT +# The one-off pods' config.json (the chart's own pods have theirs) +"${K[@]}" apply -f ../spike/server-common.yaml >/dev/null + echo "== stopping everything on the old store" "${K[@]}" -n inbuxa scale deploy --all --replicas=0 >/dev/null 2>&1 || true "${K[@]}" -n inbuxa wait --for=delete pod -l app.kubernetes.io/component=server --timeout=180s >/dev/null 2>&1 || true diff --git a/lab/e2e.sh b/lab/e2e.sh index 6dfa8e8..07f79f2 100755 --- a/lab/e2e.sh +++ b/lab/e2e.sh @@ -10,7 +10,13 @@ # JMAP, SMTP out to an outside server, CalDAV, CardDAV, the webmail across # both replicas and the console # -# and how long each part took; CHAOS=1 adds the chaos tests after it. CI +# and how long each part took; CHAOS=1 adds the chaos tests after it. +# +# INSTALL_WITH=argocd or flux (K-L10) installs the chart through that +# controller instead of Helm (lab/gitops/gitops.sh), then after the client +# checks runs lab/gitops/gitops-test.sh (steady, self-heal, an upgrade under +# open webmail tabs) and the 27.1 crossing from CROSSING_FROM (versions.env) +# as changes to the controller's spec. CI # runs this on a fresh runner; on the bench # it builds the lab from scratch too, so take the lab down first # (lab/down.sh) unless you mean to reuse it. @@ -79,18 +85,31 @@ rm -f "$STATE/s3-access-key" | "${K[@]}" apply -f - >/dev/null umask 022 -echo "== the chart: seeded, two replicas of everything" -s=$SECONDS -helm --kube-context "kind-$KIND_CLUSTER" -n inbuxa upgrade --install inbuxa "$CHART" \ - -f "$here/chart/values-lab.yaml" -f "$here/chart/values-frontends.yaml" \ - --set server.image.repository=ghcr.io/inbuxa/inbuxa-server --set server.image.tag="$SERVER" \ - --set server.image.pullPolicy=IfNotPresent \ - --set webmail.image.repository=ghcr.io/inbuxa/inbuxa-webmail --set webmail.image.tag="$WEBMAIL" \ - --set webmail.image.pullPolicy=IfNotPresent \ - --set console.image.repository=ghcr.io/inbuxa/inbuxa-admin --set console.image.tag="$CONSOLE" \ - --set console.image.pullPolicy=IfNotPresent \ - --wait --timeout 15m >/dev/null -"${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}' +if [ -n "${INSTALL_WITH:-}" ]; then + echo "== $INSTALL_WITH, and the chart as a Helm repository in the cluster" + s=$SECONDS + "$here/gitops/gitops.sh" controller "$INSTALL_WITH" + CHART_REPO="$CHART_REPO" "$here/gitops/gitops.sh" repo + lap "$INSTALL_WITH and the chart repository" "$s" + echo "== the chart through $INSTALL_WITH: seeded, two replicas of everything" + s=$SECONDS + SERVER="$SERVER" WEBMAIL="$WEBMAIL" CONSOLE="$CONSOLE" "$here/gitops/gitops.sh" apply "$INSTALL_WITH" >/dev/null + "$here/gitops/gitops.sh" wait "$INSTALL_WITH" "" 1200 | sed 's/^/ /' + "${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}' +else + echo "== the chart: seeded, two replicas of everything" + s=$SECONDS + helm --kube-context "kind-$KIND_CLUSTER" -n inbuxa upgrade --install inbuxa "$CHART" \ + -f "$here/chart/values-lab.yaml" -f "$here/chart/values-frontends.yaml" \ + --set server.image.repository=ghcr.io/inbuxa/inbuxa-server --set server.image.tag="$SERVER" \ + --set server.image.pullPolicy=IfNotPresent \ + --set webmail.image.repository=ghcr.io/inbuxa/inbuxa-webmail --set webmail.image.tag="$WEBMAIL" \ + --set webmail.image.pullPolicy=IfNotPresent \ + --set console.image.repository=ghcr.io/inbuxa/inbuxa-admin --set console.image.tag="$CONSOLE" \ + --set console.image.pullPolicy=IfNotPresent \ + --wait --timeout 15m >/dev/null + "${K[@]}" get pods --no-headers -o wide | awk '{print " " $1, $3, $7}' +fi lap "chart install (seed, server, webmail, console)" "$s" echo "== the client checks (chart/mailcheck.sh)" @@ -106,5 +125,20 @@ if [ -n "${CHAOS:-}" ]; then lap "chaos: pod kill, drain, PostgreSQL failover, NATS restart, S3 outage" "$s" fi +# INSTALL_WITH: then the controller's own tests, and the crossing +if [ -n "${INSTALL_WITH:-}" ]; then + echo "== $INSTALL_WITH (gitops/gitops-test.sh)" + s=$SECONDS + SERVER="$SERVER" WEBMAIL="$WEBMAIL" CONSOLE="$CONSOLE" STATE="$STATE" \ + "$here/gitops/gitops-test.sh" "$INSTALL_WITH" 3>"$STATE/gitops.env" + lap "$INSTALL_WITH: steady, self-heal, upgrade under open tabs" "$s" + echo "== the 27.1 crossing from $CROSSING_FROM through $INSTALL_WITH (chart/crossing-test.sh)" + s=$SECONDS + # shellcheck disable=SC2046 + env $(cat "$STATE/gitops.env") GITOPS="$INSTALL_WITH" OLD="$CROSSING_FROM" TO="$SERVER" STATE="$STATE" \ + CHART="$CHART" "$here/chart/crossing-test.sh" + lap "$INSTALL_WITH: the crossing from $CROSSING_FROM, refused, then acknowledged" "$s" +fi + lap "**total**" echo "== passed in $(( SECONDS - t0 ))s" diff --git a/lab/gitops/gitops-test.sh b/lab/gitops/gitops-test.sh new file mode 100755 index 0000000..b7a91cf --- /dev/null +++ b/lab/gitops/gitops-test.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L10: the chart under a GitOps controller, after it installed it +# (INSTALL_WITH=TOOL lab/e2e.sh does both). TOOL is argocd or flux. +# +# 1. steady: the app stays in sync across refreshes; nothing is re-applied +# and no Secret changes (a chart that renders differently each time +# never settles under Argo CD) +# 2. self-heal: the webmail scaled down by hand comes back to two +# 3. an upgrade, 27.1.0-lab.1 to -lab.2 with every image pinned by digest +# as a chart release pins them: everything rolls, the upgrade hook +# runs, and webmail tabs open across it are never signed out +# 4. after it: the same Secrets, mail works, and steady again +# +# lab/gitops/gitops-test.sh TOOL +# +# Wants SERVER, WEBMAIL and CONSOLE, the tags the install used. Leaves the +# app on -lab.2 with the digests, and prints them as KEY=value lines on fd 3 +# when it's open (e2e.sh passes them to the crossing). +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +. "$here/../versions.env" +TOOL="${1:?argocd or flux}" +K=(kubectl --context "kind-$KIND_CLUSTER") +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +G=("$here/gitops.sh") +: "${SERVER:?}" "${WEBMAIL:?}" "${CONSOLE:?}" +WORK="$(mktemp -d)" +trap '"${K[@]}" -n lab-sink delete pod gitops-tabs --ignore-not-found --wait=false >/dev/null 2>&1 || true; rm -rf "$WORK"' EXIT +fail() { echo "!! $*" >&2; exit 1; } + +# Every Secret in the namespace by name, uid and a hash of its data; Helm's +# own release records change on every upgrade by design, so they're left out +secrets() { + "${K[@]}" -n inbuxa get secrets -o json | python3 -c ' +import hashlib, json, sys +for s in sorted(json.load(sys.stdin)["items"], key=lambda s: s["metadata"]["name"]): + if s.get("type") == "helm.sh/release.v1": + continue + h = hashlib.sha256(json.dumps(s.get("data", {}), sort_keys=True).encode()).hexdigest()[:16] + print(s["metadata"]["name"], s["metadata"]["uid"], h)' +} +refresh() { + if [ "$TOOL" = argocd ]; then + "${K[@]}" -n argocd annotate application inbuxa argocd.argoproj.io/refresh=hard --overwrite >/dev/null + else + "${K[@]}" -n flux-system annotate helmrelease inbuxa reconcile.fluxcd.io/requestedAt="$(date +%s%N)" --overwrite >/dev/null + fi +} +steady() { + local before after + before="$("${G[@]}" deployed "$TOOL")" + for i in 1 2 3; do + refresh + sleep 30 + "${G[@]}" state "$TOOL" > "$WORK/state" + python3 -c 'import json, sys; sys.exit(0 if json.load(open(sys.argv[1]))["ok"] else 1)' "$WORK/state" \ + || { cat "$WORK/state" >&2; fail "out of sync or unhealthy on refresh $i"; } + done + after="$("${G[@]}" deployed "$TOOL")" + [ "$before" = "$after" ] || fail "$TOOL applied the chart again with nothing changed ($before, then $after)" + secrets > "$WORK/secrets.now" + diff "$WORK/secrets" "$WORK/secrets.now" >/dev/null || + { diff "$WORK/secrets" "$WORK/secrets.now" | sed 's/^/ /'; fail "a Secret changed"; } +} +digest_of() { docker buildx imagetools inspect "ghcr.io/inbuxa/$1:$2" --format '{{json .Manifest.Digest}}' | tr -d '"'; } +lb() { "${K[@]}" -n inbuxa get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}'; } + +"${G[@]}" wait "$TOOL" "" 600 >/dev/null +secrets > "$WORK/secrets" +grep -q '^inbuxa-webmail' "$WORK/secrets" || fail "no webmail Secret" +echo " $(wc -l < "$WORK/secrets") Secrets: $(awk '{print $1}' "$WORK/secrets" | tr '\n' ' ')" + +echo "== 1. steady: three refreshes" +steady +echo " in sync each time; nothing re-applied; every Secret as it was" + +echo "== 2. self-heal: the webmail scaled to 1 by hand" +"${K[@]}" -n inbuxa scale deploy/inbuxa-webmail --replicas=1 >/dev/null +for _ in $(seq 1 60); do + [ "$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.replicas}')" = 2 ] && break + sleep 5 +done +[ "$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.replicas}')" = 2 ] || fail "$TOOL didn't put the webmail back" +"${K[@]}" -n inbuxa rollout status deploy/inbuxa-webmail --timeout=180s >/dev/null +"${G[@]}" wait "$TOOL" "" 300 >/dev/null +echo " back to 2" + +echo "== 3. upgrade to 27.1.0-lab.2, every image by digest, under open webmail tabs" +S_D="$(digest_of inbuxa-server "$SERVER")"; W_D="$(digest_of inbuxa-webmail "$WEBMAIL")"; C_D="$(digest_of inbuxa-admin "$CONSOLE")" +for d in "$S_D" "$W_D" "$C_D"; do [[ "$d" =~ ^sha256: ]] || fail "couldn't read a digest: $d"; done +"${K[@]}" create namespace lab-sink --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${K[@]}" -n lab-sink create configmap webmail-tabs --from-file="$here/../chart/webmail-tabs.py" --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${K[@]}" -n lab-sink create secret generic webmail-tabs --from-file=password="$STATE/alice.password" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null +"${K[@]}" -n lab-sink delete pod gitops-tabs --ignore-not-found --wait >/dev/null +"${K[@]}" -n lab-sink apply -f - >/dev/null </dev/null +sleep 10 +[ "$("${K[@]}" -n lab-sink get pod gitops-tabs -o jsonpath='{.status.phase}')" = Running ] || + { "${K[@]}" -n lab-sink logs gitops-tabs | tail -20 >&2; fail "the tabs stopped before the upgrade"; } + +since="$("${G[@]}" op "$TOOL")" +start=$SECONDS +CHART_VERSION=27.1.0-lab.2 SERVER="$S_D" WEBMAIL="$W_D" CONSOLE="$C_D" "${G[@]}" apply "$TOOL" | sed 's/^/ /' +"${G[@]}" wait "$TOOL" "$since" 1500 > "$WORK/state" +for d in front mta core webmail console; do + "${K[@]}" -n inbuxa rollout status "deploy/inbuxa-$d" --timeout=600s >/dev/null +done +echo " in after $(( SECONDS - start ))s: $(cat "$WORK/state")" +images="$("${K[@]}" -n inbuxa get deploy -o jsonpath='{range .items[*]}{.spec.template.spec.containers[0].image}{"\n"}{end}')" +grep -qv '@sha256:' <<<"$images" && { echo "$images" >&2; fail "a Deployment isn't on a digest"; } +sleep 20 + +"${K[@]}" -n lab-sink exec gitops-tabs -- touch /tmp/stop +"${K[@]}" -n lab-sink wait --for=jsonpath='{.status.phase}'=Succeeded pod/gitops-tabs --timeout=120s >/dev/null \ + || fail "the tabs didn't finish" +python3 - "$("${K[@]}" -n lab-sink logs gitops-tabs | tail -1)" <<'PY' +import json, sys +r = json.loads(sys.argv[1]) +print(f" tabs: {r['requests']} requests, {r['failed']} failed, signed in {r['signins']} time(s)") +for e in r["errors"][:10]: + print(f" {e}") +if r["signins"] != 1: + print("!! the upgrade signed the tabs out", file=sys.stderr) + sys.exit(1) +PY + +echo "== 4. after the upgrade" +secrets > "$WORK/secrets.now" +diff "$WORK/secrets" "$WORK/secrets.now" >/dev/null || + { diff "$WORK/secrets" "$WORK/secrets.now" | sed 's/^/ /'; fail "the upgrade changed a Secret"; } +echo " every Secret as it was" +ACCOUNT=alice@lab.test PASSWORD_FILE="$STATE/alice.password" timeout 150 python3 "$here/../chart/mailtest.py" "$(lb)" +steady +echo " steady again" +if { true >&3; } 2>/dev/null; then + printf 'CHART_VERSION=27.1.0-lab.2\nWEBMAIL=%s\nCONSOLE=%s\n' "$W_D" "$C_D" >&3 +fi +echo "== $TOOL: installed, steady, self-healed and upgraded; no Secret changed, nobody signed out" diff --git a/lab/gitops/gitops.sh b/lab/gitops/gitops.sh new file mode 100755 index 0000000..8c2e4b7 --- /dev/null +++ b/lab/gitops/gitops.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-L10: the chart installed and upgraded by a GitOps controller instead of +# by Helm. TOOL is argocd or flux. +# +# lab/gitops/gitops.sh controller TOOL Argo CD (core) or Flux, from the +# pinned manifest (versions.env) +# lab/gitops/gitops.sh repo the chart as a Helm repository in +# the cluster: versions 27.1.0-lab.1 +# and -lab.2, the same chart +# lab/gitops/gitops.sh apply TOOL the Application or HelmRelease, as +# the variables below say; prints +# what it changed +# lab/gitops/gitops.sh wait TOOL [SINCE] until the controller says the +# current spec is in and healthy +# lab/gitops/gitops.sh refused TOOL [SINCE] +# until it says the current spec failed +# lab/gitops/gitops.sh suspend TOOL no syncing until the next apply +# lab/gitops/gitops.sh state TOOL one line: what the controller says +# +# apply reads CHART_VERSION (27.1.0-lab.1), SERVER, WEBMAIL and CONSOLE +# (each a tag or sha256:...), SEED (true) and ACK (the layout to +# acknowledge, if any). Self-heal is on in both: Argo CD's selfHeal, Flux's +# drift detection. +# +# Secrets: under Argo CD the chart renders without the cluster, so +# generateSecrets is off and the webmail's and the metrics Secrets are made +# here first, as the docs will tell people to. Flux runs Helm itself, so the +# chart makes and keeps its own, as under Helm. +# +# The chart is the checkout's (CHART_REPO, as e2e.sh); the lab's values files +# go inside the package (lab/), as values files in a chart repository do. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +. "$here/../versions.env" +CHART_REPO="${CHART_REPO:-$here/../../../inbuxa-charts}" +K=(kubectl --context "kind-$KIND_CLUSTER") +REPO_URL=http://charts.lab-charts.svc:8080 +fail() { echo "!! $*" >&2; exit 1; } + +controller() { + local tool="$1" url sum work + work="$(mktemp -d)" + case "$tool" in + argocd) + url="https://raw.githubusercontent.com/argoproj/argo-cd/$ARGOCD_VERSION/manifests/core-install.yaml" + sum="$ARGOCD_CORE_SHA256" ;; + flux) + url="https://github.com/fluxcd/flux2/releases/download/$FLUX_VERSION/install.yaml" + sum="$FLUX_INSTALL_SHA256" ;; + *) fail "argocd or flux" ;; + esac + curl -fsSL --retry 3 --proto '=https' -o "$work/install.yaml" "$url" + echo "$sum $work/install.yaml" | sha256sum -c --quiet - || fail "$url isn't the pinned manifest" + if [ "$tool" = argocd ]; then + "${K[@]}" create namespace argocd --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + # Server-side: Argo CD's CRDs are too big for the last-applied annotation + "${K[@]}" -n argocd apply --server-side --force-conflicts -f "$work/install.yaml" >/dev/null + "${K[@]}" -n argocd rollout status deploy/argocd-repo-server --timeout=300s >/dev/null + "${K[@]}" -n argocd rollout status statefulset/argocd-application-controller --timeout=300s >/dev/null + # The core install has no API server, which is what makes the default + # project elsewhere + "${K[@]}" -n argocd apply -f - >/dev/null <<'EOF' +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: {name: default, namespace: argocd} +spec: + sourceRepos: ["*"] + destinations: [{server: "*", namespace: "*"}] + clusterResourceWhitelist: [{group: "*", kind: "*"}] +EOF + echo " Argo CD $ARGOCD_VERSION (core)" + else + "${K[@]}" apply --server-side --force-conflicts -f "$work/install.yaml" >/dev/null + for d in source-controller helm-controller; do + "${K[@]}" -n flux-system rollout status "deploy/$d" --timeout=300s >/dev/null + done + echo " Flux $FLUX_VERSION" + fi + rm -rf "$work" +} + +repo() { + local work v + work="$(mktemp -d)" + mkdir -p "$work/charts" + for v in 27.1.0-lab.1 27.1.0-lab.2; do + rm -rf "$work/inbuxa" && cp -r "$CHART_REPO/charts/inbuxa" "$work/inbuxa" + mkdir -p "$work/inbuxa/lab" + cp "$here/../chart/values-lab.yaml" "$here/../chart/values-frontends.yaml" "$work/inbuxa/lab/" + helm package "$work/inbuxa" --version "$v" -d "$work/charts" >/dev/null + done + helm repo index "$work/charts" --url "$REPO_URL" + "${K[@]}" create namespace lab-charts --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + "${K[@]}" label namespace lab-charts pod-security.kubernetes.io/enforce=restricted --overwrite >/dev/null + "${K[@]}" -n lab-charts create configmap charts --from-file="$work/charts" --dry-run=client -o yaml \ + | "${K[@]}" apply --server-side --force-conflicts -f - >/dev/null + "${K[@]}" -n lab-charts apply -f - >/dev/null </dev/null + "${K[@]}" -n lab-charts rollout status deploy/charts --timeout=180s >/dev/null + echo " $REPO_URL: $(cd "$work/charts" && ls *.tgz | tr '\n' ' ')" + rm -rf "$work" +} + +# Under Argo CD: the Secrets the chart would otherwise make, made once +secrets() { + local r + r() { python3 -c 'import secrets, string, sys; a = string.ascii_letters + string.digits; print("".join(secrets.choice(a) for _ in range(int(sys.argv[1]))), end="")' "$1"; } + if ! "${K[@]}" -n inbuxa get secret inbuxa-webmail-keys >/dev/null 2>&1; then + "${K[@]}" -n inbuxa create secret generic inbuxa-webmail-keys \ + --from-literal=app-secret="$(r 48)" --from-literal=oauth-client-secret="$(r 48)" >/dev/null + fi + if ! "${K[@]}" -n inbuxa get secret inbuxa-metrics-auth >/dev/null 2>&1; then + "${K[@]}" -n inbuxa create secret generic inbuxa-metrics-auth \ + --from-literal=username=prometheus --from-literal=password="$(r 40)" >/dev/null + fi +} + +# The values on top of the chart's and the lab's files, as JSON (which is YAML) +overrides() { + python3 - "$1" <<'PY' +import json, os, sys +tool = sys.argv[1] +def image(repo, ref): + i = {"repository": repo, "pullPolicy": "IfNotPresent"} + if ref.startswith("sha256:"): + i.update(digest=ref, tag="") + else: + i.update(tag=ref, digest="") + return i +v = { + "server": { + "image": image("ghcr.io/inbuxa/inbuxa-server", os.environ["SERVER"]), + "seed": {"enabled": os.environ.get("SEED", "true") == "true"}, + "metrics": {"prometheus": {"enabled": True}}, + }, + "webmail": {"image": image("ghcr.io/inbuxa/inbuxa-webmail", os.environ["WEBMAIL"])}, + "console": {"image": image("ghcr.io/inbuxa/inbuxa-admin", os.environ["CONSOLE"])}, +} +if os.environ.get("ACK"): + v["server"]["upgrade"] = {"acknowledgeLayout": int(os.environ["ACK"])} +if tool == "argocd": + v["generateSecrets"] = False + v["webmail"]["existingSecret"] = "inbuxa-webmail-keys" + v["server"]["metrics"]["prometheus"]["existingSecret"] = "inbuxa-metrics-auth" +print(json.dumps(v)) +PY +} + +apply() { + local tool="$1" version="${CHART_VERSION:-27.1.0-lab.1}" values + : "${SERVER:?}" "${WEBMAIL:?}" "${CONSOLE:?}" + values="$(overrides "$tool")" + if [ "$tool" = argocd ]; then + secrets + # The finalizer: deleting the Application deletes what it made + "${K[@]}" apply -f - </dev/null + else + "${K[@]}" -n flux-system patch helmrelease inbuxa --type merge -p '{"spec": {"suspend": true}}' >/dev/null + fi +} + +# One JSON object: what the controller says about the current spec. +# ok: in and healthy; failed: it gave up on it; op: the last sync +# (Argo CD) or the spec generation looked at (Flux), to tell an answer +# about a new spec from the last one; deployed: when the last sync or +# Helm release was, which a steady app doesn't change +state() { + local tool="$1" ns=argocd kind=application + [ "$tool" = flux ] && ns=flux-system kind=helmrelease + "${K[@]}" -n "$ns" get "$kind" inbuxa -o json | python3 -c ' +import json, sys +tool, a = sys.argv[1], json.load(sys.stdin) +s, spec = a.get("status", {}), a["spec"] +if tool == "argocd": + op = s.get("operationState", {}) + phase = op.get("phase", "") + want = spec["source"]["targetRevision"] + synced = s.get("sync", {}).get("status") == "Synced" and s.get("sync", {}).get("revision") == want + healthy = s.get("health", {}).get("status") == "Healthy" + print(json.dumps({"ok": synced and healthy and phase == "Succeeded", "failed": phase in ("Failed", "Error"), + "op": op.get("startedAt", ""), "deployed": op.get("startedAt", ""), "sync": s.get("sync", {}).get("status"), + "health": s.get("health", {}).get("status"), "phase": phase, + "revision": s.get("sync", {}).get("revision"), "message": op.get("message", "")[:300]})) +else: + cond = {c["type"]: c for c in s.get("conditions", [])} + ready = cond.get("Ready", {}) + current = s.get("observedGeneration") == a["metadata"]["generation"] + hist = s.get("history", []) + want = spec["chart"]["spec"]["version"] + # Flux adds +N to the version of a chart it rebuilt with valuesFiles + got = (hist[0].get("chartVersion") or "").split("+")[0] if hist else None + ok = current and ready.get("status") == "True" and got == want + failed = current and ready.get("status") == "False" and ( + cond.get("Stalled", {}).get("status") == "True" or ready.get("reason", "").endswith("Failed")) + print(json.dumps({"ok": ok, "failed": failed, "op": "gen:" + str(s.get("observedGeneration")), + "deployed": hist[0].get("lastDeployed", "") if hist else "", + "ready": ready.get("status"), "reason": ready.get("reason"), + "version": got, + "actions": len(hist), "message": ready.get("message", "")[:300]})) +' "$tool" +} + +# wait|refused TOOL [SINCE] [TIMEOUT]: SINCE is the op the state had before +# the apply, so the answer is about this spec, not the last one +await() { + local want="$1" tool="$2" since="${3:-}" timeout="${4:-1500}" end s + end=$(( SECONDS + timeout )) + while [ "$SECONDS" -lt "$end" ]; do + s="$(state "$tool" 2>/dev/null || echo '{}')" + if python3 -c ' +import json, sys +s, want, since = json.loads(sys.argv[1]), sys.argv[2], sys.argv[3] +new = s.get("op", "") != since or not since +sys.exit(0 if new and s.get(want) else 1)' "$s" "$want" "$since"; then + echo "$s" + return 0 + fi + if [ "$want" = failed ] && python3 -c ' +import json, sys +s, since = json.loads(sys.argv[1]), sys.argv[2] +sys.exit(0 if s.get("ok") and (s.get("op", "") != since or not since) else 1)' "$s" "$since"; then + echo "$s" >&2 + fail "$tool took the spec" + fi + if [ "$want" = ok ] && python3 -c ' +import json, sys +s, since = json.loads(sys.argv[1]), sys.argv[2] +sys.exit(0 if s.get("failed") and (s.get("op", "") != since or not since) else 1)' "$s" "$since"; then + echo "$s" >&2 + fail "$tool gave up on the spec" + fi + sleep 5 + done + echo "$s" >&2 + fail "$tool: not $want after ${timeout}s" +} + +field() { state "$1" | python3 -c 'import json, sys; print(json.load(sys.stdin)[sys.argv[1]])' "$2"; } + +case "${1:-}" in + controller) controller "$2" ;; + repo) repo ;; + apply) apply "$2" ;; + wait) await ok "$2" "${3:-}" "${4:-1500}" ;; + refused) await failed "$2" "${3:-}" "${4:-900}" ;; + suspend) suspend "$2" ;; + state) state "$2" ;; + op) field "$2" op ;; + deployed) field "$2" deployed ;; + *) sed -n '4,33p' "$0"; exit 2 ;; +esac diff --git a/lab/versions.env b/lab/versions.env index 37711a5..6aaeb02 100644 --- a/lab/versions.env +++ b/lab/versions.env @@ -7,3 +7,13 @@ KYVERNO_CHART=3.9.2 # Kyverno 1.19.2 GARAGE_IMAGE=docker.io/dxflrs/garage:v2.5.0 PG_IMAGE=ghcr.io/cloudnative-pg/postgresql:17 SERVER_IMAGE=${SERVER_IMAGE:-ghcr.io/inbuxa/inbuxa-server:27.1-rc.27} +# K-L10: the GitOps controllers, each as its project's install manifest, +# checked by SHA-256 before it's applied. Flux publishes the sum +# (flux__checksums.txt); Argo CD doesn't, so its sum is the one taken +# when the pin was set. +ARGOCD_VERSION=v3.6.2 +ARGOCD_CORE_SHA256=7d39e823fa236666a512bdd294f307360a59f7865e6936fed49fd7b57d348a3f +FLUX_VERSION=v2.10.1 +FLUX_INSTALL_SHA256=4db2fb9ce179622327d3a9a36a765bbfc0329fd50690e8d37da7dd0918b09392 +# The 2026.x release the GitOps runs cross from (chart/crossing-test.sh) +CROSSING_FROM=2026.11.18.3