From f2f13c336773af043fd8285088e2e5627480e6eb Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 19 Nov 2026 11:30:05 -0700 Subject: [PATCH] metrics-test and alerts-test: scraping, OTLP and the chart's alerts (K-C10) metrics-test.sh: every server pod refuses /metrics/prometheus without the password and with a wrong one, answers the chart's with the alert gauges, renewed its cluster lease in the last 150 s, is listed by the headless metrics Service, and pushes OTLP (HAProxy on the kind network stands in for the collector) with the token from the headers Secret. alerts-test.sh: Prometheus v3.5.0 on its own (no operator), scraping as the chart's ServiceMonitor does, with the chart's rules and every for cut to 0s. Quiet on a healthy install; then each alert from real breakage: a stopped server process, a pod cut off from Postgres, mail to a host that never answers (wait.lab.test, added to CoreDNS for the test), mail to example.com's null MX, a certificate that runs out in two days. The store latency alert with a threshold any read crosses. Puts everything back. admin-jmap.py and queue-clean.py: JMAP as the lab admin, and removing the mail a test queued. --- lab/chart/admin-jmap.py | 26 +++ lab/chart/alerts-test.sh | 371 ++++++++++++++++++++++++++++++++++++++ lab/chart/metrics-test.sh | 144 +++++++++++++++ lab/chart/queue-clean.py | 23 +++ 4 files changed, 564 insertions(+) create mode 100755 lab/chart/admin-jmap.py create mode 100755 lab/chart/alerts-test.sh create mode 100755 lab/chart/metrics-test.sh create mode 100755 lab/chart/queue-clean.py diff --git a/lab/chart/admin-jmap.py b/lab/chart/admin-jmap.py new file mode 100755 index 0000000..41bc223 --- /dev/null +++ b/lab/chart/admin-jmap.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""JMAP method calls as admin@lab.test, read as JSON from stdin. + + admin-jmap.py < calls.json + +Signs in like the console (as blocked-ips.py) and prints the +methodResponses as JSON. Exits non-zero on an HTTP error. +""" +import base64, hashlib, http.client, json, secrets, sys, urllib.parse +server, pwfile = sys.argv[1:] +def request(method, path, headers=None, body=None): + u = urllib.parse.urlsplit(server); c = http.client.HTTPConnection(u.hostname, u.port, timeout=60) + c.request(method, path, body=body, headers=headers or {}); r = c.getresponse(); d = r.read(); c.close(); return r, d +v = secrets.token_urlsafe(48); ch = base64.urlsafe_b64encode(hashlib.sha256(v.encode()).digest()).rstrip(b"=").decode() +red = "https://admin.lab.test/oauth/callback" +r, d = request("POST", "/api/auth", {"content-type": "application/json"}, json.dumps({"type": "authCode", "accountName": "admin@lab.test", + "accountSecret": open(pwfile).read().strip(), "clientId": "inbuxa-admin", "redirectUri": red, "codeChallenge": ch, "codeChallengeMethod": "S256", "state": "x"})) +code = json.loads(d)["client_code"] +r, d = request("POST", "/auth/token", {"content-type": "application/x-www-form-urlencoded"}, urllib.parse.urlencode({"grant_type": "authorization_code", + "code": code, "redirect_uri": red, "client_id": "inbuxa-admin", "code_verifier": v})) +tok = json.loads(d)["access_token"] +r, d = request("POST", "/jmap", {"content-type": "application/json", "authorization": "Bearer " + tok}, + json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"], "methodCalls": json.load(sys.stdin)})) +if r.status != 200: + sys.exit(f"jmap: HTTP {r.status}: {d[:300]!r}") +print(json.dumps(json.loads(d)["methodResponses"])) diff --git a/lab/chart/alerts-test.sh b/lab/chart/alerts-test.sh new file mode 100755 index 0000000..32428f3 --- /dev/null +++ b/lab/chart/alerts-test.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C10: the chart's alert rules, in a real Prometheus, quiet on a healthy +# install and firing when things break. +# +# Prometheus runs on its own in the monitoring namespace (no operator): it +# scrapes the server pods as the chart's ServiceMonitor does, and loads the +# rules from the chart's PrometheusRule, every `for` cut to 0s so a test +# doesn't wait out the hold times. +# +# 1. quiet: the lab as it is, with the chart's thresholds: nothing fires. +# 2. breakage, each checked for the alert it should raise: +# - a server process stopped (SIGSTOP on its node): InbuxaServerDown +# - a front pod cut off from Postgres: InbuxaClusterLeaseNotRenewed +# - mail to a host that never answers (wait.lab.test, added to the +# lab's CoreDNS for the test) waits in the queue: +# InbuxaQueueLarge, InbuxaQueueMessageWaiting (thresholds lowered) +# - mail to example.com (a null MX) bounces: InbuxaDeliveriesFailing +# - a certificate that runs out in two days: InbuxaCertificateExpiring +# - InbuxaDataStoreSlow with a threshold any store time crosses: the +# expression reads real series. +# The kube-state-metrics alerts are left out: the lab doesn't run it. +# +# Puts everything back: the chart, CoreDNS, the network policy, the stopped +# process, the queued test mail, the certificate, and the monitoring +# namespace. +# +# SERVER_TAG=kc10 WEBMAIL_TAG=kw3 lab/chart/alerts-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +CTX=kind-inbuxa-lab +K=(kubectl --context "$CTX" -n inbuxa) +M=(kubectl --context "$CTX" -n monitoring) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +SERVER_TAG="${SERVER_TAG:?the server image tag in kind}" +PROM_IMAGE="${PROM_IMAGE:-prom/prometheus:v3.5.0}" +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +PORT="${PORT:-28080}" +EXTRA=() +[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG") +METRICS=(--set server.metrics.prometheus.enabled=true) +fails=0 +work=$(mktemp -d) +stopped="" # node and pid of a stopped server process +cut="" # the pod cut off from Postgres +fwd="" +cert="" # the test certificate's id +corefile="" # CoreDNS's Corefile before the test's name was added +START=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +install() { # [helm args] + helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \ + -f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \ + "${EXTRA[@]}" "$@" --wait --timeout 10m >/dev/null + for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done +} +cleanup() { + [ -n "$corefile" ] && dns_restore + [ -n "$stopped" ] && docker exec "${stopped% *}" kill -CONT "${stopped#* }" 2>/dev/null || true + kubectl --context "$CTX" -n data delete networkpolicy lab-cut-off --ignore-not-found >/dev/null 2>&1 || true + [ -n "$cut" ] && "${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off- >/dev/null 2>&1 || true + if [ -n "$fwd" ] || forward; then + python3 queue-clean.py "http://127.0.0.1:$PORT" "$STATE/admin.password" alice@lab.test "$START" >/dev/null || + echo "!! couldn't remove the test mail from the queue" >&2 + [ -n "$cert" ] && { echo "[[\"x:Certificate/set\", {\"destroy\": [\"$cert\"]}, \"d\"]]" | + admin_jmap >/dev/null || echo "!! couldn't remove the test certificate $cert" >&2; } + fi + [ -n "$fwd" ] && kill "$fwd" 2>/dev/null || true + kubectl --context "$CTX" delete namespace monitoring --ignore-not-found --wait=false >/dev/null 2>&1 || true + install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2 + rm -rf "$work" +} +trap cleanup EXIT + +check() { # + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi +} +prom() { # : GET from Prometheus + "${M[@]}" exec deploy/prometheus -- wget -qO- "http://localhost:9090$1" +} +firing() { # the Inbuxa alerts firing, sorted, one per line + prom /api/v1/alerts | python3 -c ' +import json, sys +names = {a["labels"]["alertname"] for a in json.load(sys.stdin)["data"]["alerts"] + if a["state"] == "firing" and a["labels"]["alertname"].startswith("Inbuxa")} +if names: + print("\n".join(sorted(names)))' +} +wait_for() { # : until it fires + local t=0 + while [ "$t" -lt "$2" ]; do + firing | grep -qx "$1" && { echo "ok $1 fired (${t}s)"; return; } + sleep 5; t=$((t + 5)) + done + echo "FAIL $1 didn't fire in $2s"; fails=$((fails + 1)) +} +rules() { # [helm args]: the chart's rules as a Prometheus rules file, every for 0s + helm template inbuxa "$CHART" -n inbuxa -f values-lab.yaml --set server.image.tag="$SERVER_TAG" \ + "${METRICS[@]}" --set server.metrics.prometheus.serviceMonitor.enabled=true \ + --set server.metrics.prometheus.rules.enabled=true \ + --set server.metrics.prometheus.rules.kubeStateMetrics=false "$@" \ + --show-only templates/server/metrics.yaml | python3 -c ' +import sys, yaml +for doc in yaml.safe_load_all(sys.stdin): + if doc and doc["kind"] == "PrometheusRule": + for group in doc["spec"]["groups"]: + for rule in group["rules"]: + rule["for"] = "0s" + print(yaml.safe_dump({"groups": doc["spec"]["groups"]}, sort_keys=False))' +} +load_rules() { # [helm args]: new rules into Prometheus, reloaded + rules "$@" >"$work/rules.yml" + "${M[@]}" create configmap prometheus-rules --from-file=rules.yml="$work/rules.yml" \ + --dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null + # The kubelet takes up to a minute to update the mounted file + for _ in $(seq 40); do + "${M[@]}" exec deploy/prometheus -- cat /etc/prometheus/rules/rules.yml 2>/dev/null | + cmp -s - "$work/rules.yml" && break + sleep 3 + done + "${M[@]}" exec deploy/prometheus -- wget -qO- --post-data= http://localhost:9090/-/reload >/dev/null +} +forward() { + "${K[@]}" port-forward svc/inbuxa-http "$PORT:8080" >/dev/null 2>&1 & + fwd=$! + for _ in $(seq 20); do curl -s -o /dev/null "http://127.0.0.1:$PORT/healthz/live" && return 0; sleep 1; done + return 1 +} +# wait.lab.test, for mail that has to wait: an address in TEST-NET-2 that +# nothing answers, so delivery fails for now and is retried. Added to the +# lab's CoreDNS for the test (its reload plugin picks it up, which can +# take a couple of minutes) and taken out +# after. +dns_add() { + corefile=$(kubectl --context "$CTX" -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}') + printf '%s' "$corefile" >"$work/Corefile.orig" + python3 - "$work/Corefile.orig" >"$work/Corefile" <<'EOF2' +import sys +s = open(sys.argv[1]).read() +anchor = " forward . /etc/resolv.conf" +assert anchor in s +print(s.replace(anchor, " hosts {\n 198.51.100.1 wait.lab.test\n fallthrough\n }\n" + anchor, 1), end="") +EOF2 + kubectl --context "$CTX" -n kube-system create configmap coredns --from-file=Corefile="$work/Corefile" \ + --dry-run=client -o yaml | kubectl --context "$CTX" -n kube-system apply -f - >/dev/null 2>&1 +} +dns_restore() { + kubectl --context "$CTX" -n kube-system create configmap coredns --from-file=Corefile="$work/Corefile.orig" \ + --dry-run=client -o yaml | kubectl --context "$CTX" -n kube-system apply -f - >/dev/null 2>&1 || + echo "!! couldn't restore CoreDNS's Corefile: it's in $work/Corefile.orig" >&2 +} +dns_resolves() { # whether a server pod resolves wait.lab.test yet + "${K[@]}" exec deploy/inbuxa-core -c server -- sh -c 'getent hosts wait.lab.test || curl -s -m 2 -o /dev/null http://wait.lab.test; [ $? -ne 6 ]' >/dev/null 2>&1 +} +admin_jmap() { # JMAP calls on stdin, as admin@lab.test + python3 admin-jmap.py "http://127.0.0.1:$PORT" "$STATE/admin.password" +} +send() { # : messages from alice, over submissions + LB="$LB" PASSWORD_FILE="$STATE/alice.password" python3 - "$@" <<'EOF' +import os, smtplib, ssl, sys +from email.message import EmailMessage +from email.utils import formatdate, make_msgid +tls = ssl.create_default_context(); tls.check_hostname = False; tls.verify_mode = ssl.CERT_NONE +password = open(os.environ["PASSWORD_FILE"]).read().strip() +with smtplib.SMTP_SSL(os.environ["LB"], 465, context=tls, timeout=30) as s: + s.login("alice@lab.test", password) + for i in range(int(sys.argv[1])): + m = EmailMessage() + m["From"] = "alice@lab.test"; m["To"] = sys.argv[2] + m["Subject"] = f"alerts-test {i}"; m["Date"] = formatdate() + m["Message-ID"] = make_msgid(domain="lab.test") + m.set_content("Sent by the inbuxa-k8s-lab alerts test.\n") + s.send_message(m) +EOF +} + +echo "== the chart, with scraping on" +install "${METRICS[@]}" +LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + +echo "== Prometheus" +kubectl --context "$CTX" create namespace monitoring --dry-run=client -o yaml | kubectl --context "$CTX" apply -f - >/dev/null +"${M[@]}" create secret generic inbuxa-scrape \ + --from-literal=password="$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.password}' | base64 -d)" \ + --dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null +rules >"$work/rules.yml" +"${M[@]}" create configmap prometheus-rules --from-file=rules.yml="$work/rules.yml" \ + --dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null +kubectl --context "$CTX" apply -f - >/dev/null </dev/null +for _ in $(seq 40); do prom /-/ready >/dev/null 2>&1 && break; sleep 3; done +for _ in $(seq 30); do + up=$(prom '/api/v1/query?query=sum(up%7Bjob%3D%22inbuxa-metrics%22%7D)' 2>/dev/null | python3 -c ' +import json, sys +try: r = json.load(sys.stdin)["data"]["result"] +except Exception: r = [] +print(int(float(r[0]["value"][1])) if r else 0)') + [ "$up" = 5 ] && break + sleep 3 +done +check "Prometheus scrapes the five server pods" "$up" 5 +rules_loaded=$(prom /api/v1/rules | python3 -c 'import json, sys; print(sum(len(g["rules"]) for g in json.load(sys.stdin)["data"]["groups"]))') +check "rules loaded" "$rules_loaded" 7 + +echo "== 1. quiet while nothing is wrong" +sleep 60 +check "alerts firing on a healthy install" "$(firing | paste -sd " ")" "" + +echo "== 2. breakage" +forward || { echo "!! no port-forward to the server" >&2; exit 1; } + +echo "-- a certificate that runs out in two days" +openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 2 \ + -subj "/CN=alerts.lab.test" -addext "subjectAltName=DNS:alerts.lab.test" \ + -keyout "$work/cert.key" -out "$work/cert.crt" 2>/dev/null +cert=$(python3 - "$work/cert.crt" "$work/cert.key" <<'EOF2' | admin_jmap | python3 -c ' +import json, sys; r = json.load(sys.stdin)[0][1]; print(r.get("created", {}).get("c", {}).get("id", ""))' +import json, sys +print(json.dumps([["x:Certificate/set", {"create": {"c": { + "certificate": {"@type": "Text", "value": open(sys.argv[1]).read()}, + "privateKey": {"@type": "Text", "secret": open(sys.argv[2]).read()}}}}, "s"]])) +EOF2 +) +[ -n "$cert" ] || { echo "FAIL couldn't store the test certificate"; fails=$((fails + 1)); } +wait_for InbuxaCertificateExpiring 90 +# Lower thresholds: two queued messages, a minute's wait, any store time, +# two bounces +load_rules --set server.metrics.prometheus.rules.queueMessages=1 \ + --set-json server.metrics.prometheus.rules.queueOldestHours=0.015 \ + --set-json server.metrics.prometheus.rules.storeLatencyMs=0.001 \ + --set-json server.metrics.prometheus.rules.deliveryFailureRatio=0.01 \ + --set server.metrics.prometheus.rules.deliveryFailureMinimum=2 + +echo "-- a server process stopped" +victim=$("${K[@]}" get pods -l inbuxa.org/group=mta -o jsonpath='{.items[0].metadata.name}') +node=$("${K[@]}" get pod "$victim" -o jsonpath='{.spec.nodeName}') +cid=$("${K[@]}" get pod "$victim" -o jsonpath='{.status.containerStatuses[0].containerID}' | sed 's|.*://||') +pid=$(docker exec "$node" crictl inspect --output go-template --template '{{.info.pid}}' "$cid") +docker exec "$node" kill -STOP "$pid" +stopped="$node $pid" +wait_for InbuxaServerDown 45 +docker exec "$node" kill -CONT "$pid" +stopped="" + +echo "-- mail that waits, and mail that bounces" +dns_add +for _ in $(seq 60); do dns_resolves && break; sleep 3; done +dns_resolves || { echo "FAIL wait.lab.test doesn't resolve"; fails=$((fails + 1)); } +# The stopped pod is back in a moment; a connection that lands on it +# before then is tried again +for attempt in 1 2 3 4 5 6; do + # wait.lab.test never answers, which is retried; example.com publishes a + # null MX (RFC 7505), which bounces at once + send 3 "lab-wait@wait.lab.test" 2>/dev/null && send 3 "lab-bounce@example.com" 2>/dev/null && break + [ "$attempt" = 6 ] && { echo "FAIL couldn't send the test mail"; fails=$((fails + 1)); } + sleep 10 +done + +echo "-- a front pod cut off from Postgres" +cut=$("${K[@]}" get pods -l inbuxa.org/group=front -o jsonpath='{.items[0].metadata.name}') +cut_ip=$("${K[@]}" get pod "$cut" -o jsonpath='{.status.podIP}') +"${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off=true >/dev/null +kubectl --context "$CTX" apply -f - >/dev/null <<'EOF' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: lab-cut-off, namespace: data} +spec: + podSelector: {matchLabels: {cnpg.io/cluster: inbuxa-pg}} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: cnpg-system}} + - podSelector: {matchLabels: {cnpg.io/cluster: inbuxa-pg}} + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: inbuxa}} + podSelector: + matchExpressions: [{key: lab.inbuxa.org/cut-off, operator: DoesNotExist}] +EOF +# The policy stops new connections; end the ones the pod holds +sleep 5 +kubectl --context "$CTX" -n data exec inbuxa-pg-1 -c postgres -- psql -qtA -U postgres -c \ + "select count(pg_terminate_backend(pid)) from pg_stat_activity where client_addr = '$cut_ip'" >/dev/null + + +# The lease alert needs five minutes without a renewal, the queue gauges +# the core's next count (every five minutes) +wait_for InbuxaDataStoreSlow 60 +wait_for InbuxaDeliveriesFailing 180 +wait_for InbuxaQueueLarge 360 +wait_for InbuxaQueueMessageWaiting 360 +wait_for InbuxaClusterLeaseNotRenewed 420 + +echo "-- back to normal" +kubectl --context "$CTX" -n data delete networkpolicy lab-cut-off >/dev/null +"${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off- >/dev/null +cut="" + +echo +if [ "$fails" -eq 0 ]; then echo "alerts-test: all passed"; else echo "alerts-test: $fails failed"; exit 1; fi diff --git a/lab/chart/metrics-test.sh b/lab/chart/metrics-test.sh new file mode 100755 index 0000000..91b2bbe --- /dev/null +++ b/lab/chart/metrics-test.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C10: metrics. Installs with server.metrics.prometheus and .otlp on, then +# checks, for every server pod: +# - /metrics/prometheus refuses no password and a wrong one, and answers +# the chart's password with the gauges the alerts read; +# - its cluster lease was renewed in the last two and a half minutes; +# - the headless metrics Service lists it; +# - it pushes OTLP to a collector, with the header from the Secret (the +# collector sees the nodes' addresses, so this counts pushes). +# The collector is HAProxy on the kind network, answering 200 and logging +# each request with its Authorization header. +# +# Leaves the chart as it found it, and removes the collector and the Secret. +# +# SERVER_TAG=kc10 WEBMAIL_TAG=kw3 lab/chart/metrics-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +K=(kubectl --context kind-inbuxa-lab -n inbuxa) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +SERVER_TAG="${SERVER_TAG:?the server image tag in kind}" +HAPROXY_IMAGE="${HAPROXY_IMAGE:-haproxy:3.0-alpine}" +EXTRA=() +[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG") +SINK=inbuxa-lab-otlp +HEADERS=inbuxa-lab-otlp +TOKEN="Bearer lab-otlp-$RANDOM$RANDOM" +fails=0 + +install() { # [helm args]: the lab's install, with the webmail and console + helm --kube-context kind-inbuxa-lab -n inbuxa upgrade inbuxa "$CHART" \ + -f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \ + "${EXTRA[@]}" "$@" --wait --timeout 10m >/dev/null + for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done +} +work=$(mktemp -d) +cleanup() { + install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2 + "${K[@]}" delete secret "$HEADERS" --ignore-not-found >/dev/null + docker rm -f "$SINK" >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT + +settle() { # until the last rollout's old pods are gone + for _ in $(seq 60); do + [ -z "$("${K[@]}" get pods -l app.kubernetes.io/component=server -o jsonpath='{.items[?(@.metadata.deletionTimestamp)].metadata.name}')" ] && return + sleep 3 + done +} +check() { # + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi +} +scrape() { # [curl args]: the status code, the body in $work/body + "${K[@]}" exec deploy/inbuxa-core -c server -- \ + curl -s -m 10 -o /dev/stdout -w '\n%{http_code}' "${@:2}" "http://$1:8080/metrics/prometheus" >"$work/out" || true + sed '$d' "$work/out" >"$work/body" + tail -n1 "$work/out" +} +gauge() { # : its value in $work/body, as an integer + awk -v n="$1" '$1 == n { printf "%d", $2 }' "$work/body" +} + +echo "== the collector" +cat >"$work/haproxy.cfg" <<'EOF' +global + log stdout format raw local0 +defaults + mode http + log global + timeout client 30s + timeout server 30s + timeout connect 5s +frontend otlp + bind :4318 + http-request capture req.hdr(authorization) len 64 + log-format "%ci %HM %HP %hr" + http-request return status 200 +EOF +docker rm -f "$SINK" >/dev/null 2>&1 || true +docker run -d --name "$SINK" --network kind -v "$work/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro" "$HAPROXY_IMAGE" >/dev/null +SINK_IP=$(docker inspect "$SINK" -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') +[ -n "$SINK_IP" ] || { docker logs "$SINK" >&2; echo "!! the collector didn't start" >&2; exit 1; } +printf 'authorization=%s\n' "$TOKEN" >"$work/headers" +"${K[@]}" create secret generic "$HEADERS" --from-file=headers="$work/headers" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + +echo "== install with metrics on" +install --set server.metrics.prometheus.enabled=true \ + --set server.metrics.otlp.enabled=true \ + --set server.metrics.otlp.endpoint="http://$SINK_IP:4318" \ + --set server.metrics.otlp.intervalSeconds=5 \ + --set server.metrics.otlp.headersSecret="$HEADERS" +settle +PW=$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.password}' | base64 -d) +USER=$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.username}' | base64 -d) +mapfile -t PODS < <("${K[@]}" get pods -l app.kubernetes.io/component=server \ + -o jsonpath='{range .items[*]}{.metadata.name} {.status.podIP}{"\n"}{end}') +check "five server pods" "${#PODS[@]}" 5 + +echo "== the metrics Service lists every pod" +listed=$("${K[@]}" get endpointslices -l kubernetes.io/service-name=inbuxa-metrics \ + -o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\n"}{end}' | sort -u | wc -l) +check "inbuxa-metrics endpoints" "$listed" 5 + +echo "== scraping" +first_ip=$(echo "${PODS[0]}" | cut -d' ' -f2) +check "no password: refused" "$(scrape "$first_ip")" 401 +check "wrong password: refused" "$(scrape "$first_ip" -u "$USER:not-it")" 401 +now=$(date +%s) +for p in "${PODS[@]}"; do + name=${p% *}; ip=${p#* } + check "$name: the chart's password" "$(scrape "$ip" -u "$USER:$PW")" 200 + for m in queue_count queue_oldest_message cluster_lease_renewed tls_earliest_expiry server_memory; do + grep -q "^$m " "$work/body" || { echo "FAIL $name: no $m"; fails=$((fails + 1)); } + done + renewed=$(gauge cluster_lease_renewed) + age=$((now - renewed)) + if [ "$renewed" -gt 0 ] && [ "$age" -le 150 ]; then + echo "ok $name: lease renewed ${age}s ago" + else + echo "FAIL $name: lease renewed at $renewed (${age}s ago)"; fails=$((fails + 1)) + fi +done + +echo "== OTLP pushes, with the header" +sleep 20 +docker logs "$SINK" 2>/dev/null | grep " POST /v1/metrics " >"$work/pushes" || true +# Pods reach the kind network through their node's address, so the +# collector sees one address per node with server pods, and every pod's +# pushes, about one per pod per interval +nodes=$("${K[@]}" get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | wc -l) +pushers=$(awk '{print $1}' "$work/pushes" | sort -u | wc -l) +check "nodes the pushes came from" "$pushers" "$nodes" +count=$(wc -l <"$work/pushes") +if [ "$count" -ge 15 ]; then echo "ok $count pushes in the first 20s or so"; else echo "FAIL only $count pushes"; fails=$((fails + 1)); fi +bad=$(grep -vcF "{$TOKEN}" "$work/pushes" || true) +check "pushes without the token" "$bad" 0 + +echo +if [ "$fails" -eq 0 ]; then echo "metrics-test: all passed"; else echo "metrics-test: $fails failed"; exit 1; fi diff --git a/lab/chart/queue-clean.py b/lab/chart/queue-clean.py new file mode 100755 index 0000000..1f0733e --- /dev/null +++ b/lab/chart/queue-clean.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Removes the queued messages a test sent: from a sender, since a time. + + queue-clean.py + +Prints how many it removed. +""" +import json, subprocess, sys +server, pwfile, sender, since = sys.argv[1:] +here = __file__.rsplit("/", 1)[0] +def jmap(calls): + out = subprocess.run([sys.executable, f"{here}/admin-jmap.py", server, pwfile], input=json.dumps(calls), + capture_output=True, text=True, check=True).stdout + return json.loads(out) +got = jmap([["x:QueuedMessage/query", {}, "q"], + ["x:QueuedMessage/get", {"#ids": {"resultOf": "q", "name": "x:QueuedMessage/query", "path": "/ids"}, + "properties": ["returnPath", "createdAt"]}, "g"]]) +ids = [m["id"] for m in got[1][1]["list"] if m.get("returnPath") == sender and m.get("createdAt", "") >= since] +if ids: + res = jmap([["x:QueuedMessage/set", {"destroy": ids}, "d"]])[0][1] + if res.get("notDestroyed"): + sys.exit(f"not removed: {json.dumps(res['notDestroyed'])[:300]}") +print(len(ids)) -- 2.54.0