diff --git a/lab/chart/admin-jmap.py b/lab/chart/admin-jmap.py new file mode 100755 index 0000000..41bc223 --- /dev/null +++ b/lab/chart/admin-jmap.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""JMAP method calls as admin@lab.test, read as JSON from stdin. + + admin-jmap.py < calls.json + +Signs in like the console (as blocked-ips.py) and prints the +methodResponses as JSON. Exits non-zero on an HTTP error. +""" +import base64, hashlib, http.client, json, secrets, sys, urllib.parse +server, pwfile = sys.argv[1:] +def request(method, path, headers=None, body=None): + u = urllib.parse.urlsplit(server); c = http.client.HTTPConnection(u.hostname, u.port, timeout=60) + c.request(method, path, body=body, headers=headers or {}); r = c.getresponse(); d = r.read(); c.close(); return r, d +v = secrets.token_urlsafe(48); ch = base64.urlsafe_b64encode(hashlib.sha256(v.encode()).digest()).rstrip(b"=").decode() +red = "https://admin.lab.test/oauth/callback" +r, d = request("POST", "/api/auth", {"content-type": "application/json"}, json.dumps({"type": "authCode", "accountName": "admin@lab.test", + "accountSecret": open(pwfile).read().strip(), "clientId": "inbuxa-admin", "redirectUri": red, "codeChallenge": ch, "codeChallengeMethod": "S256", "state": "x"})) +code = json.loads(d)["client_code"] +r, d = request("POST", "/auth/token", {"content-type": "application/x-www-form-urlencoded"}, urllib.parse.urlencode({"grant_type": "authorization_code", + "code": code, "redirect_uri": red, "client_id": "inbuxa-admin", "code_verifier": v})) +tok = json.loads(d)["access_token"] +r, d = request("POST", "/jmap", {"content-type": "application/json", "authorization": "Bearer " + tok}, + json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"], "methodCalls": json.load(sys.stdin)})) +if r.status != 200: + sys.exit(f"jmap: HTTP {r.status}: {d[:300]!r}") +print(json.dumps(json.loads(d)["methodResponses"])) diff --git a/lab/chart/alerts-test.sh b/lab/chart/alerts-test.sh new file mode 100755 index 0000000..32428f3 --- /dev/null +++ b/lab/chart/alerts-test.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C10: the chart's alert rules, in a real Prometheus, quiet on a healthy +# install and firing when things break. +# +# Prometheus runs on its own in the monitoring namespace (no operator): it +# scrapes the server pods as the chart's ServiceMonitor does, and loads the +# rules from the chart's PrometheusRule, every `for` cut to 0s so a test +# doesn't wait out the hold times. +# +# 1. quiet: the lab as it is, with the chart's thresholds: nothing fires. +# 2. breakage, each checked for the alert it should raise: +# - a server process stopped (SIGSTOP on its node): InbuxaServerDown +# - a front pod cut off from Postgres: InbuxaClusterLeaseNotRenewed +# - mail to a host that never answers (wait.lab.test, added to the +# lab's CoreDNS for the test) waits in the queue: +# InbuxaQueueLarge, InbuxaQueueMessageWaiting (thresholds lowered) +# - mail to example.com (a null MX) bounces: InbuxaDeliveriesFailing +# - a certificate that runs out in two days: InbuxaCertificateExpiring +# - InbuxaDataStoreSlow with a threshold any store time crosses: the +# expression reads real series. +# The kube-state-metrics alerts are left out: the lab doesn't run it. +# +# Puts everything back: the chart, CoreDNS, the network policy, the stopped +# process, the queued test mail, the certificate, and the monitoring +# namespace. +# +# SERVER_TAG=kc10 WEBMAIL_TAG=kw3 lab/chart/alerts-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +CTX=kind-inbuxa-lab +K=(kubectl --context "$CTX" -n inbuxa) +M=(kubectl --context "$CTX" -n monitoring) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +SERVER_TAG="${SERVER_TAG:?the server image tag in kind}" +PROM_IMAGE="${PROM_IMAGE:-prom/prometheus:v3.5.0}" +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +PORT="${PORT:-28080}" +EXTRA=() +[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG") +METRICS=(--set server.metrics.prometheus.enabled=true) +fails=0 +work=$(mktemp -d) +stopped="" # node and pid of a stopped server process +cut="" # the pod cut off from Postgres +fwd="" +cert="" # the test certificate's id +corefile="" # CoreDNS's Corefile before the test's name was added +START=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +install() { # [helm args] + helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \ + -f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \ + "${EXTRA[@]}" "$@" --wait --timeout 10m >/dev/null + for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done +} +cleanup() { + [ -n "$corefile" ] && dns_restore + [ -n "$stopped" ] && docker exec "${stopped% *}" kill -CONT "${stopped#* }" 2>/dev/null || true + kubectl --context "$CTX" -n data delete networkpolicy lab-cut-off --ignore-not-found >/dev/null 2>&1 || true + [ -n "$cut" ] && "${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off- >/dev/null 2>&1 || true + if [ -n "$fwd" ] || forward; then + python3 queue-clean.py "http://127.0.0.1:$PORT" "$STATE/admin.password" alice@lab.test "$START" >/dev/null || + echo "!! couldn't remove the test mail from the queue" >&2 + [ -n "$cert" ] && { echo "[[\"x:Certificate/set\", {\"destroy\": [\"$cert\"]}, \"d\"]]" | + admin_jmap >/dev/null || echo "!! couldn't remove the test certificate $cert" >&2; } + fi + [ -n "$fwd" ] && kill "$fwd" 2>/dev/null || true + kubectl --context "$CTX" delete namespace monitoring --ignore-not-found --wait=false >/dev/null 2>&1 || true + install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2 + rm -rf "$work" +} +trap cleanup EXIT + +check() { # + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi +} +prom() { # : GET from Prometheus + "${M[@]}" exec deploy/prometheus -- wget -qO- "http://localhost:9090$1" +} +firing() { # the Inbuxa alerts firing, sorted, one per line + prom /api/v1/alerts | python3 -c ' +import json, sys +names = {a["labels"]["alertname"] for a in json.load(sys.stdin)["data"]["alerts"] + if a["state"] == "firing" and a["labels"]["alertname"].startswith("Inbuxa")} +if names: + print("\n".join(sorted(names)))' +} +wait_for() { # : until it fires + local t=0 + while [ "$t" -lt "$2" ]; do + firing | grep -qx "$1" && { echo "ok $1 fired (${t}s)"; return; } + sleep 5; t=$((t + 5)) + done + echo "FAIL $1 didn't fire in $2s"; fails=$((fails + 1)) +} +rules() { # [helm args]: the chart's rules as a Prometheus rules file, every for 0s + helm template inbuxa "$CHART" -n inbuxa -f values-lab.yaml --set server.image.tag="$SERVER_TAG" \ + "${METRICS[@]}" --set server.metrics.prometheus.serviceMonitor.enabled=true \ + --set server.metrics.prometheus.rules.enabled=true \ + --set server.metrics.prometheus.rules.kubeStateMetrics=false "$@" \ + --show-only templates/server/metrics.yaml | python3 -c ' +import sys, yaml +for doc in yaml.safe_load_all(sys.stdin): + if doc and doc["kind"] == "PrometheusRule": + for group in doc["spec"]["groups"]: + for rule in group["rules"]: + rule["for"] = "0s" + print(yaml.safe_dump({"groups": doc["spec"]["groups"]}, sort_keys=False))' +} +load_rules() { # [helm args]: new rules into Prometheus, reloaded + rules "$@" >"$work/rules.yml" + "${M[@]}" create configmap prometheus-rules --from-file=rules.yml="$work/rules.yml" \ + --dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null + # The kubelet takes up to a minute to update the mounted file + for _ in $(seq 40); do + "${M[@]}" exec deploy/prometheus -- cat /etc/prometheus/rules/rules.yml 2>/dev/null | + cmp -s - "$work/rules.yml" && break + sleep 3 + done + "${M[@]}" exec deploy/prometheus -- wget -qO- --post-data= http://localhost:9090/-/reload >/dev/null +} +forward() { + "${K[@]}" port-forward svc/inbuxa-http "$PORT:8080" >/dev/null 2>&1 & + fwd=$! + for _ in $(seq 20); do curl -s -o /dev/null "http://127.0.0.1:$PORT/healthz/live" && return 0; sleep 1; done + return 1 +} +# wait.lab.test, for mail that has to wait: an address in TEST-NET-2 that +# nothing answers, so delivery fails for now and is retried. Added to the +# lab's CoreDNS for the test (its reload plugin picks it up, which can +# take a couple of minutes) and taken out +# after. +dns_add() { + corefile=$(kubectl --context "$CTX" -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}') + printf '%s' "$corefile" >"$work/Corefile.orig" + python3 - "$work/Corefile.orig" >"$work/Corefile" <<'EOF2' +import sys +s = open(sys.argv[1]).read() +anchor = " forward . /etc/resolv.conf" +assert anchor in s +print(s.replace(anchor, " hosts {\n 198.51.100.1 wait.lab.test\n fallthrough\n }\n" + anchor, 1), end="") +EOF2 + kubectl --context "$CTX" -n kube-system create configmap coredns --from-file=Corefile="$work/Corefile" \ + --dry-run=client -o yaml | kubectl --context "$CTX" -n kube-system apply -f - >/dev/null 2>&1 +} +dns_restore() { + kubectl --context "$CTX" -n kube-system create configmap coredns --from-file=Corefile="$work/Corefile.orig" \ + --dry-run=client -o yaml | kubectl --context "$CTX" -n kube-system apply -f - >/dev/null 2>&1 || + echo "!! couldn't restore CoreDNS's Corefile: it's in $work/Corefile.orig" >&2 +} +dns_resolves() { # whether a server pod resolves wait.lab.test yet + "${K[@]}" exec deploy/inbuxa-core -c server -- sh -c 'getent hosts wait.lab.test || curl -s -m 2 -o /dev/null http://wait.lab.test; [ $? -ne 6 ]' >/dev/null 2>&1 +} +admin_jmap() { # JMAP calls on stdin, as admin@lab.test + python3 admin-jmap.py "http://127.0.0.1:$PORT" "$STATE/admin.password" +} +send() { # : messages from alice, over submissions + LB="$LB" PASSWORD_FILE="$STATE/alice.password" python3 - "$@" <<'EOF' +import os, smtplib, ssl, sys +from email.message import EmailMessage +from email.utils import formatdate, make_msgid +tls = ssl.create_default_context(); tls.check_hostname = False; tls.verify_mode = ssl.CERT_NONE +password = open(os.environ["PASSWORD_FILE"]).read().strip() +with smtplib.SMTP_SSL(os.environ["LB"], 465, context=tls, timeout=30) as s: + s.login("alice@lab.test", password) + for i in range(int(sys.argv[1])): + m = EmailMessage() + m["From"] = "alice@lab.test"; m["To"] = sys.argv[2] + m["Subject"] = f"alerts-test {i}"; m["Date"] = formatdate() + m["Message-ID"] = make_msgid(domain="lab.test") + m.set_content("Sent by the inbuxa-k8s-lab alerts test.\n") + s.send_message(m) +EOF +} + +echo "== the chart, with scraping on" +install "${METRICS[@]}" +LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + +echo "== Prometheus" +kubectl --context "$CTX" create namespace monitoring --dry-run=client -o yaml | kubectl --context "$CTX" apply -f - >/dev/null +"${M[@]}" create secret generic inbuxa-scrape \ + --from-literal=password="$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.password}' | base64 -d)" \ + --dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null +rules >"$work/rules.yml" +"${M[@]}" create configmap prometheus-rules --from-file=rules.yml="$work/rules.yml" \ + --dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null +kubectl --context "$CTX" apply -f - >/dev/null </dev/null +for _ in $(seq 40); do prom /-/ready >/dev/null 2>&1 && break; sleep 3; done +for _ in $(seq 30); do + up=$(prom '/api/v1/query?query=sum(up%7Bjob%3D%22inbuxa-metrics%22%7D)' 2>/dev/null | python3 -c ' +import json, sys +try: r = json.load(sys.stdin)["data"]["result"] +except Exception: r = [] +print(int(float(r[0]["value"][1])) if r else 0)') + [ "$up" = 5 ] && break + sleep 3 +done +check "Prometheus scrapes the five server pods" "$up" 5 +rules_loaded=$(prom /api/v1/rules | python3 -c 'import json, sys; print(sum(len(g["rules"]) for g in json.load(sys.stdin)["data"]["groups"]))') +check "rules loaded" "$rules_loaded" 7 + +echo "== 1. quiet while nothing is wrong" +sleep 60 +check "alerts firing on a healthy install" "$(firing | paste -sd " ")" "" + +echo "== 2. breakage" +forward || { echo "!! no port-forward to the server" >&2; exit 1; } + +echo "-- a certificate that runs out in two days" +openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 2 \ + -subj "/CN=alerts.lab.test" -addext "subjectAltName=DNS:alerts.lab.test" \ + -keyout "$work/cert.key" -out "$work/cert.crt" 2>/dev/null +cert=$(python3 - "$work/cert.crt" "$work/cert.key" <<'EOF2' | admin_jmap | python3 -c ' +import json, sys; r = json.load(sys.stdin)[0][1]; print(r.get("created", {}).get("c", {}).get("id", ""))' +import json, sys +print(json.dumps([["x:Certificate/set", {"create": {"c": { + "certificate": {"@type": "Text", "value": open(sys.argv[1]).read()}, + "privateKey": {"@type": "Text", "secret": open(sys.argv[2]).read()}}}}, "s"]])) +EOF2 +) +[ -n "$cert" ] || { echo "FAIL couldn't store the test certificate"; fails=$((fails + 1)); } +wait_for InbuxaCertificateExpiring 90 +# Lower thresholds: two queued messages, a minute's wait, any store time, +# two bounces +load_rules --set server.metrics.prometheus.rules.queueMessages=1 \ + --set-json server.metrics.prometheus.rules.queueOldestHours=0.015 \ + --set-json server.metrics.prometheus.rules.storeLatencyMs=0.001 \ + --set-json server.metrics.prometheus.rules.deliveryFailureRatio=0.01 \ + --set server.metrics.prometheus.rules.deliveryFailureMinimum=2 + +echo "-- a server process stopped" +victim=$("${K[@]}" get pods -l inbuxa.org/group=mta -o jsonpath='{.items[0].metadata.name}') +node=$("${K[@]}" get pod "$victim" -o jsonpath='{.spec.nodeName}') +cid=$("${K[@]}" get pod "$victim" -o jsonpath='{.status.containerStatuses[0].containerID}' | sed 's|.*://||') +pid=$(docker exec "$node" crictl inspect --output go-template --template '{{.info.pid}}' "$cid") +docker exec "$node" kill -STOP "$pid" +stopped="$node $pid" +wait_for InbuxaServerDown 45 +docker exec "$node" kill -CONT "$pid" +stopped="" + +echo "-- mail that waits, and mail that bounces" +dns_add +for _ in $(seq 60); do dns_resolves && break; sleep 3; done +dns_resolves || { echo "FAIL wait.lab.test doesn't resolve"; fails=$((fails + 1)); } +# The stopped pod is back in a moment; a connection that lands on it +# before then is tried again +for attempt in 1 2 3 4 5 6; do + # wait.lab.test never answers, which is retried; example.com publishes a + # null MX (RFC 7505), which bounces at once + send 3 "lab-wait@wait.lab.test" 2>/dev/null && send 3 "lab-bounce@example.com" 2>/dev/null && break + [ "$attempt" = 6 ] && { echo "FAIL couldn't send the test mail"; fails=$((fails + 1)); } + sleep 10 +done + +echo "-- a front pod cut off from Postgres" +cut=$("${K[@]}" get pods -l inbuxa.org/group=front -o jsonpath='{.items[0].metadata.name}') +cut_ip=$("${K[@]}" get pod "$cut" -o jsonpath='{.status.podIP}') +"${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off=true >/dev/null +kubectl --context "$CTX" apply -f - >/dev/null <<'EOF' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: {name: lab-cut-off, namespace: data} +spec: + podSelector: {matchLabels: {cnpg.io/cluster: inbuxa-pg}} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: cnpg-system}} + - podSelector: {matchLabels: {cnpg.io/cluster: inbuxa-pg}} + - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: inbuxa}} + podSelector: + matchExpressions: [{key: lab.inbuxa.org/cut-off, operator: DoesNotExist}] +EOF +# The policy stops new connections; end the ones the pod holds +sleep 5 +kubectl --context "$CTX" -n data exec inbuxa-pg-1 -c postgres -- psql -qtA -U postgres -c \ + "select count(pg_terminate_backend(pid)) from pg_stat_activity where client_addr = '$cut_ip'" >/dev/null + + +# The lease alert needs five minutes without a renewal, the queue gauges +# the core's next count (every five minutes) +wait_for InbuxaDataStoreSlow 60 +wait_for InbuxaDeliveriesFailing 180 +wait_for InbuxaQueueLarge 360 +wait_for InbuxaQueueMessageWaiting 360 +wait_for InbuxaClusterLeaseNotRenewed 420 + +echo "-- back to normal" +kubectl --context "$CTX" -n data delete networkpolicy lab-cut-off >/dev/null +"${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off- >/dev/null +cut="" + +echo +if [ "$fails" -eq 0 ]; then echo "alerts-test: all passed"; else echo "alerts-test: $fails failed"; exit 1; fi diff --git a/lab/chart/metrics-test.sh b/lab/chart/metrics-test.sh new file mode 100755 index 0000000..91b2bbe --- /dev/null +++ b/lab/chart/metrics-test.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Coffey Labs LLC +# SPDX-License-Identifier: Apache-2.0 +# +# K-C10: metrics. Installs with server.metrics.prometheus and .otlp on, then +# checks, for every server pod: +# - /metrics/prometheus refuses no password and a wrong one, and answers +# the chart's password with the gauges the alerts read; +# - its cluster lease was renewed in the last two and a half minutes; +# - the headless metrics Service lists it; +# - it pushes OTLP to a collector, with the header from the Secret (the +# collector sees the nodes' addresses, so this counts pushes). +# The collector is HAProxy on the kind network, answering 200 and logging +# each request with its Authorization header. +# +# Leaves the chart as it found it, and removes the collector and the Secret. +# +# SERVER_TAG=kc10 WEBMAIL_TAG=kw3 lab/chart/metrics-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +K=(kubectl --context kind-inbuxa-lab -n inbuxa) +CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}" +SERVER_TAG="${SERVER_TAG:?the server image tag in kind}" +HAPROXY_IMAGE="${HAPROXY_IMAGE:-haproxy:3.0-alpine}" +EXTRA=() +[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG") +SINK=inbuxa-lab-otlp +HEADERS=inbuxa-lab-otlp +TOKEN="Bearer lab-otlp-$RANDOM$RANDOM" +fails=0 + +install() { # [helm args]: the lab's install, with the webmail and console + helm --kube-context kind-inbuxa-lab -n inbuxa upgrade inbuxa "$CHART" \ + -f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \ + "${EXTRA[@]}" "$@" --wait --timeout 10m >/dev/null + for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done +} +work=$(mktemp -d) +cleanup() { + install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2 + "${K[@]}" delete secret "$HEADERS" --ignore-not-found >/dev/null + docker rm -f "$SINK" >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT + +settle() { # until the last rollout's old pods are gone + for _ in $(seq 60); do + [ -z "$("${K[@]}" get pods -l app.kubernetes.io/component=server -o jsonpath='{.items[?(@.metadata.deletionTimestamp)].metadata.name}')" ] && return + sleep 3 + done +} +check() { # + if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi +} +scrape() { # [curl args]: the status code, the body in $work/body + "${K[@]}" exec deploy/inbuxa-core -c server -- \ + curl -s -m 10 -o /dev/stdout -w '\n%{http_code}' "${@:2}" "http://$1:8080/metrics/prometheus" >"$work/out" || true + sed '$d' "$work/out" >"$work/body" + tail -n1 "$work/out" +} +gauge() { # : its value in $work/body, as an integer + awk -v n="$1" '$1 == n { printf "%d", $2 }' "$work/body" +} + +echo "== the collector" +cat >"$work/haproxy.cfg" <<'EOF' +global + log stdout format raw local0 +defaults + mode http + log global + timeout client 30s + timeout server 30s + timeout connect 5s +frontend otlp + bind :4318 + http-request capture req.hdr(authorization) len 64 + log-format "%ci %HM %HP %hr" + http-request return status 200 +EOF +docker rm -f "$SINK" >/dev/null 2>&1 || true +docker run -d --name "$SINK" --network kind -v "$work/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro" "$HAPROXY_IMAGE" >/dev/null +SINK_IP=$(docker inspect "$SINK" -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}') +[ -n "$SINK_IP" ] || { docker logs "$SINK" >&2; echo "!! the collector didn't start" >&2; exit 1; } +printf 'authorization=%s\n' "$TOKEN" >"$work/headers" +"${K[@]}" create secret generic "$HEADERS" --from-file=headers="$work/headers" \ + --dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null + +echo "== install with metrics on" +install --set server.metrics.prometheus.enabled=true \ + --set server.metrics.otlp.enabled=true \ + --set server.metrics.otlp.endpoint="http://$SINK_IP:4318" \ + --set server.metrics.otlp.intervalSeconds=5 \ + --set server.metrics.otlp.headersSecret="$HEADERS" +settle +PW=$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.password}' | base64 -d) +USER=$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.username}' | base64 -d) +mapfile -t PODS < <("${K[@]}" get pods -l app.kubernetes.io/component=server \ + -o jsonpath='{range .items[*]}{.metadata.name} {.status.podIP}{"\n"}{end}') +check "five server pods" "${#PODS[@]}" 5 + +echo "== the metrics Service lists every pod" +listed=$("${K[@]}" get endpointslices -l kubernetes.io/service-name=inbuxa-metrics \ + -o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\n"}{end}' | sort -u | wc -l) +check "inbuxa-metrics endpoints" "$listed" 5 + +echo "== scraping" +first_ip=$(echo "${PODS[0]}" | cut -d' ' -f2) +check "no password: refused" "$(scrape "$first_ip")" 401 +check "wrong password: refused" "$(scrape "$first_ip" -u "$USER:not-it")" 401 +now=$(date +%s) +for p in "${PODS[@]}"; do + name=${p% *}; ip=${p#* } + check "$name: the chart's password" "$(scrape "$ip" -u "$USER:$PW")" 200 + for m in queue_count queue_oldest_message cluster_lease_renewed tls_earliest_expiry server_memory; do + grep -q "^$m " "$work/body" || { echo "FAIL $name: no $m"; fails=$((fails + 1)); } + done + renewed=$(gauge cluster_lease_renewed) + age=$((now - renewed)) + if [ "$renewed" -gt 0 ] && [ "$age" -le 150 ]; then + echo "ok $name: lease renewed ${age}s ago" + else + echo "FAIL $name: lease renewed at $renewed (${age}s ago)"; fails=$((fails + 1)) + fi +done + +echo "== OTLP pushes, with the header" +sleep 20 +docker logs "$SINK" 2>/dev/null | grep " POST /v1/metrics " >"$work/pushes" || true +# Pods reach the kind network through their node's address, so the +# collector sees one address per node with server pods, and every pod's +# pushes, about one per pod per interval +nodes=$("${K[@]}" get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | wc -l) +pushers=$(awk '{print $1}' "$work/pushes" | sort -u | wc -l) +check "nodes the pushes came from" "$pushers" "$nodes" +count=$(wc -l <"$work/pushes") +if [ "$count" -ge 15 ]; then echo "ok $count pushes in the first 20s or so"; else echo "FAIL only $count pushes"; fails=$((fails + 1)); fi +bad=$(grep -vcF "{$TOKEN}" "$work/pushes" || true) +check "pushes without the token" "$bad" 0 + +echo +if [ "$fails" -eq 0 ]; then echo "metrics-test: all passed"; else echo "metrics-test: $fails failed"; exit 1; fi diff --git a/lab/chart/queue-clean.py b/lab/chart/queue-clean.py new file mode 100755 index 0000000..1f0733e --- /dev/null +++ b/lab/chart/queue-clean.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Removes the queued messages a test sent: from a sender, since a time. + + queue-clean.py + +Prints how many it removed. +""" +import json, subprocess, sys +server, pwfile, sender, since = sys.argv[1:] +here = __file__.rsplit("/", 1)[0] +def jmap(calls): + out = subprocess.run([sys.executable, f"{here}/admin-jmap.py", server, pwfile], input=json.dumps(calls), + capture_output=True, text=True, check=True).stdout + return json.loads(out) +got = jmap([["x:QueuedMessage/query", {}, "q"], + ["x:QueuedMessage/get", {"#ids": {"resultOf": "q", "name": "x:QueuedMessage/query", "path": "/ids"}, + "properties": ["returnPath", "createdAt"]}, "g"]]) +ids = [m["id"] for m in got[1][1]["list"] if m.get("returnPath") == sender and m.get("createdAt", "") >= since] +if ids: + res = jmap([["x:QueuedMessage/set", {"destroy": ids}, "d"]])[0][1] + if res.get("notDestroyed"): + sys.exit(f"not removed: {json.dumps(res['notDestroyed'])[:300]}") +print(len(ids))