metrics-test and alerts-test: scraping, OTLP and the chart's alerts (K-C10) #28

Merged
jcoffey-dev merged 1 commits from feat/metrics-test into main 2026-11-19 18:35:10 +00:00
4 changed files with 564 additions and 0 deletions

No files matched your search

+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env python3
"""JMAP method calls as [email protected], read as JSON from stdin.
admin-jmap.py <server> <admin password file> < calls.json
Signs in like the console (as blocked-ips.py) and prints the
methodResponses as JSON. Exits non-zero on an HTTP error.
"""
import base64, hashlib, http.client, json, secrets, sys, urllib.parse
server, pwfile = sys.argv[1:]
def request(method, path, headers=None, body=None):
u = urllib.parse.urlsplit(server); c = http.client.HTTPConnection(u.hostname, u.port, timeout=60)
c.request(method, path, body=body, headers=headers or {}); r = c.getresponse(); d = r.read(); c.close(); return r, d
v = secrets.token_urlsafe(48); ch = base64.urlsafe_b64encode(hashlib.sha256(v.encode()).digest()).rstrip(b"=").decode()
red = "https://admin.lab.test/oauth/callback"
r, d = request("POST", "/api/auth", {"content-type": "application/json"}, json.dumps({"type": "authCode", "accountName": "[email protected]",
"accountSecret": open(pwfile).read().strip(), "clientId": "inbuxa-admin", "redirectUri": red, "codeChallenge": ch, "codeChallengeMethod": "S256", "state": "x"}))
code = json.loads(d)["client_code"]
r, d = request("POST", "/auth/token", {"content-type": "application/x-www-form-urlencoded"}, urllib.parse.urlencode({"grant_type": "authorization_code",
"code": code, "redirect_uri": red, "client_id": "inbuxa-admin", "code_verifier": v}))
tok = json.loads(d)["access_token"]
r, d = request("POST", "/jmap", {"content-type": "application/json", "authorization": "Bearer " + tok},
json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"], "methodCalls": json.load(sys.stdin)}))
if r.status != 200:
sys.exit(f"jmap: HTTP {r.status}: {d[:300]!r}")
print(json.dumps(json.loads(d)["methodResponses"]))
+371
View File
@@ -0,0 +1,371 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-C10: the chart's alert rules, in a real Prometheus, quiet on a healthy
# install and firing when things break.
#
# Prometheus runs on its own in the monitoring namespace (no operator): it
# scrapes the server pods as the chart's ServiceMonitor does, and loads the
# rules from the chart's PrometheusRule, every `for` cut to 0s so a test
# doesn't wait out the hold times.
#
# 1. quiet: the lab as it is, with the chart's thresholds: nothing fires.
# 2. breakage, each checked for the alert it should raise:
# - a server process stopped (SIGSTOP on its node): InbuxaServerDown
# - a front pod cut off from Postgres: InbuxaClusterLeaseNotRenewed
# - mail to a host that never answers (wait.lab.test, added to the
# lab's CoreDNS for the test) waits in the queue:
# InbuxaQueueLarge, InbuxaQueueMessageWaiting (thresholds lowered)
# - mail to example.com (a null MX) bounces: InbuxaDeliveriesFailing
# - a certificate that runs out in two days: InbuxaCertificateExpiring
# - InbuxaDataStoreSlow with a threshold any store time crosses: the
# expression reads real series.
# The kube-state-metrics alerts are left out: the lab doesn't run it.
#
# Puts everything back: the chart, CoreDNS, the network policy, the stopped
# process, the queued test mail, the certificate, and the monitoring
# namespace.
#
# SERVER_TAG=kc10 WEBMAIL_TAG=kw3 lab/chart/alerts-test.sh
set -euo pipefail
cd "$(dirname "$0")"
CTX=kind-inbuxa-lab
K=(kubectl --context "$CTX" -n inbuxa)
M=(kubectl --context "$CTX" -n monitoring)
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
SERVER_TAG="${SERVER_TAG:?the server image tag in kind}"
PROM_IMAGE="${PROM_IMAGE:-prom/prometheus:v3.5.0}"
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
PORT="${PORT:-28080}"
EXTRA=()
[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG")
METRICS=(--set server.metrics.prometheus.enabled=true)
fails=0
work=$(mktemp -d)
stopped="" # node and pid of a stopped server process
cut="" # the pod cut off from Postgres
fwd=""
cert="" # the test certificate's id
corefile="" # CoreDNS's Corefile before the test's name was added
START=$(date -u +%Y-%m-%dT%H:%M:%SZ)
install() { # [helm args]
helm --kube-context "$CTX" -n inbuxa upgrade inbuxa "$CHART" \
-f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \
"${EXTRA[@]}" "$@" --wait --timeout 10m >/dev/null
for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done
}
cleanup() {
[ -n "$corefile" ] && dns_restore
[ -n "$stopped" ] && docker exec "${stopped% *}" kill -CONT "${stopped#* }" 2>/dev/null || true
kubectl --context "$CTX" -n data delete networkpolicy lab-cut-off --ignore-not-found >/dev/null 2>&1 || true
[ -n "$cut" ] && "${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off- >/dev/null 2>&1 || true
if [ -n "$fwd" ] || forward; then
python3 queue-clean.py "http://127.0.0.1:$PORT" "$STATE/admin.password" [email protected] "$START" >/dev/null ||
echo "!! couldn't remove the test mail from the queue" >&2
[ -n "$cert" ] && { echo "[[\"x:Certificate/set\", {\"destroy\": [\"$cert\"]}, \"d\"]]" |
admin_jmap >/dev/null || echo "!! couldn't remove the test certificate $cert" >&2; }
fi
[ -n "$fwd" ] && kill "$fwd" 2>/dev/null || true
kubectl --context "$CTX" delete namespace monitoring --ignore-not-found --wait=false >/dev/null 2>&1 || true
install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2
rm -rf "$work"
}
trap cleanup EXIT
check() { # <name> <got> <want>
if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi
}
prom() { # <path>: GET from Prometheus
"${M[@]}" exec deploy/prometheus -- wget -qO- "http://localhost:9090$1"
}
firing() { # the Inbuxa alerts firing, sorted, one per line
prom /api/v1/alerts | python3 -c '
import json, sys
names = {a["labels"]["alertname"] for a in json.load(sys.stdin)["data"]["alerts"]
if a["state"] == "firing" and a["labels"]["alertname"].startswith("Inbuxa")}
if names:
print("\n".join(sorted(names)))'
}
wait_for() { # <alert> <seconds>: until it fires
local t=0
while [ "$t" -lt "$2" ]; do
firing | grep -qx "$1" && { echo "ok $1 fired (${t}s)"; return; }
sleep 5; t=$((t + 5))
done
echo "FAIL $1 didn't fire in $2s"; fails=$((fails + 1))
}
rules() { # [helm args]: the chart's rules as a Prometheus rules file, every for 0s
helm template inbuxa "$CHART" -n inbuxa -f values-lab.yaml --set server.image.tag="$SERVER_TAG" \
"${METRICS[@]}" --set server.metrics.prometheus.serviceMonitor.enabled=true \
--set server.metrics.prometheus.rules.enabled=true \
--set server.metrics.prometheus.rules.kubeStateMetrics=false "$@" \
--show-only templates/server/metrics.yaml | python3 -c '
import sys, yaml
for doc in yaml.safe_load_all(sys.stdin):
if doc and doc["kind"] == "PrometheusRule":
for group in doc["spec"]["groups"]:
for rule in group["rules"]:
rule["for"] = "0s"
print(yaml.safe_dump({"groups": doc["spec"]["groups"]}, sort_keys=False))'
}
load_rules() { # [helm args]: new rules into Prometheus, reloaded
rules "$@" >"$work/rules.yml"
"${M[@]}" create configmap prometheus-rules --from-file=rules.yml="$work/rules.yml" \
--dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null
# The kubelet takes up to a minute to update the mounted file
for _ in $(seq 40); do
"${M[@]}" exec deploy/prometheus -- cat /etc/prometheus/rules/rules.yml 2>/dev/null |
cmp -s - "$work/rules.yml" && break
sleep 3
done
"${M[@]}" exec deploy/prometheus -- wget -qO- --post-data= http://localhost:9090/-/reload >/dev/null
}
forward() {
"${K[@]}" port-forward svc/inbuxa-http "$PORT:8080" >/dev/null 2>&1 &
fwd=$!
for _ in $(seq 20); do curl -s -o /dev/null "http://127.0.0.1:$PORT/healthz/live" && return 0; sleep 1; done
return 1
}
# wait.lab.test, for mail that has to wait: an address in TEST-NET-2 that
# nothing answers, so delivery fails for now and is retried. Added to the
# lab's CoreDNS for the test (its reload plugin picks it up, which can
# take a couple of minutes) and taken out
# after.
dns_add() {
corefile=$(kubectl --context "$CTX" -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}')
printf '%s' "$corefile" >"$work/Corefile.orig"
python3 - "$work/Corefile.orig" >"$work/Corefile" <<'EOF2'
import sys
s = open(sys.argv[1]).read()
anchor = " forward . /etc/resolv.conf"
assert anchor in s
print(s.replace(anchor, " hosts {\n 198.51.100.1 wait.lab.test\n fallthrough\n }\n" + anchor, 1), end="")
EOF2
kubectl --context "$CTX" -n kube-system create configmap coredns --from-file=Corefile="$work/Corefile" \
--dry-run=client -o yaml | kubectl --context "$CTX" -n kube-system apply -f - >/dev/null 2>&1
}
dns_restore() {
kubectl --context "$CTX" -n kube-system create configmap coredns --from-file=Corefile="$work/Corefile.orig" \
--dry-run=client -o yaml | kubectl --context "$CTX" -n kube-system apply -f - >/dev/null 2>&1 ||
echo "!! couldn't restore CoreDNS's Corefile: it's in $work/Corefile.orig" >&2
}
dns_resolves() { # whether a server pod resolves wait.lab.test yet
"${K[@]}" exec deploy/inbuxa-core -c server -- sh -c 'getent hosts wait.lab.test || curl -s -m 2 -o /dev/null http://wait.lab.test; [ $? -ne 6 ]' >/dev/null 2>&1
}
admin_jmap() { # JMAP calls on stdin, as [email protected]
python3 admin-jmap.py "http://127.0.0.1:$PORT" "$STATE/admin.password"
}
send() { # <count> <recipient>: messages from alice, over submissions
LB="$LB" PASSWORD_FILE="$STATE/alice.password" python3 - "$@" <<'EOF'
import os, smtplib, ssl, sys
from email.message import EmailMessage
from email.utils import formatdate, make_msgid
tls = ssl.create_default_context(); tls.check_hostname = False; tls.verify_mode = ssl.CERT_NONE
password = open(os.environ["PASSWORD_FILE"]).read().strip()
with smtplib.SMTP_SSL(os.environ["LB"], 465, context=tls, timeout=30) as s:
s.login("[email protected]", password)
for i in range(int(sys.argv[1])):
m = EmailMessage()
m["From"] = "[email protected]"; m["To"] = sys.argv[2]
m["Subject"] = f"alerts-test {i}"; m["Date"] = formatdate()
m["Message-ID"] = make_msgid(domain="lab.test")
m.set_content("Sent by the inbuxa-k8s-lab alerts test.\n")
s.send_message(m)
EOF
}
echo "== the chart, with scraping on"
install "${METRICS[@]}"
LB=$("${K[@]}" get svc inbuxa-mail -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
echo "== Prometheus"
kubectl --context "$CTX" create namespace monitoring --dry-run=client -o yaml | kubectl --context "$CTX" apply -f - >/dev/null
"${M[@]}" create secret generic inbuxa-scrape \
--from-literal=password="$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.password}' | base64 -d)" \
--dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null
rules >"$work/rules.yml"
"${M[@]}" create configmap prometheus-rules --from-file=rules.yml="$work/rules.yml" \
--dry-run=client -o yaml | "${M[@]}" apply -f - >/dev/null
kubectl --context "$CTX" apply -f - >/dev/null <<EOF
apiVersion: v1
kind: ServiceAccount
metadata: {name: prometheus, namespace: monitoring}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: {name: lab-prometheus, namespace: inbuxa}
rules:
- apiGroups: [""]
resources: [services, pods, endpoints]
verbs: [get, list, watch]
- apiGroups: [discovery.k8s.io]
resources: [endpointslices]
verbs: [get, list, watch]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: {name: lab-prometheus, namespace: inbuxa}
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: lab-prometheus}
subjects: [{kind: ServiceAccount, name: prometheus, namespace: monitoring}]
---
apiVersion: v1
kind: ConfigMap
metadata: {name: prometheus-config, namespace: monitoring}
data:
prometheus.yml: |
global: {scrape_interval: 5s, scrape_timeout: 4s, evaluation_interval: 5s}
rule_files: [/etc/prometheus/rules/rules.yml]
scrape_configs:
# What the chart's ServiceMonitor asks the operator for
- job_name: inbuxa
metrics_path: /metrics/prometheus
basic_auth: {username: prometheus, password_file: /etc/prometheus/scrape/password}
kubernetes_sd_configs: [{role: endpointslice, namespaces: {names: [inbuxa]}}]
relabel_configs:
- {source_labels: [__meta_kubernetes_service_label_inbuxa_org_metrics], regex: "true", action: keep}
- {source_labels: [__meta_kubernetes_endpointslice_port_name], regex: http, action: keep}
- {source_labels: [__meta_kubernetes_service_name], target_label: job}
- {source_labels: [__meta_kubernetes_namespace], target_label: namespace}
- {source_labels: [__meta_kubernetes_pod_name], target_label: pod}
- {source_labels: [__meta_kubernetes_pod_label_inbuxa_org_group], target_label: inbuxa_org_group}
---
apiVersion: apps/v1
kind: Deployment
metadata: {name: prometheus, namespace: monitoring}
spec:
replicas: 1
selector: {matchLabels: {app: prometheus}}
template:
metadata: {labels: {app: prometheus}}
spec:
serviceAccountName: prometheus
securityContext: {runAsNonRoot: true, runAsUser: 65534, fsGroup: 65534, seccompProfile: {type: RuntimeDefault}}
containers:
- name: prometheus
image: $PROM_IMAGE
imagePullPolicy: Never
args: [--config.file=/etc/prometheus/prometheus.yml, --storage.tsdb.path=/prometheus, --web.enable-lifecycle]
securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: [ALL]}}
volumeMounts:
- {name: config, mountPath: /etc/prometheus/prometheus.yml, subPath: prometheus.yml}
- {name: rules, mountPath: /etc/prometheus/rules}
- {name: scrape, mountPath: /etc/prometheus/scrape}
- {name: data, mountPath: /prometheus}
volumes:
- {name: config, configMap: {name: prometheus-config}}
- {name: rules, configMap: {name: prometheus-rules}}
- {name: scrape, secret: {secretName: inbuxa-scrape}}
- {name: data, emptyDir: {}}
EOF
"${M[@]}" rollout status deploy/prometheus --timeout 3m >/dev/null
for _ in $(seq 40); do prom /-/ready >/dev/null 2>&1 && break; sleep 3; done
for _ in $(seq 30); do
up=$(prom '/api/v1/query?query=sum(up%7Bjob%3D%22inbuxa-metrics%22%7D)' 2>/dev/null | python3 -c '
import json, sys
try: r = json.load(sys.stdin)["data"]["result"]
except Exception: r = []
print(int(float(r[0]["value"][1])) if r else 0)')
[ "$up" = 5 ] && break
sleep 3
done
check "Prometheus scrapes the five server pods" "$up" 5
rules_loaded=$(prom /api/v1/rules | python3 -c 'import json, sys; print(sum(len(g["rules"]) for g in json.load(sys.stdin)["data"]["groups"]))')
check "rules loaded" "$rules_loaded" 7
echo "== 1. quiet while nothing is wrong"
sleep 60
check "alerts firing on a healthy install" "$(firing | paste -sd " ")" ""
echo "== 2. breakage"
forward || { echo "!! no port-forward to the server" >&2; exit 1; }
echo "-- a certificate that runs out in two days"
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 2 \
-subj "/CN=alerts.lab.test" -addext "subjectAltName=DNS:alerts.lab.test" \
-keyout "$work/cert.key" -out "$work/cert.crt" 2>/dev/null
cert=$(python3 - "$work/cert.crt" "$work/cert.key" <<'EOF2' | admin_jmap | python3 -c '
import json, sys; r = json.load(sys.stdin)[0][1]; print(r.get("created", {}).get("c", {}).get("id", ""))'
import json, sys
print(json.dumps([["x:Certificate/set", {"create": {"c": {
"certificate": {"@type": "Text", "value": open(sys.argv[1]).read()},
"privateKey": {"@type": "Text", "secret": open(sys.argv[2]).read()}}}}, "s"]]))
EOF2
)
[ -n "$cert" ] || { echo "FAIL couldn't store the test certificate"; fails=$((fails + 1)); }
wait_for InbuxaCertificateExpiring 90
# Lower thresholds: two queued messages, a minute's wait, any store time,
# two bounces
load_rules --set server.metrics.prometheus.rules.queueMessages=1 \
--set-json server.metrics.prometheus.rules.queueOldestHours=0.015 \
--set-json server.metrics.prometheus.rules.storeLatencyMs=0.001 \
--set-json server.metrics.prometheus.rules.deliveryFailureRatio=0.01 \
--set server.metrics.prometheus.rules.deliveryFailureMinimum=2
echo "-- a server process stopped"
victim=$("${K[@]}" get pods -l inbuxa.org/group=mta -o jsonpath='{.items[0].metadata.name}')
node=$("${K[@]}" get pod "$victim" -o jsonpath='{.spec.nodeName}')
cid=$("${K[@]}" get pod "$victim" -o jsonpath='{.status.containerStatuses[0].containerID}' | sed 's|.*://||')
pid=$(docker exec "$node" crictl inspect --output go-template --template '{{.info.pid}}' "$cid")
docker exec "$node" kill -STOP "$pid"
stopped="$node $pid"
wait_for InbuxaServerDown 45
docker exec "$node" kill -CONT "$pid"
stopped=""
echo "-- mail that waits, and mail that bounces"
dns_add
for _ in $(seq 60); do dns_resolves && break; sleep 3; done
dns_resolves || { echo "FAIL wait.lab.test doesn't resolve"; fails=$((fails + 1)); }
# The stopped pod is back in a moment; a connection that lands on it
# before then is tried again
for attempt in 1 2 3 4 5 6; do
# wait.lab.test never answers, which is retried; example.com publishes a
# null MX (RFC 7505), which bounces at once
send 3 "[email protected]" 2>/dev/null && send 3 "[email protected]" 2>/dev/null && break
[ "$attempt" = 6 ] && { echo "FAIL couldn't send the test mail"; fails=$((fails + 1)); }
sleep 10
done
echo "-- a front pod cut off from Postgres"
cut=$("${K[@]}" get pods -l inbuxa.org/group=front -o jsonpath='{.items[0].metadata.name}')
cut_ip=$("${K[@]}" get pod "$cut" -o jsonpath='{.status.podIP}')
"${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off=true >/dev/null
kubectl --context "$CTX" apply -f - >/dev/null <<'EOF'
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: {name: lab-cut-off, namespace: data}
spec:
podSelector: {matchLabels: {cnpg.io/cluster: inbuxa-pg}}
policyTypes: [Ingress]
ingress:
- from:
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: cnpg-system}}
- podSelector: {matchLabels: {cnpg.io/cluster: inbuxa-pg}}
- namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: inbuxa}}
podSelector:
matchExpressions: [{key: lab.inbuxa.org/cut-off, operator: DoesNotExist}]
EOF
# The policy stops new connections; end the ones the pod holds
sleep 5
kubectl --context "$CTX" -n data exec inbuxa-pg-1 -c postgres -- psql -qtA -U postgres -c \
"select count(pg_terminate_backend(pid)) from pg_stat_activity where client_addr = '$cut_ip'" >/dev/null
# The lease alert needs five minutes without a renewal, the queue gauges
# the core's next count (every five minutes)
wait_for InbuxaDataStoreSlow 60
wait_for InbuxaDeliveriesFailing 180
wait_for InbuxaQueueLarge 360
wait_for InbuxaQueueMessageWaiting 360
wait_for InbuxaClusterLeaseNotRenewed 420
echo "-- back to normal"
kubectl --context "$CTX" -n data delete networkpolicy lab-cut-off >/dev/null
"${K[@]}" label pod "$cut" lab.inbuxa.org/cut-off- >/dev/null
cut=""
echo
if [ "$fails" -eq 0 ]; then echo "alerts-test: all passed"; else echo "alerts-test: $fails failed"; exit 1; fi
+144
View File
@@ -0,0 +1,144 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: 2026 Coffey Labs LLC
# SPDX-License-Identifier: Apache-2.0
#
# K-C10: metrics. Installs with server.metrics.prometheus and .otlp on, then
# checks, for every server pod:
# - /metrics/prometheus refuses no password and a wrong one, and answers
# the chart's password with the gauges the alerts read;
# - its cluster lease was renewed in the last two and a half minutes;
# - the headless metrics Service lists it;
# - it pushes OTLP to a collector, with the header from the Secret (the
# collector sees the nodes' addresses, so this counts pushes).
# The collector is HAProxy on the kind network, answering 200 and logging
# each request with its Authorization header.
#
# Leaves the chart as it found it, and removes the collector and the Secret.
#
# SERVER_TAG=kc10 WEBMAIL_TAG=kw3 lab/chart/metrics-test.sh
set -euo pipefail
cd "$(dirname "$0")"
K=(kubectl --context kind-inbuxa-lab -n inbuxa)
CHART="${CHART:-../../../inbuxa-charts/charts/inbuxa}"
SERVER_TAG="${SERVER_TAG:?the server image tag in kind}"
HAPROXY_IMAGE="${HAPROXY_IMAGE:-haproxy:3.0-alpine}"
EXTRA=()
[ -n "${WEBMAIL_TAG:-}" ] && EXTRA+=(--set webmail.image.tag="$WEBMAIL_TAG")
SINK=inbuxa-lab-otlp
HEADERS=inbuxa-lab-otlp
TOKEN="Bearer lab-otlp-$RANDOM$RANDOM"
fails=0
install() { # [helm args]: the lab's install, with the webmail and console
helm --kube-context kind-inbuxa-lab -n inbuxa upgrade inbuxa "$CHART" \
-f values-lab.yaml -f values-frontends.yaml --set server.image.tag="$SERVER_TAG" \
"${EXTRA[@]}" "$@" --wait --timeout 10m >/dev/null
for d in front mta core; do "${K[@]}" rollout status deploy/inbuxa-$d --timeout 5m >/dev/null; done
}
work=$(mktemp -d)
cleanup() {
install >/dev/null 2>&1 || echo "!! couldn't put the chart back: run frontends-test.sh" >&2
"${K[@]}" delete secret "$HEADERS" --ignore-not-found >/dev/null
docker rm -f "$SINK" >/dev/null 2>&1 || true
rm -rf "$work"
}
trap cleanup EXIT
settle() { # until the last rollout's old pods are gone
for _ in $(seq 60); do
[ -z "$("${K[@]}" get pods -l app.kubernetes.io/component=server -o jsonpath='{.items[?(@.metadata.deletionTimestamp)].metadata.name}')" ] && return
sleep 3
done
}
check() { # <name> <got> <want>
if [ "$2" = "$3" ]; then echo "ok $1"; else echo "FAIL $1: got $2, want $3"; fails=$((fails + 1)); fi
}
scrape() { # <pod ip> [curl args]: the status code, the body in $work/body
"${K[@]}" exec deploy/inbuxa-core -c server -- \
curl -s -m 10 -o /dev/stdout -w '\n%{http_code}' "${@:2}" "http://$1:8080/metrics/prometheus" >"$work/out" || true
sed '$d' "$work/out" >"$work/body"
tail -n1 "$work/out"
}
gauge() { # <name>: its value in $work/body, as an integer
awk -v n="$1" '$1 == n { printf "%d", $2 }' "$work/body"
}
echo "== the collector"
cat >"$work/haproxy.cfg" <<'EOF'
global
log stdout format raw local0
defaults
mode http
log global
timeout client 30s
timeout server 30s
timeout connect 5s
frontend otlp
bind :4318
http-request capture req.hdr(authorization) len 64
log-format "%ci %HM %HP %hr"
http-request return status 200
EOF
docker rm -f "$SINK" >/dev/null 2>&1 || true
docker run -d --name "$SINK" --network kind -v "$work/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro" "$HAPROXY_IMAGE" >/dev/null
SINK_IP=$(docker inspect "$SINK" -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
[ -n "$SINK_IP" ] || { docker logs "$SINK" >&2; echo "!! the collector didn't start" >&2; exit 1; }
printf 'authorization=%s\n' "$TOKEN" >"$work/headers"
"${K[@]}" create secret generic "$HEADERS" --from-file=headers="$work/headers" \
--dry-run=client -o yaml | "${K[@]}" apply -f - >/dev/null
echo "== install with metrics on"
install --set server.metrics.prometheus.enabled=true \
--set server.metrics.otlp.enabled=true \
--set server.metrics.otlp.endpoint="http://$SINK_IP:4318" \
--set server.metrics.otlp.intervalSeconds=5 \
--set server.metrics.otlp.headersSecret="$HEADERS"
settle
PW=$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.password}' | base64 -d)
USER=$("${K[@]}" get secret inbuxa-metrics -o jsonpath='{.data.username}' | base64 -d)
mapfile -t PODS < <("${K[@]}" get pods -l app.kubernetes.io/component=server \
-o jsonpath='{range .items[*]}{.metadata.name} {.status.podIP}{"\n"}{end}')
check "five server pods" "${#PODS[@]}" 5
echo "== the metrics Service lists every pod"
listed=$("${K[@]}" get endpointslices -l kubernetes.io/service-name=inbuxa-metrics \
-o jsonpath='{range .items[*].endpoints[*]}{.addresses[0]}{"\n"}{end}' | sort -u | wc -l)
check "inbuxa-metrics endpoints" "$listed" 5
echo "== scraping"
first_ip=$(echo "${PODS[0]}" | cut -d' ' -f2)
check "no password: refused" "$(scrape "$first_ip")" 401
check "wrong password: refused" "$(scrape "$first_ip" -u "$USER:not-it")" 401
now=$(date +%s)
for p in "${PODS[@]}"; do
name=${p% *}; ip=${p#* }
check "$name: the chart's password" "$(scrape "$ip" -u "$USER:$PW")" 200
for m in queue_count queue_oldest_message cluster_lease_renewed tls_earliest_expiry server_memory; do
grep -q "^$m " "$work/body" || { echo "FAIL $name: no $m"; fails=$((fails + 1)); }
done
renewed=$(gauge cluster_lease_renewed)
age=$((now - renewed))
if [ "$renewed" -gt 0 ] && [ "$age" -le 150 ]; then
echo "ok $name: lease renewed ${age}s ago"
else
echo "FAIL $name: lease renewed at $renewed (${age}s ago)"; fails=$((fails + 1))
fi
done
echo "== OTLP pushes, with the header"
sleep 20
docker logs "$SINK" 2>/dev/null | grep " POST /v1/metrics " >"$work/pushes" || true
# Pods reach the kind network through their node's address, so the
# collector sees one address per node with server pods, and every pod's
# pushes, about one per pod per interval
nodes=$("${K[@]}" get pods -l app.kubernetes.io/component=server -o jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}' | sort -u | wc -l)
pushers=$(awk '{print $1}' "$work/pushes" | sort -u | wc -l)
check "nodes the pushes came from" "$pushers" "$nodes"
count=$(wc -l <"$work/pushes")
if [ "$count" -ge 15 ]; then echo "ok $count pushes in the first 20s or so"; else echo "FAIL only $count pushes"; fails=$((fails + 1)); fi
bad=$(grep -vcF "{$TOKEN}" "$work/pushes" || true)
check "pushes without the token" "$bad" 0
echo
if [ "$fails" -eq 0 ]; then echo "metrics-test: all passed"; else echo "metrics-test: $fails failed"; exit 1; fi
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env python3
"""Removes the queued messages a test sent: from a sender, since a time.
queue-clean.py <server> <admin password file> <sender> <since, ISO 8601 UTC>
Prints how many it removed.
"""
import json, subprocess, sys
server, pwfile, sender, since = sys.argv[1:]
here = __file__.rsplit("/", 1)[0]
def jmap(calls):
out = subprocess.run([sys.executable, f"{here}/admin-jmap.py", server, pwfile], input=json.dumps(calls),
capture_output=True, text=True, check=True).stdout
return json.loads(out)
got = jmap([["x:QueuedMessage/query", {}, "q"],
["x:QueuedMessage/get", {"#ids": {"resultOf": "q", "name": "x:QueuedMessage/query", "path": "/ids"},
"properties": ["returnPath", "createdAt"]}, "g"]])
ids = [m["id"] for m in got[1][1]["list"] if m.get("returnPath") == sender and m.get("createdAt", "") >= since]
if ids:
res = jmap([["x:QueuedMessage/set", {"destroy": ids}, "d"]])[0][1]
if res.get("notDestroyed"):
sys.exit(f"not removed: {json.dumps(res['notDestroyed'])[:300]}")
print(len(ids))