From dc2c7a54b16d6eb58480538c359c6d4049018625 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Wed, 18 Nov 2026 19:46:24 -0700 Subject: [PATCH] ratelimit-test: sign-in limits with several replicas (K-W3) The webmail's own limit counts per replica and per browser its proxy names; the server counts failed sign-ins across its pods and bans after about 100; through a proxy, each client counts, not the proxy. Bans the test causes are lifted at the end (blocked-ips.py). --- lab/chart/blocked-ips.py | 34 +++++++++++ lab/chart/probe.js | 49 +++++++++++++++ lab/chart/ratelimit-test.sh | 115 ++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100755 lab/chart/blocked-ips.py create mode 100644 lab/chart/probe.js create mode 100755 lab/chart/ratelimit-test.sh diff --git a/lab/chart/blocked-ips.py b/lab/chart/blocked-ips.py new file mode 100755 index 0000000..144b877 --- /dev/null +++ b/lab/chart/blocked-ips.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""The server's blocked addresses, as admin@lab.test. + + blocked-ips.py list + blocked-ips.py unblock ... + +Signs in like the console (authCode with PKCE), then x:BlockedIp/get or +x:BlockedIp/set destroy. list prints one address per line. +""" +import base64, hashlib, http.client, json, secrets, sys, urllib.parse +server, pwfile, action, *ips = sys.argv[1:] +def request(method, path, headers=None, body=None): + u = urllib.parse.urlsplit(server); c = http.client.HTTPConnection(u.hostname, u.port, timeout=60) + c.request(method, path, body=body, headers=headers or {}); r = c.getresponse(); d = r.read(); c.close(); return r, d +v = secrets.token_urlsafe(48); ch = base64.urlsafe_b64encode(hashlib.sha256(v.encode()).digest()).rstrip(b"=").decode() +red = "https://admin.lab.test/oauth/callback" +r, d = request("POST", "/api/auth", {"content-type": "application/json"}, json.dumps({"type": "authCode", "accountName": "admin@lab.test", + "accountSecret": open(pwfile).read().strip(), "clientId": "inbuxa-admin", "redirectUri": red, "codeChallenge": ch, "codeChallengeMethod": "S256", "state": "x"})) +code = json.loads(d)["client_code"] +r, d = request("POST", "/auth/token", {"content-type": "application/x-www-form-urlencoded"}, urllib.parse.urlencode({"grant_type": "authorization_code", + "code": code, "redirect_uri": red, "client_id": "inbuxa-admin", "code_verifier": v})) +tok = json.loads(d)["access_token"] +def jmap(calls): + r, d = request("POST", "/jmap", {"content-type": "application/json", "authorization": "Bearer " + tok}, + json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"], "methodCalls": calls})) + return json.loads(d)["methodResponses"] +res = jmap([["x:BlockedIp/get", {"ids": None}, "g"]]) +items = res[0][1].get("list", []) +if action == "list": + for i in items: print(i.get("address"), i.get("reason"), i.get("expiresAt")) +else: + ids = [i["id"] for i in items if any(ip in json.dumps(i) for ip in ips)] + if ids: print(json.dumps(jmap([["x:BlockedIp/set", {"destroy": ids}, "d"]])[0][1].get("destroyed"))) + else: print("nothing to unblock") diff --git a/lab/chart/probe.js b/lab/chart/probe.js new file mode 100644 index 0000000..c512ba3 --- /dev/null +++ b/lab/chart/probe.js @@ -0,0 +1,49 @@ +// Sign-in failures and bad tokens, from a pod, straight at server pods. +// +// node -e "$(cat probe.js)" ... +// +// password wrong passwords for one account, kw3-$RUN@lab.test +// password-xff wrong passwords for a different account each time, each +// "through a proxy" for a different client (X-Forwarded-For) +// bearer a token the server never issued +// +// Targets take turns, PACE_MS apart (the server also limits unauthenticated +// requests to 100 a minute per address). Prints each run of identical answers. +const [mode, n, ...targets] = process.argv.slice(1); +const http = require("http"); +function once(target, i) { + const [host, port] = target.split(":"); + return new Promise((resolve) => { + const opts = { host, port: +port, timeout: 15000, headers: {} }; + let body = null; + if (mode === "bearer") Object.assign(opts, { method: "GET", path: "/jmap/session", headers: { authorization: "Bearer not-a-token-" + i } }); + else { + const run = process.env.RUN || "0"; + const account = mode === "password-xff" ? `kw3-${run}-${i}@lab.test` : `kw3-${run}@lab.test`; + body = JSON.stringify({ type: "authCode", accountName: account, accountSecret: "wrong-" + i, clientId: "inbuxa-admin", + redirectUri: "https://admin.lab.test/oauth/callback", codeChallenge: "x".repeat(43), codeChallengeMethod: "S256", state: "s" }); + opts.method = "POST"; + opts.path = "/api/auth"; + opts.headers = { "content-type": "application/json", "content-length": Buffer.byteLength(body) }; + if (mode === "password-xff") opts.headers["x-forwarded-for"] = `198.51.100.${(i % 250) + 1}`; + } + const req = http.request(opts, (res) => { + let d = ""; + res.on("data", (c) => (d += c)); + res.on("end", () => resolve(res.statusCode + " " + d.slice(0, 40).replace(/\s+/g, " "))); + }); + req.on("timeout", () => { req.destroy(); resolve("timeout"); }); + req.on("error", (e) => resolve("error " + e.code)); + if (body) req.write(body); + req.end(); + }); +} +(async () => { + let last = null, from = 0; + for (let i = 0; i < +n; i++) { + if (i) await new Promise((r) => setTimeout(r, +(process.env.PACE_MS || 0))); + const r = await once(targets[i % targets.length], i); + if (r !== last) { if (last !== null) console.log(`#${from}-#${i - 1}: ${last}`); last = r; from = i; } + } + console.log(`#${from}-#${+n - 1}: ${last}`); +})(); diff --git a/lab/chart/ratelimit-test.sh b/lab/chart/ratelimit-test.sh new file mode 100755 index 0000000..8f7249b --- /dev/null +++ b/lab/chart/ratelimit-test.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Sign-in limits with several replicas (K-W3), on the lab's install with the +# webmail (frontends-test.sh first): +# +# 1. the webmail's own limit counts per replica: one replica refuses, the +# other still starts a sign-in +# 2. the webmail counts the browser its proxy names, not the proxy: with +# one client refused, another through the same replica still gets in +# 3. the server counts failed sign-ins across its pods, and bans the +# address after about 100 (the default), whichever pod they reached +# 4. through a proxy, the server counts each client, not the proxy: 110 +# failures for 110 clients ban nobody, the proxy least of all +# +# That the webmail passes that address on to the server as X-Forwarded-For +# is its own test (inbuxa-webmail server/src/clientcontext.test.ts); step 4's +# proxy is a pod on the network the webmail's are on, which the server +# believes. +# +# Addresses the test gets banned are unblocked at the end. +# +# lab/chart/ratelimit-test.sh +set -euo pipefail + +cd "$(dirname "$0")" +K=(kubectl --context kind-inbuxa-lab) +STATE="${STATE:-$HOME/.cache/inbuxa-lab}" +PROBE_IMAGE="${PROBE_IMAGE:-$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.template.spec.containers[0].image}')}" +PIDS=() +cleanup() { for p in "${PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; } +trap cleanup EXIT +fail() { echo "!! $*" >&2; exit 1; } +forward() { # + "${K[@]}" -n inbuxa port-forward "$2" "$1:$3" >/dev/null 2>&1 & + PIDS+=($!) +} +blocked() { python3 blocked-ips.py http://127.0.0.1:18090 "$STATE/admin.password" list | sort; } +probe() { # : from a pod in "data", outside the chart's pods + "${K[@]}" -n data run "kw3-probe-$RANDOM" --rm -i --quiet --restart=Never --image="$PROBE_IMAGE" \ + --image-pull-policy=IfNotPresent --env="RUN=$RUN" --env="PACE_MS=${PACE_MS:-700}" --command -- node -e "$(cat probe.js)" "$@" +} + +mapfile -t webmail < <("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=webmail \ + --field-selector status.phase=Running -o name | sort) +mapfile -t front < <("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,inbuxa.org/group=front \ + --field-selector status.phase=Running -o jsonpath='{range .items[*]}{.status.podIP}:8080{"\n"}{end}') +[ "${#webmail[@]}" -ge 2 ] && [ "${#front[@]}" -ge 2 ] || fail "needs two webmail and two front pods" +forward 18090 svc/inbuxa-http 8080 +forward 18081 "${webmail[0]}" 8080 +forward 18082 "${webmail[1]}" 8080 +for port in 18090 18081 18082; do + for _ in $(seq 30); do curl -s -o /dev/null -m 2 "http://127.0.0.1:$port/" && break; sleep 1; done +done +before=$(blocked) +# The server's counts last a day: a new account name each run +RUN=$(date +%s) +unblock_new() { # what's been banned since $before + local new + new=$(comm -13 <(echo "$before") <(blocked) | awk '{print $1}') + [ -z "$new" ] || python3 blocked-ips.py http://127.0.0.1:18090 "$STATE/admin.password" unblock $new >/dev/null +} + +echo "== 1. the webmail's limit counts per replica" +RUN=$RUN python3 - <<'EOF' +import http.client, os, sys +# One client for this run, as an ingress would name it, so the lab's own +# address keeps its allowance for the other tests +client = f"192.0.2.{int(os.environ['RUN']) % 250 + 1}" +# A refusal sends the browser back with signin_error=rate_limited; a start +# sends it to the server's page +def start(port, who=None): + c = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + c.request("GET", "/api/auth/oauth/start?username=alice%40lab.test", headers={"x-forwarded-for": who or client}) + r = c.getresponse(); r.read(); c.close() + return r.getheader("location") or f"status {r.status}" +def started(where): + return "rate_limited" not in where and "/login" in where +for i in range(1, 400): + if "rate_limited" in start(18081): + break +else: + sys.exit("!! replica A never refused") +if i < 150: + sys.exit(f"!! replica A refused {client} at {i}: counted with other clients (the proxy?)") +print(f"replica A: {client}'s sign-in {i} refused (rate_limited)") +if not started(where := start(18082)): + sys.exit(f"!! replica B: {where}, expected it to start the sign-in") +print(f"replica B: still starts {client}'s sign-in, to the server's page") +other = f"192.0.2.{(int(os.environ['RUN']) + 1) % 250 + 1}" +if not started(where := start(18081, other)): + sys.exit(f"!! replica A refused {other} too ({where}): it counted the proxy, not the browser") +print(f"== 2. the webmail counts the browser its proxy names\nreplica A: still starts {other}'s sign-in") +EOF + +echo "== 3. the server counts failed sign-ins across its pods" +out=$(probe password 110 "${front[@]}") +echo "$out" | sed 's/^/ /' +first=$(grep -m1 ': 403' <<<"$out" | sed 's/^#\([0-9]*\)-.*/\1/') +# Counted per pod, the ban would take about 200; at or under the limit of +# 100 means the pods share one count (fewer if kind reused an address that +# already had failures today) +[ -n "$first" ] && [ "$first" -le 105 ] \ + || fail "expected the ban by about 100 failures across both pods, got: ${first:-none}" +echo "banned after $first failures split across ${#front[@]} pods" +unblock_new + +echo "== 4. through a proxy, each client counts, not the proxy" +mid=$(blocked) +out=$(probe password-xff 110 "${front[@]}") +echo "$out" | sed 's/^/ /' +grep -q ': 403' <<<"$out" && fail "a proxy was banned for its clients' failures" +[ "$(blocked)" = "$mid" ] || fail "something new was banned: $(comm -13 <(echo "$mid") <(blocked))" +echo "110 failures for 110 clients: nobody banned" + +unblock_new +echo "== all pass" -- 2.54.0