ratelimit-test: sign-in limits with several replicas (K-W3) #23
No files matched your search
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The server's blocked addresses, as [email protected].
|
||||
|
||||
blocked-ips.py <server> <admin password file> list
|
||||
blocked-ips.py <server> <admin password file> unblock <ip>...
|
||||
|
||||
Signs in like the console (authCode with PKCE), then x:BlockedIp/get or
|
||||
x:BlockedIp/set destroy. list prints one address per line.
|
||||
"""
|
||||
import base64, hashlib, http.client, json, secrets, sys, urllib.parse
|
||||
server, pwfile, action, *ips = sys.argv[1:]
|
||||
def request(method, path, headers=None, body=None):
|
||||
u = urllib.parse.urlsplit(server); c = http.client.HTTPConnection(u.hostname, u.port, timeout=60)
|
||||
c.request(method, path, body=body, headers=headers or {}); r = c.getresponse(); d = r.read(); c.close(); return r, d
|
||||
v = secrets.token_urlsafe(48); ch = base64.urlsafe_b64encode(hashlib.sha256(v.encode()).digest()).rstrip(b"=").decode()
|
||||
red = "https://admin.lab.test/oauth/callback"
|
||||
r, d = request("POST", "/api/auth", {"content-type": "application/json"}, json.dumps({"type": "authCode", "accountName": "[email protected]",
|
||||
"accountSecret": open(pwfile).read().strip(), "clientId": "inbuxa-admin", "redirectUri": red, "codeChallenge": ch, "codeChallengeMethod": "S256", "state": "x"}))
|
||||
code = json.loads(d)["client_code"]
|
||||
r, d = request("POST", "/auth/token", {"content-type": "application/x-www-form-urlencoded"}, urllib.parse.urlencode({"grant_type": "authorization_code",
|
||||
"code": code, "redirect_uri": red, "client_id": "inbuxa-admin", "code_verifier": v}))
|
||||
tok = json.loads(d)["access_token"]
|
||||
def jmap(calls):
|
||||
r, d = request("POST", "/jmap", {"content-type": "application/json", "authorization": "Bearer " + tok},
|
||||
json.dumps({"using": ["urn:ietf:params:jmap:core", "urn:inbuxa:jmap:registry"], "methodCalls": calls}))
|
||||
return json.loads(d)["methodResponses"]
|
||||
res = jmap([["x:BlockedIp/get", {"ids": None}, "g"]])
|
||||
items = res[0][1].get("list", [])
|
||||
if action == "list":
|
||||
for i in items: print(i.get("address"), i.get("reason"), i.get("expiresAt"))
|
||||
else:
|
||||
ids = [i["id"] for i in items if any(ip in json.dumps(i) for ip in ips)]
|
||||
if ids: print(json.dumps(jmap([["x:BlockedIp/set", {"destroy": ids}, "d"]])[0][1].get("destroyed")))
|
||||
else: print("nothing to unblock")
|
||||
@@ -0,0 +1,49 @@
|
||||
// Sign-in failures and bad tokens, from a pod, straight at server pods.
|
||||
//
|
||||
// node -e "$(cat probe.js)" <mode> <n> <host:port>...
|
||||
//
|
||||
// password wrong passwords for one account, [email protected]
|
||||
// password-xff wrong passwords for a different account each time, each
|
||||
// "through a proxy" for a different client (X-Forwarded-For)
|
||||
// bearer a token the server never issued
|
||||
//
|
||||
// Targets take turns, PACE_MS apart (the server also limits unauthenticated
|
||||
// requests to 100 a minute per address). Prints each run of identical answers.
|
||||
const [mode, n, ...targets] = process.argv.slice(1);
|
||||
const http = require("http");
|
||||
function once(target, i) {
|
||||
const [host, port] = target.split(":");
|
||||
return new Promise((resolve) => {
|
||||
const opts = { host, port: +port, timeout: 15000, headers: {} };
|
||||
let body = null;
|
||||
if (mode === "bearer") Object.assign(opts, { method: "GET", path: "/jmap/session", headers: { authorization: "Bearer not-a-token-" + i } });
|
||||
else {
|
||||
const run = process.env.RUN || "0";
|
||||
const account = mode === "password-xff" ? `kw3-${run}-${i}@lab.test` : `kw3-${run}@lab.test`;
|
||||
body = JSON.stringify({ type: "authCode", accountName: account, accountSecret: "wrong-" + i, clientId: "inbuxa-admin",
|
||||
redirectUri: "https://admin.lab.test/oauth/callback", codeChallenge: "x".repeat(43), codeChallengeMethod: "S256", state: "s" });
|
||||
opts.method = "POST";
|
||||
opts.path = "/api/auth";
|
||||
opts.headers = { "content-type": "application/json", "content-length": Buffer.byteLength(body) };
|
||||
if (mode === "password-xff") opts.headers["x-forwarded-for"] = `198.51.100.${(i % 250) + 1}`;
|
||||
}
|
||||
const req = http.request(opts, (res) => {
|
||||
let d = "";
|
||||
res.on("data", (c) => (d += c));
|
||||
res.on("end", () => resolve(res.statusCode + " " + d.slice(0, 40).replace(/\s+/g, " ")));
|
||||
});
|
||||
req.on("timeout", () => { req.destroy(); resolve("timeout"); });
|
||||
req.on("error", (e) => resolve("error " + e.code));
|
||||
if (body) req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
(async () => {
|
||||
let last = null, from = 0;
|
||||
for (let i = 0; i < +n; i++) {
|
||||
if (i) await new Promise((r) => setTimeout(r, +(process.env.PACE_MS || 0)));
|
||||
const r = await once(targets[i % targets.length], i);
|
||||
if (r !== last) { if (last !== null) console.log(`#${from}-#${i - 1}: ${last}`); last = r; from = i; }
|
||||
}
|
||||
console.log(`#${from}-#${+n - 1}: ${last}`);
|
||||
})();
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sign-in limits with several replicas (K-W3), on the lab's install with the
|
||||
# webmail (frontends-test.sh first):
|
||||
#
|
||||
# 1. the webmail's own limit counts per replica: one replica refuses, the
|
||||
# other still starts a sign-in
|
||||
# 2. the webmail counts the browser its proxy names, not the proxy: with
|
||||
# one client refused, another through the same replica still gets in
|
||||
# 3. the server counts failed sign-ins across its pods, and bans the
|
||||
# address after about 100 (the default), whichever pod they reached
|
||||
# 4. through a proxy, the server counts each client, not the proxy: 110
|
||||
# failures for 110 clients ban nobody, the proxy least of all
|
||||
#
|
||||
# That the webmail passes that address on to the server as X-Forwarded-For
|
||||
# is its own test (inbuxa-webmail server/src/clientcontext.test.ts); step 4's
|
||||
# proxy is a pod on the network the webmail's are on, which the server
|
||||
# believes.
|
||||
#
|
||||
# Addresses the test gets banned are unblocked at the end.
|
||||
#
|
||||
# lab/chart/ratelimit-test.sh
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
K=(kubectl --context kind-inbuxa-lab)
|
||||
STATE="${STATE:-$HOME/.cache/inbuxa-lab}"
|
||||
PROBE_IMAGE="${PROBE_IMAGE:-$("${K[@]}" -n inbuxa get deploy inbuxa-webmail -o jsonpath='{.spec.template.spec.containers[0].image}')}"
|
||||
PIDS=()
|
||||
cleanup() { for p in "${PIDS[@]}"; do kill "$p" 2>/dev/null || true; done; }
|
||||
trap cleanup EXIT
|
||||
fail() { echo "!! $*" >&2; exit 1; }
|
||||
forward() { # <local port> <target> <remote port>
|
||||
"${K[@]}" -n inbuxa port-forward "$2" "$1:$3" >/dev/null 2>&1 &
|
||||
PIDS+=($!)
|
||||
}
|
||||
blocked() { python3 blocked-ips.py http://127.0.0.1:18090 "$STATE/admin.password" list | sort; }
|
||||
probe() { # <mode> <n> <targets...>: from a pod in "data", outside the chart's pods
|
||||
"${K[@]}" -n data run "kw3-probe-$RANDOM" --rm -i --quiet --restart=Never --image="$PROBE_IMAGE" \
|
||||
--image-pull-policy=IfNotPresent --env="RUN=$RUN" --env="PACE_MS=${PACE_MS:-700}" --command -- node -e "$(cat probe.js)" "$@"
|
||||
}
|
||||
|
||||
mapfile -t webmail < <("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=webmail \
|
||||
--field-selector status.phase=Running -o name | sort)
|
||||
mapfile -t front < <("${K[@]}" -n inbuxa get pods -l app.kubernetes.io/component=server,inbuxa.org/group=front \
|
||||
--field-selector status.phase=Running -o jsonpath='{range .items[*]}{.status.podIP}:8080{"\n"}{end}')
|
||||
[ "${#webmail[@]}" -ge 2 ] && [ "${#front[@]}" -ge 2 ] || fail "needs two webmail and two front pods"
|
||||
forward 18090 svc/inbuxa-http 8080
|
||||
forward 18081 "${webmail[0]}" 8080
|
||||
forward 18082 "${webmail[1]}" 8080
|
||||
for port in 18090 18081 18082; do
|
||||
for _ in $(seq 30); do curl -s -o /dev/null -m 2 "http://127.0.0.1:$port/" && break; sleep 1; done
|
||||
done
|
||||
before=$(blocked)
|
||||
# The server's counts last a day: a new account name each run
|
||||
RUN=$(date +%s)
|
||||
unblock_new() { # what's been banned since $before
|
||||
local new
|
||||
new=$(comm -13 <(echo "$before") <(blocked) | awk '{print $1}')
|
||||
[ -z "$new" ] || python3 blocked-ips.py http://127.0.0.1:18090 "$STATE/admin.password" unblock $new >/dev/null
|
||||
}
|
||||
|
||||
echo "== 1. the webmail's limit counts per replica"
|
||||
RUN=$RUN python3 - <<'EOF'
|
||||
import http.client, os, sys
|
||||
# One client for this run, as an ingress would name it, so the lab's own
|
||||
# address keeps its allowance for the other tests
|
||||
client = f"192.0.2.{int(os.environ['RUN']) % 250 + 1}"
|
||||
# A refusal sends the browser back with signin_error=rate_limited; a start
|
||||
# sends it to the server's page
|
||||
def start(port, who=None):
|
||||
c = http.client.HTTPConnection("127.0.0.1", port, timeout=30)
|
||||
c.request("GET", "/api/auth/oauth/start?username=alice%40lab.test", headers={"x-forwarded-for": who or client})
|
||||
r = c.getresponse(); r.read(); c.close()
|
||||
return r.getheader("location") or f"status {r.status}"
|
||||
def started(where):
|
||||
return "rate_limited" not in where and "/login" in where
|
||||
for i in range(1, 400):
|
||||
if "rate_limited" in start(18081):
|
||||
break
|
||||
else:
|
||||
sys.exit("!! replica A never refused")
|
||||
if i < 150:
|
||||
sys.exit(f"!! replica A refused {client} at {i}: counted with other clients (the proxy?)")
|
||||
print(f"replica A: {client}'s sign-in {i} refused (rate_limited)")
|
||||
if not started(where := start(18082)):
|
||||
sys.exit(f"!! replica B: {where}, expected it to start the sign-in")
|
||||
print(f"replica B: still starts {client}'s sign-in, to the server's page")
|
||||
other = f"192.0.2.{(int(os.environ['RUN']) + 1) % 250 + 1}"
|
||||
if not started(where := start(18081, other)):
|
||||
sys.exit(f"!! replica A refused {other} too ({where}): it counted the proxy, not the browser")
|
||||
print(f"== 2. the webmail counts the browser its proxy names\nreplica A: still starts {other}'s sign-in")
|
||||
EOF
|
||||
|
||||
echo "== 3. the server counts failed sign-ins across its pods"
|
||||
out=$(probe password 110 "${front[@]}")
|
||||
echo "$out" | sed 's/^/ /'
|
||||
first=$(grep -m1 ': 403' <<<"$out" | sed 's/^#\([0-9]*\)-.*/\1/')
|
||||
# Counted per pod, the ban would take about 200; at or under the limit of
|
||||
# 100 means the pods share one count (fewer if kind reused an address that
|
||||
# already had failures today)
|
||||
[ -n "$first" ] && [ "$first" -le 105 ] \
|
||||
|| fail "expected the ban by about 100 failures across both pods, got: ${first:-none}"
|
||||
echo "banned after $first failures split across ${#front[@]} pods"
|
||||
unblock_new
|
||||
|
||||
echo "== 4. through a proxy, each client counts, not the proxy"
|
||||
mid=$(blocked)
|
||||
out=$(probe password-xff 110 "${front[@]}")
|
||||
echo "$out" | sed 's/^/ /'
|
||||
grep -q ': 403' <<<"$out" && fail "a proxy was banned for its clients' failures"
|
||||
[ "$(blocked)" = "$mid" ] || fail "something new was banned: $(comm -13 <(echo "$mid") <(blocked))"
|
||||
echo "110 failures for 110 clients: nobody banned"
|
||||
|
||||
unblock_new
|
||||
echo "== all pass"
|
||||
Reference in new issue
Block a user