From 8388c9708955082ac111627a13ed112bdefbc37c Mon Sep 17 00:00:00 2001 From: John Coffey Date: Fri, 20 Nov 2026 00:23:27 -0700 Subject: [PATCH] Drain on SIGTERM instead of dropping everything (K-W5) On SIGTERM the server exited at once: a JMAP request in flight failed in the browser, and a request reaching the pod as it went hung until it timed out. Now it drains (server/src/drain.ts): 1. /api/health answers 503 and every response says Connection: close, so load balancers and keep-alive clients move to another replica; requests are still served, for SHUTDOWN_DELAY (3 s). 2. No new connections, and every live-update stream (fan-out, relay and raw relay) is ended cleanly, so each tab's EventSource reconnects to a replica that's staying. 3. Requests in flight get SHUTDOWN_TIMEOUT (5 s) to finish; then what's left is closed and the process exits. A second signal stops at once. Tested in the lab (inbuxa-k8s-lab webmail-restart-test.sh): a rolling restart under three open tabs and a JMAP request every 200 ms. The released webmail had a request hang for 30 s; with this, 182-183 requests and none failed, three runs out of three, every stream ending cleanly and reconnecting. --- .env.example | 9 +++++ server/src/app.ts | 19 ++++++++- server/src/config.ts | 4 ++ server/src/drain.test.ts | 78 +++++++++++++++++++++++++++++++++++++ server/src/drain.ts | 84 ++++++++++++++++++++++++++++++++++++++++ server/src/index.ts | 13 ++++++- 6 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 server/src/drain.test.ts create mode 100644 server/src/drain.ts diff --git a/.env.example b/.env.example index 80a59f8..0fdc29f 100644 --- a/.env.example +++ b/.env.example @@ -79,6 +79,15 @@ SESSION_FILE=./data/sessions.json # Upstream timeouts / limits UPSTREAM_TIMEOUT=30000 + +# On SIGTERM (docker stop, a Kubernetes rollout) the server drains instead of +# dropping everything: it keeps serving for SHUTDOWN_DELAY ms while load +# balancers move on, then stops taking connections and ends each tab's +# live-update stream cleanly (the tab reconnects to another replica), and +# gives requests in flight SHUTDOWN_TIMEOUT ms to finish. The defaults fit +# inside docker stop's 10 s. +SHUTDOWN_DELAY=3000 +SHUTDOWN_TIMEOUT=5000 MAX_UPLOAD_BYTES=52428800 # Remote-image privacy proxy (Gmail-style). Set to 0 to load remote images directly. diff --git a/server/src/app.ts b/server/src/app.ts index 56100cd..0b6efd4 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -7,6 +7,7 @@ import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; import { RESPONSE_ALREADY_SENT } from "@hono/node-server/utils/response"; import { ALIVE_FRAME, attach as pushAttach, attachRelay as pushAttachRelay, prepare as pushPrepare, receive as pushReceive, pushStatus } from "./push.js"; +import { isDraining, onStreamsEnd } from "./drain.js"; import { getConnInfo } from "@hono/node-server/conninfo"; import { config } from "./config.js"; import { sourceDocument } from "./source.js"; @@ -522,13 +523,22 @@ export function createApp(basePath = config.basePath): Hono { // The browser's address, for the mail server (clientcontext.ts) app.use("*", (c, next) => withClientIp(clientIp(c), next)); app.use("*", securityHeaders); + // K-W5: while draining, every response closes its connection, so + // keep-alive clients and proxies open their next one to another replica + app.use("*", async (c, next) => { + await next(); + if (isDraining() && !c.res.headers.get("content-type")?.startsWith("text/event-stream")) c.header("Connection", "close"); + }); app.use("*", compressResponses(basePath)); const api = new Hono(); api.use("*", csrfGuard); api.use("*", smallBodies); - api.get("/health", (c) => c.json({ ok: true, name: config.appName, version: config.version, push: pushStatus() })); + // 503 while draining (K-W5): a readiness probe or load balancer stops + // sending here before the server stops taking connections + api.get("/health", (c) => c.json({ ok: !isDraining(), name: config.appName, version: config.version, push: pushStatus(), + ...(isDraining() ? { draining: true } : {}) }, isDraining() ? 503 : 200)); /* * Stalwart's push delivery. Authenticated by the token in the path -- 32 @@ -1396,11 +1406,16 @@ export function createApp(basePath = config.basePath): Hono { out.writeHead(200, SSE_HEADERS); out.flushHeaders(); out.write(": subscribed\n\n" + ALIVE_FRAME); + // Ended cleanly when the server drains; the tab reconnects elsewhere + out.on("close", onStreamsEnd(() => out.end())); return RESPONSE_ALREADY_SENT; } if (config.rawPushRelay) return relayPushRaw(c, url, session.authorization, session.username); const controller = new AbortController(); c.req.raw.signal.addEventListener("abort", () => controller.abort()); + // Draining ends the upstream, which ends this stream cleanly (pull()) + const offDrain = onStreamsEnd(() => controller.abort()); + controller.signal.addEventListener("abort", offDrain); const upstreamRes = upstreamFetch(url, { headers: { authorization: session.authorization, accept: "text/event-stream" }, signal: controller.signal, @@ -1705,6 +1720,8 @@ export function relayPushRaw(c: Context, url: string, authorization: string // The browser already has a 200, so an upstream failure ends the stream; // EventSource reports that as an error and the client retries with backoff. const fail = () => { out.end(); }; + // K-W5: a draining server ends this stream the same way + out.on("close", onStreamsEnd(fail)); /* * Once this account's subscription verifies, the upstream request goes and * the browser stream below is served by fan-out instead. Two things have diff --git a/server/src/config.ts b/server/src/config.ts index 181bdd4..e754b4e 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -395,6 +395,10 @@ export const config = { /** True when this instance has asserted, and verified, that it is immutable. */ immutable, upstreamTimeout: int("UPSTREAM_TIMEOUT", 30_000), + /** K-W5: on SIGTERM, how long to keep serving while load balancers move on (ms). */ + shutdownDelay: int("SHUTDOWN_DELAY", 3_000), + /** K-W5: then how long requests in flight get to finish (ms). */ + shutdownTimeout: int("SHUTDOWN_TIMEOUT", 5_000), maxUploadBytes: int("MAX_UPLOAD_BYTES", 50 * 1024 * 1024), imageProxy: bool("IMAGE_PROXY", true), /* diff --git a/server/src/drain.test.ts b/server/src/drain.test.ts new file mode 100644 index 0000000..e9c1434 --- /dev/null +++ b/server/src/drain.test.ts @@ -0,0 +1,78 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer, request } from "node:http"; +import type { AddressInfo } from "node:net"; + +const { drain, endStreams, isDraining, onStreamsEnd, resetDrain } = await import("./drain.js"); +const { createApp } = await import("./app.js"); + +test("health answers 503 while draining, with Connection: close", async () => { + resetDrain(); + const app = createApp(); + let res = await app.request("/api/health"); + assert.equal(res.status, 200); + assert.equal(res.headers.get("connection"), null); + const server = createServer(); + const done = drain(server, 50, 50, () => {}); + assert.equal(isDraining(), true); + res = await app.request("/api/health"); + assert.equal(res.status, 503); + assert.equal(((await res.json()) as { draining?: boolean }).draining, true); + assert.equal(res.headers.get("connection"), "close"); + await done; + resetDrain(); +}); + +test("every live stream is ended once, and one that ended first is left alone", () => { + resetDrain(); + let a = 0, b = 0; + onStreamsEnd(() => { a++; }); + const off = onStreamsEnd(() => { b++; }); + off(); + assert.equal(endStreams(), 1); + assert.equal(endStreams(), 0); + assert.deepEqual([a, b], [1, 0]); + resetDrain(); +}); + +test("a stream opened after the streams ended is ended at once", async () => { + resetDrain(); + endStreams(); + let ended = false; + onStreamsEnd(() => { ended = true; }); + await new Promise((r) => setTimeout(r, 0)); + assert.equal(ended, true); + resetDrain(); +}); + +test("a request in flight finishes before the server stops", async () => { + resetDrain(); + const server = createServer((_req, res) => { setTimeout(() => res.end("done"), 150); }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const { port } = server.address() as AddressInfo; + const body = new Promise((resolve, reject) => { + request({ host: "127.0.0.1", port, path: "/" }, (res) => { + let d = ""; res.on("data", (c) => { d += c; }); res.on("end", () => resolve(d)); + }).on("error", reject).end(); + }); + await new Promise((r) => setTimeout(r, 20)); + await drain(server, 0, 2_000, () => {}); + assert.equal(await body, "done"); + resetDrain(); +}); + +test("a request still open at the timeout is closed, and the drain still returns", async () => { + resetDrain(); + const server = createServer(() => { /* never answers */ }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const { port } = server.address() as AddressInfo; + const failed = new Promise((resolve) => { + request({ host: "127.0.0.1", port, path: "/" }, () => resolve(false)).on("error", () => resolve(true)).end(); + }); + await new Promise((r) => setTimeout(r, 20)); + const t = Date.now(); + await drain(server, 0, 100, () => {}); + assert.ok(Date.now() - t < 1_000); + assert.equal(await failed, true); + resetDrain(); +}); diff --git a/server/src/drain.ts b/server/src/drain.ts new file mode 100644 index 0000000..e025f72 --- /dev/null +++ b/server/src/drain.ts @@ -0,0 +1,84 @@ +/* + * Shutting down without anyone noticing (K-W5): on SIGTERM the server + * drains instead of exiting on the spot. + * + * 1. draining: /api/health answers 503, and every response says + * `Connection: close`, so load balancers and keep-alive clients move to + * another replica. Requests are still served. + * 2. after SHUTDOWN_DELAY (default 3 s, the time a load balancer takes to + * notice), no new connections, and every live-update stream is ended + * cleanly: each tab's EventSource reconnects, through the load + * balancer, to a replica that's staying. + * 3. requests already in flight finish, for up to SHUTDOWN_TIMEOUT + * (default 5 s); whatever is left is closed, and the process exits. + * + * The defaults fit inside both `docker stop` (10 s) and Kubernetes' default + * grace period (30 s). + */ +import type { Server } from "node:http"; + +let draining = false; +const closers = new Set<() => void>(); + +/** True from SIGTERM on. */ +export function isDraining(): boolean { + return draining; +} + +/** + * Runs `close` when the streams are ended (step 2), or at once if they + * already have been. Returns the call that takes it off the list, for a + * stream that ends first. + */ +export function onStreamsEnd(close: () => void): () => void { + if (streamsEnded) { + queueMicrotask(close); + return () => {}; + } + closers.add(close); + return () => { closers.delete(close); }; +} + +let streamsEnded = false; + +/** Step 2's stream half: every registered stream ends. Exported for tests. */ +export function endStreams(): number { + streamsEnded = true; + const all = [...closers]; + closers.clear(); + for (const close of all) { + try { close(); } catch { /* one stream's trouble isn't the others' */ } + } + return all.length; +} + +/** For tests: back to serving. */ +export function resetDrain(): void { + draining = false; + streamsEnded = false; + closers.clear(); +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** The whole sequence; resolves once the server has stopped. */ +export async function drain(server: Server, delayMs: number, timeoutMs: number, log: (msg: string) => void): Promise { + draining = true; + log(`draining: ${delayMs} ms for load balancers to move on`); + if (delayMs > 0) await sleep(delayMs); + + const closed = new Promise((resolve) => server.close(() => resolve())); + const streams = endStreams(); + server.closeIdleConnections(); + log(`no new connections; ${streams} live-update stream(s) ended`); + + // A connection a request just finished on goes idle, and would hold the + // server open (keep-alive) until the timeout: close those as they come + const sweep = setInterval(() => server.closeIdleConnections(), 100); + const finished = await Promise.race([closed.then(() => true), sleep(timeoutMs).then(() => false)]); + clearInterval(sweep); + if (!finished) { + log(`requests still open after ${timeoutMs} ms: closing them`); + server.closeAllConnections(); + } +} diff --git a/server/src/index.ts b/server/src/index.ts index 51cf79f..e2381ad 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,6 +1,7 @@ import { serve } from "@hono/node-server"; import { config } from "./config.js"; import { createApp, sessions } from "./app.js"; +import { drain } from "./drain.js"; async function main() { await sessions.init(); @@ -24,9 +25,19 @@ async function main() { process.exit(1); }); + // Drained, not dropped (see drain.ts): requests in flight finish and each + // tab's live-update stream reconnects to a replica that's staying. A + // second signal while draining means now. + let stopping = false; const shutdown = async (signal: string) => { + if (stopping) { + console.log(`[ihasmail] ${signal} again: stopping now`); + process.exit(0); + } + stopping = true; console.log(`[ihasmail] ${signal} received, shutting down`); - server.close(); + await drain(server as import("node:http").Server, config.shutdownDelay, config.shutdownTimeout, + (msg) => console.log(`[ihasmail] ${msg}`)); await sessions.close(); process.exit(0); }; -- 2.54.0