Drain on SIGTERM instead of dropping everything (K-W5) #132
No files matched your search
@@ -79,6 +79,15 @@ SESSION_FILE=./data/sessions.json
|
||||
|
||||
# Upstream timeouts / limits
|
||||
UPSTREAM_TIMEOUT=30000
|
||||
|
||||
# On SIGTERM (docker stop, a Kubernetes rollout) the server drains instead of
|
||||
# dropping everything: it keeps serving for SHUTDOWN_DELAY ms while load
|
||||
# balancers move on, then stops taking connections and ends each tab's
|
||||
# live-update stream cleanly (the tab reconnects to another replica), and
|
||||
# gives requests in flight SHUTDOWN_TIMEOUT ms to finish. The defaults fit
|
||||
# inside docker stop's 10 s.
|
||||
SHUTDOWN_DELAY=3000
|
||||
SHUTDOWN_TIMEOUT=5000
|
||||
MAX_UPLOAD_BYTES=52428800
|
||||
|
||||
# Remote-image privacy proxy (Gmail-style). Set to 0 to load remote images directly.
|
||||
|
||||
+18
-1
@@ -7,6 +7,7 @@ import { request as httpRequest } from "node:http";
|
||||
import { request as httpsRequest } from "node:https";
|
||||
import { RESPONSE_ALREADY_SENT } from "@hono/node-server/utils/response";
|
||||
import { ALIVE_FRAME, attach as pushAttach, attachRelay as pushAttachRelay, prepare as pushPrepare, receive as pushReceive, pushStatus } from "./push.js";
|
||||
import { isDraining, onStreamsEnd } from "./drain.js";
|
||||
import { getConnInfo } from "@hono/node-server/conninfo";
|
||||
import { config } from "./config.js";
|
||||
import { sourceDocument } from "./source.js";
|
||||
@@ -522,13 +523,22 @@ export function createApp(basePath = config.basePath): Hono<Env> {
|
||||
// The browser's address, for the mail server (clientcontext.ts)
|
||||
app.use("*", (c, next) => withClientIp(clientIp(c), next));
|
||||
app.use("*", securityHeaders);
|
||||
// K-W5: while draining, every response closes its connection, so
|
||||
// keep-alive clients and proxies open their next one to another replica
|
||||
app.use("*", async (c, next) => {
|
||||
await next();
|
||||
if (isDraining() && !c.res.headers.get("content-type")?.startsWith("text/event-stream")) c.header("Connection", "close");
|
||||
});
|
||||
app.use("*", compressResponses(basePath));
|
||||
|
||||
const api = new Hono<Env>();
|
||||
api.use("*", csrfGuard);
|
||||
api.use("*", smallBodies);
|
||||
|
||||
api.get("/health", (c) => c.json({ ok: true, name: config.appName, version: config.version, push: pushStatus() }));
|
||||
// 503 while draining (K-W5): a readiness probe or load balancer stops
|
||||
// sending here before the server stops taking connections
|
||||
api.get("/health", (c) => c.json({ ok: !isDraining(), name: config.appName, version: config.version, push: pushStatus(),
|
||||
...(isDraining() ? { draining: true } : {}) }, isDraining() ? 503 : 200));
|
||||
|
||||
/*
|
||||
* Stalwart's push delivery. Authenticated by the token in the path -- 32
|
||||
@@ -1396,11 +1406,16 @@ export function createApp(basePath = config.basePath): Hono<Env> {
|
||||
out.writeHead(200, SSE_HEADERS);
|
||||
out.flushHeaders();
|
||||
out.write(": subscribed\n\n" + ALIVE_FRAME);
|
||||
// Ended cleanly when the server drains; the tab reconnects elsewhere
|
||||
out.on("close", onStreamsEnd(() => out.end()));
|
||||
return RESPONSE_ALREADY_SENT;
|
||||
}
|
||||
if (config.rawPushRelay) return relayPushRaw(c, url, session.authorization, session.username);
|
||||
const controller = new AbortController();
|
||||
c.req.raw.signal.addEventListener("abort", () => controller.abort());
|
||||
// Draining ends the upstream, which ends this stream cleanly (pull())
|
||||
const offDrain = onStreamsEnd(() => controller.abort());
|
||||
controller.signal.addEventListener("abort", offDrain);
|
||||
const upstreamRes = upstreamFetch(url, {
|
||||
headers: { authorization: session.authorization, accept: "text/event-stream" },
|
||||
signal: controller.signal,
|
||||
@@ -1705,6 +1720,8 @@ export function relayPushRaw(c: Context<Env>, url: string, authorization: string
|
||||
// The browser already has a 200, so an upstream failure ends the stream;
|
||||
// EventSource reports that as an error and the client retries with backoff.
|
||||
const fail = () => { out.end(); };
|
||||
// K-W5: a draining server ends this stream the same way
|
||||
out.on("close", onStreamsEnd(fail));
|
||||
/*
|
||||
* Once this account's subscription verifies, the upstream request goes and
|
||||
* the browser stream below is served by fan-out instead. Two things have
|
||||
|
||||
@@ -395,6 +395,10 @@ export const config = {
|
||||
/** True when this instance has asserted, and verified, that it is immutable. */
|
||||
immutable,
|
||||
upstreamTimeout: int("UPSTREAM_TIMEOUT", 30_000),
|
||||
/** K-W5: on SIGTERM, how long to keep serving while load balancers move on (ms). */
|
||||
shutdownDelay: int("SHUTDOWN_DELAY", 3_000),
|
||||
/** K-W5: then how long requests in flight get to finish (ms). */
|
||||
shutdownTimeout: int("SHUTDOWN_TIMEOUT", 5_000),
|
||||
maxUploadBytes: int("MAX_UPLOAD_BYTES", 50 * 1024 * 1024),
|
||||
imageProxy: bool("IMAGE_PROXY", true),
|
||||
/*
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer, request } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
const { drain, endStreams, isDraining, onStreamsEnd, resetDrain } = await import("./drain.js");
|
||||
const { createApp } = await import("./app.js");
|
||||
|
||||
test("health answers 503 while draining, with Connection: close", async () => {
|
||||
resetDrain();
|
||||
const app = createApp();
|
||||
let res = await app.request("/api/health");
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get("connection"), null);
|
||||
const server = createServer();
|
||||
const done = drain(server, 50, 50, () => {});
|
||||
assert.equal(isDraining(), true);
|
||||
res = await app.request("/api/health");
|
||||
assert.equal(res.status, 503);
|
||||
assert.equal(((await res.json()) as { draining?: boolean }).draining, true);
|
||||
assert.equal(res.headers.get("connection"), "close");
|
||||
await done;
|
||||
resetDrain();
|
||||
});
|
||||
|
||||
test("every live stream is ended once, and one that ended first is left alone", () => {
|
||||
resetDrain();
|
||||
let a = 0, b = 0;
|
||||
onStreamsEnd(() => { a++; });
|
||||
const off = onStreamsEnd(() => { b++; });
|
||||
off();
|
||||
assert.equal(endStreams(), 1);
|
||||
assert.equal(endStreams(), 0);
|
||||
assert.deepEqual([a, b], [1, 0]);
|
||||
resetDrain();
|
||||
});
|
||||
|
||||
test("a stream opened after the streams ended is ended at once", async () => {
|
||||
resetDrain();
|
||||
endStreams();
|
||||
let ended = false;
|
||||
onStreamsEnd(() => { ended = true; });
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
assert.equal(ended, true);
|
||||
resetDrain();
|
||||
});
|
||||
|
||||
test("a request in flight finishes before the server stops", async () => {
|
||||
resetDrain();
|
||||
const server = createServer((_req, res) => { setTimeout(() => res.end("done"), 150); });
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
const body = new Promise<string>((resolve, reject) => {
|
||||
request({ host: "127.0.0.1", port, path: "/" }, (res) => {
|
||||
let d = ""; res.on("data", (c) => { d += c; }); res.on("end", () => resolve(d));
|
||||
}).on("error", reject).end();
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
await drain(server, 0, 2_000, () => {});
|
||||
assert.equal(await body, "done");
|
||||
resetDrain();
|
||||
});
|
||||
|
||||
test("a request still open at the timeout is closed, and the drain still returns", async () => {
|
||||
resetDrain();
|
||||
const server = createServer(() => { /* never answers */ });
|
||||
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||
const { port } = server.address() as AddressInfo;
|
||||
const failed = new Promise<boolean>((resolve) => {
|
||||
request({ host: "127.0.0.1", port, path: "/" }, () => resolve(false)).on("error", () => resolve(true)).end();
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
const t = Date.now();
|
||||
await drain(server, 0, 100, () => {});
|
||||
assert.ok(Date.now() - t < 1_000);
|
||||
assert.equal(await failed, true);
|
||||
resetDrain();
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Shutting down without anyone noticing (K-W5): on SIGTERM the server
|
||||
* drains instead of exiting on the spot.
|
||||
*
|
||||
* 1. draining: /api/health answers 503, and every response says
|
||||
* `Connection: close`, so load balancers and keep-alive clients move to
|
||||
* another replica. Requests are still served.
|
||||
* 2. after SHUTDOWN_DELAY (default 3 s, the time a load balancer takes to
|
||||
* notice), no new connections, and every live-update stream is ended
|
||||
* cleanly: each tab's EventSource reconnects, through the load
|
||||
* balancer, to a replica that's staying.
|
||||
* 3. requests already in flight finish, for up to SHUTDOWN_TIMEOUT
|
||||
* (default 5 s); whatever is left is closed, and the process exits.
|
||||
*
|
||||
* The defaults fit inside both `docker stop` (10 s) and Kubernetes' default
|
||||
* grace period (30 s).
|
||||
*/
|
||||
import type { Server } from "node:http";
|
||||
|
||||
let draining = false;
|
||||
const closers = new Set<() => void>();
|
||||
|
||||
/** True from SIGTERM on. */
|
||||
export function isDraining(): boolean {
|
||||
return draining;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `close` when the streams are ended (step 2), or at once if they
|
||||
* already have been. Returns the call that takes it off the list, for a
|
||||
* stream that ends first.
|
||||
*/
|
||||
export function onStreamsEnd(close: () => void): () => void {
|
||||
if (streamsEnded) {
|
||||
queueMicrotask(close);
|
||||
return () => {};
|
||||
}
|
||||
closers.add(close);
|
||||
return () => { closers.delete(close); };
|
||||
}
|
||||
|
||||
let streamsEnded = false;
|
||||
|
||||
/** Step 2's stream half: every registered stream ends. Exported for tests. */
|
||||
export function endStreams(): number {
|
||||
streamsEnded = true;
|
||||
const all = [...closers];
|
||||
closers.clear();
|
||||
for (const close of all) {
|
||||
try { close(); } catch { /* one stream's trouble isn't the others' */ }
|
||||
}
|
||||
return all.length;
|
||||
}
|
||||
|
||||
/** For tests: back to serving. */
|
||||
export function resetDrain(): void {
|
||||
draining = false;
|
||||
streamsEnded = false;
|
||||
closers.clear();
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/** The whole sequence; resolves once the server has stopped. */
|
||||
export async function drain(server: Server, delayMs: number, timeoutMs: number, log: (msg: string) => void): Promise<void> {
|
||||
draining = true;
|
||||
log(`draining: ${delayMs} ms for load balancers to move on`);
|
||||
if (delayMs > 0) await sleep(delayMs);
|
||||
|
||||
const closed = new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
const streams = endStreams();
|
||||
server.closeIdleConnections();
|
||||
log(`no new connections; ${streams} live-update stream(s) ended`);
|
||||
|
||||
// A connection a request just finished on goes idle, and would hold the
|
||||
// server open (keep-alive) until the timeout: close those as they come
|
||||
const sweep = setInterval(() => server.closeIdleConnections(), 100);
|
||||
const finished = await Promise.race([closed.then(() => true), sleep(timeoutMs).then(() => false)]);
|
||||
clearInterval(sweep);
|
||||
if (!finished) {
|
||||
log(`requests still open after ${timeoutMs} ms: closing them`);
|
||||
server.closeAllConnections();
|
||||
}
|
||||
}
|
||||
+12
-1
@@ -1,6 +1,7 @@
|
||||
import { serve } from "@hono/node-server";
|
||||
import { config } from "./config.js";
|
||||
import { createApp, sessions } from "./app.js";
|
||||
import { drain } from "./drain.js";
|
||||
|
||||
async function main() {
|
||||
await sessions.init();
|
||||
@@ -24,9 +25,19 @@ async function main() {
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Drained, not dropped (see drain.ts): requests in flight finish and each
|
||||
// tab's live-update stream reconnects to a replica that's staying. A
|
||||
// second signal while draining means now.
|
||||
let stopping = false;
|
||||
const shutdown = async (signal: string) => {
|
||||
if (stopping) {
|
||||
console.log(`[ihasmail] ${signal} again: stopping now`);
|
||||
process.exit(0);
|
||||
}
|
||||
stopping = true;
|
||||
console.log(`[ihasmail] ${signal} received, shutting down`);
|
||||
server.close();
|
||||
await drain(server as import("node:http").Server, config.shutdownDelay, config.shutdownTimeout,
|
||||
(msg) => console.log(`[ihasmail] ${msg}`));
|
||||
await sessions.close();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user