diff --git a/KNOWN-ISSUES.md b/KNOWN-ISSUES.md index dfcc22c..4152fd4 100644 --- a/KNOWN-ISSUES.md +++ b/KNOWN-ISSUES.md @@ -21,6 +21,8 @@ works the same way — and dropped where 0.15 was the whole subject. Support for 0.15 was removed on 2026-08-26; the last release that runs on it is tagged [`stalwart-0.15-support`](https://github.com/LINUXexpert-org/ihasmail/releases/tag/stalwart-0.15-support). +- **A compressing hop in front of Stalwart truncated every blob download, and nothing said so.** Node decompresses a gzip response before the code ever sees the body, but leaves the `content-length` header describing the *compressed* bytes. The blob proxy copied that header onto the longer body it forwarded, so the browser stopped reading exactly that many bytes in and called the download complete. Reported on [#76](https://github.com/LINUXexpert-org/ihasmail/issues/76) against a Coolify deployment, where Traefik's compress middleware only engages above 1 KiB: filter rules one and two were fine and the third pushed the script past the threshold, after which it came back cut off mid-rule — 384 bytes of a 1.3 KB script. The size threshold is what made it look like a race. This is the *second* cause behind that issue, and the first fix did not touch it: a truncated script is neither unknown nor empty, so the "refuse to save from a baseline we could not read" guard never fired — the script parsed, just with rules missing, and the next save wrote the short version back over the real one. Every blob download shared the fault, not just Sieve: message source, vCards, signature HTML, attachments being forwarded, and the `settings.json` sync. Settings degraded honestly by luck rather than design — a truncated file fails `JSON.parse`, which is caught and leaves the local cache in charge — so it stopped syncing between devices instead of being overwritten. The proxy now asks upstream for `identity` and, for a hop that compresses anyway, forwards no length at all rather than one describing different bytes. The image proxy is unaffected: it uses `node:http` directly, sends no `accept-encoding`, and never decompresses. + - **Delete all spam destroys, and does not pass through Deleted Items** — this is the point of the feature and the thing worth checking on a real server, since a folder that empties into another folder has solved nothing. `Email/set destroy`, walked a page at a time so it survives `maxObjectsInSet` the way emptying Deleted Items already had to. **Confirmed live on 0.16.19 (2026-08-26)**: Junk Mail emptied and Deleted Items stayed empty afterwards. There is no undo, which is why all three entry points share one dialog that says so. Only Deleted Items and Junk Mail can be emptied this way, enforced in the store rather than only hidden in the menus. - **Sharing a mail folder is accepted and does nothing.** `Mailbox/set` with a `shareWith` map is applied, `Mailbox/get` reads it back, and the folder never appears for the account it was shared with — **confirmed live on 0.16.19 (2026-08-27)** with a folder shared read-only to another account on the same server, which never saw it. Stalwart's own sharing documentation lists calendars, address books and file storage; mail folders are not among them. Nothing reports a failure at any point, which is the whole problem: the share is stored, so a client that trusts what it reads back shows it as live for ever. The entry point is withdrawn. A folder that is *already* shared still offers **Stop sharing**, because a share nobody can see is exactly the one you want to be able to clear, and there is no other way to. File sharing is unaffected and works end to end. - **Address book sharing works, and was briefly withdrawn by mistake.** It was taken out alongside mail folders on 2026-08-27 on a report that it behaved the same way; the report was mistaken and the feature was put back the same day. Nothing was ever shown to be wrong with it, and Stalwart documents address books as shareable. Recorded because the withdrawal is in the history and would otherwise read as a finding. Shared books now appear in the Contacts pane under "Shared with me" rather than behind an account switch, and their contacts are offered when addressing a message. diff --git a/server/src/app.test.ts b/server/src/app.test.ts index 97aaefa..207a629 100644 --- a/server/src/app.test.ts +++ b/server/src/app.test.ts @@ -35,3 +35,56 @@ test("image proxy refuses private targets", async () => { const res = await app.request("/api/image?url=http://127.0.0.1/x"); assert.equal(res.status, 401); }); + +test("a compressed upstream blob is not forwarded with the compressed length", async () => { + const { forwardedContentLength } = await import("./app.js"); + // gzip: the body we forward has already been decompressed, so the length on + // the wire describes different bytes and must not be copied (issue #76). + const gz = new Headers({ "content-encoding": "gzip", "content-length": "384" }); + assert.equal(forwardedContentLength(gz), null); + // identity, spelled out or absent: the length describes the body we send. + assert.equal(forwardedContentLength(new Headers({ "content-encoding": "identity", "content-length": "1157" })), "1157"); + assert.equal(forwardedContentLength(new Headers({ "content-length": "1157" })), "1157"); + assert.equal(forwardedContentLength(new Headers({ "content-encoding": "BR", "content-length": "384" })), null); + // Nothing to forward is not an error. + assert.equal(forwardedContentLength(new Headers()), null); +}); + +test("a Sieve script larger than a compressing hop's threshold survives the proxy", async () => { + const http = await import("node:http"); + const zlib = await import("node:zlib"); + const { forwardedContentLength } = await import("./app.js"); + + const script = + "# ihasmail filters v1 - edit with care; rules are stored in the `# rule:` comments\nrequire [\"fileinto\"];\n\n" + + ["a", "b", "c"] + .map( + (k) => + `# rule:{"id":"r${k}","name":"From ${k}@example.com","enabled":true,"join":"allof","tests":[{"type":"header","header":"from","op":"contains","value":"${k}@example.com"}],"actions":[{"type":"fileinto","mailbox":"INBOX/${k}"}]}\n` + + `if header :contains "from" "${k}@example.com"\n{\n fileinto "INBOX/${k}";\n}\n\n`, + ) + .join(""); + const gz = zlib.gzipSync(Buffer.from(script)); + assert.ok(gz.length < Buffer.byteLength(script), "the script has to compress for this test to mean anything"); + + // A hop that compresses regardless of what we asked for. + const origin = http.createServer((_req, res) => { + res.writeHead(200, { "content-type": "application/sieve", "content-encoding": "gzip", "content-length": String(gz.length) }); + res.end(gz); + }); + await new Promise((r) => origin.listen(0, () => r())); + const port = (origin.address() as { port: number }).port; + + try { + const up = await fetch(`http://127.0.0.1:${port}/`); + // What the blob route forwards. + const headers = new Headers({ "content-type": "application/sieve; charset=utf-8" }); + const cl = forwardedContentLength(up.headers); + if (cl) headers.set("Content-Length", cl); + const out = new Response(await up.arrayBuffer(), { status: 200, headers }); + assert.equal(out.headers.get("content-length"), null); + assert.equal(await out.text(), script); + } finally { + origin.close(); + } +}); diff --git a/server/src/app.ts b/server/src/app.ts index 6797394..38bbf00 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -519,14 +519,17 @@ export function createApp(): Hono { const upstream = await getUpstreamSession(session.id, session.authorization); const url = absoluteUpstream(expandTemplate(upstream.downloadUrl, { accountId, blobId, name, type: accept })); const res = await fetch(url, { - headers: { authorization: session.authorization }, + // Ask for the bytes as they are. undici would otherwise negotiate gzip + // on our behalf and hand back a decompressed body whose content-length + // header still describes the compressed one -- see forwardedContentLength. + headers: { authorization: session.authorization, "accept-encoding": "identity" }, signal: AbortSignal.timeout(Math.max(config.upstreamTimeout, 5 * 60_000)), }); if (!res.ok) return c.json({ error: "not_found" }, res.status === 404 ? 404 : 502); const headers = new Headers(); const type = sanitizeContentType(res.headers.get("content-type") ?? accept); headers.set("Content-Type", type); - const cl = res.headers.get("content-length"); + const cl = forwardedContentLength(res.headers); if (cl) headers.set("Content-Length", cl); const safeInline = inline && isInlineSafe(type); headers.set( @@ -651,6 +654,32 @@ function passthrough(res: Response): Response { return new Response(res.body, { status: res.status, headers }); } +/** + * The upstream content-length, but only when it describes the bytes we are + * about to forward. + * + * A compressed response is decompressed for us before we ever see the body -- + * undici does it transparently -- while the content-length header is left + * describing the *compressed* length. Copying it onto the longer body we then + * send makes the browser stop reading exactly that many bytes in and call the + * download complete, so the file arrives silently truncated. + * + * That is the second half of issue #76. A hop in front of Stalwart compressed + * responses over 1 KiB, so a Sieve script stayed intact until the third rule + * pushed it past the threshold and it came back cut off mid-rule. Nothing + * reported an error: the script parsed, just with rules missing, and saving + * wrote that shortened version back over the real one. + * + * We ask for `identity` above so the usual case still carries a length the + * browser can show progress against; this is the guard for a hop that + * compresses anyway. + */ +export function forwardedContentLength(headers: Headers): string | null { + const encoding = headers.get("content-encoding")?.trim().toLowerCase(); + if (encoding && encoding !== "identity") return null; + return headers.get("content-length"); +} + function sanitizeContentType(ct: string): string { const lower = ct.split(";")[0]!.trim().toLowerCase(); // Never let the browser render HTML/SVG/XML/JS served from the blob endpoint.