Merge pull request #241 from Coffey-Labs/feat/domain-server-mapping

Choose the Stalwart by the domain somebody signs in with

.env.example

@@ -93,3 +93,17 @@ SOURCE_URL=https://github.com/Coffey-Labs/ihasmail
 #
 # Read once at startup: editing a policy means restarting the container.
 # Docs: https://docs.ihasmail.org/configure/#settings-your-installation-decides
+
+# ---- Several Stalwart servers (optional) ----
+#
+# Choose the upstream by the domain someone signs in with. STALWART_URL above
+# stays required and stays the default; this only adds domains that go
+# elsewhere. See the shipped stalwart-servers.example.json, and mount it
+# read-only:
+#
+#   -v /srv/ihasmail/servers.json:/etc/ihasmail/servers.json:ro
+#
+# STALWART_SERVERS_FILE=/etc/ihasmail/servers.json
+#
+# An unlisted domain, or a username with no domain, goes to STALWART_URL. A
+# listed domain never falls back. Read once at startup: editing means a restart.

README.md

@@ -117,6 +117,47 @@ nowhere to live across a restart. Removing it means moving the session upstream
 into a token Stalwart itself issues and can revoke, which is what the OAuth work
 in [ROADMAP.md](ROADMAP.md) is for.
 
+### Several Stalwart servers
+
+One ihasmail can front more than one Stalwart, choosing by the domain somebody
+signs in with. **`STALWART_URL` stays required and stays the default**, so an
+installation that sets nothing else behaves exactly as it always has.
+
+```bash
+-e STALWART_SERVERS_FILE=/etc/ihasmail/servers.json \
+-v /srv/ihasmail/servers.json:/etc/ihasmail/servers.json:ro
+```
+
+```json
+{
+  "example.com": "https://mail.example.com",
+  "customer-b.test": "https://jmap.customer-b.test"
+}
+```
+
+[`stalwart-servers.example.json`](stalwart-servers.example.json) is that file
+with the rules written in it.
+
+A domain nobody listed — and a bare username, which Stalwart accepts and which
+has no domain at all — goes to `STALWART_URL`. **A listed domain never falls
+back.** If its server is unreachable that sign-in fails rather than retrying
+against the default, because falling back would authenticate somebody against a
+server their domain was deliberately routed away from; if the same account name
+existed there they would land in another tenant's mailbox.
+
+Read once at startup, so editing it means restarting the container. Malformed
+JSON, a duplicate domain once lower-cased, or a value that is not an `http(s)`
+URL stops the server rather than failing quietly at somebody's sign-in. The
+servers themselves are not contacted at boot — a mapping is a routing table,
+not a health check, and one customer's outage must not stop ihasmail starting
+for everybody else.
+
+This is one server per *person*, chosen at sign-in. Several servers at once for
+one person, with unified or cross-account views, is not supported: JMAP account
+ids are only unique within a server, so it would mean namespacing ids through
+the proxy. Reading somebody else's mail, calendars or files on the *same* server
+already works through JMAP sharing.
+
 ### Settings the installation decides
 
 A deployment can seed and lock user settings, which is what a school wanting

server/src/account.ts

@@ -65,7 +65,7 @@ function accountId(ctx: Ctx): string {
 type Invocation = [string, Record<string, unknown>, string];
 
 async function jmap(ctx: Ctx, methodCalls: Invocation[]): Promise<{ methodResponses?: [string, unknown, string][] }> {
-  const res = await fetch(absoluteUpstream(ctx.session.apiUrl), {
+  const res = await fetch(absoluteUpstream(ctx.session.apiUrl, ctx.session.baseUrl), {
     method: "POST",
     headers: { authorization: ctx.authorization, "content-type": "application/json", accept: "application/json" },
     body: JSON.stringify({ using: [JMAP_CORE, STALWART_CAP], methodCalls }),

server/src/app.ts

@@ -16,6 +16,7 @@ import {
   forgetUpstreamSession,
   getAccountInfo,
   getUpstreamSession,
+  upstreamFor,
   localizeSession,
 } from "./upstream.js";
 import {
@@ -237,7 +238,7 @@ export function createApp(basePath = config.basePath): Hono<Env> {
     const effectivePassword = totp ? `${password}$${totp}` : password;
     const authorization = `Basic ${Buffer.from(`${username}:${effectivePassword}`, "utf8").toString("base64")}`;
     try {
-      const upstream = await fetchUpstreamSession(authorization);
+      const upstream = await fetchUpstreamSession(authorization, upstreamFor(username));
       // ihasmail requires Stalwart 0.16 or newer. Refuse here, once and
       // clearly, rather than signing someone in and letting Files, the account
       // locale and self-service credentials each fail in their own way with
@@ -311,7 +312,7 @@ export function createApp(basePath = config.basePath): Hono<Env> {
   api.get("/auth/session", requireSession, async (c) => {
     const session = c.get("session");
     try {
-      const upstream = await getUpstreamSession(session.id, session.authorization, c.req.query("refresh") === "1");
+      const upstream = await getUpstreamSession(session.id, session.authorization, upstreamFor(session.username), c.req.query("refresh") === "1");
       const info = await getAccountInfo(session.id, session.authorization, upstream);
       return c.json(localizeSession(upstream, sessionExtras(session, info)));
     } catch (err) {
@@ -528,8 +529,8 @@ export function createApp(basePath = config.basePath): Hono<Env> {
       return c.json({ error: "unsupported_media_type" }, 415);
     }
     try {
-      const upstream = await getUpstreamSession(session.id, session.authorization);
-      const res = await fetch(absoluteUpstream(upstream.apiUrl), {
+      const upstream = await getUpstreamSession(session.id, session.authorization, upstreamFor(session.username));
+      const res = await fetch(absoluteUpstream(upstream.apiUrl, upstream.baseUrl), {
         method: "POST",
         headers: {
           authorization: session.authorization,
@@ -562,8 +563,8 @@ export function createApp(basePath = config.basePath): Hono<Env> {
     // suggestion; count the bytes as they go past.
     const body = c.req.raw.body ? c.req.raw.body.pipeThrough(byteCap(config.maxUploadBytes)) : null;
     try {
-      const upstream = await getUpstreamSession(session.id, session.authorization);
-      const url = absoluteUpstream(expandTemplate(upstream.uploadUrl, { accountId }));
+      const upstream = await getUpstreamSession(session.id, session.authorization, upstreamFor(session.username));
+      const url = absoluteUpstream(expandTemplate(upstream.uploadUrl, { accountId }), upstream.baseUrl);
       const res = await fetch(url, {
         method: "POST",
         headers: {
@@ -588,8 +589,8 @@ export function createApp(basePath = config.basePath): Hono<Env> {
     const accept = c.req.query("accept") ?? "application/octet-stream";
     const inline = c.req.query("inline") === "1";
     try {
-      const upstream = await getUpstreamSession(session.id, session.authorization);
-      const url = absoluteUpstream(expandTemplate(upstream.downloadUrl, { accountId, blobId, name, type: accept }));
+      const upstream = await getUpstreamSession(session.id, session.authorization, upstreamFor(session.username));
+      const url = absoluteUpstream(expandTemplate(upstream.downloadUrl, { accountId, blobId, name, type: accept }), upstream.baseUrl);
       const res = await fetch(url, {
         // Ask for the bytes as they are. undici would otherwise negotiate gzip
         // on our behalf and hand back a decompressed body whose content-length
@@ -639,8 +640,8 @@ export function createApp(basePath = config.basePath): Hono<Env> {
     const closeafter = c.req.query("closeafter") ?? "no";
     const ping = c.req.query("ping") ?? "30";
     try {
-      const upstream = await getUpstreamSession(session.id, session.authorization);
-      const url = absoluteUpstream(expandTemplate(upstream.eventSourceUrl, { types, closeafter, ping }));
+      const upstream = await getUpstreamSession(session.id, session.authorization, upstreamFor(session.username));
+      const url = absoluteUpstream(expandTemplate(upstream.eventSourceUrl, { types, closeafter, ping }), upstream.baseUrl);
       const controller = new AbortController();
       c.req.raw.signal.addEventListener("abort", () => controller.abort());
       const res = await fetch(url, {

server/src/config.ts

@@ -187,6 +187,59 @@ function readSettingsPolicy(): { defaults: Record<string, unknown>; enforced: Re
   };
 }
 
+/**
+ * Which Stalwart a domain signs in to.
+ *
+ * `STALWART_URL` stays required and stays the default; this only adds domains
+ * that go somewhere else (#238). An installation that sets nothing behaves
+ * exactly as it always has.
+ *
+ * Read once at boot and never written, so it mounts read-only and costs
+ * nothing in immutability -- the same shape as the settings policy.
+ *
+ * Servers are deliberately **not** probed here. A mapping is a routing table,
+ * not a health check, and refusing to boot because one of five customers is
+ * having an outage would take the other four down with it. What happens when
+ * one is unreachable is a sign-in question, answered in #239.
+ */
+function readStalwartServers(): Record<string, string> {
+  const file = process.env.STALWART_SERVERS_FILE;
+  if (!file) return {};
+  if (!existsSync(file)) throw new Error(`STALWART_SERVERS_FILE does not exist: ${file}`);
+
+  let raw: unknown;
+  try {
+    raw = JSON.parse(readFileSync(file, "utf8"));
+  } catch (err) {
+    throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): ${(err as Error).message}`);
+  }
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+    throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): expected an object of domain to URL`);
+  }
+
+  const out: Record<string, string> = {};
+  for (const [rawDomain, rawUrl] of Object.entries(raw as Record<string, unknown>)) {
+    /* Lower-cased and stripped of the root dot, because that is how a domain
+       taken off a username will arrive and comparing them any other way means
+       a mapping that silently never matches. */
+    const domain = rawDomain.trim().toLowerCase().replace(/\.$/, "");
+    if (!domain) throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): a domain key is empty`);
+    if (domain in out) throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): "${domain}" appears twice once normalised`);
+    if (typeof rawUrl !== "string") throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): "${domain}" is not a URL`);
+    let parsed: URL;
+    try {
+      parsed = new URL(rawUrl);
+    } catch {
+      throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): "${domain}" is not an absolute URL`);
+    }
+    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+      throw new Error(`Invalid STALWART_SERVERS_FILE (${file}): "${domain}" must be http or https`);
+    }
+    out[domain] = rawUrl.replace(/\/+$/, "");
+  }
+  return out;
+}
+
 export const config = {
   isProd,
   appName: env("APP_NAME", "ihasmail"),
@@ -222,6 +275,7 @@ export const config = {
    */
   basePath: normalizeBasePath(process.env.BASE_PATH),
   stalwartUrl,
+  stalwartServers: readStalwartServers(),
   appSecret,
   trustProxy: bool("TRUST_PROXY", true),
   /**

server/src/domainrouting.test.ts

@@ -0,0 +1,67 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+process.env.STALWART_URL = "https://default.example";
+
+const { upstreamFor } = await import("./upstream.js");
+const { config } = await import("./config.js");
+
+/**
+ * Which Stalwart a username goes to (#238).
+ *
+ * `STALWART_URL` is required and is the default. The mapping only adds domains
+ * that go elsewhere, so an installation with no mapping behaves exactly as it
+ * always has -- which is what these first cases pin.
+ */
+
+test("with no mapping at all, everything goes to the default", () => {
+  assert.deepEqual(config.stalwartServers, {});
+  assert.equal(upstreamFor("[email protected]"), "https://default.example");
+  assert.equal(upstreamFor("[email protected]"), "https://default.example");
+});
+
+test("a bare username has no domain to map, so it goes to the default", () => {
+  // Stalwart accepts a login with no domain at all.
+  assert.equal(upstreamFor("demo"), "https://default.example");
+  assert.equal(upstreamFor(""), "https://default.example");
+});
+
+test("a mapped domain goes to its own server", () => {
+  config.stalwartServers["mapped.test"] = "https://mail.mapped.test";
+  try {
+    assert.equal(upstreamFor("[email protected]"), "https://mail.mapped.test");
+  } finally {
+    delete config.stalwartServers["mapped.test"];
+  }
+});
+
+test("an unmapped domain still goes to the default while others are mapped", () => {
+  config.stalwartServers["mapped.test"] = "https://mail.mapped.test";
+  try {
+    assert.equal(upstreamFor("[email protected]"), "https://default.example");
+  } finally {
+    delete config.stalwartServers["mapped.test"];
+  }
+});
+
+test("the domain is matched however it was typed", () => {
+  // Keys are normalised on load; the username has to be normalised the same
+  // way or a mapping silently never matches.
+  config.stalwartServers["mapped.test"] = "https://mail.mapped.test";
+  try {
+    assert.equal(upstreamFor("[email protected]"), "https://mail.mapped.test");
+    assert.equal(upstreamFor("[email protected]."), "https://mail.mapped.test", "root dot");
+    assert.equal(upstreamFor("someone@ mapped.test "), "https://mail.mapped.test", "stray spaces");
+  } finally {
+    delete config.stalwartServers["mapped.test"];
+  }
+});
+
+test("an address with an @ in the local part maps on the last one", () => {
+  config.stalwartServers["mapped.test"] = "https://mail.mapped.test";
+  try {
+    assert.equal(upstreamFor('"odd@name"@mapped.test'), "https://mail.mapped.test");
+  } finally {
+    delete config.stalwartServers["mapped.test"];
+  }
+});

server/src/settingspolicy.test.ts

@@ -53,3 +53,30 @@ test("the example's commentary cannot be mistaken for a section", () => {
     assert.ok(real.has(key) || key.startsWith("_"), `unexpected top-level key ${key}`);
   }
 });
+
+/**
+ * The shipped server-mapping example, checked the same way and for the same
+ * reason: an example that no longer loads is worse than no example, because
+ * the first experience of the feature is a server that refuses to start.
+ */
+const SERVERS = fileURLToPath(new URL("../../stalwart-servers.example.json", import.meta.url));
+
+test("the example server mapping is valid JSON", () => {
+  assert.doesNotThrow(() => JSON.parse(readFileSync(SERVERS, "utf8")));
+});
+
+test("every entry in the example mapping is a domain and an http(s) URL", () => {
+  const m = JSON.parse(readFileSync(SERVERS, "utf8")) as Record<string, unknown>;
+  const seen = new Set<string>();
+  for (const [key, value] of Object.entries(m)) {
+    if (key.startsWith("_")) continue;
+    const domain = key.trim().toLowerCase().replace(/\.$/, "");
+    assert.ok(domain, "a domain key is empty");
+    assert.ok(!seen.has(domain), `${domain} appears twice once normalised`);
+    seen.add(domain);
+    assert.equal(typeof value, "string", `${domain} is not a string`);
+    const url = new URL(value as string);
+    assert.ok(url.protocol === "http:" || url.protocol === "https:", `${domain} must be http or https`);
+  }
+  assert.ok(seen.size > 0, "the example should show at least one mapping");
+});

server/src/upstream.ts

@@ -10,6 +10,15 @@ export interface UpstreamSession {
   uploadUrl: string;
   eventSourceUrl: string;
   state: string;
+  /**
+   * Which Stalwart this document came from.
+   *
+   * Recorded rather than looked up again, because the relative URLs inside it
+   * -- apiUrl, uploadUrl and the rest -- only mean anything against the server
+   * that issued them. Anything holding a session already knows where to send
+   * the next request. Not part of the JMAP session resource; ours.
+   */
+  baseUrl: string;
 }
 
 export class UpstreamError extends Error {
@@ -24,16 +33,37 @@ export class UpstreamError extends Error {
 const sessionCache = new Map<string, { session: UpstreamSession; fetchedAt: number }>();
 const SESSION_CACHE_MS = 5 * 60_000;
 
-export function wellKnownUrl(): string {
-  return `${config.stalwartUrl}/.well-known/jmap`;
+/**
+ * The Stalwart a username belongs to.
+ *
+ * `STALWART_URL` is the default and is always the answer for a domain nobody
+ * mapped -- and for a bare username, which Stalwart accepts and which has no
+ * domain to map (#238).
+ *
+ * A *mapped* domain never falls back. If its server is unreachable that
+ * sign-in fails, because falling back would authenticate somebody against a
+ * server their domain was deliberately routed away from -- and if the same
+ * account name exists there, they would land in another tenant's mailbox. The
+ * fallback is a decision about unmapped domains, taken before any network
+ * call, not a recovery path.
+ */
+export function upstreamFor(username: string): string {
+  const at = username.lastIndexOf("@");
+  if (at < 0) return config.stalwartUrl;
+  const domain = username.slice(at + 1).trim().toLowerCase().replace(/\.$/, "");
+  return config.stalwartServers[domain] ?? config.stalwartUrl;
+}
+
+export function wellKnownUrl(base: string = config.stalwartUrl): string {
+  return `${base}/.well-known/jmap`;
 }
 
 /**
  * Fetch the JMAP session resource from Stalwart using the given Authorization
  * header. Throws UpstreamError(401) on bad credentials.
  */
-export async function fetchUpstreamSession(authorization: string): Promise<UpstreamSession> {
-  const res = await fetch(wellKnownUrl(), {
+export async function fetchUpstreamSession(authorization: string, base: string = config.stalwartUrl): Promise<UpstreamSession> {
+  const res = await fetch(wellKnownUrl(base), {
     headers: { authorization, accept: "application/json" },
     redirect: "follow",
     signal: AbortSignal.timeout(config.upstreamTimeout),
@@ -46,13 +76,13 @@ export async function fetchUpstreamSession(authorization: string): Promise<Upstr
   }
   const session = (await res.json()) as UpstreamSession;
   if (!session.apiUrl) throw new UpstreamError("Upstream returned an invalid JMAP session", 502);
-  return session;
+  return { ...session, baseUrl: base };
 }
 
-export async function getUpstreamSession(sessionId: string, authorization: string, force = false) {
+export async function getUpstreamSession(sessionId: string, authorization: string, base: string = config.stalwartUrl, force = false) {
   const cached = sessionCache.get(sessionId);
   if (!force && cached && Date.now() - cached.fetchedAt < SESSION_CACHE_MS) return cached.session;
-  const session = await fetchUpstreamSession(authorization);
+  const session = await fetchUpstreamSession(authorization, base);
   sessionCache.set(sessionId, { session, fetchedAt: Date.now() });
   return session;
 }
@@ -210,9 +240,9 @@ function localeOf(call: [string, Record<string, unknown>, string] | undefined):
  * Which edition the server is running. Stalwart deliberately does not publish
  * its version number to clients, but 0.16 does report its edition here.
  */
-async function fetchEdition(authorization: string): Promise<string | null> {
+async function fetchEdition(authorization: string, base: string): Promise<string | null> {
   try {
-    const res = await fetch(`${config.stalwartUrl}/api/account`, {
+    const res = await fetch(`${base}/api/account`, {
       headers: { authorization, accept: "application/json" },
       signal: AbortSignal.timeout(config.upstreamTimeout),
     });
@@ -230,7 +260,7 @@ export async function getAccountInfo(sessionId: string, authorization: string, s
   let info = EMPTY_INFO;
   try {
     info = await fetchAccountInfo(authorization, session);
-    info = { ...info, edition: await fetchEdition(authorization) };
+    info = { ...info, edition: await fetchEdition(authorization, session.baseUrl) };
   } catch {
     /* all of this is a nicety - never fail the session over it */
   }
@@ -258,9 +288,9 @@ export function localizeSession(s: UpstreamSession, extras: Record<string, unkno
 }
 
 /** Resolve a possibly-relative upstream URL template against STALWART_URL. */
-export function absoluteUpstream(url: string): string {
+export function absoluteUpstream(url: string, base: string = config.stalwartUrl): string {
   try {
-    return new URL(url, config.stalwartUrl).toString();
+    return new URL(url, base).toString();
   } catch {
     return url;
   }

stalwart-servers.example.json

@@ -0,0 +1,25 @@
+{
+  "_comment": [
+    "Optional: which Stalwart a domain signs in to.",
+    "",
+    "STALWART_URL stays required and stays the default. This file only adds",
+    "domains that go somewhere else -- delete it and nothing changes.",
+    "",
+    "Point at it with STALWART_SERVERS_FILE=/etc/ihasmail/servers.json and mount",
+    "it read-only. Read once at startup, so editing it means restarting.",
+    "",
+    "A domain that is not listed here, and a bare username with no domain at",
+    "all, go to STALWART_URL. A domain that IS listed never falls back: if its",
+    "server is unreachable that sign-in fails, because falling back would",
+    "authenticate somebody against a server their domain was routed away from.",
+    "",
+    "Keys are lower-cased and stripped of a trailing dot when read. Malformed",
+    "JSON, a duplicate domain, or a value that is not an http(s) URL stops the",
+    "server at startup rather than failing quietly at somebody's sign-in.",
+    "",
+    "Docs: https://docs.ihasmail.org/configure/#several-stalwart-servers"
+  ],
+
+  "example.com": "https://mail.example.com",
+  "customer-b.test": "https://jmap.customer-b.test"
+}