Measure signatures in bytes, so the oversize fallback actually saves

Stalwart accepts a signature of `value.len() < 2048`, and that is Rust's len():
2047 bytes of UTF-8. Every check here counted JavaScript `.length` instead,
which is UTF-16 units and agrees only for ASCII — an accent is one unit and two
bytes, CJK three, an emoji two units and four.

That alone would let a non-Latin signature we judged to fit come back rejected.
But the fallback that is supposed to rescue an oversize signature was broken
outright, for everyone: it truncated to `budget - 1` characters and appended an
ellipsis, one character but three bytes, so the result was always 2047
characters and 2049 bytes. Every marker signature Stalwart was ever offered was
two bytes too long, ASCII included. That is why this flow has been sitting in
the README as implemented but unconfirmed — the first person to exceed 2 KB
would have hit it.

Cutting the source text and rendering afterwards, rather than slicing the
rendered string, also means a cut can no longer land inside an HTML entity, and
stepping through code points means it cannot split a surrogate pair.

The old tests used ASCII only, which is how this survived; the new ones weigh
the encoded form.

web/src/lib/__tests__/signatureHtml.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { buildMarkerSignature, compactHtml, markerOf, SIGNATURE_LIMIT } from "../signatureHtml";
+import { buildMarkerSignature, byteLength, compactHtml, markerOf, signatureTooLong, SIGNATURE_LIMIT } from "../signatureHtml";
 
 describe("signature compaction", () => {
   it("strips office cruft and non-essential styles but keeps colours and links", () => {
@@ -24,3 +24,59 @@ describe("signature compaction", () => {
     expect(markerOf("<div>plain</div>")).toBeNull();
   });
 });
+
+/**
+ * Stalwart's cap is `value.len() < 2048` on a Rust string — 2047 bytes of
+ * UTF-8. Measuring with JavaScript's `.length` counts UTF-16 units instead,
+ * which agrees only for ASCII: an accent is one unit and two bytes, CJK three,
+ * an emoji two units and four. Every check has to weigh the encoded form or a
+ * signature we judged to fit comes back rejected.
+ */
+describe("signature size is measured in bytes", () => {
+  const sigOf = (html: string) => buildMarkerSignature("blob123", html);
+
+  it("counts multi-byte characters at their encoded size", () => {
+    expect(byteLength("hello")).toBe(5);
+    expect(byteLength("Grüße")).toBe(7); // two 2-byte characters
+    expect(byteLength("日本語")).toBe(9); // three 3-byte characters
+    expect(byteLength("🎉")).toBe(4); // one 4-byte character, two UTF-16 units
+  });
+
+  it("spots a signature that fits in characters but not in bytes", () => {
+    // Comfortably under the limit counted as characters, well over it as bytes.
+    const cjk = "日".repeat(1200);
+    expect(cjk.length).toBeLessThan(SIGNATURE_LIMIT);
+    expect(signatureTooLong(cjk, cjk)).toBe(true);
+  });
+
+  it("keeps a marker signature within the byte limit for non-ASCII text", () => {
+    for (const filler of ["ü", "日", "🎉", "x"]) {
+      const m = sigOf(`<div>${filler.repeat(3000)}</div>`);
+      expect(byteLength(m.htmlSignature), `html for ${filler}`).toBeLessThanOrEqual(SIGNATURE_LIMIT);
+      expect(byteLength(m.textSignature), `text for ${filler}`).toBeLessThanOrEqual(SIGNATURE_LIMIT);
+      expect(markerOf(m.htmlSignature)).toEqual({ blobId: "blob123", type: "text/html" });
+    }
+  });
+
+  it("never truncates through a surrogate pair", () => {
+    const m = sigOf(`<div>${"🎉".repeat(3000)}</div>`);
+    // A split pair leaves a lone surrogate, which encodes as U+FFFD.
+    expect(m.htmlSignature).not.toContain("�");
+    expect(m.textSignature).not.toContain("�");
+    expect(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])/.test(m.textSignature)).toBe(false);
+  });
+
+  it("never truncates through an HTML entity", () => {
+    // Escaping turns each of these into a 5-character entity; cutting the
+    // rendered string could leave "&am" behind.
+    const m = sigOf(`<div>${"a &amp; b <c> ".repeat(400)}</div>`);
+    expect(m.htmlSignature).not.toMatch(/&[a-z]*$/i);
+    expect(m.htmlSignature.replace(/&(amp|lt|gt|quot|#39);/g, "")).not.toContain("&");
+  });
+
+  it("leaves a signature that already fits completely alone", () => {
+    const m = sigOf("<div>Grüße, John</div>");
+    expect(m.textSignature).toBe("Grüße, John");
+    expect(m.htmlSignature).not.toContain("…");
+  });
+});

web/src/lib/signatureHtml.ts

@@ -6,8 +6,45 @@
  */
 import { escapeHtml, htmlToText } from "./text";
 
+/**
+ * Stalwart accepts a signature of `value.len() < 2048` — and that is Rust's
+ * `len()`, so the limit is 2047 **bytes of UTF-8**, not characters. A string's
+ * `.length` in JavaScript counts UTF-16 units, which matches only for ASCII: an
+ * accent is one unit but two bytes, CJK three, an emoji two units and four. So
+ * every check here weighs the encoded form, or a signature we judged to fit
+ * would come back rejected.
+ */
 export const SIGNATURE_LIMIT = 2047;
 
+const encoder = new TextEncoder();
+
+export function byteLength(s: string): number {
+  return encoder.encode(s).length;
+}
+
+/**
+ * Render `text` into at most `budget` bytes, appending an ellipsis if it had to
+ * be cut. Cutting the *source* text and rendering afterwards — rather than
+ * slicing the rendered string — means a cut can never land inside an HTML
+ * entity or a `<br>`; stepping through code points means it never splits a
+ * surrogate pair either. Binary search keeps it to a handful of encodes.
+ */
+function renderWithinBytes(text: string, budget: number, render: (t: string) => string): string {
+  const whole = render(text);
+  if (byteLength(whole) <= budget) return whole;
+  const ellipsis = "…";
+  if (budget < byteLength(ellipsis)) return "";
+  const chars = Array.from(text);
+  let lo = 0;
+  let hi = chars.length;
+  while (lo < hi) {
+    const mid = Math.ceil((lo + hi) / 2);
+    if (byteLength(render(chars.slice(0, mid).join("")) + ellipsis) <= budget) lo = mid;
+    else hi = mid - 1;
+  }
+  return render(chars.slice(0, lo).join("")) + ellipsis;
+}
+
 const KEEP_STYLES = new Set(["color", "background-color", "font-weight", "font-style", "text-decoration", "font-size", "font-family", "text-align", "vertical-align", "width", "height", "max-width", "border", "border-left", "padding-left", "margin"]);
 const KEEP_ATTRS = new Set(["href", "src", "alt", "width", "height", "target", "style", "title", "colspan", "rowspan", "cellpadding", "cellspacing", "border", "align", "valign"]);
 const DROP_TAGS = new Set(["META", "STYLE", "SCRIPT", "LINK", "TITLE", "HEAD", "O:P", "XML", "NOSCRIPT", "IFRAME", "OBJECT", "EMBED", "FORM", "INPUT", "BUTTON"]);
@@ -96,8 +133,15 @@ export function markerOf(htmlSignature: string | null | undefined): { blobId: st
 export function buildMarkerSignature(blobId: string, fullHtml: string): { htmlSignature: string; textSignature: string } {
   const text = htmlToText(fullHtml);
   const marker = `<!--ihasmail:sig=${blobId}:text/html-->`;
-  const budget = SIGNATURE_LIMIT - marker.length - 11; // <div></div>
-  let fallback = escapeHtml(text).replace(/\n/g, "<br>");
-  if (fallback.length > budget) fallback = `${fallback.slice(0, Math.max(0, budget - 1))}…`;
-  return { htmlSignature: `${marker}<div>${fallback}</div>`, textSignature: text.length > SIGNATURE_LIMIT ? `${text.slice(0, SIGNATURE_LIMIT - 1)}…` : text };
+  const budget = SIGNATURE_LIMIT - byteLength(marker) - "<div></div>".length;
+  const fallback = renderWithinBytes(text, budget, (t) => escapeHtml(t).replace(/\n/g, "<br>"));
+  return {
+    htmlSignature: `${marker}<div>${fallback}</div>`,
+    textSignature: renderWithinBytes(text, SIGNATURE_LIMIT, (t) => t),
+  };
+}
+
+/** Whether a signature would be refused by the server as it stands. */
+export function signatureTooLong(htmlSignature: string, textSignature: string): boolean {
+  return byteLength(htmlSignature) > SIGNATURE_LIMIT || byteLength(textSignature) > SIGNATURE_LIMIT;
 }

web/src/views/settings/IdentitiesSettings.tsx

@@ -10,7 +10,7 @@ import { parseAddressList, formatAddressList } from "@/lib/address";
 import { htmlToText } from "@/lib/text";
 import { sanitizeEditorHtml } from "@/lib/html";
 import { externalizeDataImages, storeSignatureHtml, uploadSignatureImage } from "@/lib/signatureImages";
-import { buildMarkerSignature, compactHtml, SIGNATURE_LIMIT } from "@/lib/signatureHtml";
+import { buildMarkerSignature, byteLength, compactHtml, signatureTooLong, SIGNATURE_LIMIT } from "@/lib/signatureHtml";
 
 export function IdentitiesSettings() {
   const identities = useMail((s) => s.identities);
@@ -57,8 +57,9 @@ function IdentityDialog({ identity, onClose }: { identity: Partial<Identity>; on
   const [busy, setBusy] = useState(false);
   const ref = useRef<RichEditorHandle>(null);
   const compact = compactHtml(sanitizeEditorHtml(html));
-  const sigLen = compact.length;
-  const tooLong = sigLen > SIGNATURE_LIMIT || htmlToText(compact).length > SIGNATURE_LIMIT;
+  // The server's limit is on encoded bytes, so that is what to count and show.
+  const sigLen = byteLength(compact);
+  const tooLong = signatureTooLong(compact, htmlToText(compact));
   const save = async () => {
     setBusy(true);
     try {
@@ -67,7 +68,7 @@ function IdentityDialog({ identity, onClose }: { identity: Partial<Identity>; on
       const clean = compactHtml(externalized);
       let htmlSignature = clean;
       let textSignature = htmlToText(clean);
-      if (clean.length > SIGNATURE_LIMIT || textSignature.length > SIGNATURE_LIMIT) {
+      if (signatureTooLong(clean, textSignature)) {
         const blobId = await storeSignatureHtml(clean);
         ({ htmlSignature, textSignature } = buildMarkerSignature(blobId, clean));
       }
@@ -103,7 +104,7 @@ function IdentityDialog({ identity, onClose }: { identity: Partial<Identity>; on
           <span className="hint">Images are stored in your Files (folder “ihasmail”) and embedded when you send.</span>
           <span className="hint nowrap" style={tooLong ? { color: "var(--warn)", fontWeight: 600 } : undefined}>{sigLen.toLocaleString()} / {SIGNATURE_LIMIT.toLocaleString()}</span>
         </div>
-        {tooLong && <div className="warn-box mt-8">This signature is larger than the server's {SIGNATURE_LIMIT}-character limit. ihasmail will keep the full version in your Files and store a short text fallback on the server — other mail clients will see the plain-text version.</div>}
+        {tooLong && <div className="warn-box mt-8">This signature is larger than the server's {SIGNATURE_LIMIT}-byte limit. ihasmail will keep the full version in your Files and store a short text fallback on the server — other mail clients will see the plain-text version.</div>}
       </div>
     </Dialog>
   );