Show the language model's opinion on a message #15

Merged
jcoffey-dev merged 1 commits from feature/llm-opinion into main 2026-09-23 05:39:19 +00:00
17 changed files with 319 additions and 0 deletions
+2
View File
@@ -268,6 +268,8 @@ export interface Email {
"header:Received:asText:all"?: string[] | null;
"header:X-Spam-Status:asText"?: string | null;
"header:X-Spam-Result:asText"?: string | null;
/** inbuxa: the language model's opinion, when AI spam classification is on (lib/llmOpinion). */
"header:X-Spam-LLM:asText"?: string | null;
}
export interface Thread {
+52
View File
@@ -0,0 +1,52 @@
import { describe, expect, it } from "vitest";
import { LLM_HEADER_PROP, llmOpinion, parseLlmOpinion } from "@/lib/llmOpinion";
/*
* The header as inbuxa-server writes it (crates/features/src/ai/answer.rs):
* `X-Spam-LLM: <TAG>`, optionally followed by the explanation in one pair of
* parentheses, folded at 78 columns.
*/
describe("parseLlmOpinion", () => {
it("reads category, confidence and explanation", () => {
expect(parseLlmOpinion("LLM_UNSOLICITED_HIGH (Promotes a product the reader never asked about)")).toEqual({
tag: "LLM_UNSOLICITED_HIGH",
category: "Unsolicited",
confidence: "High",
explanation: "Promotes a product the reader never asked about",
});
});
it("reads a tag with no confidence and no explanation", () => {
expect(parseLlmOpinion("LLM_LEGITIMATE")).toEqual({
tag: "LLM_LEGITIMATE",
category: "Legitimate",
confidence: null,
explanation: null,
});
});
it("keeps an operator's multi-word category whole", () => {
const o = parseLlmOpinion("LLM_COLD_OUTREACH_MEDIUM");
expect(o?.category).toBe("Cold outreach");
expect(o?.confidence).toBe("Medium");
// An unknown last word is part of the category, not a confidence.
expect(parseLlmOpinion("LLM_COLD_OUTREACH")?.category).toBe("Cold outreach");
expect(parseLlmOpinion("LLM_COLD_OUTREACH")?.confidence).toBeNull();
});
it("unfolds a folded header and keeps inner parentheses", () => {
const o = parseLlmOpinion("LLM_HARMFUL_LOW (Asks for a password\r\n (urgently) via a link)");
expect(o?.explanation).toBe("Asks for a password (urgently) via a link");
});
it("returns null for anything that isn't the server's tag", () => {
for (const raw of [null, undefined, "", " ", "Yes, score=6.7", "LLM_", "llm_unsolicited_high", "X LLM_SPAM"]) {
expect(parseLlmOpinion(raw), String(raw)).toBeNull();
}
});
it("reads the JMAP property a full message carries", () => {
expect(llmOpinion({ [LLM_HEADER_PROP]: "LLM_LEGITIMATE_HIGH" })?.category).toBe("Legitimate");
expect(llmOpinion({})).toBeNull();
});
});
+80
View File
@@ -0,0 +1,80 @@
/**
* inbuxa: the language model's opinion, read back off the message.
*
* When the server's AI spam classification is on (inbuxa-server,
* docs/spec/features/ai-spam-classification.md), it writes the model's answer
* into an `X-Spam-LLM` header at delivery:
*
* X-Spam-LLM: LLM_UNSOLICITED_HIGH (Promotes a product the reader never asked about)
*
* a tag, then optionally the model's explanation in parentheses. The tag is
* `LLM_` + category, or `LLM_` + category + `_` + confidence, uppercased with
* anything outside A-Z and 0-9 turned into `_`. The explanation is already
* sanitized by the server and may arrive as encoded words, which the JMAP
* `asText` form decodes.
*
* Like `spamScore`, nothing here judges anything: it only reads what the
* server wrote. It is one signal the spam filter weighed among many, and the
* UI says so.
*/
/** The JMAP property that carries the header, decoded and unfolded. */
export const LLM_HEADER_PROP = "header:X-Spam-LLM:asText" as const;
/**
* Confidence words the fork's default prompt uses. A tag ending in one of
* these is read as category + confidence; anything else is all category,
* since an operator's own categories may contain underscores.
*/
const CONFIDENCES = new Set(["LOW", "MEDIUM", "HIGH"]);
export interface LlmOpinion {
/** The tag as the server wrote it, e.g. `LLM_UNSOLICITED_HIGH`. */
tag: string;
/** Readable category, e.g. `Unsolicited`. */
category: string;
/** Readable confidence, e.g. `High`, where the tag carried one. */
confidence: string | null;
/** The model's own explanation, as plain text, where there is one. */
explanation: string | null;
}
/** `UNSOLICITED_BULK` -> `Unsolicited bulk`. */
function readable(words: string[]): string {
const s = words.join(" ").toLowerCase();
return s.charAt(0).toUpperCase() + s.slice(1);
}
/** Headers arrive folded, so tabs and newlines are whitespace like any other. */
function flatten(v: string | null | undefined): string {
return (v ?? "").replace(/\s+/g, " ").trim();
}
export function parseLlmOpinion(raw: string | null | undefined): LlmOpinion | null {
const s = flatten(raw);
const m = /^(LLM_[A-Z0-9_]+)(?:\s+(.*))?$/.exec(s);
if (!m) return null;
const tag = m[1]!;
const parts = tag.slice("LLM_".length).split("_").filter(Boolean);
if (parts.length === 0) return null;
let confidence: string | null = null;
if (parts.length > 1 && CONFIDENCES.has(parts[parts.length - 1]!)) {
confidence = readable([parts.pop()!]);
}
let explanation: string | null = null;
const rest = (m[2] ?? "").trim();
if (rest) {
// The server wraps the explanation in one pair of parentheses.
const inner = rest.startsWith("(") && rest.endsWith(")") ? rest.slice(1, -1).trim() : rest;
explanation = inner || null;
}
return { tag, category: readable(parts), confidence, explanation };
}
/** The opinion on a message, if the server recorded one. */
export function llmOpinion(email: { [LLM_HEADER_PROP]?: string | null }): LlmOpinion | null {
return parseLlmOpinion(email[LLM_HEADER_PROP]);
}
+3
View File
@@ -1740,6 +1740,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Zur Bestätigung {phrase} eingeben",
"Turn off legacy protocols": "Ältere Mailprotokolle ausschalten",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Ältere Mailprotokolle sind für Ihre Organisation ausgeschaltet. Nur {app} und JMAP-Apps können sich anmelden.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Einschätzung des Sprachmodells",
"One of several signals the spam filter weighed": "Eines von mehreren Signalen, die der Spamfilter berücksichtigt hat",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1713,6 +1713,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Para confirmar, escriba {phrase}",
"Turn off legacy protocols": "Desactivar los protocolos de correo heredados",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Los protocolos de correo heredados están desactivados para su organización. Solo {app} y las aplicaciones JMAP pueden iniciar sesión.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Opinión del modelo de lenguaje",
"One of several signals the spam filter weighed": "Una de varias señales que el filtro de spam ha tenido en cuenta",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1718,6 +1718,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Pour confirmer, saisissez {phrase}",
"Turn off legacy protocols": "Désactiver les protocoles de messagerie historiques",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Les protocoles de messagerie historiques sont désactivés pour votre organisation. Seuls {app} et les applications JMAP peuvent se connecter.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Avis du modèle de langage",
"One of several signals the spam filter weighed": "Un signal parmi d'autres pris en compte par le filtre antispam",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1721,6 +1721,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "確認のため {phrase} と入力してください",
"Turn off legacy protocols": "従来のメールプロトコルをオフにする",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "組織では従来のメールプロトコルがオフになっています。サインインできるのは {app} と JMAP アプリのみです。",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "言語モデルの見解",
"One of several signals the spam filter weighed": "迷惑メールフィルターが考慮した複数の判断材料のひとつ",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1710,6 +1710,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Typ ter bevestiging {phrase}",
"Turn off legacy protocols": "Verouderde mailprotocollen uitschakelen",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Verouderde mailprotocollen zijn uitgeschakeld voor uw organisatie. Alleen {app} en JMAP-apps kunnen inloggen.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Oordeel van het taalmodel",
"One of several signals the spam filter weighed": "Een van meerdere signalen die het spamfilter heeft meegewogen",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1716,6 +1716,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Para confirmar, digite {phrase}",
"Turn off legacy protocols": "Desativar os protocolos de e-mail legados",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Os protocolos de e-mail legados estão desativados para sua organização. Só {app} e aplicativos JMAP podem entrar.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Opinião do modelo de linguagem",
"One of several signals the spam filter weighed": "Um dos vários sinais considerados pelo filtro de spam",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1715,6 +1715,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Для подтверждения введите {phrase}",
"Turn off legacy protocols": "Отключить устаревшие почтовые протоколы",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Устаревшие почтовые протоколы отключены для вашей организации. Входить могут только {app} и приложения JMAP.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Мнение языковой модели",
"One of several signals the spam filter weighed": "Один из нескольких признаков, которые учёл спам-фильтр",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1709,6 +1709,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "Для підтвердження введіть {phrase}",
"Turn off legacy protocols": "Вимкнути застарілі поштові протоколи",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "Застарілі поштові протоколи вимкнено для вашої організації. Входити можуть лише {app} і програми JMAP.",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "Думка мовної моделі",
"One of several signals the spam filter weighed": "Одна з кількох ознак, які врахував спам-фільтр",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1720,6 +1720,9 @@ export const catalog: Catalog = {
"To confirm, type {phrase}": "请输入 {phrase} 以确认",
"Turn off legacy protocols": "关闭传统邮件协议",
"Legacy mail protocols are off for your organization. Only {app} and JMAP apps can sign in.": "您的组织已关闭传统邮件协议。只有 {app} 和 JMAP 应用可以登录。",
// ── Spam filter: the language model's opinion (inbuxa) ──────────
"Language model's opinion": "语言模型的判断",
"One of several signals the spam filter weighed": "垃圾邮件过滤考虑的多个信号之一",
},
plurals: {
// ── Administration: legacy mail protocols (INBUXA) ──────────────
+3
View File
@@ -1,4 +1,5 @@
import { SPAM_HEADER_PROPS } from "@/lib/spamScore";
import { LLM_HEADER_PROP } from "@/lib/llmOpinion";
/*
@@ -67,6 +68,8 @@ export const FULL_PROPS = [
"header:Precedence:asText",
"header:Authentication-Results:asText",
...SPAM_HEADER_PROPS,
// inbuxa: the language model's opinion, when the server wrote one
LLM_HEADER_PROP,
];
export const BODY_PROPS = ["partId", "blobId", "size", "name", "type", "charset", "disposition", "cid", "language", "location", "subParts", "headers"];
+7
View File
@@ -2387,6 +2387,13 @@ button.dp-open:disabled { cursor: default; opacity: .5; }
.spam-weight.bad { color: var(--danger); }
.spam-weight.good { color: var(--success); }
/* inbuxa: the language model's opinion, in the details and above a message in
Junk (views/mail/LlmOpinion.tsx). The explanation is the model's own words,
so it keeps its line breaks out and wraps rather than widening the pane. */
.llm-opinion { display: flex; flex-direction: column; gap: 4px; }
.llm-heading { display: inline-flex; flex-wrap: wrap; gap: .4em; align-items: baseline; }
.llm-explanation { overflow-wrap: anywhere; }
/* The placeholder reference under a template's body. */
.placeholder-list { display: grid; grid-template-columns: auto 1fr; gap: 4px 12px; align-items: baseline; }
.placeholder-row { display: contents; }
+67
View File
@@ -0,0 +1,67 @@
import { Bot } from "lucide-react";
import type { LlmOpinion } from "@/lib/llmOpinion";
import { t as translate } from "@/lib/i18n";
/*
* inbuxa: the language model's opinion on a message, where the server's AI
* spam classification recorded one (lib/llmOpinion).
*
* Two rules, both from the server's spec. It is always labeled as one signal
* the spam filter weighed among several, never as the reason a message was
* filed where it was: the model can add a bounded amount to the score and no
* more. And the explanation is the model's own output, so it is only ever
* rendered as text.
*
* The category and confidence come from the server's configuration and aren't
* translated; only the two framing strings are.
*/
function Verdict({ opinion }: { opinion: LlmOpinion }) {
return (
<>
<strong>{opinion.category}</strong>
{opinion.confidence && <span className="hint">{` · ${opinion.confidence}`}</span>}
</>
);
}
/** In the message details, beside the spam filter's own working. */
export function LlmOpinionDetail({ opinion }: { opinion: LlmOpinion }) {
return (
<div className="llm-opinion">
<div>
<Verdict opinion={opinion} />
</div>
{opinion.explanation && <div className="llm-explanation">{opinion.explanation}</div>}
<div className="hint">{translate("One of several signals the spam filter weighed")}</div>
</div>
);
}
/** Above a message that's in Junk. */
export function LlmOpinionBanner({ opinion }: { opinion: LlmOpinion }) {
return (
<div className="remote-banner llm-banner" role="note" style={{ margin: "0 16px 8px" }}>
<Bot size={16} />
<span className="grow llm-opinion">
<span className="llm-heading">
<span>{translate("Language model's opinion")}</span>
<span>
<Verdict opinion={opinion} />
</span>
</span>
{opinion.explanation && <span className="llm-explanation">{opinion.explanation}</span>}
<span className="hint">{translate("One of several signals the spam filter weighed")}</span>
</span>
</div>
);
}
/** The banner shows only for a message in Junk that carries an opinion. */
export function llmBannerOpinion(
opinion: LlmOpinion | null,
mailboxIds: Record<string, boolean>,
junkId: string | null | undefined,
): LlmOpinion | null {
return opinion && junkId && mailboxIds[junkId] ? opinion : null;
}
+8
View File
@@ -15,6 +15,8 @@ import { emlFilename } from "@/lib/text/emlName";
import { isTnef, parseTnef, type TnefAttachment } from "@/lib/tnef";
import { internalDomains, isExternalSender, linkVerdict } from "@/lib/warnings";
import { spamReport, type SpamReport } from "@/lib/spamScore";
import { llmOpinion } from "@/lib/llmOpinion";
import { LlmOpinionBanner, LlmOpinionDetail, llmBannerOpinion } from "./LlmOpinion";
import { formatFullDate, formatListDate, formatSize } from "@/lib/format";
import { displayName, domainOf, formatAddress } from "@/lib/address";
import { EMAIL_BASE_CSS, TEXT_EMAIL_CSS, hasHtmlAlternative, htmlDeclaresColors, markKeptSurfaces, sanitizeEmailHtml } from "@/lib/text/html";
@@ -187,6 +189,10 @@ export const MessageView = memo(function MessageView({ email: e, expanded, wasUn
const receiptRequested = Boolean(e["header:Disposition-Notification-To:asAddresses"]?.length);
const authFailed = /\b(dkim|spf|dmarc)=fail\b/i.test(e["header:Authentication-Results:asText"] ?? "");
const spam = useMemo(() => spamReport(e), [e]);
// inbuxa: the language model's opinion, where the server's AI spam classification wrote one
const llm = useMemo(() => llmOpinion(e), [e]);
const junkId = useMail((st) => st.roleId("junk"));
const llmBanner = llmBannerOpinion(llm, e.mailboxIds, junkId);
const identities = useMail((st) => st.identities);
/*
* Only computed when the warning is on, because the domains it compares
@@ -374,6 +380,7 @@ export const MessageView = memo(function MessageView({ email: e, expanded, wasUn
{e["header:List-Id:asText"] && <><dt>{translate("List")}</dt><dd>{e["header:List-Id:asText"]}</dd></>}
<dt>{translate("Size")}</dt><dd>{formatSize(e.size)}</dd>
{spam && <><dt>{translate("Spam filter")}</dt><dd><SpamSummary report={spam} /></dd></>}
{llm && <><dt>{translate("Language model's opinion")}</dt><dd><LlmOpinionDetail opinion={llm} /></dd></>}
{receiptRequested && <><dt>{translate("Receipt")}</dt><dd>{receipt.offer ? translate("Requested, to {address}. Never sent automatically.", { address: receipt.to!.email }) : translate(refusalText(receipt.refusal!))}</dd></>}
</dl>
)}
@@ -425,6 +432,7 @@ export const MessageView = memo(function MessageView({ email: e, expanded, wasUn
</div>
)}
<SignatureBanner state={signature} />
{llmBanner && <LlmOpinionBanner opinion={llmBanner} />}
{externalSender && (
<div className="remote-banner external-banner" style={{ margin: "0 16px 8px" }}>
<ShieldAlert size={16} />
@@ -0,0 +1,73 @@
import { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { LlmOpinionBanner, LlmOpinionDetail, llmBannerOpinion } from "../LlmOpinion";
import { parseLlmOpinion, type LlmOpinion } from "@/lib/llmOpinion";
(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
/*
* The framing is the feature: the model's opinion is always one signal among
* several, never presented as why a message is where it is, and its
* explanation is model output, so it must never be rendered as markup.
*/
const opinion = (raw: string) => parseLlmOpinion(raw) as LlmOpinion;
describe("the language model's opinion", () => {
let host: HTMLDivElement;
let root: Root;
const render = async (node: React.ReactNode) => {
await act(async () => {
root.render(node);
});
};
beforeEach(() => {
host = document.createElement("div");
document.body.appendChild(host);
root = createRoot(host);
});
afterEach(async () => {
await act(async () => root.unmount());
host.remove();
});
it("shows category, confidence and explanation, as one signal of several", async () => {
await render(<LlmOpinionDetail opinion={opinion("LLM_UNSOLICITED_HIGH (Sells something unasked)")} />);
expect(host.textContent).toContain("Unsolicited");
expect(host.textContent).toContain("High");
expect(host.textContent).toContain("Sells something unasked");
expect(host.textContent).toContain("One of several signals the spam filter weighed");
});
it("renders the explanation as text, never markup", async () => {
await render(<LlmOpinionDetail opinion={opinion('LLM_HARMFUL_HIGH (<img src=x onerror="alert(1)"> <b>bold</b>)')} />);
expect(host.querySelector("img")).toBeNull();
expect(host.querySelector("b")).toBeNull();
expect(host.textContent).toContain('<img src=x onerror="alert(1)">');
});
it("leaves out what the header didn't carry", async () => {
await render(<LlmOpinionDetail opinion={opinion("LLM_LEGITIMATE")} />);
expect(host.querySelector(".llm-explanation")).toBeNull();
expect(host.textContent).not.toContain("·");
});
it("banners a message in Junk with the same framing", async () => {
await render(<LlmOpinionBanner opinion={opinion("LLM_UNSOLICITED_MEDIUM (Bulk newsletter)")} />);
expect(host.textContent).toContain("Language model's opinion");
expect(host.textContent).toContain("Unsolicited");
expect(host.textContent).toContain("Bulk newsletter");
expect(host.textContent).toContain("One of several signals the spam filter weighed");
});
it("banners only a message that's in Junk and carries an opinion", () => {
const o = opinion("LLM_UNSOLICITED_HIGH");
expect(llmBannerOpinion(o, { junk1: true }, "junk1")).toBe(o);
expect(llmBannerOpinion(o, { inbox1: true }, "junk1")).toBeNull();
expect(llmBannerOpinion(o, { junk1: true }, null)).toBeNull();
expect(llmBannerOpinion(null, { junk1: true }, "junk1")).toBeNull();
});
});