From 2e6c2341528be2169212756c95f130acab6a4acd Mon Sep 17 00:00:00 2001 From: John Coffey Date: Sat, 19 Sep 2026 06:21:08 -0700 Subject: [PATCH] AI spam classification: default to 2 KiB of text and a +2.0 cap on the model's tag, as calibrated for low-end CPU-only instances The calibration harness (tests/src/system/ai_calibration.rs, ignored) sends what the classifier sends to a real local model and scores its answers with the classifier's parser. --- crates/features/src/ai/limits.rs | 4 +- crates/spam-filter/src/analysis/score.rs | 2 +- tests/src/system/ai.rs | 4 +- tests/src/system/ai_calibration.rs | 124 +++++++++++++++++++++++ tests/src/system/mod.rs | 1 + 5 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 tests/src/system/ai_calibration.rs diff --git a/crates/features/src/ai/limits.rs b/crates/features/src/ai/limits.rs index da4de99..01fb7ea 100644 --- a/crates/features/src/ai/limits.rs +++ b/crates/features/src/ai/limits.rs @@ -31,11 +31,11 @@ pub struct AiLimits { impl Default for AiLimits { fn default() -> Self { AiLimits { - spam_max_added: 5.0, + spam_max_added: 2.0, spam_max_subtracted: 1.0, spam_call_ceiling: Duration::from_millis(20_000), max_concurrent_calls: 4, - max_content_bytes: 16_384, + max_content_bytes: 2_048, failure_backoff: Duration::from_millis(60_000), user_calls_per_hour: 60, } diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index 217a149..042048b 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -59,7 +59,7 @@ impl SpamFilterAnalyzeScore for Server { // inbuxa: AI-13: the model's tag moves the score only so far, and // never discards or rejects on its own if inbuxa_features::ai::answer::is_llm_tag(tag) { - let (max_added, max_subtracted) = ctx.result.llm_bounds.unwrap_or((5.0, 1.0)); + let (max_added, max_subtracted) = ctx.result.llm_bounds.unwrap_or((2.0, 1.0)); let score = match self.core.spam.lists.scores.get(tag) { Some(SpamFilterAction::Allow(score)) => { inbuxa_features::ai::answer::clamp(*score, max_added, max_subtracted) diff --git a/tests/src/system/ai.rs b/tests/src/system/ai.rs index ffe9dd8..b4c7a29 100644 --- a/tests/src/system/ai.rs +++ b/tests/src/system/ai.rs @@ -276,7 +276,7 @@ pub async fn test(test: &mut TestServer) { deliver(&[USER], "Long", &"a".repeat(100 * 1024)).await; let (_, long) = stub.last(); let text = long["messages"][1]["content"].as_str().unwrap(); - assert!(text.len() < 16_384 + 256 && text.contains("[truncated]"), "test 7"); + assert!(text.len() < 2_048 + 256 && text.contains("[truncated]"), "test 7"); // Acceptance test 12: a hostile explanation and a planted header // AI-12 reads the first line only, so the hostile part rides on a lone CR @@ -324,7 +324,7 @@ pub async fn test(test: &mut TestServer) { .await; admin.reload_settings().await; for (answer, expected) in [ - ("Unsolicited,High,x", "LLM_UNSOLICITED_HIGH (5.00)"), + ("Unsolicited,High,x", "LLM_UNSOLICITED_HIGH (2.00)"), ("Legitimate,High,x", "LLM_LEGITIMATE_HIGH (-1.00)"), ("Harmful,High,x", "LLM_HARMFUL_HIGH (0.00)"), ] { diff --git a/tests/src/system/ai_calibration.rs b/tests/src/system/ai_calibration.rs new file mode 100644 index 0000000..08dafef --- /dev/null +++ b/tests/src/system/ai_calibration.rs @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! Calibrating a real model against labelled mail: how often it answers in +//! the configured format, how often its category is right, and how long it +//! takes. It sends exactly what the classifier sends (the fork's framing, +//! markers and limits, `inbuxa_features::ai::request`) and reads the answer +//! exactly as the classifier does (`ai::answer::parse`). +//! +//! Not part of any suite. Run with: +//! +//! - `CALIB_URL`: the model server's chat completions URL; +//! - `CALIB_SAMPLE`: a file of `labelpath` lines, `label` `spam` or `ham`; +//! - `CALIB_OUT`: where to write one JSON line per message; +//! - `CALIB_LIMIT` (optional): stop after this many messages; +//! - `CALIB_PROMPT` (optional): a prompt other than the default; +//! - `CALIB_MAX_BYTES` (optional): text sent per message, default 2048, as the server's. + +use inbuxa_features::ai::{answer, request}; +use mail_parser::MessageParser; +use serde_json::json; +use std::{ + io::Write, + time::{Duration, Instant}, +}; + +/// The fork's default prompt (spec, "Default prompt"). +pub const DEFAULT_PROMPT: &str = "Classify the email below as one of: Unsolicited, Commercial, \ +Harmful, Legitimate. Unsolicited: bulk mail the recipient didn't ask for. Commercial: selling \ +something. Harmful: phishing, fraud or malware. Legitimate: anything else. Then give your \ +confidence: High, Medium or Low. Answer on one line as Category,Confidence,Reason with a reason \ +of at most 20 words."; + +#[ignore] +#[tokio::test(flavor = "multi_thread")] +pub async fn ai_calibration() { + let url = std::env::var("CALIB_URL").expect("CALIB_URL"); + let sample = std::fs::read_to_string(std::env::var("CALIB_SAMPLE").expect("CALIB_SAMPLE")) + .expect("sample file"); + let limit = std::env::var("CALIB_LIMIT") + .ok() + .and_then(|l| l.parse::().ok()) + .unwrap_or(usize::MAX); + let prompt = std::env::var("CALIB_PROMPT").unwrap_or_else(|_| DEFAULT_PROMPT.to_string()); + let max_bytes = std::env::var("CALIB_MAX_BYTES") + .ok() + .and_then(|l| l.parse::().ok()) + .unwrap_or(2_048); + let mut out = std::fs::File::create(std::env::var("CALIB_OUT").expect("CALIB_OUT")) + .expect("output file"); + + let categories = ["Unsolicited", "Commercial", "Harmful", "Legitimate"].map(String::from); + let confidence = ["High", "Medium", "Low"].map(String::from); + let rules = answer::Rules { + separator: ",", + pos_category: 0, + pos_confidence: Some(1), + pos_explanation: Some(2), + categories: &categories, + confidence: &confidence, + }; + let system = request::system_text(&prompt); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .unwrap(); + + for line in sample.lines().filter(|l| !l.is_empty()).take(limit) { + let (label, path) = line.split_once('\t').expect("labelpath"); + let raw = std::fs::read(path).expect("message"); + let Some(message) = MessageParser::new().parse(&raw) else { + continue; + }; + let subject = message.subject().unwrap_or_default().to_string(); + let text = (0..message.text_body.len()) + .filter_map(|i| message.body_text(i)) + .collect::>() + .join("\n\n"); + let user = request::classification_text(&subject, &text, max_bytes, &request::nonce()); + let body = request::body( + request::Kind::Chat, + "calibration", + Some(&system), + &user, + 0.5, + request::CLASSIFY_MAX_TOKENS, + ); + + let started = Instant::now(); + let reply = match client + .post(&url) + .header("content-type", "application/json") + .body(body.to_string()) + .send() + .await + { + Ok(response) => response.bytes().await.ok(), + Err(_) => None, + }; + let elapsed = started.elapsed().as_millis() as u64; + let answer = reply + .as_deref() + .and_then(|bytes| request::answer(request::Kind::Chat, bytes)); + let tag = answer + .as_deref() + .and_then(|a| answer::parse(a, &rules)) + .map(|c| c.tag); + writeln!( + out, + "{}", + json!({ + "label": label, + "path": path, + "answer": answer, + "tag": tag, + "ms": elapsed, + }) + ) + .unwrap(); + } +} diff --git a/tests/src/system/mod.rs b/tests/src/system/mod.rs index 310e270..ef2e989 100644 --- a/tests/src/system/mod.rs +++ b/tests/src/system/mod.rs @@ -7,6 +7,7 @@ pub mod antispam; pub mod authentication; pub mod ai; +pub mod ai_calibration; pub mod authorization; pub mod branding; pub mod crypto;