diff --git a/.gitea/workflows/upstream-watch.yml b/.gitea/workflows/upstream-watch.yml index 1dc8e47..8ae43c9 100644 --- a/.gitea/workflows/upstream-watch.yml +++ b/.gitea/workflows/upstream-watch.yml @@ -12,6 +12,9 @@ # An issue is opened once per release: an existing one with the same title, # open or closed, stops a second. # +# It also watches spam-filter, whose rules the server bundles +# (resources/spam-filter/), and opens an issue for a newer release. +# # Daily 06:17 UTC; run it by hand with workflow_dispatch. name: upstream-watch @@ -64,7 +67,7 @@ jobs: and key(r["tag_name"]) > key(base)), key=lambda r: key(r["tag_name"])) if not newer: - print(f"Up to date: {base} is the newest upstream release."); sys.exit(0) + print(f"Up to date: {base} is the newest upstream release.") # Titles and bodies stay free of the upstream project's name, as the # rest of the fork's user-visible text does. @@ -85,4 +88,35 @@ jobs: "add any new third-party notices to `THIRD-PARTY.md`, then merge `upstream` into `main`.") issue = call("POST", f"{api}/issues", {"title": title, "body": body}) print(f"{tag}: opened #{issue['number']}.") + + # The spam filter rules bundled with the server (resources/spam-filter/): + # an issue when spam-filter publishes a newer release than the one + # BUNDLED_SPAM_RULES_VERSION names on main. + src = call("GET", f"{api}/contents/crates/common/src/manager/spam_rules.rs?ref=main") + import base64 + text = base64.b64decode(src["content"]).decode() + m = re.search(r'BUNDLED_SPAM_RULES_VERSION: &str = "(\d+\.\d+\.\d+)"', text) + if not m: + print("Can't read BUNDLED_SPAM_RULES_VERSION from spam_rules.rs", file=sys.stderr); sys.exit(1) + bundled = "v" + m.group(1) + rels = call("GET", "https://api.github.com/repos/stalwartlabs/spam-filter/releases?per_page=30", token=None) + newer = sorted((r for r in rels + if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"]) + and key(r["tag_name"]) > key(bundled)), + key=lambda r: key(r["tag_name"])) + if not newer: + print(f"Up to date: the bundled spam rules are {bundled}, the newest release."); sys.exit(0) + latest = newer[-1] + tag = latest["tag_name"] + title = f"Update the bundled spam rules to {tag}" + existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=bundled+spam+rules&limit=50")} + if title in existing: + print(f"spam rules {tag}: issue already exists."); sys.exit(0) + body = (f"spam-filter published {tag} on {latest['published_at'][:10]}. " + f"The server bundles {bundled}.\n\n" + "Update it as resources/spam-filter/README.md describes: take the rules file " + f"from the {tag} release (by tag, not `latest`), set BUNDLED_SPAM_RULES_VERSION, " + "and run the antispam test.") + issue = call("POST", f"{api}/issues", {"title": title, "body": body}) + print(f"spam rules {tag}: opened #{issue['number']}.") PY diff --git a/THIRD-PARTY.md b/THIRD-PARTY.md index 499061a..8ca017a 100644 --- a/THIRD-PARTY.md +++ b/THIRD-PARTY.md @@ -24,7 +24,7 @@ carry their own license files. | `crates/common/src/network/acme/directory.rs`, `crates/common/src/network/acme/jose.rs`, `crates/common/src/network/acme/order.rs` | [rustls-acme](https://github.com/FlorianUekermann/rustls-acme) (MIT or Apache-2.0) | Copyright (c) Florian Uekermann | | `crates/types/src/id.rs` | [crockford](https://github.com/archer884/crockford) (MIT or Apache-2.0) | Copyright (c) 2017 J/A | | `crates/nlp/src/tokenizers/types.rs` | test cases from [linkify](https://github.com/robinst/linkify) (MIT or Apache-2.0) | Copyright (c) 2017 Robin Stocker | -| `tests/resources/smtp/antispam/spam-filter-rules.json.gz` | the published rules of [spam-filter](https://github.com/stalwartlabs/spam-filter) v3.0.2, unmodified, for the spam filter's tests (MIT or Apache-2.0) | Copyright (C) 2024, Stalwart Labs LLC | +| `resources/spam-filter/spam-filter-rules.json.gz` | the published rules of [spam-filter](https://github.com/stalwartlabs/spam-filter) v3.0.2, unmodified, built into the server as its default spam rules (MIT or Apache-2.0) | Copyright (C) 2024, Stalwart Labs LLC | Each notice above applies with this permission notice: diff --git a/crates/common/src/config/mailstore/spamfilter.rs b/crates/common/src/config/mailstore/spamfilter.rs index 882ceff..779e234 100644 --- a/crates/common/src/config/mailstore/spamfilter.rs +++ b/crates/common/src/config/mailstore/spamfilter.rs @@ -243,7 +243,8 @@ impl SpamFilterConfig { spam_threshold: spam.score_spam.into_inner() as f32, }, grey_list_expiry: spam.greylist_for.map(|d| d.into_inner().as_secs()), - spam_rules_url: spam.spam_filter_rules_url, + // inbuxa: unset, empty or upstream's old default means the bundled rules + spam_rules_url: crate::manager::spam_rules::rules_url(spam.spam_filter_rules_url), url_client: utils::http::http_client_builder(true) .pool_max_idle_per_host(0) .redirect(reqwest::redirect::Policy::none()) diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index e90ab4b..3c5554d 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -530,13 +530,22 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { use store::write::BatchBuilder; use types::id::Id; - if bp.registry.count_object(ObjectType::SpamRule).await? == 0 - && bp - .registry + // inbuxa: rules are always to hand, since a copy ships with the server + // (spam_rules). They load on first boot, and again when the bundled + // version differs from the one last loaded, which only adds what's + // missing: new tags and rules, never a changed score. + let rules_url = super::spam_rules::rules_url( + bp.registry .object::(Id::singleton()) .await? - .is_none_or(|spam| spam.spam_filter_rules_url.is_some()) - { + .and_then(|spam| spam.spam_filter_rules_url), + ); + let bundled_is_new = rules_url.is_none() + && super::spam_rules::applied_version(&bp.data_store) + .await? + .as_deref() + != Some(super::spam_rules::BUNDLED_SPAM_RULES_VERSION); + if bp.registry.count_object(ObjectType::SpamRule).await? == 0 || bundled_is_new { let mut batch = BatchBuilder::new(); batch.schedule_task(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance { maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules, diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index f2a30df..865ae0a 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -22,6 +22,7 @@ pub mod console; pub mod defaults; pub mod first_party; pub mod restore; +pub mod spam_rules; // inbuxa: rules bundled with the server pub const SPAM_TRAINER_KEY: &[u8] = "INBUXA_SPAM_TRAIN_DATA.lz4".as_bytes(); pub const SPAM_CLASSIFIER_KEY: &[u8] = "INBUXA_SPAM_CLASSIFIER_MODEL.lz4".as_bytes(); diff --git a/crates/common/src/manager/spam_rules.rs b/crates/common/src/manager/spam_rules.rs new file mode 100644 index 0000000..acac5e2 --- /dev/null +++ b/crates/common/src/manager/spam_rules.rs @@ -0,0 +1,103 @@ +/* + * SPDX-FileCopyrightText: 2026 Coffey Labs + * + * SPDX-License-Identifier: AGPL-3.0-only + */ + +//! inbuxa: the spam filter rules that ship with the server. +//! +//! Upstream fetches its latest published rules from GitHub at run time, so +//! scoring changes with a release nobody here tested and depends on reaching +//! it. The fork embeds a pinned copy (resources/spam-filter/, with its version +//! and license) and uses it whenever no other source is configured. The rules +//! URL remains an operator override (`https://` or `file://`). +//! +//! Loading rules only ever adds what's missing, never changes an existing rule +//! or score. They load on first boot, and again whenever the bundled version +//! differs from the one last applied, so an upgrade brings new tags (the AI +//! classifier's `LLM_*` scores, say) to an install that already had rules. + +use std::io::Read; +use store::{ + SUBSPACE_INBUXA, Store, ValueKey, + write::{AnyClass, BatchBuilder, ValueClass}, +}; +use trc::AddContext; + +/// The version of spam-filter the embedded rules come from. +pub const BUNDLED_SPAM_RULES_VERSION: &str = "3.0.2"; + +static BUNDLED_SPAM_RULES: &[u8] = + include_bytes!("../../../../resources/spam-filter/spam-filter-rules.json.gz"); + +/// Upstream's default rules source, the value every install created before +/// the rules were bundled has saved. Read only to treat it as unset. +const LEGACY_DEFAULT_URL: &str = + "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz"; + +/// The URL to fetch rules from, or `None` for the bundled rules. An empty +/// setting and upstream's old default both mean the bundled rules. +pub fn rules_url(configured: Option) -> Option { + configured.filter(|url| !url.trim().is_empty() && url != LEGACY_DEFAULT_URL) +} + +/// The bundled rules, uncompressed: the same JSON the rules URL serves. +pub fn bundled_rules() -> Result, String> { + let mut json = Vec::new(); + mail_auth::flate2::read::GzDecoder::new(BUNDLED_SPAM_RULES) + .read_to_end(&mut json) + .map_err(|err| format!("Failed to decompress the bundled spam rules: {err}"))?; + Ok(json) +} + +fn applied_key() -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: b"Sr".to_vec(), + }) +} + +/// The bundled version last loaded into the registry, if any. +pub async fn applied_version(data: &Store) -> trc::Result> { + data.get_value::(ValueKey::from(applied_key())) + .await + .caused_by(trc::location!()) +} + +/// Records that the bundled rules of this version have been loaded. +pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + batch.set(applied_key(), version.as_bytes().to_vec()); + data.write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upstream_default_and_empty_mean_bundled() { + assert_eq!(rules_url(None), None); + assert_eq!(rules_url(Some(String::new())), None); + assert_eq!(rules_url(Some(" ".into())), None); + assert_eq!(rules_url(Some(LEGACY_DEFAULT_URL.into())), None); + assert_eq!( + rules_url(Some("file:///srv/rules.json.gz".into())).as_deref(), + Some("file:///srv/rules.json.gz") + ); + } + + #[test] + fn bundled_rules_parse_and_score_the_ai_tags() { + let rules: serde_json::Value = serde_json::from_slice(&bundled_rules().unwrap()).unwrap(); + let tags = rules["SpamTag"].as_array().unwrap(); + for (tag, score) in [("LLM_UNSOLICITED_HIGH", 3.0), ("LLM_LEGITIMATE_HIGH", -3.0)] { + let found = tags.iter().find(|t| t["tag"] == tag).unwrap(); + assert_eq!(found["score"].as_f64(), Some(score), "{tag}"); + } + assert!(!rules["SpamRule"].as_array().unwrap().is_empty()); + } +} diff --git a/crates/registry/src/schema/structs_impl.rs b/crates/registry/src/schema/structs_impl.rs index cf45958..0b7fc8e 100644 --- a/crates/registry/src/schema/structs_impl.rs +++ b/crates/registry/src/schema/structs_impl.rs @@ -40215,7 +40215,7 @@ impl Default for SpamSettings { score_reject: Float::new(0.0f64), score_spam: Float::new(5.0f64), trust_replies: true, - spam_filter_rules_url: Some("https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz".to_string()), + spam_filter_rules_url: None, } } } diff --git a/crates/services/src/task_manager/spam_classifier.rs b/crates/services/src/task_manager/spam_classifier.rs index 502fb9b..9744cb1 100644 --- a/crates/services/src/task_manager/spam_classifier.rs +++ b/crates/services/src/task_manager/spam_classifier.rs @@ -2,13 +2,15 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::task_manager::{TaskFailureType, TaskResult}; use common::{ Server, ipc::{BroadcastEvent, RegistryChange}, - manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource}, + manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource, spam_rules}, }; use registry::{ schema::{ @@ -106,6 +108,7 @@ struct RuleUpdateResult { async fn update_spam_rules(server: &Server) -> trc::Result { let started = Instant::now(); + let bundled = server.core.spam.spam_rules_url.is_none(); let rules = match fetch_spam_rules(server).await { Ok(rules) => rules, Err(err) => { @@ -289,29 +292,36 @@ async fn update_spam_rules(server: &Server) -> trc::Result { Elapsed = started.elapsed(), ); + // inbuxa: so the next start knows these bundled rules are in + if bundled { + spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_VERSION) + .await?; + } + Ok(TaskResult::Success(vec![])) } async fn fetch_spam_rules(server: &Server) -> Result { - let Some(rules_url) = server.core.spam.spam_rules_url.as_ref() else { - return Err(RuleUpdateError { - typ: TaskFailureType::Permanent, - reason: "Spam rules resource URL not configured".to_string(), - }); - }; - let rules_json: AHashMap> = - fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500) + // inbuxa: no URL means the rules bundled with the server + let bytes = match server.core.spam.spam_rules_url.as_ref() { + Some(rules_url) => fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500) .await .map_err(|reason| RuleUpdateError { typ: TaskFailureType::Temporary, reason, + }), + None => spam_rules::bundled_rules().map_err(|reason| RuleUpdateError { + typ: TaskFailureType::Permanent, + reason, + }), + }; + let rules_json: AHashMap> = + bytes.and_then(|bytes| { + serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError { + typ: TaskFailureType::Permanent, + reason: format!("Failed to parse spam rules JSON: {err}"), }) - .and_then(|bytes| { - serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError { - typ: TaskFailureType::Permanent, - reason: format!("Failed to parse spam rules JSON: {err}"), - }) - })?; + })?; let mut rules = Rules::default(); for (object_type, values) in rules_json { diff --git a/docs/spec/features/ai-spam-classification.md b/docs/spec/features/ai-spam-classification.md index 52621f7..10f22fc 100644 --- a/docs/spec/features/ai-spam-classification.md +++ b/docs/spec/features/ai-spam-classification.md @@ -102,7 +102,10 @@ Permissions: `sysSpamLlmGet`, `sysSpamLlmUpdate`. - **Tags and scores.** The classifier's tags are ordinary spam tags, scored by `x:SpamTag` entries like every other tag: `Score` (a number), `Discard` or `Reject`. The documented defaults are `LLM_UNSOLICITED_HIGH` 3.0 and - `LLM_LEGITIMATE_HIGH` −3.0. A tag with no entry scores 0. + `LLM_LEGITIMATE_HIGH` −3.0. A tag with no entry scores 0. The server ships + those entries in its bundled spam rules (`resources/spam-filter/`), loaded + on first boot and again when the bundled version changes, so an install + that predates them gains them on upgrade (added 2026-09-23). - **`interactAi`** permission ("Interact with AI models"): lets an account's own Sieve scripts call `llm_prompt`. This repository's default roles give it to users, tenant administrators and superusers diff --git a/resources/schema/schema.json.gz b/resources/schema/schema.json.gz index 16a537a..4dbad15 100644 Binary files a/resources/schema/schema.json.gz and b/resources/schema/schema.json.gz differ diff --git a/resources/schema/schema.json.sha256 b/resources/schema/schema.json.sha256 index 4186a89..82c3111 100644 --- a/resources/schema/schema.json.sha256 +++ b/resources/schema/schema.json.sha256 @@ -1 +1 @@ -rWqJwNJkqgKsbC1eqcEmmIAMtJPmnlDlThR2ZtW_N1c \ No newline at end of file +LXwX4bO6norwJeWWcibbt1Wxd-jZAHO5TRgLXl3dFzk \ No newline at end of file diff --git a/resources/spam-filter/README.md b/resources/spam-filter/README.md new file mode 100644 index 0000000..d346d99 --- /dev/null +++ b/resources/spam-filter/README.md @@ -0,0 +1,32 @@ +# Bundled spam filter rules + +`spam-filter-rules.json.gz` is the published rules file of +[spam-filter](https://github.com/stalwartlabs/spam-filter) **v3.0.2**, +unmodified. The server embeds it (`crates/common/src/manager/spam_rules.rs`) +and loads it whenever no other rules source is configured, so a release +scores mail with the rules it was tested with, offline and with nothing to +fetch. The rules URL setting stays an operator override. + +The rules are dual-licensed MIT or Apache-2.0, Copyright (C) 2024, Stalwart +Labs LLC; the fork takes them under MIT, with the notice in `THIRD-PARTY.md`. + +They include the scores for the AI classifier's tags (`LLM_*`, 3.0 for the +high-confidence spam categories, −3.0 for legitimate), which match +`docs/spec/features/ai-spam-classification.md`. + +## Updating + +The `upstream-watch` workflow opens an issue when spam-filter publishes a +newer release. To take it: + +1. Download `spam-filter-rules.json.gz` from that release, pinned by tag + (`releases/download/vX.Y.Z/…`, not `latest`), over this file. +2. Set `BUNDLED_SPAM_RULES_VERSION` in `spam_rules.rs` and the version in + this README and in `THIRD-PARTY.md`. +3. Run the antispam test (`STORE=RocksDb RUST_MIN_STACK=16777216 cargo test + -p tests --lib -- smtp::inbound::antispam::antispam --exact`) and fix + expectations the new rules change, knowingly. + +On the next start each server loads the new version once. Loading only adds +rules and tags that are missing; it never changes an existing one, so an +operator's own adjustments survive. diff --git a/tests/resources/smtp/antispam/spam-filter-rules.json.gz b/resources/spam-filter/spam-filter-rules.json.gz similarity index 100% rename from tests/resources/smtp/antispam/spam-filter-rules.json.gz rename to resources/spam-filter/spam-filter-rules.json.gz diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index a960156..c26bbb7 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -86,19 +86,10 @@ async fn antispam() { .registry_create_object(SpamSettings { score_spam: Float::new(5.0), // inbuxa: the rules carry the scores the expectations are written - // against, so they're pinned (spam-filter v3.0.2, beside the test - // cases) rather than read from a developer's own checkout, which - // left every score at zero. SPAM_RULES_URL still overrides. - spam_filter_rules_url: std::env::var("SPAM_RULES_URL") - .unwrap_or_else(|_| { - concat!( - "file://", - env!("CARGO_MANIFEST_DIR"), - "/resources/smtp/antispam/spam-filter-rules.json.gz" - ) - .to_string() - }) - .into(), + // against. Unset, the server uses the rules bundled with it + // (resources/spam-filter/), the path production takes; + // SPAM_RULES_URL tests another set. + spam_filter_rules_url: std::env::var("SPAM_RULES_URL").ok(), ..Default::default() }) .await; diff --git a/tools/fork/name-allowlist.txt b/tools/fork/name-allowlist.txt index 8a80b6e..5ebd7d0 100644 --- a/tools/fork/name-allowlist.txt +++ b/tools/fork/name-allowlist.txt @@ -23,6 +23,6 @@ crates/migration/src/lib.rs "STALWART_SPAM_CLASSIFIER_MODEL.lz4" crates/migration/src/lib.rs "STALWART_SPAM_TRAIN_DATA.lz4" crates/types/src/branding.rs "STALWART" -# OPEN, not yet decided (2026-09-22): upstream's published spam-filter rules, -# which the server downloads at runtime from this address. -crates/registry/src/schema/structs_impl.rs "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz" +# Upstream's old default rules source, read only to treat it as unset: the +# server uses the rules bundled with it (resources/spam-filter/). +crates/common/src/manager/spam_rules.rs "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz" diff --git a/tools/fork/renames.py b/tools/fork/renames.py index db38d9d..770849a 100644 --- a/tools/fork/renames.py +++ b/tools/fork/renames.py @@ -46,6 +46,10 @@ TEXT_RENAMES = [ # that must match their containers and identity provider (database users, # passwords, an OIDC audience), and name their databases explicitly. ('"stalwart".to_string()', '"inbuxa".to_string()', ('crates',)), + # The spam filter rules ship with the server (common::manager::spam_rules); + # upstream's default of fetching its latest from GitHub becomes unset. + ('spam_filter_rules_url: Some("https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz".to_string()),', + 'spam_filter_rules_url: None,', ('crates',)), ] ROOTS = ('crates', 'tests', 'resources') SKIP_SUFFIXES = {'.md', '.txt'} @@ -58,6 +62,10 @@ SCHEMA_HASH = Path('resources/schema/schema.json.sha256') SCHEMA_RENAMES = [ ('"stalwart"', '"inbuxa"'), ('vnd.stalwart', 'vnd.inbuxa'), + # The bundled spam rules: no default URL, and say what empty means. + ('"spamFilterRulesUrl":"https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz",', ''), + ('"URL to download spam filter rules from"', + '"URL to download spam filter rules from. Empty uses the rules bundled with the server."'), ]