diff --git a/.gitea/workflows/spam-rules-watch.yml b/.gitea/workflows/spam-rules-watch.yml deleted file mode 100644 index eb9ecdf6..00000000 --- a/.gitea/workflows/spam-rules-watch.yml +++ /dev/null @@ -1,83 +0,0 @@ -# Watch spam-filter, whose rules the server bundles (resources/spam-filter/), -# and open an issue when it publishes a newer release than the one -# BUNDLED_SPAM_RULES_VERSION names on main, so it waits in the tracker until -# someone takes it in as resources/spam-filter/README.md describes. -# -# Reads metadata only: the releases list from GitHub's API and spam_rules.rs -# from Gitea's. Drafts and pre-releases are ignored. An issue is opened once -# per release: an existing one with the same title, open or closed, stops a -# second. -# -# This was upstream-watch, which also opened an issue for each new release of -# the server the fork was built on. 0.16.27 was the last import (2026-10-27), -# so that half is gone. -# -# Daily 06:17 UTC; run it by hand with workflow_dispatch. -name: spam-rules-watch - -on: - schedule: - - cron: '17 6 * * *' - workflow_dispatch: - -concurrency: - group: spam-rules-watch - cancel-in-progress: false - -jobs: - spam-rules-watch: - runs-on: light - container: - image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim - env: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ github.repository }} - steps: - - shell: bash - run: | - python3 - <<'PY' - import base64, json, os, re, sys, urllib.request - - api = f"{os.environ['CI_SERVER_INTERNAL']}/api/v1/repos/{os.environ['REPO']}" - def call(method, url, body=None, token=os.environ["TOKEN"]): - headers = {"Content-Type": "application/json", "User-Agent": "inbuxa-spam-rules-watch"} - if token: - headers["Authorization"] = f"token {token}" - req = urllib.request.Request(url, method=method, headers=headers, - data=json.dumps(body).encode() if body is not None else None) - with urllib.request.urlopen(req, timeout=30) as r: - return json.load(r) - SEMVER = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") - def key(tag): - return tuple(int(x) for x in SEMVER.match(tag).groups()) - - # The spam filter rules bundled with the server (resources/spam-filter/): - # an issue when spam-filter publishes a newer release than the one - # BUNDLED_SPAM_RULES_VERSION names on main. - src = call("GET", f"{api}/contents/crates/common/src/manager/spam_rules.rs?ref=main") - text = base64.b64decode(src["content"]).decode() - m = re.search(r'BUNDLED_SPAM_RULES_VERSION: &str = "(\d+\.\d+\.\d+)"', text) - if not m: - print("Can't read BUNDLED_SPAM_RULES_VERSION from spam_rules.rs", file=sys.stderr); sys.exit(1) - bundled = "v" + m.group(1) - rels = call("GET", "https://api.github.com/repos/stalwartlabs/spam-filter/releases?per_page=30", token=None) - newer = sorted((r for r in rels - if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"]) - and key(r["tag_name"]) > key(bundled)), - key=lambda r: key(r["tag_name"])) - if not newer: - print(f"Up to date: the bundled spam rules are {bundled}, the newest release."); sys.exit(0) - latest = newer[-1] - tag = latest["tag_name"] - title = f"Update the bundled spam rules to {tag}" - existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=bundled+spam+rules&limit=50")} - if title in existing: - print(f"spam rules {tag}: issue already exists."); sys.exit(0) - body = (f"spam-filter published {tag} on {latest['published_at'][:10]}. " - f"The server bundles {bundled}.\n\n" - "Update it as resources/spam-filter/README.md describes: take the rules file " - f"from the {tag} release (by tag, not `latest`), set BUNDLED_SPAM_RULES_VERSION, " - "and run the antispam test.") - issue = call("POST", f"{api}/issues", {"title": title, "body": body}) - print(f"spam rules {tag}: opened #{issue['number']}.") - PY diff --git a/THIRD-PARTY.md b/THIRD-PARTY.md index 8ca017a7..ac299df7 100644 --- a/THIRD-PARTY.md +++ b/THIRD-PARTY.md @@ -24,7 +24,7 @@ carry their own license files. | `crates/common/src/network/acme/directory.rs`, `crates/common/src/network/acme/jose.rs`, `crates/common/src/network/acme/order.rs` | [rustls-acme](https://github.com/FlorianUekermann/rustls-acme) (MIT or Apache-2.0) | Copyright (c) Florian Uekermann | | `crates/types/src/id.rs` | [crockford](https://github.com/archer884/crockford) (MIT or Apache-2.0) | Copyright (c) 2017 J/A | | `crates/nlp/src/tokenizers/types.rs` | test cases from [linkify](https://github.com/robinst/linkify) (MIT or Apache-2.0) | Copyright (c) 2017 Robin Stocker | -| `resources/spam-filter/spam-filter-rules.json.gz` | the published rules of [spam-filter](https://github.com/stalwartlabs/spam-filter) v3.0.2, unmodified, built into the server as its default spam rules (MIT or Apache-2.0) | Copyright (C) 2024, Stalwart Labs LLC | +| `resources/spam-filter/spam-filter-rules.json.gz` | the published rules of [spam-filter](https://github.com/stalwartlabs/spam-filter) v3.0.2, with the fork's changes listed in `resources/spam-filter/README.md`, built into the server as its default spam rules (MIT or Apache-2.0) | Copyright (C) 2024, Stalwart Labs LLC | Each notice above applies with this permission notice: diff --git a/crates/common/src/manager/spam_rules.rs b/crates/common/src/manager/spam_rules.rs index 32f9a1e5..289021af 100644 --- a/crates/common/src/manager/spam_rules.rs +++ b/crates/common/src/manager/spam_rules.rs @@ -29,13 +29,14 @@ use store::{ }; use trc::AddContext; -/// The version of spam-filter the embedded rules come from. -pub const BUNDLED_SPAM_RULES_VERSION: &str = "3.0.2"; +/// The version of the embedded rules: the spam-filter release they started +/// from, then the fork's own revision of them (resources/spam-filter/README.md). +pub const BUNDLED_SPAM_RULES_VERSION: &str = "3.0.2-inbuxa.1"; /// What's recorded once the bundled rules are loaded: their version, then the /// fork's own generation of the update, so a change to how an update applies /// runs it once more. Generation 2 fingerprints (upstream v0.16.24). -pub const BUNDLED_SPAM_RULES_APPLIED: &str = "3.0.2+2"; +pub const BUNDLED_SPAM_RULES_APPLIED: &str = "3.0.2-inbuxa.1+2"; static BUNDLED_SPAM_RULES: &[u8] = include_bytes!("../../../../resources/spam-filter/spam-filter-rules.json.gz"); @@ -217,10 +218,15 @@ mod tests { } #[test] - fn bundled_rules_parse_and_score_the_ai_tags() { + fn bundled_rules_parse_and_score_the_ai_and_dkim2_tags() { let rules: serde_json::Value = serde_json::from_slice(&bundled_rules().unwrap()).unwrap(); let tags = rules["SpamTag"].as_array().unwrap(); - for (tag, score) in [("LLM_UNSOLICITED_HIGH", 3.0), ("LLM_LEGITIMATE_HIGH", -3.0)] { + for (tag, score) in [ + ("LLM_UNSOLICITED_HIGH", 3.0), + ("LLM_LEGITIMATE_HIGH", -3.0), + ("DKIM2_ALLOW", -0.2), + ("DKIM2_REJECT", 1.0), + ] { let found = tags.iter().find(|t| t["tag"] == tag).unwrap(); assert_eq!(found["score"].as_f64(), Some(score), "{tag}"); } diff --git a/resources/spam-filter/README.md b/resources/spam-filter/README.md index 08b471ad..fb7c644b 100644 --- a/resources/spam-filter/README.md +++ b/resources/spam-filter/README.md @@ -1,31 +1,43 @@ # Bundled spam filter rules -`spam-filter-rules.json.gz` is the published rules file of -[spam-filter](https://github.com/stalwartlabs/spam-filter) **v3.0.2**, -unmodified. The server embeds it (`crates/common/src/manager/spam_rules.rs`) -and loads it whenever no other rules source is configured, so a release -scores mail with the rules it was tested with, offline and with nothing to -fetch. The rules URL setting stays an operator override. +`spam-filter-rules.json.gz` holds the server's default spam rules. The server +embeds it (`crates/common/src/manager/spam_rules.rs`) and loads it whenever no +other rules source is configured, so a release scores mail with the rules it +was tested with, offline and with nothing to fetch. The rules URL setting +stays an operator override. -The rules are dual-licensed MIT or Apache-2.0, Copyright (C) 2024, Stalwart -Labs LLC; the fork takes them under MIT, with the notice in `THIRD-PARTY.md`. +The file started as the published rules of +[spam-filter](https://github.com/stalwartlabs/spam-filter) **v3.0.2**. Since +2026-10-28 the fork maintains it here: spam-filter's later releases follow +upstream's 1.0 server (its `AI_*` tag names, its lookup settings), which this +server doesn't use, so they aren't taken in. `BUNDLED_SPAM_RULES_VERSION` +names the release the rules started from and the fork's revision of them, +for example `3.0.2-inbuxa.1`. + +The v3.0.2 rules are dual-licensed MIT or Apache-2.0, Copyright (C) 2024, +Stalwart Labs LLC; the fork takes them under MIT, with the notice in +`THIRD-PARTY.md`. They include the scores for the AI classifier's tags (`LLM_*`, 3.0 for the high-confidence spam categories, −3.0 for legitimate), which match `docs/spec/features/ai-spam-classification.md`. -## Updating +## Changes since v3.0.2 -The `spam-rules-watch` workflow opens an issue when spam-filter publishes a -newer release. To take it: +| Revision | Change | +|---|---| +| `inbuxa.1` | Scores for the DKIM2 result tags the server already emits (`crates/spam-filter/src/analysis/dmarc.rs`): `DKIM2_ALLOW` −0.2, `DKIM2_REJECT` 1.0, and `DKIM2_NA`, `DKIM2_PERMFAIL`, `DKIM2_TEMPFAIL` 0.0. The values are spam-filter v3.0.3's. | -1. Download `spam-filter-rules.json.gz` from that release, pinned by tag - (`releases/download/vX.Y.Z/…`, not `latest`), over this file. -2. Set `BUNDLED_SPAM_RULES_VERSION` in `spam_rules.rs` and the version in - this README and in `THIRD-PARTY.md`. +## Changing the rules + +1. Edit the JSON (`gunzip -c` it; it's one line, keys in their original + order) and write it back compact, then compress it without a name or + timestamp so the file is reproducible: `gzip -n -9`. +2. Bump the revision in `BUNDLED_SPAM_RULES_VERSION` and + `BUNDLED_SPAM_RULES_APPLIED` (`spam_rules.rs`), and add a row above. 3. Run the antispam test (`STORE=RocksDb RUST_MIN_STACK=16777216 cargo test -p tests --lib -- smtp::inbound::antispam::antispam --exact`) and fix - expectations the new rules change, knowingly. + expectations the change affects, knowingly. On the next start each server loads the new version once. Loading only adds rules and tags that are missing; it never changes an existing one, so an diff --git a/resources/spam-filter/spam-filter-rules.json.gz b/resources/spam-filter/spam-filter-rules.json.gz index 6deb9d2b..6ebc658b 100644 Binary files a/resources/spam-filter/spam-filter-rules.json.gz and b/resources/spam-filter/spam-filter-rules.json.gz differ