inbuxa/inbuxa-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 23 Wiki Activity

Merge upstream v0.16.24

Browse Source 
Eight conflicted files resolved, plus the lock file and the schema:

- crates/services/src/task_manager/spam_classifier.rs: upstream's rules
  update now replaces existing rules, DNSBL servers, lookups and file
  extensions, keeping only whether each is on. Taken, with one difference:
  an object an admin edited is kept as it is. Every object an update writes
  is fingerprinted (content without `enable`, SHA-256, stored under
  SUBSPACE_INBUXA "Sf"), and only one that still matches is replaced.
  Scores are never replaced, as upstream has it. The AU-1.10 summary record
  now names what was added, replaced and kept, and the bundled rules are
  marked applied only when the update fully succeeded, so a failure runs
  again on the next start. The marker becomes "3.0.2+2", which runs the
  update once on upgrade to fingerprint every rule still as bundled.
- crates/common/src/network/autoconfig/autodiscover.rs: upstream's rewrite
  (implicit TLS first, labeled SSL), with the per-protocol switches (LP-7,
  LP-14a) passed in as a filter.
- crates/store/src/backend/mysql/{search,write}.rs: upstream's chunked
  deletes (no unbounded first DELETE, stop on a short chunk, halve the
  chunk on the new chunk-too-large errors) inside the fork's query timeout.
- crates/smtp/src/lib.rs: the fork's queue spawn kept. It already fixed the
  stall upstream fixes here (a node without outboundMta stops accepting
  mail at about 1024 queued messages), and follows role changes live.
- crates/jmap/src/registry/mapping/bootstrap.rs: the log path stays
  /var/log/inbuxa/; upstream's PowerDNS mapping taken.
- crates/main/Cargo.toml: the AGPL-only license kept, version 0.16.24.
- tests/src/jmap/principal/get.rs: the fork's capabilities kept.
- resources/schema/schema.json.gz: merged as JSON; upstream relabeled the
  vendor Sieve extensions "(Stalwart)", kept as "(vnd.inbuxa)".
- Cargo.lock: upstream's, with the fork's crates added by Cargo.

Also:

- tests/src/smtp/inbound/spam_rules_kept.rs: an edited rule survives an
  update, an unedited one is updated, rules from before fingerprints are
  handled, and the audit summary says so. Upstream's own spam_rules test
  passes unchanged.
- tests/src/smtp/reporting/reschedule.rs moves to port 19058; upstream's
  new spam_rules test took 19057.
- tools/fork/renames.py renames the "(Stalwart)" labels and the default
  log path, so neither conflicts again.
- tools/fork/notice-check.py compares against the newest snapshot in the
  checked-out history instead of the upstream branch head, so moving the
  branch no longer fails other open pull requests.
- tests/src/directory/issuer.rs (since v0.16.23) stays out, and is on the
  build check's known list: it tests issuer-based directory routing, which
  the fork doesn't have (DIR-2).
- Strip report: docs/fork/strip-reports/v0.16.24.{md,json}.
This commit is contained in:
jcoffey-dev
2026-09-28 06:30:20 -07:00
parent de275bac60 f59b084ce5
commit b2453d066b
94 changed files with 4206 additions and 1065 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/src/smtp/inbound/milter.rs
+73 -2
View File
@@ -28,9 +28,12 @@ use registry::{
schema::{
enums::{self, MtaStage},
prelude::{ObjectType, Property},
structs::{Expression, MtaHook, MtaMilter, MtaStageRcpt},
structs::{
Expression, ExpressionMatch, MtaHook, MtaMilter, MtaStageData, MtaStageRcpt,
SieveSystemScript,
},
},
types::map::Map,
types::{list::List, map::Map},
};
use serde::Deserialize;
use smtp::{
@@ -238,6 +241,34 @@ async fn mta_hook_session() {
..Default::default()
})
.await;
admin
.registry_create_object(MtaStageData {
script: Expression {
match_: List::from_iter([ExpressionMatch {
if_: "sender = '[email protected]'".into(),
then: "'tag_quarantine'".into(),
}]),
else_: "false".into(),
},
..Default::default()
})
.await;
admin
.registry_create_object(SieveSystemScript {
contents: concat!(
"require [\"editheader\"];\n",
"addheader \"X-Sieve\" \"Seen\";\n",
"if exists \"X-Quarantine\" {\n",
" deleteheader \"Subject\";\n",
" addheader \"Subject\" \"INFECTED\";\n",
"}\n"
)
.into(),
description: None,
is_active: true,
name: "tag_quarantine".into(),
})
.await;
admin.reload_settings().await;
test.reload_core();
test.expect_reload_settings().await;
@@ -355,6 +386,41 @@ async fn mta_hook_session() {
.await
.assert_contains("X-Spam: Yes")
.assert_contains("123456");
// Test quarantine
session
.send_message(
"[email protected]",
&["[email protected]"],
"test:no_dkim",
"250 2.0.0",
)
.await;
test.expect_message()
.await
.read_lines(&test)
.await
.assert_contains("X-Quarantine: true")
.assert_contains("Subject: Is dinner ready?")
.assert_contains("Are you hungry yet?");
// Test that a DATA stage Sieve script sees and preserves hook modifications
session
.send_message(
"[email protected]",
&["[email protected]"],
"test:no_dkim",
"250 2.0.0",
)
.await;
test.expect_message()
.await
.read_lines(&test)
.await
.assert_contains("X-Quarantine: true")
.assert_contains("X-Sieve: Seen")
.assert_contains("Subject: INFECTED")
.assert_contains("Are you hungry yet?");
}
#[test]
@@ -865,6 +931,11 @@ fn handle_mta_hook(request: Request, tests: Arc<Vec<HeaderTest>>) -> hooks::Resp
response: None,
modifications: vec![],
},
"quarantine" | "quarantine_only" => hooks::Response {
action: hooks::Action::Quarantine,
response: None,
modifications: vec![],
},
"temp_fail" => hooks::Response {
action: hooks::Action::Reject,
response: SmtpResponse {
tests/src/smtp/inbound/mod.rs
+2
View File
@@ -36,6 +36,8 @@ pub mod rcpt;
pub mod rewrite;
pub mod scripts;
pub mod sign;
pub mod spam_rules;
pub mod spam_rules_kept; // inbuxa: spam rules updates keep admin edits
pub mod throttle;
pub mod vrfy;
tests/src/smtp/inbound/spam_rules.rs
+364
View File
@@ -0,0 +1,364 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::utils::server::{TestServer, TestServerBuilder};
use registry::{
schema::{
enums::TaskSpamFilterMaintenanceType,
prelude::{Object, ObjectType},
structs::{
HttpLookup, MemoryLookupKey, MemoryLookupKeyValue, SpamDnsblServer, SpamFileExtension,
SpamRule, SpamSettings, SpamTag, SpamTagScore, Task, TaskSpamFilterMaintenance,
TaskStatus,
},
},
types::{ObjectImpl, float::Float},
};
use serde_json::{Value, json};
use std::fs;
use store::registry::RegistryObject;
#[tokio::test(flavor = "multi_thread")]
async fn spam_rules_update() {
let test = TestServerBuilder::new("spam_rules_update_test")
.await
.with_http_listener(19057)
.await
.build()
.await;
let admin = test.account("admin");
let rules_path = test.temp_dir.path.join("spam-filter-rules.json");
admin
.registry_create_object(SpamSettings {
spam_filter_rules_url: format!("file://{}", rules_path.display()).into(),
..Default::default()
})
.await;
admin
.registry_create_object(MemoryLookupKeyValue {
namespace: "test-keys".into(),
key: "conflict.org".into(),
value: "local".into(),
is_glob_pattern: false,
})
.await;
admin.reload_settings().await;
update_rules(&test, &release_1()).await;
for name in ["STWT_TEST_RULE", "STWT_TEST_DISABLED"] {
assert!(
stored::<SpamRule>(&test, "name", name)
.await
.unwrap()
.object
.enable()
);
}
assert!(
stored::<SpamDnsblServer>(&test, "name", "STWT_TEST_DNSBL")
.await
.unwrap()
.object
.enable()
);
assert!(
stored::<MemoryLookupKey>(&test, "key", "conflict.org")
.await
.is_none()
);
let disabled_rule = stored::<SpamRule>(&test, "name", "STWT_TEST_DISABLED")
.await
.unwrap();
admin
.registry_update_object(
ObjectType::SpamRule,
disabled_rule.id.id(),
json!({"enable": false}),
)
.await;
let local_tag = stored::<SpamTag>(&test, "tag", "TEST_TAG_LOCAL")
.await
.unwrap();
admin
.registry_update_object(
ObjectType::SpamTag,
local_tag.id.id(),
json!({"score": 2.0}),
)
.await;
let feed = stored::<HttpLookup>(&test, "namespace", "stwt_test_feed")
.await
.unwrap();
admin
.registry_update_object(
ObjectType::HttpLookup,
feed.id.id(),
json!({"enable": false}),
)
.await;
let release_2 = release_2();
for _ in 0..2 {
update_rules(&test, &release_2).await;
let rule = stored::<SpamRule>(&test, "name", "STWT_TEST_RULE")
.await
.unwrap()
.object;
assert!(rule.enable());
assert_eq!(object_json(&rule)["priority"], 400);
let rule = stored::<SpamRule>(&test, "name", "STWT_TEST_DISABLED")
.await
.unwrap()
.object;
assert!(!rule.enable());
assert_eq!(object_json(&rule)["priority"], 450);
assert!(
stored::<SpamRule>(&test, "name", "STWT_TEST_NEW")
.await
.is_some()
);
for upstream in release_2["SpamRule"].as_array().unwrap() {
let mut upstream: SpamRule = serde_json::from_value(upstream.clone()).unwrap();
let local = stored::<SpamRule>(
&test,
"name",
object_json(&upstream)["name"].as_str().unwrap(),
)
.await
.unwrap()
.object;
upstream.set_enable(local.enable());
assert_eq!(local, upstream);
}
for upstream in release_2["SpamDnsblServer"].as_array().unwrap() {
let upstream: SpamDnsblServer = serde_json::from_value(upstream.clone()).unwrap();
let local = stored::<SpamDnsblServer>(
&test,
"name",
object_json(&upstream)["name"].as_str().unwrap(),
)
.await
.unwrap()
.object;
assert_eq!(local, upstream);
}
for (tag, score) in [("TEST_TAG", 6.5), ("TEST_TAG_LOCAL", 2.0)] {
assert_eq!(
stored::<SpamTag>(&test, "tag", tag).await.unwrap().object,
SpamTag::Score(SpamTagScore {
tag: tag.into(),
score: Float::new(score),
})
);
}
let feed = stored::<HttpLookup>(&test, "namespace", "stwt_test_feed")
.await
.unwrap()
.object;
assert!(!feed.enable);
assert_eq!(feed.max_entries, 2000);
assert!(
stored::<MemoryLookupKey>(&test, "key", "example.org")
.await
.unwrap()
.object
.is_glob_pattern
);
assert!(
stored::<MemoryLookupKey>(&test, "key", "conflict.org")
.await
.is_none()
);
assert_eq!(
stored::<MemoryLookupKeyValue>(&test, "key", "conflict.org")
.await
.unwrap()
.object
.value,
"local"
);
assert!(
stored::<SpamFileExtension>(&test, "extension", "tst")
.await
.unwrap()
.object
.is_bad
);
}
}
async fn update_rules(test: &TestServer, rules: &Value) {
fs::write(
test.temp_dir.path.join("spam-filter-rules.json"),
rules.to_string(),
)
.unwrap();
test.account("admin")
.registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
status: TaskStatus::now(),
}))
.await;
test.wait_for_tasks().await;
}
async fn stored<T: ObjectImpl + From<Object>>(
test: &TestServer,
property: &str,
value: &str,
) -> Option<RegistryObject<T>> {
test.server
.registry()
.list::<T>()
.await
.unwrap()
.into_iter()
.find(|item| object_json(&item.object)[property] == value)
}
fn object_json<T: ObjectImpl>(object: &T) -> Value {
serde_json::to_value(object).unwrap()
}
fn release_1() -> Value {
json!({
"SpamRule": [
{
"@type": "Any",
"name": "STWT_TEST_RULE",
"enable": true,
"priority": 500,
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
},
{
"@type": "Any",
"name": "STWT_TEST_DISABLED",
"enable": true,
"priority": 500,
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
}
],
"SpamDnsblServer": [
{
"@type": "Domain",
"name": "STWT_TEST_DNSBL",
"enable": true,
"tag": {"else": "false", "match": {
"0": {"if": "octets[3] == 1", "then": "'URIBL_BLOCKED'"},
"1": {"if": "octets[3] == 2", "then": "'URIBL_BLACK'"},
"2": {"if": "octets[3] == 4", "then": "'URIBL_GREY'"},
"3": {"if": "octets[3] == 8", "then": "'URIBL_RED'"}
}},
"zone": {"else": "value + '.multi.uribl.com'", "match": {}}
}
],
"SpamTag": [
{"@type": "Score", "tag": "TEST_TAG", "score": 6.5},
{"@type": "Score", "tag": "TEST_TAG_LOCAL", "score": 1.0}
],
"HttpLookup": [
{
"namespace": "stwt_test_feed",
"url": "http://127.0.0.1:1/feed.txt",
"format": {"@type": "List"},
"enable": true,
"isGzipped": false,
"maxSize": 1048576,
"maxEntries": 1000,
"maxEntrySize": 512,
"refresh": 43200000,
"retry": 3600000,
"timeout": 30000
}
],
"MemoryLookupKey": [
{"namespace": "test-keys", "key": "example.org", "isGlobPattern": false},
{"namespace": "test-keys", "key": "conflict.org", "isGlobPattern": false}
],
"SpamFileExtension": [
{"extension": "tst", "contentTypes": {}, "isArchive": false, "isBad": false, "isNz": false}
]
})
}
fn release_2() -> Value {
json!({
"SpamRule": [
{
"@type": "Any",
"name": "STWT_TEST_RULE",
"enable": true,
"priority": 400,
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
},
{
"@type": "Any",
"name": "STWT_TEST_DISABLED",
"enable": true,
"priority": 450,
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
},
{
"@type": "Any",
"name": "STWT_TEST_NEW",
"enable": true,
"priority": 500,
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
}
],
"SpamDnsblServer": [
{
"@type": "Domain",
"name": "STWT_TEST_DNSBL",
"enable": true,
"tag": {"else": "false", "match": {
"0": {"if": "octets[0] != 127", "then": "false"},
"1": {"if": "octets[3] == 1", "then": "'URIBL_BLOCKED'"},
"2": {"if": "bit_and(octets[3], 2) != 0", "then": "'URIBL_BLACK'"},
"3": {"if": "bit_and(octets[3], 4) != 0", "then": "'URIBL_GREY'"},
"4": {"if": "bit_and(octets[3], 8) != 0", "then": "'URIBL_RED'"}
}},
"zone": {"else": "value + '.multi.uribl.com'", "match": {}}
}
],
"SpamTag": [
{"@type": "Score", "tag": "TEST_TAG", "score": 4.5},
{"@type": "Score", "tag": "TEST_TAG_LOCAL", "score": 1.0}
],
"HttpLookup": [
{
"namespace": "stwt_test_feed",
"url": "http://127.0.0.1:1/feed.txt",
"format": {"@type": "List"},
"enable": true,
"isGzipped": false,
"maxSize": 1048576,
"maxEntries": 2000,
"maxEntrySize": 512,
"refresh": 43200000,
"retry": 3600000,
"timeout": 30000
}
],
"MemoryLookupKey": [
{"namespace": "test-keys", "key": "example.org", "isGlobPattern": true},
{"namespace": "test-keys", "key": "conflict.org", "isGlobPattern": false}
],
"SpamFileExtension": [
{"extension": "tst", "contentTypes": {}, "isArchive": false, "isBad": true, "isNz": false}
]
})
}
tests/src/smtp/inbound/spam_rules_kept.rs
+204
View File
@@ -0,0 +1,204 @@
/*
* SPDX-FileCopyrightText: 2026 Coffey Labs
*
* SPDX-License-Identifier: AGPL-3.0-only
*/
//! A spam rules update brings unedited rules up to date and leaves an admin's
//! edits alone (common::manager::spam_rules), including on an install whose
//! rules were loaded before updates fingerprinted what they wrote. Each
//! update records what it replaced and what it kept in one audit record
//! (AU-1.10).
use crate::utils::server::{TestServer, TestServerBuilder};
use registry::{
schema::{
enums::TaskSpamFilterMaintenanceType,
prelude::ObjectType,
structs::{SpamRule, SpamSettings, Task, TaskSpamFilterMaintenance, TaskStatus},
},
types::ObjectImpl,
};
use serde_json::{Value, json};
use std::fs;
const USING: &[&str] = &["urn:ietf:params:jmap:core", "urn:inbuxa:jmap"];
#[tokio::test(flavor = "multi_thread")]
async fn spam_rules_keep_admin_edits() {
let test = TestServerBuilder::new("spam_rules_kept_test")
.await
.with_http_listener(19059)
.await
.build()
.await;
let admin = test.account("admin");
let rules_path = test.temp_dir.path.join("spam-filter-rules.json");
admin
.registry_create_object(SpamSettings {
spam_filter_rules_url: format!("file://{}", rules_path.display()).into(),
..Default::default()
})
.await;
// Two rules an admin made before any update ran: one exactly as the
// release has it, as an install from before fingerprints would, and one
// different from it, as an edited one would be.
admin.registry_create_object(rule("INBX_PRESET", 500)).await;
admin
.registry_create_object(rule("INBX_PRESET_EDITED", 300))
.await;
admin.reload_settings().await;
update_rules(&test, &release(500)).await;
assert_eq!(priority(&test, "INBX_UNTOUCHED").await, 500);
assert_eq!(priority(&test, "INBX_PRESET_EDITED").await, 300);
// An admin edits one rule the update wrote, and switches another off.
let edited = id_of(&test, "INBX_EDITED").await;
admin
.registry_update_object(ObjectType::SpamRule, edited, json!({"priority": 300}))
.await;
let switched_off = id_of(&test, "INBX_SWITCHED_OFF").await;
admin
.registry_update_object(ObjectType::SpamRule, switched_off, json!({"enable": false}))
.await;
for run in 0..2 {
update_rules(&test, &release(400)).await;
for (name, expected) in [
("INBX_UNTOUCHED", 400),
("INBX_SWITCHED_OFF", 400),
("INBX_PRESET", 400),
("INBX_EDITED", 300),
("INBX_PRESET_EDITED", 300),
] {
assert_eq!(priority(&test, name).await, expected, "{name}, run {run}");
}
assert!(
!stored(&test, "INBX_SWITCHED_OFF").await.enable(),
"run {run}: switching a rule off was undone"
);
}
// Two updates changed something: the first and the first of release 400.
// The second run of 400 changed nothing, so it isn't recorded.
let records = audit(&test, json!({"targetKind": "x:SpamRule"})).await;
let details = records
.iter()
.filter_map(|record| record["details"].as_str())
.filter(|details| details.starts_with("Rules update"))
.collect::<Vec<_>>();
assert_eq!(details.len(), 2, "{details:?}");
assert!(
details[0].contains("replaced 3 SpamRule")
&& details[0].contains("kept as edited locally 2 SpamRule"),
"{}",
details[0]
);
assert!(details[1].contains("added 3 SpamRule"), "{}", details[1]);
}
fn rule(name: &str, priority: i64) -> SpamRule {
serde_json::from_value(rule_json(name, priority)).unwrap()
}
fn rule_json(name: &str, priority: i64) -> Value {
json!({
"@type": "Any",
"name": name,
"enable": true,
"priority": priority,
"condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
})
}
fn release(priority: i64) -> Value {
let rules = [
"INBX_UNTOUCHED",
"INBX_EDITED",
"INBX_SWITCHED_OFF",
"INBX_PRESET",
"INBX_PRESET_EDITED",
]
.into_iter()
.map(|name| rule_json(name, priority))
.collect::<Vec<_>>();
json!({ "SpamRule": rules })
}
async fn update_rules(test: &TestServer, rules: &Value) {
fs::write(
test.temp_dir.path.join("spam-filter-rules.json"),
rules.to_string(),
)
.unwrap();
test.account("admin")
.registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
status: TaskStatus::now(),
}))
.await;
test.wait_for_tasks().await;
}
async fn stored(test: &TestServer, name: &str) -> SpamRule {
test.server
.registry()
.list::<SpamRule>()
.await
.unwrap()
.into_iter()
.find(|item| object_json(&item.object)["name"] == name)
.unwrap_or_else(|| panic!("no rule {name}"))
.object
}
async fn id_of(test: &TestServer, name: &str) -> types::id::Id {
test.server
.registry()
.list::<SpamRule>()
.await
.unwrap()
.into_iter()
.find(|item| object_json(&item.object)["name"] == name)
.unwrap_or_else(|| panic!("no rule {name}"))
.id
.id()
}
async fn priority(test: &TestServer, name: &str) -> i64 {
object_json(&stored(test, name).await)["priority"]
.as_i64()
.unwrap()
}
fn object_json<T: ObjectImpl>(object: &T) -> Value {
serde_json::to_value(object).unwrap()
}
/// Audit records matching `filter`, newest first.
async fn audit(test: &TestServer, filter: Value) -> Vec<Value> {
let admin = test.account("admin");
let response = admin
.jmap_request(
USING,
json!([
["inbuxa:AuditEvent/query", {
"accountId": admin.id_string(), "filter": filter, "limit": 100
}, "q"],
["inbuxa:AuditEvent/get", {
"accountId": admin.id_string(),
"#ids": {"resultOf": "q", "name": "inbuxa:AuditEvent/query", "path": "/ids"}
}, "g"]
]),
)
.await;
response
.0
.pointer("/methodResponses/1/1/list")
.and_then(Value::as_array)
.cloned()
.unwrap_or_else(|| panic!("audit query failed: {}", response.0))
}
tests/src/smtp/reporting/reschedule.rs
+1 -1
View File
@@ -51,7 +51,7 @@ use utils::snowflake::SnowflakeIdGenerator;
async fn report_reschedule() {
let mut test = TestServerBuilder::new("smtp_report_reschedule")
.await
.with_http_listener(19057)
.with_http_listener(19058)
.await
.capture_queue()
.build()