Merge main (upstream v0.16.24) into feature/compliance-roles

The schema conflicted as a binary file: taken from main and the one
edit here re-applied (sysComplianceGet after sysLegalHoldExport). The
import kept the permission count at 673, so the new id stays 673.

Retested on the merged tree in its own target directory: the
compliance and system suites pass. One earlier system run failed in
purge.rs (an imported blob not found) and didn't recur.

tests/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tests"
-version = "0.16.23"
+version = "0.16.24"
 edition = "2024"
 
 [features]

tests/src/directory/mod.rs

@@ -16,6 +16,9 @@ pub mod per_domain; // inbuxa: per-domain directories
 pub mod sql;
 pub mod synchronization;
 pub mod unavailable;
+// inbuxa: upstream's issuer.rs (since v0.16.23) is left out. It tests routing a
+// token that names no address by its issuer, which upstream ships in
+// Enterprise; here such a token gets the server's default directory (DIR-2).
 
 #[tokio::test(flavor = "multi_thread")]
 pub async fn directory_tests() {

tests/src/imap/antispam.rs

@@ -129,6 +129,21 @@ pub async fn test(test: &TestServer) {
     assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 11);
     assert_eq!(samples.iter().filter(|x| x.1.is_spam).count(), 11);
 
+    let last_id = samples.iter().map(|(id, _)| id.id()).max().unwrap();
+    for _ in 0..2 {
+        admin
+            .registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
+                maintenance_type: TaskSpamFilterMaintenanceType::Train,
+                status: TaskStatus::now(),
+            }))
+            .await;
+        test.wait_for_tasks().await;
+        let model = spam_classifier_model(&test.server).await;
+        assert_eq!(model.reservoir.ham.total_seen, 11);
+        assert_eq!(model.reservoir.spam.total_seen, 11);
+        assert_eq!(model.last_id, last_id);
+    }
+
     // Global spam samples should not appear in the account
     let samples = account.spam_training_samples().await;
     assert_eq!(samples.iter().filter(|x| !x.1.is_spam).count(), 11);

tests/src/imap/copy_move.rs

@@ -174,4 +174,53 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) {
     imap.assert_read(Type::Tagged, ResponseType::Ok)
         .await
         .assert_contains("COPYUID");
+
+    move_store_race(imap, imap_check).await;
+}
+
+async fn move_store_race(imap: &mut ImapConnection, imap_check: &mut ImapConnection) {
+    const RACE_MESSAGES: usize = 10;
+    const RACE_ROUNDS: usize = 20;
+
+    // A MOVE that loses a race with a STORE on the same messages in another
+    // session is retried instead of failing with CONTACTADMIN
+    imap.send_ok("CREATE \"Race Left\"").await;
+    imap.send_ok("CREATE \"Race Right\"").await;
+    for i in 0..RACE_MESSAGES {
+        imap.append(
+            "Race Left",
+            &format!("From: [email protected]\r\nSubject: Move race {i}\r\n\r\nrace\r\n"),
+        )
+        .await;
+    }
+
+    for round in 0..RACE_ROUNDS {
+        let (src, dest, store) = if round % 2 == 0 {
+            ("Race Left", "Race Right", "UID STORE 1:* +FLAGS (\\Seen)")
+        } else {
+            ("Race Right", "Race Left", "UID STORE 1:* -FLAGS (\\Seen)")
+        };
+        imap.send_ok(&format!("SELECT \"{src}\"")).await;
+        imap_check.send_ok(&format!("SELECT \"{src}\"")).await;
+
+        // The STORE may lose the race instead, so only the MOVE is checked
+        imap.send(&format!("UID MOVE 1:* \"{dest}\"")).await;
+        imap_check.send(store).await;
+        let (moved, _) = tokio::join!(
+            imap.assert_read(Type::Tagged, ResponseType::Ok),
+            imap_check.read(Type::Tagged)
+        );
+        moved.assert_contains("COPYUID");
+
+        imap.send(&format!("STATUS \"{dest}\" (MESSAGES)")).await;
+        imap.assert_read(Type::Tagged, ResponseType::Ok)
+            .await
+            .assert_contains(&format!("(MESSAGES {RACE_MESSAGES})"));
+    }
+
+    // Restore the state the following tests expect
+    imap.send_ok("SELECT \"Burrata al Tartufo\"").await;
+    imap_check.send_ok("SELECT \"Burrata al Tartufo\"").await;
+    imap.send_ok("DELETE \"Race Left\"").await;
+    imap.send_ok("DELETE \"Race Right\"").await;
 }

tests/src/jmap/core/blob.rs

@@ -367,6 +367,82 @@ pub async fn test(test: &TestServer) {
     }
     test.blob_expire_all().await;
 
+    let inbox_id = Id::from(INBOX_ID).to_string();
+    let response = account
+        .jmap_method_calls(json!([
+          [
+            "Blob/upload",
+            {
+              "accountId": account.id_string(),
+              "create": {
+                "m0": {
+                  "data": [
+                    {
+                      "data:asText": concat!(
+                          "From: [email protected]\r\n",
+                          "To: [email protected]\r\n",
+                          "Subject: Blob reference import\r\n",
+                          "\r\n",
+                          "Imported through a Blob/upload creation id."
+                      )
+                    }
+                  ],
+                  "type": "message/rfc822"
+                }
+              }
+            },
+            "U0"
+          ],
+          [
+            "Email/import",
+            {
+              "accountId": account.id_string(),
+              "emails": {
+                "i0": {
+                  "blobId": "#m0",
+                  "mailboxIds": { (inbox_id.as_str()): true }
+                }
+              }
+            },
+            "I0"
+          ],
+          [
+            "Email/import",
+            {
+              "accountId": account.id_string(),
+              "emails": {
+                "i1": {
+                  "blobId": "#missing",
+                  "mailboxIds": { (inbox_id.as_str()): true }
+                }
+              }
+            },
+            "I1"
+          ]
+        ]))
+        .await;
+
+    assert_eq!(response.name_at(1), "Email/import", "{response:?}");
+    assert!(
+        response
+            .pointer("/methodResponses/1/1/created/i0/id")
+            .and_then(|v| v.as_str())
+            .is_some(),
+        "{response:?}"
+    );
+    assert!(
+        response
+            .pointer("/methodResponses/1/1/notCreated")
+            .is_none(),
+        "{response:?}"
+    );
+    assert_eq!(
+        response.error_type_at(2),
+        Some("invalidResultReference"),
+        "{response:?}"
+    );
+    test.blob_expire_all().await;
+
     // Blob/lookup
     let client = account.jmap_client().await;
     let blob_id = client

tests/src/jmap/core/push_subscription.rs

@@ -188,13 +188,37 @@ pub async fn test(test: &TestServer) {
     client.push_subscription_destroy(&push_id).await.unwrap();
 
     // Only one verification per minute is allowed
-    let push_id = client
-        .push_subscription_create("invalid", "https://127.0.0.1:19000/push", None)
+    let first_id = client
+        .push_subscription_create(
+            "first",
+            "https://127.0.0.1:19000/push?skip_checks=true",
+            None,
+        )
+        .await
+        .unwrap()
+        .take_id();
+    let verification = expect_push(&mut event_rx).await.unwrap_verification();
+    assert_eq!(verification.push_subscription_id, first_id);
+    let deferred_id = client
+        .push_subscription_create("deferred", "https://127.0.0.1:19000/push", None)
         .await
         .unwrap()
         .take_id();
     expect_nothing(&mut event_rx).await;
-    client.push_subscription_destroy(&push_id).await.unwrap();
+    let verification = expect_push_within(&mut event_rx, Duration::from_secs(8))
+        .await
+        .unwrap_verification();
+    assert_eq!(verification.push_subscription_id, deferred_id);
+    account
+        .jmap_request(
+            &["urn:ietf:params:jmap:core"],
+            json!([[
+                "PushSubscription/set",
+                { "destroy": [first_id, deferred_id] },
+                "0"
+            ]]),
+        )
+        .await;
 
     // Register push notification (with encryption)
     let push_id = client
@@ -731,7 +755,14 @@ fn assert_vapid_authorization(header: &str, expected_key: &str, expected_origin:
 }
 
 async fn expect_push(event_rx: &mut mpsc::Receiver<PushMessage>) -> PushMessage {
-    match tokio::time::timeout(Duration::from_millis(1500), event_rx.recv()).await {
+    expect_push_within(event_rx, Duration::from_millis(1500)).await
+}
+
+async fn expect_push_within(
+    event_rx: &mut mpsc::Receiver<PushMessage>,
+    wait: Duration,
+) -> PushMessage {
+    match tokio::time::timeout(wait, event_rx.recv()).await {
         Ok(Some(push)) => {
             //println!("Push received: {:?}", push);
             push

tests/src/jmap/mail/set.rs

@@ -8,7 +8,7 @@ use crate::{
     jmap::{find_values, replace_blob_ids, replace_boundaries, replace_values},
     utils::server::TestServer,
 };
-use ::email::mailbox::INBOX_ID;
+use ::email::{mailbox::INBOX_ID, message::metadata::MessageData};
 use ahash::AHashSet;
 use jmap_client::{
     Error, Set,
@@ -18,8 +18,12 @@ use jmap_client::{
     mailbox::Role,
 };
 use registry::schema::prelude::ObjectType;
-use std::{fs, path::PathBuf};
-use types::id::Id;
+use std::{fs, path::PathBuf, str::FromStr};
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use types::{collection::Collection, id::Id};
 
 pub async fn test(test: &TestServer) {
     println!("Running Email Set tests...");
@@ -29,6 +33,7 @@ pub async fn test(test: &TestServer) {
 
     create(&client, &mailbox_id).await;
     update(&client, &mailbox_id).await;
+    update_preserves_uids(test, &client, account.id().document_id(), &mailbox_id).await;
 
     test.destroy_all_mailboxes(account).await;
     test.account("[email protected]")
@@ -296,6 +301,94 @@ async fn update(client: &Client, root_mailbox_id: &str) {
         .unwrap();
 }
 
+async fn update_preserves_uids(
+    test: &TestServer,
+    client: &Client,
+    account_id: u32,
+    root_mailbox_id: &str,
+) {
+    let email_id = client
+        .email_query(
+            email::query::Filter::in_mailbox(root_mailbox_id).into(),
+            None::<Vec<_>>,
+        )
+        .await
+        .unwrap()
+        .take_ids()
+        .pop()
+        .unwrap();
+    let document_id = Id::from_str(&email_id).unwrap().document_id();
+    let test_mailbox_id = client
+        .mailbox_create("UID Test", None::<String>, Role::None)
+        .await
+        .unwrap()
+        .take_id();
+    let test_mailbox_document_id = Id::from_str(&test_mailbox_id).unwrap().document_id();
+    let uids = message_uids(test, account_id, document_id).await;
+    let inbox_uid = uids[&INBOX_ID];
+    assert_ne!(inbox_uid, 0);
+
+    // Full mailboxIds identical to the current ones plus a keyword change must keep the UID
+    let mut request = client.build();
+    request
+        .set_email()
+        .update(&email_id)
+        .mailbox_ids([root_mailbox_id])
+        .keywords(["uid-test"]);
+    request
+        .send_set_email()
+        .await
+        .unwrap()
+        .updated(&email_id)
+        .unwrap();
+    assert_eq!(
+        message_uids(test, account_id, document_id).await,
+        [(INBOX_ID, inbox_uid)].into_iter().collect()
+    );
+
+    // Full mailboxIds that keeps a mailbox and adds another must only assign a UID to the new one
+    let mut request = client.build();
+    request
+        .set_email()
+        .update(&email_id)
+        .mailbox_ids([root_mailbox_id, test_mailbox_id.as_str()]);
+    request
+        .send_set_email()
+        .await
+        .unwrap()
+        .updated(&email_id)
+        .unwrap();
+    let uids = message_uids(test, account_id, document_id).await;
+    assert_eq!(uids.len(), 2);
+    assert_eq!(uids[&INBOX_ID], inbox_uid);
+    assert_ne!(uids[&test_mailbox_document_id], 0);
+
+    client.mailbox_destroy(&test_mailbox_id, true).await.unwrap();
+}
+
+async fn message_uids(
+    test: &TestServer,
+    account_id: u32,
+    document_id: u32,
+) -> std::collections::BTreeMap<u32, u32> {
+    test.server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::Email,
+            document_id,
+        ))
+        .await
+        .unwrap()
+        .unwrap()
+        .deserialize::<MessageData>()
+        .unwrap()
+        .mailboxes
+        .iter()
+        .map(|m| (m.mailbox_id, m.uid))
+        .collect()
+}
+
 pub async fn assert_email_properties(
     client: &Client,
     message_id: &str,

tests/src/jmap/mod.rs

@@ -99,6 +99,7 @@ pub async fn jmap_tests() {
                 push_throttle: 500u64.into(),
                 websocket_throttle: 500u64.into(),
                 push_attempt_wait: 500u64.into(),
+                push_verify_timeout: 5000u64.into(),
                 ..Default::default()
             },
             &[
@@ -108,6 +109,7 @@ pub async fn jmap_tests() {
                 Property::PushThrottle,
                 Property::WebsocketThrottle,
                 Property::PushAttemptWait,
+                Property::PushVerifyTimeout,
             ],
         )
         .await;

tests/src/jmap/principal/get.rs

@@ -254,7 +254,7 @@ pub async fn test(test: &TestServer) {
             // inbuxa: MT-22, the logo that applies to the account, and
             // LP-19, whether the legacy protocols are open to it, and
             // ai-explain EX-1, whether Explain can be offered
-            "urn:inbuxa:jmap": { "logo": null, "legacyProtocols": "enabled", "aiExplain": false },
+            "urn:inbuxa:jmap": { "logo": null, "legacyProtocols": "enabled", "legacyAllowed": ["imap", "pop3", "manageSieve", "submission"], "aiExplain": false },
             "https://www.fastmail.com/dev/maskedemail": {}
           }
         }

tests/src/smtp/inbound/milter.rs

@@ -28,9 +28,12 @@ use registry::{
     schema::{
         enums::{self, MtaStage},
         prelude::{ObjectType, Property},
-        structs::{Expression, MtaHook, MtaMilter, MtaStageRcpt},
+        structs::{
+            Expression, ExpressionMatch, MtaHook, MtaMilter, MtaStageData, MtaStageRcpt,
+            SieveSystemScript,
+        },
     },
-    types::map::Map,
+    types::{list::List, map::Map},
 };
 use serde::Deserialize;
 use smtp::{
@@ -238,6 +241,34 @@ async fn mta_hook_session() {
             ..Default::default()
         })
         .await;
+    admin
+        .registry_create_object(MtaStageData {
+            script: Expression {
+                match_: List::from_iter([ExpressionMatch {
+                    if_: "sender = '[email protected]'".into(),
+                    then: "'tag_quarantine'".into(),
+                }]),
+                else_: "false".into(),
+            },
+            ..Default::default()
+        })
+        .await;
+    admin
+        .registry_create_object(SieveSystemScript {
+            contents: concat!(
+                "require [\"editheader\"];\n",
+                "addheader \"X-Sieve\" \"Seen\";\n",
+                "if exists \"X-Quarantine\" {\n",
+                "  deleteheader \"Subject\";\n",
+                "  addheader \"Subject\" \"INFECTED\";\n",
+                "}\n"
+            )
+            .into(),
+            description: None,
+            is_active: true,
+            name: "tag_quarantine".into(),
+        })
+        .await;
     admin.reload_settings().await;
     test.reload_core();
     test.expect_reload_settings().await;
@@ -355,6 +386,41 @@ async fn mta_hook_session() {
         .await
         .assert_contains("X-Spam: Yes")
         .assert_contains("123456");
+
+    // Test quarantine
+    session
+        .send_message(
+            "[email protected]",
+            &["[email protected]"],
+            "test:no_dkim",
+            "250 2.0.0",
+        )
+        .await;
+    test.expect_message()
+        .await
+        .read_lines(&test)
+        .await
+        .assert_contains("X-Quarantine: true")
+        .assert_contains("Subject: Is dinner ready?")
+        .assert_contains("Are you hungry yet?");
+
+    // Test that a DATA stage Sieve script sees and preserves hook modifications
+    session
+        .send_message(
+            "[email protected]",
+            &["[email protected]"],
+            "test:no_dkim",
+            "250 2.0.0",
+        )
+        .await;
+    test.expect_message()
+        .await
+        .read_lines(&test)
+        .await
+        .assert_contains("X-Quarantine: true")
+        .assert_contains("X-Sieve: Seen")
+        .assert_contains("Subject: INFECTED")
+        .assert_contains("Are you hungry yet?");
 }
 
 #[test]
@@ -865,6 +931,11 @@ fn handle_mta_hook(request: Request, tests: Arc<Vec<HeaderTest>>) -> hooks::Resp
             response: None,
             modifications: vec![],
         },
+        "quarantine" | "quarantine_only" => hooks::Response {
+            action: hooks::Action::Quarantine,
+            response: None,
+            modifications: vec![],
+        },
         "temp_fail" => hooks::Response {
             action: hooks::Action::Reject,
             response: SmtpResponse {

tests/src/smtp/inbound/mod.rs

@@ -36,6 +36,8 @@ pub mod rcpt;
 pub mod rewrite;
 pub mod scripts;
 pub mod sign;
+pub mod spam_rules;
+pub mod spam_rules_kept; // inbuxa: spam rules updates keep admin edits
 pub mod throttle;
 pub mod vrfy;
 

tests/src/smtp/inbound/spam_rules.rs

@@ -0,0 +1,364 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::utils::server::{TestServer, TestServerBuilder};
+use registry::{
+    schema::{
+        enums::TaskSpamFilterMaintenanceType,
+        prelude::{Object, ObjectType},
+        structs::{
+            HttpLookup, MemoryLookupKey, MemoryLookupKeyValue, SpamDnsblServer, SpamFileExtension,
+            SpamRule, SpamSettings, SpamTag, SpamTagScore, Task, TaskSpamFilterMaintenance,
+            TaskStatus,
+        },
+    },
+    types::{ObjectImpl, float::Float},
+};
+use serde_json::{Value, json};
+use std::fs;
+use store::registry::RegistryObject;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn spam_rules_update() {
+    let test = TestServerBuilder::new("spam_rules_update_test")
+        .await
+        .with_http_listener(19057)
+        .await
+        .build()
+        .await;
+    let admin = test.account("admin");
+    let rules_path = test.temp_dir.path.join("spam-filter-rules.json");
+    admin
+        .registry_create_object(SpamSettings {
+            spam_filter_rules_url: format!("file://{}", rules_path.display()).into(),
+            ..Default::default()
+        })
+        .await;
+    admin
+        .registry_create_object(MemoryLookupKeyValue {
+            namespace: "test-keys".into(),
+            key: "conflict.org".into(),
+            value: "local".into(),
+            is_glob_pattern: false,
+        })
+        .await;
+    admin.reload_settings().await;
+
+    update_rules(&test, &release_1()).await;
+
+    for name in ["STWT_TEST_RULE", "STWT_TEST_DISABLED"] {
+        assert!(
+            stored::<SpamRule>(&test, "name", name)
+                .await
+                .unwrap()
+                .object
+                .enable()
+        );
+    }
+    assert!(
+        stored::<SpamDnsblServer>(&test, "name", "STWT_TEST_DNSBL")
+            .await
+            .unwrap()
+            .object
+            .enable()
+    );
+    assert!(
+        stored::<MemoryLookupKey>(&test, "key", "conflict.org")
+            .await
+            .is_none()
+    );
+
+    let disabled_rule = stored::<SpamRule>(&test, "name", "STWT_TEST_DISABLED")
+        .await
+        .unwrap();
+    admin
+        .registry_update_object(
+            ObjectType::SpamRule,
+            disabled_rule.id.id(),
+            json!({"enable": false}),
+        )
+        .await;
+    let local_tag = stored::<SpamTag>(&test, "tag", "TEST_TAG_LOCAL")
+        .await
+        .unwrap();
+    admin
+        .registry_update_object(
+            ObjectType::SpamTag,
+            local_tag.id.id(),
+            json!({"score": 2.0}),
+        )
+        .await;
+    let feed = stored::<HttpLookup>(&test, "namespace", "stwt_test_feed")
+        .await
+        .unwrap();
+    admin
+        .registry_update_object(
+            ObjectType::HttpLookup,
+            feed.id.id(),
+            json!({"enable": false}),
+        )
+        .await;
+
+    let release_2 = release_2();
+    for _ in 0..2 {
+        update_rules(&test, &release_2).await;
+
+        let rule = stored::<SpamRule>(&test, "name", "STWT_TEST_RULE")
+            .await
+            .unwrap()
+            .object;
+        assert!(rule.enable());
+        assert_eq!(object_json(&rule)["priority"], 400);
+
+        let rule = stored::<SpamRule>(&test, "name", "STWT_TEST_DISABLED")
+            .await
+            .unwrap()
+            .object;
+        assert!(!rule.enable());
+        assert_eq!(object_json(&rule)["priority"], 450);
+
+        assert!(
+            stored::<SpamRule>(&test, "name", "STWT_TEST_NEW")
+                .await
+                .is_some()
+        );
+
+        for upstream in release_2["SpamRule"].as_array().unwrap() {
+            let mut upstream: SpamRule = serde_json::from_value(upstream.clone()).unwrap();
+            let local = stored::<SpamRule>(
+                &test,
+                "name",
+                object_json(&upstream)["name"].as_str().unwrap(),
+            )
+            .await
+            .unwrap()
+            .object;
+            upstream.set_enable(local.enable());
+            assert_eq!(local, upstream);
+        }
+        for upstream in release_2["SpamDnsblServer"].as_array().unwrap() {
+            let upstream: SpamDnsblServer = serde_json::from_value(upstream.clone()).unwrap();
+            let local = stored::<SpamDnsblServer>(
+                &test,
+                "name",
+                object_json(&upstream)["name"].as_str().unwrap(),
+            )
+            .await
+            .unwrap()
+            .object;
+            assert_eq!(local, upstream);
+        }
+
+        for (tag, score) in [("TEST_TAG", 6.5), ("TEST_TAG_LOCAL", 2.0)] {
+            assert_eq!(
+                stored::<SpamTag>(&test, "tag", tag).await.unwrap().object,
+                SpamTag::Score(SpamTagScore {
+                    tag: tag.into(),
+                    score: Float::new(score),
+                })
+            );
+        }
+
+        let feed = stored::<HttpLookup>(&test, "namespace", "stwt_test_feed")
+            .await
+            .unwrap()
+            .object;
+        assert!(!feed.enable);
+        assert_eq!(feed.max_entries, 2000);
+
+        assert!(
+            stored::<MemoryLookupKey>(&test, "key", "example.org")
+                .await
+                .unwrap()
+                .object
+                .is_glob_pattern
+        );
+        assert!(
+            stored::<MemoryLookupKey>(&test, "key", "conflict.org")
+                .await
+                .is_none()
+        );
+        assert_eq!(
+            stored::<MemoryLookupKeyValue>(&test, "key", "conflict.org")
+                .await
+                .unwrap()
+                .object
+                .value,
+            "local"
+        );
+
+        assert!(
+            stored::<SpamFileExtension>(&test, "extension", "tst")
+                .await
+                .unwrap()
+                .object
+                .is_bad
+        );
+    }
+}
+
+async fn update_rules(test: &TestServer, rules: &Value) {
+    fs::write(
+        test.temp_dir.path.join("spam-filter-rules.json"),
+        rules.to_string(),
+    )
+    .unwrap();
+    test.account("admin")
+        .registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
+            maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
+            status: TaskStatus::now(),
+        }))
+        .await;
+    test.wait_for_tasks().await;
+}
+
+async fn stored<T: ObjectImpl + From<Object>>(
+    test: &TestServer,
+    property: &str,
+    value: &str,
+) -> Option<RegistryObject<T>> {
+    test.server
+        .registry()
+        .list::<T>()
+        .await
+        .unwrap()
+        .into_iter()
+        .find(|item| object_json(&item.object)[property] == value)
+}
+
+fn object_json<T: ObjectImpl>(object: &T) -> Value {
+    serde_json::to_value(object).unwrap()
+}
+
+fn release_1() -> Value {
+    json!({
+        "SpamRule": [
+            {
+                "@type": "Any",
+                "name": "STWT_TEST_RULE",
+                "enable": true,
+                "priority": 500,
+                "condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
+            },
+            {
+                "@type": "Any",
+                "name": "STWT_TEST_DISABLED",
+                "enable": true,
+                "priority": 500,
+                "condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
+            }
+        ],
+        "SpamDnsblServer": [
+            {
+                "@type": "Domain",
+                "name": "STWT_TEST_DNSBL",
+                "enable": true,
+                "tag": {"else": "false", "match": {
+                    "0": {"if": "octets[3] == 1", "then": "'URIBL_BLOCKED'"},
+                    "1": {"if": "octets[3] == 2", "then": "'URIBL_BLACK'"},
+                    "2": {"if": "octets[3] == 4", "then": "'URIBL_GREY'"},
+                    "3": {"if": "octets[3] == 8", "then": "'URIBL_RED'"}
+                }},
+                "zone": {"else": "value + '.multi.uribl.com'", "match": {}}
+            }
+        ],
+        "SpamTag": [
+            {"@type": "Score", "tag": "TEST_TAG", "score": 6.5},
+            {"@type": "Score", "tag": "TEST_TAG_LOCAL", "score": 1.0}
+        ],
+        "HttpLookup": [
+            {
+                "namespace": "stwt_test_feed",
+                "url": "http://127.0.0.1:1/feed.txt",
+                "format": {"@type": "List"},
+                "enable": true,
+                "isGzipped": false,
+                "maxSize": 1048576,
+                "maxEntries": 1000,
+                "maxEntrySize": 512,
+                "refresh": 43200000,
+                "retry": 3600000,
+                "timeout": 30000
+            }
+        ],
+        "MemoryLookupKey": [
+            {"namespace": "test-keys", "key": "example.org", "isGlobPattern": false},
+            {"namespace": "test-keys", "key": "conflict.org", "isGlobPattern": false}
+        ],
+        "SpamFileExtension": [
+            {"extension": "tst", "contentTypes": {}, "isArchive": false, "isBad": false, "isNz": false}
+        ]
+    })
+}
+
+fn release_2() -> Value {
+    json!({
+        "SpamRule": [
+            {
+                "@type": "Any",
+                "name": "STWT_TEST_RULE",
+                "enable": true,
+                "priority": 400,
+                "condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
+            },
+            {
+                "@type": "Any",
+                "name": "STWT_TEST_DISABLED",
+                "enable": true,
+                "priority": 450,
+                "condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
+            },
+            {
+                "@type": "Any",
+                "name": "STWT_TEST_NEW",
+                "enable": true,
+                "priority": 500,
+                "condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
+            }
+        ],
+        "SpamDnsblServer": [
+            {
+                "@type": "Domain",
+                "name": "STWT_TEST_DNSBL",
+                "enable": true,
+                "tag": {"else": "false", "match": {
+                    "0": {"if": "octets[0] != 127", "then": "false"},
+                    "1": {"if": "octets[3] == 1", "then": "'URIBL_BLOCKED'"},
+                    "2": {"if": "bit_and(octets[3], 2) != 0", "then": "'URIBL_BLACK'"},
+                    "3": {"if": "bit_and(octets[3], 4) != 0", "then": "'URIBL_GREY'"},
+                    "4": {"if": "bit_and(octets[3], 8) != 0", "then": "'URIBL_RED'"}
+                }},
+                "zone": {"else": "value + '.multi.uribl.com'", "match": {}}
+            }
+        ],
+        "SpamTag": [
+            {"@type": "Score", "tag": "TEST_TAG", "score": 4.5},
+            {"@type": "Score", "tag": "TEST_TAG_LOCAL", "score": 1.0}
+        ],
+        "HttpLookup": [
+            {
+                "namespace": "stwt_test_feed",
+                "url": "http://127.0.0.1:1/feed.txt",
+                "format": {"@type": "List"},
+                "enable": true,
+                "isGzipped": false,
+                "maxSize": 1048576,
+                "maxEntries": 2000,
+                "maxEntrySize": 512,
+                "refresh": 43200000,
+                "retry": 3600000,
+                "timeout": 30000
+            }
+        ],
+        "MemoryLookupKey": [
+            {"namespace": "test-keys", "key": "example.org", "isGlobPattern": true},
+            {"namespace": "test-keys", "key": "conflict.org", "isGlobPattern": false}
+        ],
+        "SpamFileExtension": [
+            {"extension": "tst", "contentTypes": {}, "isArchive": false, "isBad": true, "isNz": false}
+        ]
+    })
+}

tests/src/smtp/inbound/spam_rules_kept.rs

@@ -0,0 +1,204 @@
+/*
+ * SPDX-FileCopyrightText: 2026 Coffey Labs
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+//! A spam rules update brings unedited rules up to date and leaves an admin's
+//! edits alone (common::manager::spam_rules), including on an install whose
+//! rules were loaded before updates fingerprinted what they wrote. Each
+//! update records what it replaced and what it kept in one audit record
+//! (AU-1.10).
+
+use crate::utils::server::{TestServer, TestServerBuilder};
+use registry::{
+    schema::{
+        enums::TaskSpamFilterMaintenanceType,
+        prelude::ObjectType,
+        structs::{SpamRule, SpamSettings, Task, TaskSpamFilterMaintenance, TaskStatus},
+    },
+    types::ObjectImpl,
+};
+use serde_json::{Value, json};
+use std::fs;
+
+const USING: &[&str] = &["urn:ietf:params:jmap:core", "urn:inbuxa:jmap"];
+
+#[tokio::test(flavor = "multi_thread")]
+async fn spam_rules_keep_admin_edits() {
+    let test = TestServerBuilder::new("spam_rules_kept_test")
+        .await
+        .with_http_listener(19059)
+        .await
+        .build()
+        .await;
+    let admin = test.account("admin");
+    let rules_path = test.temp_dir.path.join("spam-filter-rules.json");
+    admin
+        .registry_create_object(SpamSettings {
+            spam_filter_rules_url: format!("file://{}", rules_path.display()).into(),
+            ..Default::default()
+        })
+        .await;
+
+    // Two rules an admin made before any update ran: one exactly as the
+    // release has it, as an install from before fingerprints would, and one
+    // different from it, as an edited one would be.
+    admin.registry_create_object(rule("INBX_PRESET", 500)).await;
+    admin
+        .registry_create_object(rule("INBX_PRESET_EDITED", 300))
+        .await;
+    admin.reload_settings().await;
+
+    update_rules(&test, &release(500)).await;
+    assert_eq!(priority(&test, "INBX_UNTOUCHED").await, 500);
+    assert_eq!(priority(&test, "INBX_PRESET_EDITED").await, 300);
+
+    // An admin edits one rule the update wrote, and switches another off.
+    let edited = id_of(&test, "INBX_EDITED").await;
+    admin
+        .registry_update_object(ObjectType::SpamRule, edited, json!({"priority": 300}))
+        .await;
+    let switched_off = id_of(&test, "INBX_SWITCHED_OFF").await;
+    admin
+        .registry_update_object(ObjectType::SpamRule, switched_off, json!({"enable": false}))
+        .await;
+
+    for run in 0..2 {
+        update_rules(&test, &release(400)).await;
+
+        for (name, expected) in [
+            ("INBX_UNTOUCHED", 400),
+            ("INBX_SWITCHED_OFF", 400),
+            ("INBX_PRESET", 400),
+            ("INBX_EDITED", 300),
+            ("INBX_PRESET_EDITED", 300),
+        ] {
+            assert_eq!(priority(&test, name).await, expected, "{name}, run {run}");
+        }
+        assert!(
+            !stored(&test, "INBX_SWITCHED_OFF").await.enable(),
+            "run {run}: switching a rule off was undone"
+        );
+    }
+
+    // Two updates changed something: the first and the first of release 400.
+    // The second run of 400 changed nothing, so it isn't recorded.
+    let records = audit(&test, json!({"targetKind": "x:SpamRule"})).await;
+    let details = records
+        .iter()
+        .filter_map(|record| record["details"].as_str())
+        .filter(|details| details.starts_with("Rules update"))
+        .collect::<Vec<_>>();
+    assert_eq!(details.len(), 2, "{details:?}");
+    assert!(
+        details[0].contains("replaced 3 SpamRule")
+            && details[0].contains("kept as edited locally 2 SpamRule"),
+        "{}",
+        details[0]
+    );
+    assert!(details[1].contains("added 3 SpamRule"), "{}", details[1]);
+}
+
+fn rule(name: &str, priority: i64) -> SpamRule {
+    serde_json::from_value(rule_json(name, priority)).unwrap()
+}
+
+fn rule_json(name: &str, priority: i64) -> Value {
+    json!({
+        "@type": "Any",
+        "name": name,
+        "enable": true,
+        "priority": priority,
+        "condition": {"else": "false", "match": {"0": {"if": "$MISSING_ESSENTIAL_HEADERS && $SINGLE_SHORT_PART", "then": "'SHORT_PART_BAD_HEADERS'"}}}
+    })
+}
+
+fn release(priority: i64) -> Value {
+    let rules = [
+        "INBX_UNTOUCHED",
+        "INBX_EDITED",
+        "INBX_SWITCHED_OFF",
+        "INBX_PRESET",
+        "INBX_PRESET_EDITED",
+    ]
+    .into_iter()
+    .map(|name| rule_json(name, priority))
+    .collect::<Vec<_>>();
+    json!({ "SpamRule": rules })
+}
+
+async fn update_rules(test: &TestServer, rules: &Value) {
+    fs::write(
+        test.temp_dir.path.join("spam-filter-rules.json"),
+        rules.to_string(),
+    )
+    .unwrap();
+    test.account("admin")
+        .registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
+            maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
+            status: TaskStatus::now(),
+        }))
+        .await;
+    test.wait_for_tasks().await;
+}
+
+async fn stored(test: &TestServer, name: &str) -> SpamRule {
+    test.server
+        .registry()
+        .list::<SpamRule>()
+        .await
+        .unwrap()
+        .into_iter()
+        .find(|item| object_json(&item.object)["name"] == name)
+        .unwrap_or_else(|| panic!("no rule {name}"))
+        .object
+}
+
+async fn id_of(test: &TestServer, name: &str) -> types::id::Id {
+    test.server
+        .registry()
+        .list::<SpamRule>()
+        .await
+        .unwrap()
+        .into_iter()
+        .find(|item| object_json(&item.object)["name"] == name)
+        .unwrap_or_else(|| panic!("no rule {name}"))
+        .id
+        .id()
+}
+
+async fn priority(test: &TestServer, name: &str) -> i64 {
+    object_json(&stored(test, name).await)["priority"]
+        .as_i64()
+        .unwrap()
+}
+
+fn object_json<T: ObjectImpl>(object: &T) -> Value {
+    serde_json::to_value(object).unwrap()
+}
+
+/// Audit records matching `filter`, newest first.
+async fn audit(test: &TestServer, filter: Value) -> Vec<Value> {
+    let admin = test.account("admin");
+    let response = admin
+        .jmap_request(
+            USING,
+            json!([
+                ["inbuxa:AuditEvent/query", {
+                    "accountId": admin.id_string(), "filter": filter, "limit": 100
+                }, "q"],
+                ["inbuxa:AuditEvent/get", {
+                    "accountId": admin.id_string(),
+                    "#ids": {"resultOf": "q", "name": "inbuxa:AuditEvent/query", "path": "/ids"}
+                }, "g"]
+            ]),
+        )
+        .await;
+    response
+        .0
+        .pointer("/methodResponses/1/1/list")
+        .and_then(Value::as_array)
+        .cloned()
+        .unwrap_or_else(|| panic!("audit query failed: {}", response.0))
+}

tests/src/smtp/reporting/reschedule.rs

@@ -51,7 +51,7 @@ use utils::snowflake::SnowflakeIdGenerator;
 async fn report_reschedule() {
     let mut test = TestServerBuilder::new("smtp_report_reschedule")
         .await
-        .with_http_listener(19057)
+        .with_http_listener(19058)
         .await
         .capture_queue()
         .build()

tests/src/system/task.rs

@@ -6,7 +6,7 @@
  * Modified by Coffey Labs in 2026 for INBUXA.
  */
 
-use crate::utils::{account::Account, server::TestServer};
+use crate::utils::{account::Account, jmap::JmapUtils, server::TestServer};
 use registry::{
     schema::{
         enums::TaskStoreMaintenanceType,
@@ -301,14 +301,29 @@ impl Account {
         .await
     }
 
-    pub async fn tasks(&self) -> Vec<TaskId> {
+    pub async fn tasks(&self) -> Option<Vec<TaskId>> {
         let ids = self.task_ids().await;
-        let mut results = Vec::with_capacity(ids.len());
-        for id in ids {
-            let sample = self.registry_get::<Task>(id).await;
-            results.push(TaskId { id, task: sample });
+        if ids.is_empty() {
+            return Some(Vec::new());
         }
-        results
+
+        let response = self.registry_get_many(ObjectType::Task, &ids).await;
+        if response.not_found().next().is_some() {
+            return None;
+        }
+
+        Some(
+            response
+                .list()
+                .iter()
+                .map(|item| TaskId {
+                    id: item.object_id(),
+                    task: serde_json::from_str(&item.to_string()).unwrap_or_else(|err| {
+                        panic!("Failed to deserialize {item}: {err}");
+                    }),
+                })
+                .collect(),
+        )
     }
 
     async fn assert_no_tasks(&self) {
@@ -344,19 +359,28 @@ impl Account {
     ) -> Vec<TaskId> {
         let mut attempt = 0;
         loop {
-            let tasks = self.tasks().await;
-            if tasks.len() == count && tasks.iter().all(&is_expected) {
-                return tasks;
+            match self.tasks().await {
+                Some(tasks) if tasks.len() == count && tasks.iter().all(&is_expected) => {
+                    return tasks;
+                }
+                Some(tasks) => {
+                    attempt += 1;
+                    assert!(
+                        attempt < TASK_WAIT_ATTEMPTS,
+                        "Expected {} tasks, found {}: {:?}",
+                        count,
+                        tasks.len(),
+                        tasks
+                    );
+                }
+                None => {
+                    attempt += 1;
+                    assert!(
+                        attempt < TASK_WAIT_ATTEMPTS,
+                        "Task/query keeps returning ids that Task/get reports as not found"
+                    );
+                }
             }
-
-            attempt += 1;
-            assert!(
-                attempt < TASK_WAIT_ATTEMPTS,
-                "Expected {} tasks, found {}: {:?}",
-                count,
-                tasks.len(),
-                tasks
-            );
             tokio::time::sleep(TASK_WAIT_INTERVAL).await;
         }
     }