Merge upstream v0.16.24

Eight conflicted files resolved, plus the lock file and the schema:

- crates/services/src/task_manager/spam_classifier.rs: upstream's rules
  update now replaces existing rules, DNSBL servers, lookups and file
  extensions, keeping only whether each is on. Taken, with one difference:
  an object an admin edited is kept as it is. Every object an update writes
  is fingerprinted (content without `enable`, SHA-256, stored under
  SUBSPACE_INBUXA "Sf"), and only one that still matches is replaced.
  Scores are never replaced, as upstream has it. The AU-1.10 summary record
  now names what was added, replaced and kept, and the bundled rules are
  marked applied only when the update fully succeeded, so a failure runs
  again on the next start. The marker becomes "3.0.2+2", which runs the
  update once on upgrade to fingerprint every rule still as bundled.
- crates/common/src/network/autoconfig/autodiscover.rs: upstream's rewrite
  (implicit TLS first, labeled SSL), with the per-protocol switches (LP-7,
  LP-14a) passed in as a filter.
- crates/store/src/backend/mysql/{search,write}.rs: upstream's chunked
  deletes (no unbounded first DELETE, stop on a short chunk, halve the
  chunk on the new chunk-too-large errors) inside the fork's query timeout.
- crates/smtp/src/lib.rs: the fork's queue spawn kept. It already fixed the
  stall upstream fixes here (a node without outboundMta stops accepting
  mail at about 1024 queued messages), and follows role changes live.
- crates/jmap/src/registry/mapping/bootstrap.rs: the log path stays
  /var/log/inbuxa/; upstream's PowerDNS mapping taken.
- crates/main/Cargo.toml: the AGPL-only license kept, version 0.16.24.
- tests/src/jmap/principal/get.rs: the fork's capabilities kept.
- resources/schema/schema.json.gz: merged as JSON; upstream relabeled the
  vendor Sieve extensions "(Stalwart)", kept as "(vnd.inbuxa)".
- Cargo.lock: upstream's, with the fork's crates added by Cargo.

Also:

- tests/src/smtp/inbound/spam_rules_kept.rs: an edited rule survives an
  update, an unedited one is updated, rules from before fingerprints are
  handled, and the audit summary says so. Upstream's own spam_rules test
  passes unchanged.
- tests/src/smtp/reporting/reschedule.rs moves to port 19058; upstream's
  new spam_rules test took 19057.
- tools/fork/renames.py renames the "(Stalwart)" labels and the default
  log path, so neither conflicts again.
- tools/fork/notice-check.py compares against the newest snapshot in the
  checked-out history instead of the upstream branch head, so moving the
  branch no longer fails other open pull requests.
- tests/src/directory/issuer.rs (since v0.16.23) stays out, and is on the
  build check's known list: it tests issuer-based directory routing, which
  the fork doesn't have (DIR-2).
- Strip report: docs/fork/strip-reports/v0.16.24.{md,json}.

tests/src/jmap/core/blob.rs

@@ -367,6 +367,82 @@ pub async fn test(test: &TestServer) {
     }
     test.blob_expire_all().await;
 
+    let inbox_id = Id::from(INBOX_ID).to_string();
+    let response = account
+        .jmap_method_calls(json!([
+          [
+            "Blob/upload",
+            {
+              "accountId": account.id_string(),
+              "create": {
+                "m0": {
+                  "data": [
+                    {
+                      "data:asText": concat!(
+                          "From: [email protected]\r\n",
+                          "To: [email protected]\r\n",
+                          "Subject: Blob reference import\r\n",
+                          "\r\n",
+                          "Imported through a Blob/upload creation id."
+                      )
+                    }
+                  ],
+                  "type": "message/rfc822"
+                }
+              }
+            },
+            "U0"
+          ],
+          [
+            "Email/import",
+            {
+              "accountId": account.id_string(),
+              "emails": {
+                "i0": {
+                  "blobId": "#m0",
+                  "mailboxIds": { (inbox_id.as_str()): true }
+                }
+              }
+            },
+            "I0"
+          ],
+          [
+            "Email/import",
+            {
+              "accountId": account.id_string(),
+              "emails": {
+                "i1": {
+                  "blobId": "#missing",
+                  "mailboxIds": { (inbox_id.as_str()): true }
+                }
+              }
+            },
+            "I1"
+          ]
+        ]))
+        .await;
+
+    assert_eq!(response.name_at(1), "Email/import", "{response:?}");
+    assert!(
+        response
+            .pointer("/methodResponses/1/1/created/i0/id")
+            .and_then(|v| v.as_str())
+            .is_some(),
+        "{response:?}"
+    );
+    assert!(
+        response
+            .pointer("/methodResponses/1/1/notCreated")
+            .is_none(),
+        "{response:?}"
+    );
+    assert_eq!(
+        response.error_type_at(2),
+        Some("invalidResultReference"),
+        "{response:?}"
+    );
+    test.blob_expire_all().await;
+
     // Blob/lookup
     let client = account.jmap_client().await;
     let blob_id = client

tests/src/jmap/core/push_subscription.rs

@@ -188,13 +188,37 @@ pub async fn test(test: &TestServer) {
     client.push_subscription_destroy(&push_id).await.unwrap();
 
     // Only one verification per minute is allowed
-    let push_id = client
-        .push_subscription_create("invalid", "https://127.0.0.1:19000/push", None)
+    let first_id = client
+        .push_subscription_create(
+            "first",
+            "https://127.0.0.1:19000/push?skip_checks=true",
+            None,
+        )
+        .await
+        .unwrap()
+        .take_id();
+    let verification = expect_push(&mut event_rx).await.unwrap_verification();
+    assert_eq!(verification.push_subscription_id, first_id);
+    let deferred_id = client
+        .push_subscription_create("deferred", "https://127.0.0.1:19000/push", None)
         .await
         .unwrap()
         .take_id();
     expect_nothing(&mut event_rx).await;
-    client.push_subscription_destroy(&push_id).await.unwrap();
+    let verification = expect_push_within(&mut event_rx, Duration::from_secs(8))
+        .await
+        .unwrap_verification();
+    assert_eq!(verification.push_subscription_id, deferred_id);
+    account
+        .jmap_request(
+            &["urn:ietf:params:jmap:core"],
+            json!([[
+                "PushSubscription/set",
+                { "destroy": [first_id, deferred_id] },
+                "0"
+            ]]),
+        )
+        .await;
 
     // Register push notification (with encryption)
     let push_id = client
@@ -731,7 +755,14 @@ fn assert_vapid_authorization(header: &str, expected_key: &str, expected_origin:
 }
 
 async fn expect_push(event_rx: &mut mpsc::Receiver<PushMessage>) -> PushMessage {
-    match tokio::time::timeout(Duration::from_millis(1500), event_rx.recv()).await {
+    expect_push_within(event_rx, Duration::from_millis(1500)).await
+}
+
+async fn expect_push_within(
+    event_rx: &mut mpsc::Receiver<PushMessage>,
+    wait: Duration,
+) -> PushMessage {
+    match tokio::time::timeout(wait, event_rx.recv()).await {
         Ok(Some(push)) => {
             //println!("Push received: {:?}", push);
             push

tests/src/jmap/mail/set.rs

@@ -8,7 +8,7 @@ use crate::{
     jmap::{find_values, replace_blob_ids, replace_boundaries, replace_values},
     utils::server::TestServer,
 };
-use ::email::mailbox::INBOX_ID;
+use ::email::{mailbox::INBOX_ID, message::metadata::MessageData};
 use ahash::AHashSet;
 use jmap_client::{
     Error, Set,
@@ -18,8 +18,12 @@ use jmap_client::{
     mailbox::Role,
 };
 use registry::schema::prelude::ObjectType;
-use std::{fs, path::PathBuf};
-use types::id::Id;
+use std::{fs, path::PathBuf, str::FromStr};
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use types::{collection::Collection, id::Id};
 
 pub async fn test(test: &TestServer) {
     println!("Running Email Set tests...");
@@ -29,6 +33,7 @@ pub async fn test(test: &TestServer) {
 
     create(&client, &mailbox_id).await;
     update(&client, &mailbox_id).await;
+    update_preserves_uids(test, &client, account.id().document_id(), &mailbox_id).await;
 
     test.destroy_all_mailboxes(account).await;
     test.account("[email protected]")
@@ -296,6 +301,94 @@ async fn update(client: &Client, root_mailbox_id: &str) {
         .unwrap();
 }
 
+async fn update_preserves_uids(
+    test: &TestServer,
+    client: &Client,
+    account_id: u32,
+    root_mailbox_id: &str,
+) {
+    let email_id = client
+        .email_query(
+            email::query::Filter::in_mailbox(root_mailbox_id).into(),
+            None::<Vec<_>>,
+        )
+        .await
+        .unwrap()
+        .take_ids()
+        .pop()
+        .unwrap();
+    let document_id = Id::from_str(&email_id).unwrap().document_id();
+    let test_mailbox_id = client
+        .mailbox_create("UID Test", None::<String>, Role::None)
+        .await
+        .unwrap()
+        .take_id();
+    let test_mailbox_document_id = Id::from_str(&test_mailbox_id).unwrap().document_id();
+    let uids = message_uids(test, account_id, document_id).await;
+    let inbox_uid = uids[&INBOX_ID];
+    assert_ne!(inbox_uid, 0);
+
+    // Full mailboxIds identical to the current ones plus a keyword change must keep the UID
+    let mut request = client.build();
+    request
+        .set_email()
+        .update(&email_id)
+        .mailbox_ids([root_mailbox_id])
+        .keywords(["uid-test"]);
+    request
+        .send_set_email()
+        .await
+        .unwrap()
+        .updated(&email_id)
+        .unwrap();
+    assert_eq!(
+        message_uids(test, account_id, document_id).await,
+        [(INBOX_ID, inbox_uid)].into_iter().collect()
+    );
+
+    // Full mailboxIds that keeps a mailbox and adds another must only assign a UID to the new one
+    let mut request = client.build();
+    request
+        .set_email()
+        .update(&email_id)
+        .mailbox_ids([root_mailbox_id, test_mailbox_id.as_str()]);
+    request
+        .send_set_email()
+        .await
+        .unwrap()
+        .updated(&email_id)
+        .unwrap();
+    let uids = message_uids(test, account_id, document_id).await;
+    assert_eq!(uids.len(), 2);
+    assert_eq!(uids[&INBOX_ID], inbox_uid);
+    assert_ne!(uids[&test_mailbox_document_id], 0);
+
+    client.mailbox_destroy(&test_mailbox_id, true).await.unwrap();
+}
+
+async fn message_uids(
+    test: &TestServer,
+    account_id: u32,
+    document_id: u32,
+) -> std::collections::BTreeMap<u32, u32> {
+    test.server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::Email,
+            document_id,
+        ))
+        .await
+        .unwrap()
+        .unwrap()
+        .deserialize::<MessageData>()
+        .unwrap()
+        .mailboxes
+        .iter()
+        .map(|m| (m.mailbox_id, m.uid))
+        .collect()
+}
+
 pub async fn assert_email_properties(
     client: &Client,
     message_id: &str,

tests/src/jmap/mod.rs

@@ -99,6 +99,7 @@ pub async fn jmap_tests() {
                 push_throttle: 500u64.into(),
                 websocket_throttle: 500u64.into(),
                 push_attempt_wait: 500u64.into(),
+                push_verify_timeout: 5000u64.into(),
                 ..Default::default()
             },
             &[
@@ -108,6 +109,7 @@ pub async fn jmap_tests() {
                 Property::PushThrottle,
                 Property::WebsocketThrottle,
                 Property::PushAttemptWait,
+                Property::PushVerifyTimeout,
             ],
         )
         .await;