Merge upstream v0.16.24

Eight conflicted files resolved, plus the lock file and the schema:

- crates/services/src/task_manager/spam_classifier.rs: upstream's rules
  update now replaces existing rules, DNSBL servers, lookups and file
  extensions, keeping only whether each is on. Taken, with one difference:
  an object an admin edited is kept as it is. Every object an update writes
  is fingerprinted (content without `enable`, SHA-256, stored under
  SUBSPACE_INBUXA "Sf"), and only one that still matches is replaced.
  Scores are never replaced, as upstream has it. The AU-1.10 summary record
  now names what was added, replaced and kept, and the bundled rules are
  marked applied only when the update fully succeeded, so a failure runs
  again on the next start. The marker becomes "3.0.2+2", which runs the
  update once on upgrade to fingerprint every rule still as bundled.
- crates/common/src/network/autoconfig/autodiscover.rs: upstream's rewrite
  (implicit TLS first, labeled SSL), with the per-protocol switches (LP-7,
  LP-14a) passed in as a filter.
- crates/store/src/backend/mysql/{search,write}.rs: upstream's chunked
  deletes (no unbounded first DELETE, stop on a short chunk, halve the
  chunk on the new chunk-too-large errors) inside the fork's query timeout.
- crates/smtp/src/lib.rs: the fork's queue spawn kept. It already fixed the
  stall upstream fixes here (a node without outboundMta stops accepting
  mail at about 1024 queued messages), and follows role changes live.
- crates/jmap/src/registry/mapping/bootstrap.rs: the log path stays
  /var/log/inbuxa/; upstream's PowerDNS mapping taken.
- crates/main/Cargo.toml: the AGPL-only license kept, version 0.16.24.
- tests/src/jmap/principal/get.rs: the fork's capabilities kept.
- resources/schema/schema.json.gz: merged as JSON; upstream relabeled the
  vendor Sieve extensions "(Stalwart)", kept as "(vnd.inbuxa)".
- Cargo.lock: upstream's, with the fork's crates added by Cargo.

Also:

- tests/src/smtp/inbound/spam_rules_kept.rs: an edited rule survives an
  update, an unedited one is updated, rules from before fingerprints are
  handled, and the audit summary says so. Upstream's own spam_rules test
  passes unchanged.
- tests/src/smtp/reporting/reschedule.rs moves to port 19058; upstream's
  new spam_rules test took 19057.
- tools/fork/renames.py renames the "(Stalwart)" labels and the default
  log path, so neither conflicts again.
- tools/fork/notice-check.py compares against the newest snapshot in the
  checked-out history instead of the upstream branch head, so moving the
  branch no longer fails other open pull requests.
- tests/src/directory/issuer.rs (since v0.16.23) stays out, and is on the
  build check's known list: it tests issuer-based directory routing, which
  the fork doesn't have (DIR-2).
- Strip report: docs/fork/strip-reports/v0.16.24.{md,json}.

crates/services/src/state_manager/mod.rs

@@ -14,14 +14,14 @@ pub mod push;
 
 use common::ipc::PushNotification;
 use email::push::PushSubscription;
-use reqwest::Client;
+use jmap_proto::types::state::State;
 use std::{
     sync::Arc,
     time::{Duration, Instant},
 };
 use tokio::sync::mpsc;
 use types::{id::Id, type_state::DataType};
-use utils::map::bitmap::Bitmap;
+use utils::map::{bitmap::Bitmap, vec_map::VecMap};
 
 const PURGE_EVERY: Duration = Duration::from_secs(3600);
 const SEND_TIMEOUT: Duration = Duration::from_millis(500);
@@ -40,26 +40,22 @@ pub struct PushRegistration {
     member_account_ids: Vec<u32>,
     num_attempts: u32,
     last_request: Instant,
-    notifications: Vec<PushNotification>,
+    pending: PushBatch,
     in_flight: bool,
-    client: Client,
+}
+
+#[derive(Debug, Default)]
+pub struct PushBatch {
+    state_changes: VecMap<Id, VecMap<DataType, State>>,
+    notifications: Vec<PushNotification>,
 }
 
 #[derive(Debug)]
 pub enum Event {
-    Push {
-        notification: PushNotification,
-    },
-    Update {
-        account_id: u32,
-    },
-    DeliverySuccess {
-        id: Id,
-    },
-    DeliveryFailure {
-        id: Id,
-        notifications: Vec<PushNotification>,
-    },
+    Push { notification: PushNotification },
+    Update { account_id: u32 },
+    DeliverySuccess { id: Id },
+    DeliveryFailure { id: Id, failed: PushBatch },
     Reset,
 }
 
@@ -68,3 +64,130 @@ impl IpcSubscriber {
         !self.tx.is_closed()
     }
 }
+
+impl PushBatch {
+    pub fn push(&mut self, notification: PushNotification) {
+        match notification {
+            PushNotification::StateChange(state_change) => {
+                if !state_change.types.is_empty() {
+                    let states = self
+                        .state_changes
+                        .get_mut_or_insert(Id::from(state_change.account_id));
+                    for data_type in state_change.types {
+                        merge_state(states, data_type, state_change.change_id);
+                    }
+                }
+            }
+            notification => self.notifications.push(notification),
+        }
+    }
+
+    pub fn merge_failed(&mut self, failed: PushBatch) {
+        for (account_id, failed_states) in failed.state_changes {
+            let states = self.state_changes.get_mut_or_insert(account_id);
+            for (data_type, state) in failed_states {
+                if let State::Exact(change_id) = state {
+                    merge_state(states, data_type, change_id);
+                }
+            }
+        }
+
+        if !failed.notifications.is_empty() {
+            let mut notifications = failed.notifications;
+            notifications.append(&mut self.notifications);
+            self.notifications = notifications;
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.state_changes.is_empty() && self.notifications.is_empty()
+    }
+
+    pub fn clear(&mut self) {
+        self.state_changes.clear();
+        self.notifications.clear();
+    }
+}
+
+fn merge_state(states: &mut VecMap<DataType, State>, data_type: DataType, change_id: u64) {
+    match states.get_mut(&data_type) {
+        Some(State::Exact(current)) if *current >= change_id => {}
+        Some(state) => *state = State::Exact(change_id),
+        None => states.append(data_type, State::Exact(change_id)),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::PushBatch;
+    use common::ipc::{EmailPush, PushNotification};
+    use jmap_proto::types::state::State;
+    use types::{
+        id::Id,
+        type_state::{DataType, StateChange},
+    };
+    use utils::map::bitmap::Bitmap;
+
+    fn state_change<const N: usize>(change_id: u64, types: [DataType; N]) -> PushNotification {
+        PushNotification::StateChange(StateChange {
+            account_id: 1,
+            change_id,
+            types: Bitmap::from_iter(types),
+        })
+    }
+
+    fn email_push(email_id: u32) -> PushNotification {
+        PushNotification::EmailPush(EmailPush {
+            account_id: 1,
+            email_id,
+            change_id: email_id.into(),
+        })
+    }
+
+    #[test]
+    fn batch_keeps_newest_state_per_type() {
+        let mut batch = PushBatch::default();
+        batch.push(state_change(5, [DataType::Email]));
+        batch.push(state_change(7, [DataType::Email, DataType::Mailbox]));
+        batch.push(state_change(6, [DataType::Mailbox]));
+
+        let mut failed = PushBatch::default();
+        failed.push(state_change(
+            4,
+            [DataType::Email, DataType::Mailbox, DataType::Thread],
+        ));
+        batch.merge_failed(failed);
+
+        let states = batch.state_changes.get(&Id::from(1u32)).unwrap();
+        assert_eq!(states.len(), 3);
+        assert_eq!(states.get(&DataType::Email), Some(&State::Exact(7)));
+        assert_eq!(states.get(&DataType::Mailbox), Some(&State::Exact(7)));
+        assert_eq!(states.get(&DataType::Thread), Some(&State::Exact(4)));
+
+        assert!(!batch.is_empty());
+        batch.clear();
+        assert!(batch.is_empty());
+    }
+
+    #[test]
+    fn failed_notifications_are_retried_first() {
+        let mut batch = PushBatch::default();
+        batch.push(email_push(3));
+
+        let mut failed = PushBatch::default();
+        failed.push(email_push(1));
+        failed.push(email_push(2));
+        batch.merge_failed(failed);
+
+        let email_ids = batch
+            .notifications
+            .iter()
+            .filter_map(|notification| match notification {
+                PushNotification::EmailPush(email_push) => Some(email_push.email_id),
+                _ => None,
+            })
+            .collect::<Vec<_>>();
+        assert_eq!(email_ids, [1, 2, 3]);
+        assert!(batch.state_changes.is_empty());
+    }
+}