Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

crates/services/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "services"
+version = "0.16.22"
+edition = "2024"
+
+[dependencies]
+store = { path = "../store" }
+common = { path =  "../common" }
+utils = { path =  "../utils" }
+trc = { path = "../trc" }
+email = { path = "../email" }
+smtp = { path = "../smtp" }
+groupware = { path = "../groupware" }
+spam-filter = { path = "../spam-filter" }
+types = { path = "../types" }
+jmap_proto = { path = "../jmap-proto" }
+jmap-tools = { version = "0.1" }
+registry = { path =  "../registry" }
+smtp-proto = { version = "0.2", features = ["rkyv", "serde"] }
+tokio = { version = "1.53", features = ["rt"] }
+mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] }
+mail-builder = { version = "1.0" }
+aho-corasick = { version = "1.1" }
+calcard = { version = "0.3", features = ["rkyv"] }
+serde_json = "1.0"
+memory-stats = "1.2.0"
+aes-gcm = "0.11.1"
+p256 = { version = "0.13", features = ["ecdh"] }
+hkdf = "0.13"
+sha2 = "0.11"
+reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2"]}
+base64 = "0.23"
+dns-update = { version = "0.5" }
+psl = "2"
+
+[dev-dependencies]
+
+[features]
+test_mode = []
+enterprise = []
+
+[lints]
+workspace = true

crates/services/src/broadcast/mod.rs

@@ -0,0 +1,287 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::ipc::{
+    BroadcastEvent, CacheInvalidation, CalendarAlert, EmailPush, PushNotification, RegistryChange,
+};
+use registry::{
+    schema::prelude::ObjectType,
+    types::{EnumImpl, id::ObjectId},
+};
+use std::{borrow::Borrow, io::Write};
+use types::{id::Id, type_state::StateChange};
+use utils::{
+    codec::leb128::{Leb128Iterator, Leb128Writer},
+    map::bitmap::Bitmap,
+};
+
+pub mod publisher;
+pub mod subscriber;
+
+#[derive(Debug)]
+pub(crate) struct BroadcastBatch<T> {
+    messages: T,
+}
+
+const MAX_BATCH_SIZE: usize = 100;
+pub(crate) const BROADCAST_TOPIC: &str = "stwt.agora";
+
+impl BroadcastBatch<Vec<BroadcastEvent>> {
+    pub fn init() -> Self {
+        Self {
+            messages: Vec::with_capacity(MAX_BATCH_SIZE),
+        }
+    }
+
+    pub fn insert(&mut self, message: BroadcastEvent) -> bool {
+        self.messages.push(message);
+        self.messages.len() < MAX_BATCH_SIZE
+    }
+
+    pub fn serialize(&self, node_id: u16) -> Vec<u8> {
+        let mut serialized =
+            Vec::with_capacity((self.messages.len() * 10) + std::mem::size_of::<u16>());
+        let _ = serialized.write_leb128(node_id);
+        for message in &self.messages {
+            match message {
+                BroadcastEvent::PushNotification(notification) => match notification {
+                    PushNotification::StateChange(state_change) => {
+                        serialized.push(0u8);
+                        let _ = serialized.write_leb128(state_change.change_id);
+                        let _ = serialized.write_leb128(*state_change.types.as_ref());
+                        let _ = serialized.write_leb128(state_change.account_id);
+                    }
+                    PushNotification::CalendarAlert(calendar_alert) => {
+                        serialized.push(1u8);
+                        let _ = serialized.write_leb128(calendar_alert.account_id);
+                        let _ = serialized.write_leb128(calendar_alert.event_id);
+                        let _ = serialized
+                            .write_leb128(calendar_alert.recurrence_id.unwrap_or_default() as u64);
+                        let _ = serialized.write_leb128(calendar_alert.uid.len());
+                        let _ = serialized.write(calendar_alert.uid.as_bytes());
+                        let _ = serialized.write_leb128(calendar_alert.alert_id.len());
+                        let _ = serialized.write(calendar_alert.alert_id.as_bytes());
+                    }
+                    PushNotification::EmailPush(email_push) => {
+                        serialized.push(2u8);
+                        let _ = serialized.write_leb128(email_push.account_id);
+                        let _ = serialized.write_leb128(email_push.email_id);
+                        let _ = serialized.write_leb128(email_push.change_id);
+                    }
+                },
+                BroadcastEvent::PushServerUpdate(account_id) => {
+                    serialized.push(3u8);
+                    let _ = serialized.write_leb128(*account_id);
+                }
+                BroadcastEvent::RegistryChange(items) => match items {
+                    RegistryChange::Insert(id) => {
+                        serialized.push(4u8);
+                        let _ = serialized.write_leb128(id.object().to_id());
+                        let _ = serialized.write_leb128(id.id().id());
+                    }
+                    RegistryChange::Delete(id) => {
+                        serialized.push(5u8);
+                        let _ = serialized.write_leb128(id.object().to_id());
+                        let _ = serialized.write_leb128(id.id().id());
+                    }
+                    RegistryChange::Reload(object) => {
+                        serialized.push(6u8);
+                        let _ = serialized.write_leb128(object.to_id());
+                    }
+                },
+                BroadcastEvent::CacheInvalidate(items) => {
+                    serialized.push(7u8);
+                    let _ = serialized.write_leb128(items.len());
+                    for item in items {
+                        let (marker, id) = match item {
+                            CacheInvalidation::AccessToken(id) => (0u8, *id),
+                            CacheInvalidation::DavResources(id) => (1u8, *id),
+                            CacheInvalidation::Domain(id) => (2u8, *id),
+                            CacheInvalidation::Account(id) => (3u8, *id),
+                            CacheInvalidation::DkimSignature(id) => (4u8, *id),
+                            CacheInvalidation::Tenant(id) => (5u8, *id),
+                            CacheInvalidation::Role(id) => (6u8, *id),
+                            CacheInvalidation::List(id) => (7u8, *id),
+                            CacheInvalidation::DomainLogo(id) => (8u8, *id),
+                            CacheInvalidation::TenantLogo(id) => (9u8, *id),
+                            CacheInvalidation::EmailNegative {
+                                domain_id,
+                                local_part_hash,
+                            } => {
+                                serialized.push(10u8);
+                                let _ = serialized.write_leb128(*domain_id);
+                                let _ = serialized.write_leb128(*local_part_hash);
+                                continue;
+                            }
+                            CacheInvalidation::DomainNegative => (11u8, 0),
+                        };
+
+                        serialized.push(marker);
+                        let _ = serialized.write_leb128(id);
+                    }
+                }
+                BroadcastEvent::CacheInvalidateAll => {
+                    serialized.push(8u8);
+                }
+                BroadcastEvent::CacheInvalidateNegative => {
+                    serialized.push(9u8);
+                }
+                BroadcastEvent::MtaQueueStatus { is_running } => {
+                    if *is_running {
+                        serialized.push(10u8);
+                    } else {
+                        serialized.push(11u8);
+                    }
+                }
+                BroadcastEvent::QueueRefresh => {
+                    serialized.push(12u8);
+                }
+            }
+        }
+        serialized
+    }
+
+    pub fn clear(&mut self) {
+        self.messages.clear();
+    }
+}
+
+impl<T, I> BroadcastBatch<T>
+where
+    T: Iterator<Item = I> + Leb128Iterator<I>,
+    I: Borrow<u8>,
+{
+    pub fn node_id(&mut self) -> Option<u16> {
+        self.messages.next_leb128::<u16>()
+    }
+
+    pub fn next_event(&mut self) -> Result<Option<BroadcastEvent>, ()> {
+        if let Some(id) = self.messages.next() {
+            match id.borrow() {
+                0 => Ok(Some(BroadcastEvent::PushNotification(
+                    PushNotification::StateChange(StateChange {
+                        change_id: self.messages.next_leb128().ok_or(())?,
+                        types: Bitmap::from(self.messages.next_leb128::<u64>().ok_or(())?),
+                        account_id: self.messages.next_leb128().ok_or(())?,
+                    }),
+                ))),
+
+                1 => {
+                    let account_id = self.messages.next_leb128().ok_or(())?;
+                    let event_id = self.messages.next_leb128().ok_or(())?;
+                    let recurrence_id = self.messages.next_leb128::<u64>().ok_or(())? as i64;
+                    let uid_len = self.messages.next_leb128::<usize>().ok_or(())?;
+                    let mut uid_bytes = vec![0u8; uid_len];
+                    for byte in uid_bytes.iter_mut() {
+                        *byte = self.messages.next().ok_or(())?.borrow().to_owned();
+                    }
+                    let uid = String::from_utf8(uid_bytes).map_err(|_| ())?;
+                    let alert_id_len = self.messages.next_leb128::<usize>().ok_or(())?;
+                    let mut alert_id_bytes = vec![0u8; alert_id_len];
+                    for byte in alert_id_bytes.iter_mut() {
+                        *byte = self.messages.next().ok_or(())?.borrow().to_owned();
+                    }
+                    let alert_id = String::from_utf8(alert_id_bytes).map_err(|_| ())?;
+                    Ok(Some(BroadcastEvent::PushNotification(
+                        PushNotification::CalendarAlert(CalendarAlert {
+                            account_id,
+                            event_id,
+                            recurrence_id: if recurrence_id == 0 {
+                                None
+                            } else {
+                                Some(recurrence_id)
+                            },
+                            uid,
+                            alert_id,
+                        }),
+                    )))
+                }
+                2 => Ok(Some(BroadcastEvent::PushNotification(
+                    PushNotification::EmailPush(EmailPush {
+                        account_id: self.messages.next_leb128().ok_or(())?,
+                        email_id: self.messages.next_leb128().ok_or(())?,
+                        change_id: self.messages.next_leb128().ok_or(())?,
+                    }),
+                ))),
+                3 => {
+                    let account_id = self.messages.next_leb128().ok_or(())?;
+                    Ok(Some(BroadcastEvent::PushServerUpdate(account_id)))
+                }
+                4 => {
+                    let object_id = self.messages.next_leb128().ok_or(())?;
+                    let id = self.messages.next_leb128::<u64>().ok_or(())?;
+                    Ok(Some(BroadcastEvent::RegistryChange(
+                        RegistryChange::Insert(ObjectId::new(
+                            ObjectType::from_id(object_id).ok_or(())?,
+                            Id::new(id),
+                        )),
+                    )))
+                }
+                5 => {
+                    let object_id = self.messages.next_leb128().ok_or(())?;
+                    let id = self.messages.next_leb128::<u64>().ok_or(())?;
+                    Ok(Some(BroadcastEvent::RegistryChange(
+                        RegistryChange::Delete(ObjectId::new(
+                            ObjectType::from_id(object_id).ok_or(())?,
+                            Id::new(id),
+                        )),
+                    )))
+                }
+                6 => {
+                    let object_id = self.messages.next_leb128().ok_or(())?;
+                    Ok(Some(BroadcastEvent::RegistryChange(
+                        RegistryChange::Reload(ObjectType::from_id(object_id).ok_or(())?),
+                    )))
+                }
+                7 => {
+                    let count = self.messages.next_leb128::<usize>().ok_or(())?;
+                    let mut items = Vec::with_capacity(count);
+                    for _ in 0..count {
+                        let marker = self.messages.next().ok_or(())?.borrow().to_owned();
+                        let id = self.messages.next_leb128::<u32>().ok_or(())?;
+                        items.push(match marker {
+                            0 => CacheInvalidation::AccessToken(id),
+                            1 => CacheInvalidation::DavResources(id),
+                            2 => CacheInvalidation::Domain(id),
+                            3 => CacheInvalidation::Account(id),
+                            4 => CacheInvalidation::DkimSignature(id),
+                            5 => CacheInvalidation::Tenant(id),
+                            6 => CacheInvalidation::Role(id),
+                            7 => CacheInvalidation::List(id),
+                            8 => CacheInvalidation::DomainLogo(id),
+                            9 => CacheInvalidation::TenantLogo(id),
+                            10 => {
+                                let local_part_hash =
+                                    self.messages.next_leb128::<u32>().ok_or(())?;
+                                CacheInvalidation::EmailNegative {
+                                    domain_id: id,
+                                    local_part_hash,
+                                }
+                            }
+                            11 => CacheInvalidation::DomainNegative,
+                            _ => return Err(()),
+                        });
+                    }
+                    Ok(Some(BroadcastEvent::CacheInvalidate(items)))
+                }
+                8 => Ok(Some(BroadcastEvent::CacheInvalidateAll)),
+                9 => Ok(Some(BroadcastEvent::CacheInvalidateNegative)),
+                10 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: true })),
+                11 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: false })),
+                12 => Ok(Some(BroadcastEvent::QueueRefresh)),
+                _ => Err(()),
+            }
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+impl<T> BroadcastBatch<T> {
+    pub fn new(messages: T) -> Self {
+        Self { messages }
+    }
+}

crates/services/src/broadcast/publisher.rs

@@ -0,0 +1,55 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::sync::Arc;
+
+use common::{Inner, ipc::BroadcastEvent};
+use tokio::sync::mpsc;
+use trc::ClusterEvent;
+
+use super::{BROADCAST_TOPIC, BroadcastBatch};
+
+pub fn spawn_broadcast_publisher(inner: Arc<Inner>, mut event_rx: mpsc::Receiver<BroadcastEvent>) {
+    let (coordinator, this_node_id) = {
+        let _core = inner.shared_core.load();
+        let coordinator = inner.shared_core.load().storage.coordinator.clone();
+        if coordinator.is_none() {
+            return;
+        }
+        (coordinator, _core.network.node_id as u16)
+    };
+
+    tokio::spawn(async move {
+        let mut batch = BroadcastBatch::init();
+
+        trc::event!(Cluster(ClusterEvent::PublisherStart));
+
+        while let Some(event) = event_rx.recv().await {
+            batch.insert(event);
+
+            while let Ok(event) = event_rx.try_recv() {
+                if !batch.insert(event) {
+                    break;
+                }
+            }
+
+            match coordinator
+                .publish(BROADCAST_TOPIC, batch.serialize(this_node_id))
+                .await
+            {
+                Ok(_) => {
+                    batch.clear();
+                }
+                Err(err) => {
+                    batch.clear();
+                    trc::event!(Cluster(ClusterEvent::PublisherError), CausedBy = err);
+                }
+            }
+        }
+
+        trc::event!(Cluster(ClusterEvent::PublisherStop));
+    });
+}

crates/services/src/broadcast/subscriber.rs

@@ -0,0 +1,276 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch};
+use common::{
+    BuildServer, Inner,
+    ipc::{BroadcastEvent, PushEvent, PushNotification, QueueEvent, RegistryChange},
+};
+use registry::types::EnumImpl;
+use std::{sync::Arc, time::Duration};
+use tokio::sync::watch;
+use trc::{ClusterEvent, ServerEvent};
+
+pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Receiver<bool>) {
+    let this_node_id = {
+        let _core = inner.shared_core.load();
+        if _core.storage.coordinator.is_none() || _core.storage.registry.is_recovery_mode() {
+            return;
+        }
+        _core.network.node_id as u16
+    };
+
+    tokio::spawn(async move {
+        let mut retry_count = 0;
+
+        trc::event!(Cluster(ClusterEvent::SubscriberStart));
+
+        'subscribe: loop {
+            let coordinator = inner.shared_core.load().storage.coordinator.clone();
+            if coordinator.is_none() {
+                trc::event!(
+                    Cluster(ClusterEvent::SubscriberError),
+                    Details = "Coordinator is no longer configured"
+                );
+                break;
+            }
+
+            let mut stream = match coordinator.subscribe(BROADCAST_TOPIC).await {
+                Ok(stream) => {
+                    retry_count = 0;
+                    stream
+                }
+                Err(err) => {
+                    trc::event!(
+                        Cluster(ClusterEvent::SubscriberError),
+                        CausedBy = err,
+                        Details = "Failed to subscribe to channel"
+                    );
+
+                    match tokio::time::timeout(
+                        Duration::from_secs(1 << retry_count.max(6)),
+                        shutdown_rx.changed(),
+                    )
+                    .await
+                    {
+                        Ok(_) => {
+                            break;
+                        }
+                        Err(_) => {
+                            retry_count += 1;
+                            continue;
+                        }
+                    }
+                }
+            };
+
+            loop {
+                tokio::select! {
+                    message = stream.next() => {
+                        match message {
+                            Some(message) => {
+                                let mut batch = BroadcastBatch::new(message.payload().iter());
+                                let node_id = match batch.node_id() {
+                                    Some(node_id) => {
+                                        if node_id != this_node_id {
+                                            node_id
+                                        } else {
+                                            trc::event!(
+                                                Cluster(ClusterEvent::MessageSkipped),
+                                                Details = message.payload()
+                                            );
+                                            continue;
+                                        }
+                                    }
+                                    None => {
+                                        trc::event!(
+                                            Cluster(ClusterEvent::MessageInvalid),
+                                            Details = message.payload()
+                                        );
+                                        continue;
+                                    }
+                                };
+
+                                loop {
+                                    match batch.next_event() {
+                                        Ok(Some(event)) => {
+                                            trc::event!(
+                                                Cluster(ClusterEvent::MessageReceived),
+                                                From = node_id,
+                                                To = this_node_id,
+                                                Details = log_event(&event),
+                                            );
+                                            match event {
+                                                BroadcastEvent::PushNotification(notification) => {
+                                                    if inner
+                                                        .ipc
+                                                        .push_tx
+                                                        .send(PushEvent::Publish {
+                                                            notification,
+                                                            broadcast: false,
+                                                        })
+                                                        .await
+                                                        .is_err()
+                                                    {
+                                                        trc::event!(
+                                                            Server(ServerEvent::ThreadError),
+                                                            Details = "Error sending push notification.",
+                                                            CausedBy = trc::location!()
+                                                        );
+                                                    }
+                                                }
+                                                BroadcastEvent::PushServerUpdate(account_id) => {
+                                                    if inner
+                                                        .ipc
+                                                        .push_tx
+                                                        .send(PushEvent::PushServerUpdate { account_id, broadcast: false })
+                                                        .await
+                                                        .is_err()
+                                                    {
+                                                        trc::event!(
+                                                            Server(ServerEvent::ThreadError),
+                                                            Details = "Error sending reload request.",
+                                                            CausedBy = trc::location!()
+                                                        );
+                                                    }
+                                                }
+                                                BroadcastEvent::CacheInvalidate(changes) => {
+                                                    inner.build_server().invalidate_local_caches(&changes).await;
+
+                                                }
+                                                BroadcastEvent::CacheInvalidateAll => {
+                                                    inner.build_server().invalidate_all_local_caches();
+                                                }
+                                                BroadcastEvent::CacheInvalidateNegative => {
+                                                    inner.build_server().invalidate_all_local_negative_caches();
+                                                }
+                                                BroadcastEvent::MtaQueueStatus { is_running } => {
+                                                    let _ = inner
+                                                            .ipc
+                                                            .queue_tx
+                                                            .send(QueueEvent::Paused(!is_running))
+                                                            .await;
+                                                }
+                                                BroadcastEvent::QueueRefresh => {
+                                                    let core = inner.shared_core.load_full();
+                                                    if core.network.roles.outbound_mta {
+                                                        core.storage.data.invalidate_read_snapshot();
+                                                        let _ = inner
+                                                                .ipc
+                                                                .queue_tx
+                                                                .send(QueueEvent::Refresh)
+                                                                .await;
+                                                    }
+                                                }
+                                                BroadcastEvent::RegistryChange(change) => {
+                                                    let server = inner.build_server();
+                                                    server.store().invalidate_read_snapshot();
+                                                    match Box::pin(server.reload_registry(change)).await {
+                                                        Ok(result) => {
+                                                            result.log();
+                                                        }
+                                                        Err(err) => {
+                                                            trc::error!(
+                                                                err.details("Failed to reload settings")
+                                                                    .caused_by(trc::location!())
+                                                            );
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                        Ok(None) => break,
+                                        Err(_) => {
+                                            trc::event!(
+                                                Cluster(ClusterEvent::MessageInvalid),
+                                                Details = message.payload()
+                                            );
+                                            break;
+                                        }
+                                    }
+                                }
+                            }
+                            None => {
+                                trc::event!(
+                                    Cluster(ClusterEvent::SubscriberDisconnected),
+                                );
+                                break;
+                            }
+                        }
+                    },
+                    _ = shutdown_rx.changed() => {
+                        break 'subscribe;
+                    }
+                };
+            }
+        }
+
+        trc::event!(Cluster(ClusterEvent::SubscriberStop));
+    });
+}
+
+fn log_event(event: &BroadcastEvent) -> trc::Value {
+    match event {
+        BroadcastEvent::PushNotification(notification) => match notification {
+            PushNotification::StateChange(state_change) => trc::Value::Array(vec![
+                "StateChange".into(),
+                state_change.account_id.into(),
+                state_change.change_id.into(),
+                (*state_change.types.as_ref()).into(),
+            ]),
+            PushNotification::CalendarAlert(calendar_alert) => trc::Value::Array(vec![
+                "CalendarAlert".into(),
+                calendar_alert.account_id.into(),
+                calendar_alert.event_id.into(),
+                calendar_alert.recurrence_id.into(),
+                calendar_alert.uid.clone().into(),
+                calendar_alert.alert_id.clone().into(),
+            ]),
+            PushNotification::EmailPush(email_push) => trc::Value::Array(vec![
+                "EmailPush".into(),
+                email_push.account_id.into(),
+                email_push.email_id.into(),
+                email_push.change_id.into(),
+            ]),
+        },
+        BroadcastEvent::PushServerUpdate(account_id) => {
+            trc::Value::Array(vec!["PushServerUpdate".into(), (*account_id).into()])
+        }
+        BroadcastEvent::RegistryChange(change) => match change {
+            RegistryChange::Insert(id) => trc::Value::Array(vec![
+                "RegistryInsert".into(),
+                id.object().as_str().into(),
+                id.id().id().into(),
+            ]),
+            RegistryChange::Delete(id) => trc::Value::Array(vec![
+                "RegistryDelete".into(),
+                id.object().as_str().into(),
+                id.id().id().into(),
+            ]),
+            RegistryChange::Reload(object) => {
+                trc::Value::Array(vec!["RegistryReload".into(), object.as_str().into()])
+            }
+        },
+        BroadcastEvent::CacheInvalidate(items) => {
+            let mut array = Vec::with_capacity(items.len() + 1);
+            array.push("CacheInvalidation".into());
+            for item in items {
+                array.push(format!("{:?}", item).into());
+            }
+            trc::Value::Array(array)
+        }
+        BroadcastEvent::CacheInvalidateAll => "CacheInvalidateAll".into(),
+        BroadcastEvent::CacheInvalidateNegative => "CacheInvalidateNegative".into(),
+        BroadcastEvent::MtaQueueStatus { is_running } => {
+            if *is_running {
+                "MtaQueueRunning".into()
+            } else {
+                "MtaQueuePaused".into()
+            }
+        }
+        BroadcastEvent::QueueRefresh => "QueueRefresh".into(),
+    }
+}

crates/services/src/lib.rs

@@ -0,0 +1,66 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+#![warn(clippy::large_futures)]
+
+use broadcast::publisher::spawn_broadcast_publisher;
+use common::{
+    BuildServer, Inner,
+    manager::boot::{BootManager, IpcReceivers},
+};
+use state_manager::manager::spawn_push_router;
+use std::sync::Arc;
+
+use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_scheduler};
+
+pub mod broadcast;
+pub mod state_manager;
+pub mod task_manager;
+
+pub trait StartServices: Sync + Send {
+    fn start_services(&mut self) -> impl Future<Output = ()> + Send;
+}
+
+pub trait SpawnServices {
+    fn spawn_services(&mut self, inner: Arc<Inner>);
+}
+
+impl StartServices for BootManager {
+    async fn start_services(&mut self) {
+        let server = self.inner.build_server();
+        // Unpack webadmin
+        self.inner
+            .data
+            .applications
+            .unpack_all(&server, false)
+            .await;
+
+        if !server.registry().is_recovery_mode() {
+            self.ipc_rxs.spawn_services(self.inner.clone());
+        }
+    }
+}
+
+impl SpawnServices for IpcReceivers {
+    fn spawn_services(&mut self, inner: Arc<Inner>) {
+        if !inner.shared_core.load().storage.registry.is_recovery_mode() {
+            // Spawn push manager
+            spawn_push_router(inner.clone(), self.push_rx.take().unwrap());
+
+            // Spawn broadcast publisher
+            if let Some(event_rx) = self.broadcast_rx.take() {
+                // Spawn broadcast publisher
+                spawn_broadcast_publisher(inner.clone(), event_rx);
+            }
+
+            // Spawn task manager
+            spawn_task_manager(inner.clone());
+
+            // Spawn task scheduler
+            spawn_task_scheduler(inner);
+        }
+    }
+}

crates/services/src/state_manager/ece.rs

@@ -0,0 +1,251 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+use aes_gcm::{Aes128Gcm, Key, Nonce, aead::Aead};
+use hkdf::Hkdf;
+use p256::{
+    PublicKey,
+    ecdh::EphemeralSecret,
+    elliptic_curve::{rand_core::OsRng, sec1::ToEncodedPoint},
+};
+use sha2::Sha256;
+use store::rand::RngExt;
+
+/*
+
+ From https://github.com/mozilla/rust-ece (MPL-2.0 license)
+ Adapted to use 'aes-gcm' and 'p256' crates instead of 'openssl'.
+
+*/
+
+const ECE_WEBPUSH_AES128GCM_IKM_INFO_PREFIX: &str = "WebPush: info\0";
+const ECE_WEBPUSH_AES128GCM_IKM_INFO_LENGTH: usize = 144;
+const ECE_WEBPUSH_IKM_LENGTH: usize = 32;
+const ECE_WEBPUSH_PUBLIC_KEY_LENGTH: usize = 65;
+const ECE_WEBPUSH_DEFAULT_RS: u32 = 4096;
+const ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE: usize = 128;
+
+const ECE_AES128GCM_PAD_SIZE: usize = 1;
+const ECE_AES128GCM_KEY_INFO: &str = "Content-Encoding: aes128gcm\0";
+const ECE_AES128GCM_NONCE_INFO: &str = "Content-Encoding: nonce\0";
+const ECE_AES128GCM_HEADER_LENGTH: usize = 21;
+const ECE_AES_KEY_LENGTH: usize = 16;
+
+const ECE_NONCE_LENGTH: usize = 12;
+const ECE_TAG_LENGTH: usize = 16;
+
+pub(crate) const WEBPUSH_MAX_BODY_SIZE: usize = 4096;
+
+pub(crate) const ECE_WEBPUSH_MAX_PLAINTEXT_SIZE: usize = {
+    let single_record = ECE_WEBPUSH_DEFAULT_RS as usize - ECE_TAG_LENGTH;
+    let wire_budget = WEBPUSH_MAX_BODY_SIZE
+        - (ECE_AES128GCM_HEADER_LENGTH + ECE_WEBPUSH_PUBLIC_KEY_LENGTH)
+        - ECE_TAG_LENGTH;
+    let budget = if wire_budget < single_record {
+        wire_budget
+    } else {
+        single_record
+    };
+
+    budget / ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE * ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE
+        - ECE_AES128GCM_PAD_SIZE
+};
+
+pub fn ece_encrypt(
+    p256dh: &[u8],
+    client_auth_secret: &[u8],
+    mut data: &[u8],
+) -> Result<Vec<u8>, String> {
+    let salt = store::rand::rng().random::<[u8; 16]>();
+    let server_secret = EphemeralSecret::random(&mut OsRng);
+    let server_public_key = server_secret.public_key();
+    let server_public_key_bytes = server_public_key.to_encoded_point(false);
+
+    let client_public_key = PublicKey::from_sec1_bytes(p256dh).map_err(|e| e.to_string())?;
+    let shared_secret = server_secret.diffie_hellman(&client_public_key);
+
+    let ikm_info = generate_info(p256dh, server_public_key_bytes.as_bytes());
+    let ikm = hkdf_sha256(
+        client_auth_secret,
+        &shared_secret.raw_secret_bytes()[..],
+        &ikm_info,
+        ECE_WEBPUSH_IKM_LENGTH,
+    )?;
+    let key = hkdf_sha256(
+        &salt,
+        &ikm,
+        ECE_AES128GCM_KEY_INFO.as_bytes(),
+        ECE_AES_KEY_LENGTH,
+    )?;
+    let nonce = hkdf_sha256(
+        &salt,
+        &ikm,
+        ECE_AES128GCM_NONCE_INFO.as_bytes(),
+        ECE_NONCE_LENGTH,
+    )?;
+
+    // Calculate pad length
+    let mut pad_length = ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE
+        - (data.len() % ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE);
+    if pad_length < ECE_AES128GCM_PAD_SIZE {
+        pad_length += ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE;
+    }
+
+    // Split into records
+    let rs = ECE_WEBPUSH_DEFAULT_RS as usize - ECE_TAG_LENGTH;
+    let mut min_num_records = data.len() / (rs - 1);
+    if !data.len().is_multiple_of(rs - 1) {
+        min_num_records += 1;
+    }
+    let mut pad_length = std::cmp::max(pad_length, min_num_records);
+    let total_size = data.len() + pad_length;
+    let mut num_records = total_size / rs;
+    let size_of_final_record = total_size % rs;
+    if size_of_final_record > 0 {
+        num_records += 1;
+    }
+    let data_per_record = data.len() / num_records;
+    let mut extra_data = data.len() % num_records;
+    if size_of_final_record > 0 && data_per_record > size_of_final_record - 1 {
+        extra_data += data_per_record - (size_of_final_record - 1)
+    }
+    let mut sequence_number = 0;
+    let mut plain_text =
+        Vec::with_capacity(data_per_record + ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE);
+
+    // Write header
+    let key_id = server_public_key_bytes.as_bytes();
+    debug_assert_eq!(key_id.len(), ECE_WEBPUSH_PUBLIC_KEY_LENGTH);
+    let mut output = Vec::with_capacity(
+        ECE_AES128GCM_HEADER_LENGTH + key_id.len() + total_size + num_records * ECE_TAG_LENGTH,
+    );
+    output.extend_from_slice(&salt);
+    output.extend_from_slice(&ECE_WEBPUSH_DEFAULT_RS.to_be_bytes());
+    output.push(key_id.len() as u8);
+    output.extend_from_slice(key_id);
+
+    loop {
+        let records_remaining = num_records - sequence_number;
+        if records_remaining == 0 {
+            break;
+        }
+        let mut data_share = data_per_record;
+        if data_share > data.len() {
+            data_share = data.len();
+        } else if extra_data > 0 {
+            let mut extra_share = extra_data / (records_remaining - 1);
+            if !extra_data.is_multiple_of(records_remaining - 1) {
+                extra_share += 1;
+            }
+            data_share += extra_share;
+            extra_data -= extra_share;
+        }
+
+        let cur_data = &data[0..data_share];
+        data = &data[data_share..];
+        let padding = std::cmp::min(pad_length, rs - data_share);
+        pad_length -= padding;
+        let cur_sequence_number = sequence_number;
+        sequence_number += 1;
+
+        let padded_plaintext_len = cur_data.len() + padding;
+
+        plain_text.extend_from_slice(cur_data);
+        plain_text.push(if sequence_number == num_records { 2 } else { 1 });
+        plain_text.resize(padded_plaintext_len, 0);
+
+        output.extend_from_slice(&aes_gcm_128_encrypt(
+            &key,
+            &generate_iv(&nonce, cur_sequence_number),
+            &plain_text,
+        )?);
+        plain_text.clear();
+    }
+
+    Ok(output)
+}
+
+fn hkdf_sha256(salt: &[u8], secret: &[u8], info: &[u8], len: usize) -> Result<Vec<u8>, String> {
+    let (_, hk) = Hkdf::<Sha256>::extract(Some(salt), secret);
+    let mut okm = vec![0u8; len];
+    hk.expand(info, &mut okm).map_err(|e| e.to_string())?;
+    Ok(okm)
+}
+
+fn aes_gcm_128_encrypt(key: &[u8], nonce: &[u8], data: &[u8]) -> Result<Vec<u8>, String> {
+    let key: &Key<Aes128Gcm> = key
+        .try_into()
+        .map_err(|_| "Invalid AES-GCM key length".to_string())?;
+    let nonce: &Nonce<_> = nonce
+        .try_into()
+        .map_err(|_| "Invalid AES-GCM nonce length".to_string())?;
+
+    <Aes128Gcm as aes_gcm::KeyInit>::new(key)
+        .encrypt(nonce, data)
+        .map_err(|e| e.to_string())
+}
+
+fn generate_info(
+    client_public_key: &[u8],
+    server_public_key: &[u8],
+) -> [u8; ECE_WEBPUSH_AES128GCM_IKM_INFO_LENGTH] {
+    let mut info = [0u8; ECE_WEBPUSH_AES128GCM_IKM_INFO_LENGTH];
+    let prefix = ECE_WEBPUSH_AES128GCM_IKM_INFO_PREFIX.as_bytes();
+    let mut offset = prefix.len();
+    info[0..offset].copy_from_slice(prefix);
+    info[offset..offset + ECE_WEBPUSH_PUBLIC_KEY_LENGTH].copy_from_slice(client_public_key);
+    offset += ECE_WEBPUSH_PUBLIC_KEY_LENGTH;
+    info[offset..].copy_from_slice(server_public_key);
+    info
+}
+
+pub fn generate_iv(nonce: &[u8], counter: usize) -> [u8; ECE_NONCE_LENGTH] {
+    let mut iv = [0u8; ECE_NONCE_LENGTH];
+    let offset = ECE_NONCE_LENGTH - 8;
+    iv[0..offset].copy_from_slice(&nonce[0..offset]);
+    let mask = u64::from_be_bytes((&nonce[offset..]).try_into().unwrap());
+    iv[offset..].copy_from_slice(&(mask ^ (counter as u64)).to_be_bytes());
+    iv
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn encrypt_len(plaintext_len: usize) -> usize {
+        let secret = EphemeralSecret::random(&mut OsRng);
+        let p256dh = secret.public_key().to_encoded_point(false);
+
+        ece_encrypt(p256dh.as_bytes(), &[0u8; 16], &vec![b'a'; plaintext_len])
+            .expect("encryption failed")
+            .len()
+    }
+
+    #[test]
+    fn max_plaintext_stays_within_a_single_record() {
+        let header = ECE_AES128GCM_HEADER_LENGTH + ECE_WEBPUSH_PUBLIC_KEY_LENGTH;
+        let limit = WEBPUSH_MAX_BODY_SIZE;
+
+        let padded = (ECE_WEBPUSH_MAX_PLAINTEXT_SIZE + ECE_AES128GCM_PAD_SIZE)
+            .next_multiple_of(ECE_WEBPUSH_DEFAULT_PADDING_BLOCK_SIZE);
+
+        let len = encrypt_len(ECE_WEBPUSH_MAX_PLAINTEXT_SIZE);
+        assert!(
+            len <= limit,
+            "{len} exceeds the {limit} octet payload limit"
+        );
+        assert_eq!(
+            len,
+            header + padded + ECE_TAG_LENGTH,
+            "expected exactly one authentication tag, i.e. a single record"
+        );
+
+        let over = encrypt_len(ECE_WEBPUSH_MAX_PLAINTEXT_SIZE + 1);
+        assert!(
+            over > limit,
+            "{ECE_WEBPUSH_MAX_PLAINTEXT_SIZE} is not the largest single-record plaintext"
+        );
+    }
+}

crates/services/src/state_manager/email_push.rs

@@ -0,0 +1,584 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use aho_corasick::AhoCorasick;
+use common::{MessageStoreCache, Server};
+use email::{
+    cache::{MessageCacheFetch, email::MessageCacheAccess},
+    message::{
+        body::ToBodyPart,
+        headers::{HeaderToValue, IntoForm},
+        metadata::{
+            ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMessageMetadataPart,
+            ArchivedMetadataPartType, MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData,
+            MessageMetadata, MetadataHeaderName,
+        },
+    },
+    push::EmailPush,
+};
+use jmap_proto::{
+    method::query::Filter,
+    object::{
+        email::{EmailFilter, EmailProperty, EmailValue, HeaderForm},
+        push_subscription::EmailPushProperty,
+    },
+    types::date::UTCDate,
+};
+use jmap_tools::{Map, Property, Value};
+use mail_parser::{HeaderName, HeaderValue};
+use std::iter::Peekable;
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use trc::AddContext;
+use types::{
+    blob::{BlobClass, BlobId},
+    blob_hash::BlobHash,
+    collection::Collection,
+    field::EmailField,
+    id::Id,
+};
+use utils::chained_bytes::ChainedBytes;
+
+const BODY_PROPERTIES: &[EmailProperty] = &[
+    EmailProperty::PartId,
+    EmailProperty::BlobId,
+    EmailProperty::Size,
+    EmailProperty::Name,
+    EmailProperty::Type,
+    EmailProperty::Charset,
+    EmailProperty::Disposition,
+    EmailProperty::Cid,
+    EmailProperty::Language,
+    EmailProperty::Location,
+];
+
+pub async fn build_email_push_object(
+    server: &Server,
+    account_id: u32,
+    document_id: u32,
+    config: &EmailPush,
+    max_size: usize,
+) -> trc::Result<Option<(Value<'static, EmailProperty, EmailValue>, usize)>> {
+    let properties = &config.properties;
+    let Some(data) = server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::Email,
+            document_id,
+        ))
+        .await
+        .caused_by(trc::location!())?
+    else {
+        return Ok(None);
+    };
+    let data = data
+        .deserialize::<MessageData>()
+        .caused_by(trc::location!())?;
+
+    let Some(metadata_archive) = server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::property(
+            account_id,
+            Collection::Email,
+            document_id,
+            EmailField::Metadata,
+        ))
+        .await
+        .caused_by(trc::location!())?
+    else {
+        return Ok(None);
+    };
+    let metadata = metadata_archive
+        .unarchive::<MessageMetadata>()
+        .caused_by(trc::location!())?;
+    let (Some(contents), Some(root_part)) = (
+        metadata.contents.first(),
+        metadata.contents.first().and_then(|c| c.parts.first()),
+    ) else {
+        return Ok(None);
+    };
+
+    let blob_hash = BlobHash::from(&metadata.blob_hash);
+    let needs_body = properties.iter().any(|property| {
+        matches!(
+            property,
+            EmailPushProperty::TextBody
+                | EmailPushProperty::HtmlBody
+                | EmailPushProperty::Attachments
+                | EmailPushProperty::BodyStructure
+        )
+    }) || config.filter.iter().any(|filter| {
+        matches!(
+            filter,
+            Filter::Property(EmailFilter::Body(_) | EmailFilter::Text(_))
+        )
+    });
+
+    let raw_body;
+    let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref());
+    if needs_body {
+        let Some(blob) = server
+            .blob_store()
+            .get_blob(blob_hash.as_slice(), 0..usize::MAX)
+            .await
+            .caused_by(trc::location!())?
+        else {
+            return Ok(None);
+        };
+        raw_body = blob;
+        raw_message.append(
+            raw_body
+                .get(metadata.blob_body_offset.to_native() as usize..)
+                .unwrap_or_default(),
+        );
+    }
+
+    if !config.filter.is_empty() {
+        let cache = if config.filter.iter().any(|filter| {
+            matches!(
+                filter,
+                Filter::Property(
+                    EmailFilter::AllInThreadHaveKeyword(_)
+                        | EmailFilter::SomeInThreadHaveKeyword(_)
+                        | EmailFilter::NoneInThreadHaveKeyword(_)
+                )
+            )
+        }) {
+            Some(
+                server
+                    .get_cached_messages(account_id)
+                    .await
+                    .caused_by(trc::location!())?,
+            )
+        } else {
+            None
+        };
+
+        if !eval_filter_node(
+            &mut config.filter.iter().peekable(),
+            &FilterContext {
+                data: &data,
+                document_id,
+                cache: cache.as_deref(),
+                metadata,
+                contents,
+                root_part,
+                raw_message: &raw_message,
+            },
+        )
+        .unwrap_or(true)
+        {
+            return Ok(None);
+        }
+    }
+
+    let blob_id = BlobId {
+        hash: blob_hash,
+        class: BlobClass::Linked {
+            account_id,
+            collection: Collection::Email.into(),
+            document_id,
+        },
+        section: None,
+    };
+    let blob_body_offset =
+        metadata.blob_body_offset.to_native() as isize - root_part.offset_body.to_native() as isize;
+
+    let id = Id::from_parts(data.thread_id, document_id);
+    let mut email = Map::with_capacity(properties.len());
+    let mut used = 0;
+
+    for property in properties {
+        let key = EmailProperty::from(property);
+        let value: Value<'static, EmailProperty, EmailValue> = match property {
+            EmailPushProperty::Id => id.into(),
+            EmailPushProperty::ThreadId => Id::from(id.prefix_id()).into(),
+            EmailPushProperty::BlobId => blob_id.clone().into(),
+            EmailPushProperty::MailboxIds => {
+                let mut mailbox_ids = Map::with_capacity(data.mailboxes.len());
+                for mailbox in data.mailboxes.iter() {
+                    mailbox_ids.insert_unchecked(
+                        EmailProperty::IdValue(Id::from(mailbox.mailbox_id)),
+                        true,
+                    );
+                }
+                Value::Object(mailbox_ids)
+            }
+            EmailPushProperty::Keywords => {
+                let mut keywords = Map::with_capacity(2);
+                for keyword in data.keywords.iter() {
+                    keywords.insert_unchecked(EmailProperty::Keyword(keyword.clone()), true);
+                }
+                Value::Object(keywords)
+            }
+            EmailPushProperty::Size => data.size.into(),
+            EmailPushProperty::ReceivedAt => EmailValue::Date(UTCDate::from_timestamp(
+                (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64,
+            ))
+            .into(),
+            EmailPushProperty::Preview => {
+                if metadata.preview.is_empty() {
+                    continue;
+                }
+                metadata.preview.to_string().into()
+            }
+            EmailPushProperty::HasAttachment => {
+                ((metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0).into()
+            }
+            EmailPushProperty::Subject
+            | EmailPushProperty::SentAt
+            | EmailPushProperty::MessageId
+            | EmailPushProperty::InReplyTo
+            | EmailPushProperty::References
+            | EmailPushProperty::Sender
+            | EmailPushProperty::From
+            | EmailPushProperty::To
+            | EmailPushProperty::Cc
+            | EmailPushProperty::Bcc
+            | EmailPushProperty::ReplyTo => {
+                let (header_name, form) = match property {
+                    EmailPushProperty::Subject => (MetadataHeaderName::Subject, HeaderForm::Text),
+                    EmailPushProperty::SentAt => (MetadataHeaderName::Date, HeaderForm::Date),
+                    EmailPushProperty::MessageId => {
+                        (MetadataHeaderName::MessageId, HeaderForm::MessageIds)
+                    }
+                    EmailPushProperty::InReplyTo => {
+                        (MetadataHeaderName::InReplyTo, HeaderForm::MessageIds)
+                    }
+                    EmailPushProperty::References => {
+                        (MetadataHeaderName::References, HeaderForm::MessageIds)
+                    }
+                    EmailPushProperty::Sender => {
+                        (MetadataHeaderName::Sender, HeaderForm::Addresses)
+                    }
+                    EmailPushProperty::From => (MetadataHeaderName::From, HeaderForm::Addresses),
+                    EmailPushProperty::To => (MetadataHeaderName::To, HeaderForm::Addresses),
+                    EmailPushProperty::Cc => (MetadataHeaderName::Cc, HeaderForm::Addresses),
+                    EmailPushProperty::Bcc => (MetadataHeaderName::Bcc, HeaderForm::Addresses),
+                    EmailPushProperty::ReplyTo => {
+                        (MetadataHeaderName::ReplyTo, HeaderForm::Addresses)
+                    }
+                    _ => unreachable!(),
+                };
+                root_part
+                    .header_value(&header_name)
+                    .map(|value| HeaderValue::from(value).into_form(&form))
+                    .unwrap_or_default()
+            }
+            EmailPushProperty::Header(_) => root_part.header_to_value(&key, &raw_message),
+            EmailPushProperty::Headers => root_part.headers_to_value(&raw_message),
+            EmailPushProperty::TextBody
+            | EmailPushProperty::HtmlBody
+            | EmailPushProperty::Attachments => {
+                let parts = match property {
+                    EmailPushProperty::TextBody => &contents.text_body,
+                    EmailPushProperty::HtmlBody => &contents.html_body,
+                    EmailPushProperty::Attachments => &contents.attachments,
+                    _ => unreachable!(),
+                };
+                parts
+                    .iter()
+                    .map(|part_id| {
+                        contents.to_body_part(
+                            u16::from(part_id) as u32,
+                            BODY_PROPERTIES,
+                            &raw_message,
+                            &blob_id,
+                            blob_body_offset,
+                        )
+                    })
+                    .collect::<Vec<_>>()
+                    .into()
+            }
+            EmailPushProperty::BodyStructure => {
+                contents.to_body_part(0, BODY_PROPERTIES, &raw_message, &blob_id, blob_body_offset)
+            }
+            EmailPushProperty::BodyValues => Value::Object(Map::with_capacity(0)),
+        };
+
+        let entry_size = key.to_cow().len() + estimate_value_size(&value) + 4;
+        if used + entry_size < max_size {
+            used += entry_size;
+            email.insert_unchecked(key, value);
+        }
+    }
+
+    Ok(Some((email.into(), used)))
+}
+
+fn estimate_value_size(value: &Value<'_, EmailProperty, EmailValue>) -> usize {
+    match value {
+        Value::Null => 4,
+        Value::Bool(_) => 5,
+        Value::Number(_) => 12,
+        Value::Str(text) => text.len() + 2,
+        Value::Element(_) => 40,
+        Value::Array(values) => {
+            2 + values
+                .iter()
+                .map(|value| estimate_value_size(value) + 1)
+                .sum::<usize>()
+        }
+        Value::Object(map) => {
+            2 + map
+                .iter()
+                .map(|(_, value)| estimate_value_size(value) + 24)
+                .sum::<usize>()
+        }
+    }
+}
+
+struct FilterContext<'a> {
+    data: &'a MessageData,
+    document_id: u32,
+    cache: Option<&'a MessageStoreCache>,
+    metadata: &'a ArchivedMessageMetadata,
+    contents: &'a ArchivedMessageMetadataContents,
+    root_part: &'a ArchivedMessageMetadataPart,
+    raw_message: &'a ChainedBytes<'a>,
+}
+
+fn eval_filter_node<'a, I>(tokens: &mut Peekable<I>, context: &FilterContext) -> Option<bool>
+where
+    I: Iterator<Item = &'a Filter<EmailFilter>>,
+{
+    match tokens.next()? {
+        operator @ (Filter::And | Filter::Or | Filter::Not) => {
+            let mut all = true;
+            let mut any = false;
+            while let Some(token) = tokens.peek() {
+                if matches!(token, Filter::Close) {
+                    tokens.next();
+                    break;
+                }
+                if let Some(result) = eval_filter_node(tokens, context) {
+                    all &= result;
+                    any |= result;
+                }
+            }
+            Some(match operator {
+                Filter::And => all,
+                Filter::Or => any,
+                Filter::Not => !any,
+                _ => unreachable!(),
+            })
+        }
+        Filter::Property(condition) => Some(eval_filter_condition(condition, context)),
+        Filter::Close => None,
+    }
+}
+
+fn eval_filter_condition(condition: &EmailFilter, context: &FilterContext) -> bool {
+    match condition {
+        EmailFilter::InMailbox(id) => {
+            let mailbox_id = id.document_id();
+            context
+                .data
+                .mailboxes
+                .iter()
+                .any(|mailbox| mailbox.mailbox_id == mailbox_id)
+        }
+        EmailFilter::InMailboxOtherThan(ids) => context
+            .data
+            .mailboxes
+            .iter()
+            .any(|mailbox| ids.iter().all(|id| id.document_id() != mailbox.mailbox_id)),
+        EmailFilter::Before(date) => received_at(context) < date.timestamp(),
+        EmailFilter::After(date) => received_at(context) >= date.timestamp(),
+        EmailFilter::MinSize(size) => context.data.size >= *size,
+        EmailFilter::MaxSize(size) => context.data.size < *size,
+        EmailFilter::HasKeyword(keyword) => {
+            context.data.keywords.iter().any(|value| value == keyword)
+        }
+        EmailFilter::NotKeyword(keyword) => {
+            !context.data.keywords.iter().any(|value| value == keyword)
+        }
+        EmailFilter::AllInThreadHaveKeyword(keyword) => context.cache.is_some_and(|cache| {
+            cache
+                .in_thread(context.data.thread_id)
+                .all(|message| cache.has_keyword(message, keyword))
+        }),
+        EmailFilter::SomeInThreadHaveKeyword(keyword) => context.cache.is_some_and(|cache| {
+            cache
+                .in_thread(context.data.thread_id)
+                .any(|message| cache.has_keyword(message, keyword))
+        }),
+        EmailFilter::NoneInThreadHaveKeyword(keyword) => context.cache.is_some_and(|cache| {
+            !cache
+                .in_thread(context.data.thread_id)
+                .any(|message| cache.has_keyword(message, keyword))
+        }),
+        EmailFilter::HasAttachment(value) => {
+            ((context.metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0) == *value
+        }
+        EmailFilter::From(text) => ascii_matcher(text).is_some_and(|matcher| {
+            header_matches(
+                context,
+                &MetadataHeaderName::From,
+                &HeaderForm::Addresses,
+                &matcher,
+            )
+        }),
+        EmailFilter::To(text) => ascii_matcher(text).is_some_and(|matcher| {
+            header_matches(
+                context,
+                &MetadataHeaderName::To,
+                &HeaderForm::Addresses,
+                &matcher,
+            )
+        }),
+        EmailFilter::Cc(text) => ascii_matcher(text).is_some_and(|matcher| {
+            header_matches(
+                context,
+                &MetadataHeaderName::Cc,
+                &HeaderForm::Addresses,
+                &matcher,
+            )
+        }),
+        EmailFilter::Bcc(text) => ascii_matcher(text).is_some_and(|matcher| {
+            header_matches(
+                context,
+                &MetadataHeaderName::Bcc,
+                &HeaderForm::Addresses,
+                &matcher,
+            )
+        }),
+        EmailFilter::Subject(text) => ascii_matcher(text).is_some_and(|matcher| {
+            header_matches(
+                context,
+                &MetadataHeaderName::Subject,
+                &HeaderForm::Text,
+                &matcher,
+            )
+        }),
+        EmailFilter::Body(text) => {
+            ascii_matcher(text).is_some_and(|matcher| body_matches(context, &matcher))
+        }
+        EmailFilter::Text(text) => ascii_matcher(text).is_some_and(|matcher| {
+            header_matches(
+                context,
+                &MetadataHeaderName::From,
+                &HeaderForm::Addresses,
+                &matcher,
+            ) || header_matches(
+                context,
+                &MetadataHeaderName::To,
+                &HeaderForm::Addresses,
+                &matcher,
+            ) || header_matches(
+                context,
+                &MetadataHeaderName::Cc,
+                &HeaderForm::Addresses,
+                &matcher,
+            ) || header_matches(
+                context,
+                &MetadataHeaderName::Bcc,
+                &HeaderForm::Addresses,
+                &matcher,
+            ) || header_matches(
+                context,
+                &MetadataHeaderName::Subject,
+                &HeaderForm::Text,
+                &matcher,
+            ) || body_matches(context, &matcher)
+        }),
+        EmailFilter::Header(parts) => {
+            let Some(name) = parts.first() else {
+                return false;
+            };
+            let header_name = MetadataHeaderName::from(
+                HeaderName::parse(name.as_str())
+                    .unwrap_or_else(|| HeaderName::Other(name.as_str().into())),
+            );
+            match (context.root_part.header_value(&header_name), parts.get(1)) {
+                (Some(value), Some(expected)) => ascii_matcher(expected).is_some_and(|matcher| {
+                    value_contains(
+                        &HeaderValue::from(value).into_form(&HeaderForm::Raw),
+                        &matcher,
+                    )
+                }),
+                (Some(_), None) => true,
+                (None, _) => false,
+            }
+        }
+        EmailFilter::SentBefore(date) => {
+            sent_at(context).is_some_and(|sent_at| sent_at < date.timestamp())
+        }
+        EmailFilter::SentAfter(date) => {
+            sent_at(context).is_some_and(|sent_at| sent_at >= date.timestamp())
+        }
+        EmailFilter::InThread(id) => context.data.thread_id == id.document_id(),
+        EmailFilter::Id(ids) => ids.iter().any(|id| id.document_id() == context.document_id),
+        EmailFilter::_T(_) => false,
+    }
+}
+
+fn received_at(context: &FilterContext) -> i64 {
+    (context.metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64
+}
+
+fn sent_at(context: &FilterContext) -> Option<i64> {
+    match context
+        .root_part
+        .header_value(&MetadataHeaderName::Date)
+        .map(HeaderValue::from)
+    {
+        Some(HeaderValue::DateTime(datetime)) => Some(datetime.to_timestamp()),
+        _ => None,
+    }
+}
+
+fn ascii_matcher(needle: &str) -> Option<AhoCorasick> {
+    AhoCorasick::builder()
+        .ascii_case_insensitive(true)
+        .build([needle])
+        .ok()
+}
+
+fn header_matches(
+    context: &FilterContext,
+    name: &MetadataHeaderName,
+    form: &HeaderForm,
+    matcher: &AhoCorasick,
+) -> bool {
+    context
+        .root_part
+        .header_value(name)
+        .is_some_and(|value| value_contains(&HeaderValue::from(value).into_form(form), matcher))
+}
+
+fn value_contains(value: &Value<'_, EmailProperty, EmailValue>, matcher: &AhoCorasick) -> bool {
+    match value {
+        Value::Str(text) => matcher.is_match(text.as_ref()),
+        Value::Array(values) => values.iter().any(|value| value_contains(value, matcher)),
+        Value::Object(map) => map.iter().any(|(_, value)| value_contains(value, matcher)),
+        _ => false,
+    }
+}
+
+fn body_matches(context: &FilterContext, matcher: &AhoCorasick) -> bool {
+    context
+        .contents
+        .text_body
+        .iter()
+        .chain(context.contents.html_body.iter())
+        .any(|part_id| {
+            context
+                .contents
+                .parts
+                .as_ref()
+                .get(u16::from(part_id) as usize)
+                .is_some_and(|part| {
+                    matches!(
+                        part.body,
+                        ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html
+                    ) && matcher.is_match(part.decode_contents(context.raw_message).as_str())
+                })
+        })
+}

crates/services/src/state_manager/http.rs

@@ -0,0 +1,427 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{
+    Event,
+    ece::{ECE_WEBPUSH_MAX_PLAINTEXT_SIZE, WEBPUSH_MAX_BODY_SIZE, ece_encrypt},
+    email_push::build_email_push_object,
+};
+use crate::state_manager::PushRegistration;
+use calcard::jscalendar::JSCalendarDateTime;
+use common::{Server, ipc::PushNotification, network::webpush::Vapid};
+use email::push::{PushSubscription, Urgency};
+use jmap_proto::{
+    object::email::{EmailProperty, EmailValue},
+    response::status::PushObject,
+    types::state::State,
+};
+use jmap_tools::Value;
+use reqwest::{
+    Client, Url,
+    header::{AUTHORIZATION, CONTENT_ENCODING, CONTENT_TYPE},
+    redirect::Policy,
+};
+use std::net::IpAddr;
+use std::time::{Duration, Instant};
+use store::write::now;
+use tokio::sync::mpsc;
+use trc::PushSubscriptionEvent;
+use types::{id::Id, type_state::DataType};
+use utils::map::vec_map::VecMap;
+
+const MAX_ERROR_RESPONSE_LEN: usize = 1024;
+const MAX_REDIRECTS: usize = 4;
+const PUSH_OBJECT_OVERHEAD: usize = 128;
+
+#[derive(Default)]
+struct EmailPushObject {
+    emails: Vec<Value<'static, EmailProperty, EmailValue>>,
+    change_id: Option<u64>,
+    urgency: Urgency,
+    used: usize,
+}
+
+impl PushRegistration {
+    pub fn send(
+        &mut self,
+        id: Id,
+        push_tx: mpsc::Sender<Event>,
+        push_timeout: Duration,
+        server: Server,
+    ) {
+        let subscription = self.server.clone();
+        let push_client = self.client.clone();
+        let notifications = std::mem::take(&mut self.notifications);
+
+        self.in_flight = true;
+        self.last_request = Instant::now();
+
+        tokio::spawn(async move {
+            let mut changed: VecMap<Id, VecMap<DataType, State>> = VecMap::new();
+            let mut email_pushes: VecMap<Id, EmailPushObject> = VecMap::new();
+
+            let mut failed_state_change = false;
+            let mut failed_email_pushes = Vec::new();
+            let mut failed_calendar_alerts = Vec::new();
+
+            for notification in &notifications {
+                match notification {
+                    PushNotification::StateChange(state_change) => {
+                        for type_state in state_change.types {
+                            changed
+                                .get_mut_or_insert(state_change.account_id.into())
+                                .set(type_state, State::Exact(state_change.change_id));
+                        }
+                    }
+                    PushNotification::CalendarAlert(calendar_alert) => {
+                        let payload = PushObject::CalendarAlert {
+                            account_id: calendar_alert.account_id.into(),
+                            calendar_event_id: calendar_alert.event_id.into(),
+                            uid: calendar_alert.uid.clone(),
+                            recurrence_id: calendar_alert.recurrence_id.map(|timestamp| {
+                                JSCalendarDateTime::new(timestamp, true).to_rfc3339()
+                            }),
+                            alert_id: calendar_alert.alert_id.clone(),
+                        };
+                        if !http_request(
+                            &push_client,
+                            &subscription,
+                            serde_json::to_string(&payload).unwrap().into_bytes(),
+                            push_timeout,
+                            server.core.jmap.vapid.as_ref(),
+                            Urgency::Normal,
+                        )
+                        .await
+                        {
+                            failed_calendar_alerts
+                                .push((calendar_alert.account_id, calendar_alert.event_id));
+                        }
+                    }
+                    PushNotification::EmailPush(email_push) => {
+                        if let Some(config) = subscription
+                            .email_push
+                            .iter()
+                            .find(|config| config.account_id == email_push.account_id)
+                        {
+                            let emails =
+                                email_pushes.get_mut_or_insert(Id::from(email_push.account_id));
+                            let remaining = server
+                                .core
+                                .jmap
+                                .push_max_size
+                                .min(if subscription.keys.is_some() {
+                                    ECE_WEBPUSH_MAX_PLAINTEXT_SIZE
+                                } else {
+                                    WEBPUSH_MAX_BODY_SIZE
+                                })
+                                .saturating_sub(PUSH_OBJECT_OVERHEAD)
+                                .saturating_sub(emails.used);
+
+                            match build_email_push_object(
+                                &server,
+                                email_push.account_id,
+                                email_push.email_id,
+                                config,
+                                remaining,
+                            )
+                            .await
+                            {
+                                Ok(Some((object, used))) => {
+                                    emails.urgency = config.urgency;
+                                    if emails
+                                        .change_id
+                                        .is_none_or(|change_id| email_push.change_id > change_id)
+                                    {
+                                        emails.change_id = Some(email_push.change_id);
+                                    }
+                                    emails.used += used;
+                                    emails.emails.push(object);
+                                }
+                                Ok(None) => {}
+                                Err(err) => {
+                                    trc::error!(
+                                        err.details(
+                                            "Failed to build EmailPush notification object."
+                                        )
+                                    );
+                                    failed_email_pushes.push(email_push.account_id);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            if !changed.is_empty() {
+                failed_state_change = !http_request(
+                    &push_client,
+                    &subscription,
+                    serde_json::to_string(&PushObject::StateChange { changed })
+                        .unwrap()
+                        .into_bytes(),
+                    push_timeout,
+                    server.core.jmap.vapid.as_ref(),
+                    Urgency::Normal,
+                )
+                .await;
+            }
+
+            for (account_id, email_push) in email_pushes {
+                if email_push.emails.is_empty() {
+                    continue;
+                }
+
+                let payload = PushObject::EmailPush {
+                    account_id,
+                    emails: email_push.emails,
+                    state: email_push.change_id.map(State::Exact),
+                };
+
+                if !http_request(
+                    &push_client,
+                    &subscription,
+                    serde_json::to_string(&payload).unwrap().into_bytes(),
+                    push_timeout,
+                    server.core.jmap.vapid.as_ref(),
+                    email_push.urgency,
+                )
+                .await
+                {
+                    failed_email_pushes.push(account_id.document_id());
+                }
+            }
+
+            let result = if !failed_state_change
+                && failed_email_pushes.is_empty()
+                && failed_calendar_alerts.is_empty()
+            {
+                Event::DeliverySuccess { id }
+            } else {
+                let mut failed_notifications = Vec::with_capacity(
+                    failed_state_change as usize
+                        + failed_email_pushes.len()
+                        + failed_calendar_alerts.len(),
+                );
+
+                for notification in notifications {
+                    match &notification {
+                        PushNotification::StateChange(_) => {
+                            if failed_state_change {
+                                failed_notifications.push(notification);
+                            }
+                        }
+                        PushNotification::EmailPush(email_push) => {
+                            if failed_email_pushes.contains(&email_push.account_id) {
+                                failed_notifications.push(notification);
+                            }
+                        }
+                        PushNotification::CalendarAlert(calendar_alert) => {
+                            if failed_calendar_alerts
+                                .contains(&(calendar_alert.account_id, calendar_alert.event_id))
+                            {
+                                failed_notifications.push(notification);
+                            }
+                        }
+                    }
+                }
+
+                Event::DeliveryFailure {
+                    id,
+                    notifications: failed_notifications,
+                }
+            };
+
+            push_tx.send(result).await.ok();
+        });
+    }
+}
+
+pub(crate) fn build_push_client() -> Client {
+    utils::http::http_client_builder(cfg!(feature = "test_mode"))
+        .redirect(Policy::custom(|attempt| match attempt.previous().last() {
+            Some(previous) if is_same_organization(previous, attempt.url()) => {
+                if attempt.previous().len() > MAX_REDIRECTS {
+                    attempt.error("Too many redirects.")
+                } else {
+                    attempt.follow()
+                }
+            }
+            _ => attempt.stop(),
+        }))
+        .build()
+        .unwrap_or_default()
+}
+
+pub(crate) async fn http_request(
+    push_client: &Client,
+    details: &PushSubscription,
+    mut body: Vec<u8>,
+    push_timeout: Duration,
+    vapid: Option<&Vapid>,
+    urgency: Urgency,
+) -> bool {
+    let mut client = push_client
+        .post(details.url.as_str())
+        .timeout(push_timeout)
+        .header("TTL", "86400")
+        .header("Urgency", urgency.as_str());
+
+    if let Some(authorization) = vapid.and_then(|vapid| vapid.authorization(&details.url, now())) {
+        client = client.header(AUTHORIZATION, authorization);
+    }
+
+    let mut content_type = "application/json";
+    if let Some(keys) = &details.keys {
+        match ece_encrypt(&keys.p256dh, &keys.auth, &body) {
+            Ok(body_) => {
+                body = body_;
+                content_type = "application/octet-stream";
+                client = client.header(CONTENT_ENCODING, "aes128gcm");
+            }
+            Err(err) => {
+                // Do not reattempt if encryption fails.
+
+                trc::event!(
+                    PushSubscription(PushSubscriptionEvent::Error),
+                    Details = "Failed to encrypt push subscription",
+                    Url = details.url.to_string(),
+                    Reason = err
+                );
+                return true;
+            }
+        }
+    }
+
+    match client
+        .header(CONTENT_TYPE, content_type)
+        .body(body)
+        .send()
+        .await
+    {
+        Ok(response) => {
+            let status = response.status();
+
+            if status.is_success() {
+                trc::event!(
+                    PushSubscription(PushSubscriptionEvent::Success),
+                    Url = details.url.to_string()
+                );
+
+                true
+            } else {
+                let mut reason = response.text().await.unwrap_or_default();
+                reason.truncate(reason.ceil_char_boundary(MAX_ERROR_RESPONSE_LEN));
+
+                trc::event!(
+                    PushSubscription(PushSubscriptionEvent::Error),
+                    Details = "HTTP POST failed",
+                    Url = details.url.to_string(),
+                    Code = status.as_u16(),
+                    Reason = reason,
+                );
+
+                false
+            }
+        }
+        Err(err) => {
+            trc::event!(
+                PushSubscription(PushSubscriptionEvent::Error),
+                Details = "HTTP POST failed",
+                Url = details.url.to_string(),
+                Reason = err.to_string()
+            );
+
+            false
+        }
+    }
+}
+
+fn is_same_organization(previous: &Url, next: &Url) -> bool {
+    if previous.scheme() == next.scheme()
+        && let (Some(previous_host), Some(next_host)) = (previous.host_str(), next.host_str())
+    {
+        if is_ip_literal(previous_host) || is_ip_literal(next_host) {
+            previous_host == next_host
+        } else {
+            match (psl::domain_str(previous_host), psl::domain_str(next_host)) {
+                (Some(previous_domain), Some(next_domain)) => previous_domain == next_domain,
+                _ => previous_host == next_host,
+            }
+        }
+    } else {
+        false
+    }
+}
+
+fn is_ip_literal(host: &str) -> bool {
+    host.strip_prefix('[')
+        .and_then(|host| host.strip_suffix(']'))
+        .unwrap_or(host)
+        .parse::<IpAddr>()
+        .is_ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::is_same_organization;
+    use reqwest::Url;
+
+    #[test]
+    fn same_organization_redirects() {
+        for (previous, next, expected) in [
+            (
+                "https://push.example.org/a",
+                "https://push.example.org/b",
+                true,
+            ),
+            (
+                "https://push.example.org/a",
+                "https://push2.example.org/b",
+                true,
+            ),
+            ("https://push.example.org/a", "https://example.org/b", true),
+            (
+                "https://push.example.org/a",
+                "https://push.example.org:8443/b",
+                true,
+            ),
+            (
+                "https://push.example.org/a",
+                "https://push.evil.org/b",
+                false,
+            ),
+            (
+                "https://push.example.org/a",
+                "http://push.example.org/b",
+                false,
+            ),
+            (
+                "https://push.example.co.uk/a",
+                "https://evil.co.uk/b",
+                false,
+            ),
+            ("https://1.2.3.4/a", "https://1.2.3.4/b", true),
+            ("https://1.2.3.4/a", "https://5.6.7.8/b", false),
+            ("https://1.2.3.4/a", "https://5.6.3.4/b", false),
+            ("https://1.2.3.4/a", "https://127.0.0.1/b", false),
+            ("https://[2606:4700::1111]/a", "https://[::1]/b", false),
+            (
+                "https://[2606:4700::1111]/a",
+                "https://[2606:4700::1111]/b",
+                true,
+            ),
+        ] {
+            let previous = Url::parse(previous).unwrap();
+            let next = Url::parse(next).unwrap();
+
+            assert_eq!(
+                is_same_organization(&previous, &next),
+                expected,
+                "{previous} -> {next}"
+            );
+        }
+    }
+}

crates/services/src/state_manager/manager.rs

@@ -0,0 +1,190 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{Event, PURGE_EVERY, SEND_TIMEOUT, push::spawn_push_manager};
+use crate::state_manager::IpcSubscriber;
+use common::{
+    Inner,
+    ipc::{BroadcastEvent, PushEvent},
+};
+use std::{sync::Arc, time::Instant};
+use store::ahash::AHashMap;
+use tokio::sync::mpsc;
+use trc::ServerEvent;
+
+#[derive(Default)]
+struct Subscriber {
+    ipc: Vec<IpcSubscriber>,
+    is_push: bool,
+}
+
+#[allow(clippy::unwrap_or_default)]
+pub fn spawn_push_router(inner: Arc<Inner>, mut change_rx: mpsc::Receiver<PushEvent>) {
+    let push_tx = spawn_push_manager(inner.clone());
+
+    tokio::spawn(async move {
+        let mut subscribers: AHashMap<u32, Subscriber> = AHashMap::default();
+        let mut last_purge = Instant::now();
+
+        while let Some(event) = change_rx.recv().await {
+            let mut purge_needed = last_purge.elapsed() >= PURGE_EVERY;
+
+            match event {
+                PushEvent::Stop => {
+                    if push_tx.send(Event::Reset).await.is_err() {
+                        trc::event!(
+                            Server(ServerEvent::ThreadError),
+                            Details = "Error sending push reset.",
+                            CausedBy = trc::location!()
+                        );
+                    }
+                    break;
+                }
+
+                PushEvent::Subscribe {
+                    account_ids,
+                    types,
+                    tx,
+                } => {
+                    for account_id in account_ids {
+                        subscribers
+                            .entry(account_id)
+                            .or_default()
+                            .ipc
+                            .push(IpcSubscriber {
+                                types,
+                                tx: tx.clone(),
+                            });
+                    }
+                }
+
+                PushEvent::PushServerRegister { activate, expired } => {
+                    for account_id in activate {
+                        subscribers.entry(account_id).or_default().is_push = true;
+                    }
+
+                    for account_id in expired {
+                        let mut remove_account = false;
+                        if let Some(subscriber_list) = subscribers.get_mut(&account_id) {
+                            subscriber_list.is_push = false;
+                            remove_account = subscriber_list.ipc.is_empty();
+                        }
+                        if remove_account {
+                            subscribers.remove(&account_id);
+                        }
+                    }
+                }
+
+                PushEvent::Publish {
+                    notification,
+                    broadcast,
+                } => {
+                    // Publish event to cluster
+                    if broadcast
+                        && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone()
+                        && broadcast_tx
+                            .send(BroadcastEvent::PushNotification(notification.clone()))
+                            .await
+                            .is_err()
+                    {
+                        trc::event!(
+                            Server(trc::ServerEvent::ThreadError),
+                            Details = "Error sending broadcast event.",
+                            CausedBy = trc::location!()
+                        );
+                    }
+
+                    let account_id = notification.account_id();
+                    if let Some(subscribers) = subscribers.get(&account_id) {
+                        for subscriber in &subscribers.ipc {
+                            if let Some(notification) = notification.filter_types(&subscriber.types)
+                            {
+                                if subscriber.is_valid() {
+                                    let subscriber_tx = subscriber.tx.clone();
+
+                                    tokio::spawn(async move {
+                                        // Timeout after 500ms in case there is a blocked client
+                                        if subscriber_tx
+                                            .send_timeout(notification, SEND_TIMEOUT)
+                                            .await
+                                            .is_err()
+                                        {
+                                            trc::event!(
+                                                Server(ServerEvent::ThreadError),
+                                                Details =
+                                                    "Error sending state change to subscriber.",
+                                                CausedBy = trc::location!()
+                                            );
+                                        }
+                                    });
+                                } else {
+                                    purge_needed = true;
+                                }
+                            }
+                        }
+
+                        if subscribers.is_push
+                            && push_tx.send(Event::Push { notification }).await.is_err()
+                        {
+                            trc::event!(
+                                Server(ServerEvent::ThreadError),
+                                Details = "Error sending push updates.",
+                                CausedBy = trc::location!()
+                            );
+                        }
+                    }
+                }
+
+                PushEvent::PushServerUpdate {
+                    account_id,
+                    broadcast,
+                } => {
+                    // Publish event to cluster
+                    if broadcast
+                        && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone()
+                        && broadcast_tx
+                            .send(BroadcastEvent::PushServerUpdate(account_id))
+                            .await
+                            .is_err()
+                    {
+                        trc::event!(
+                            Server(trc::ServerEvent::ThreadError),
+                            Details = "Error sending broadcast event.",
+                            CausedBy = trc::location!()
+                        );
+                    }
+
+                    // Notify push manager
+                    if push_tx.send(Event::Update { account_id }).await.is_err() {
+                        trc::event!(
+                            Server(ServerEvent::ThreadError),
+                            Details = "Error sending push updates.",
+                            CausedBy = trc::location!()
+                        );
+                    }
+                }
+            }
+
+            if purge_needed {
+                let mut remove_account_ids = Vec::new();
+
+                for (account_id, subscribers) in &mut subscribers {
+                    subscribers.ipc.retain(|subscriber| subscriber.is_valid());
+
+                    if subscribers.ipc.is_empty() && !subscribers.is_push {
+                        remove_account_ids.push(*account_id);
+                    }
+                }
+
+                for remove_account_id in remove_account_ids {
+                    subscribers.remove(&remove_account_id);
+                }
+
+                last_purge = Instant::now();
+            }
+        }
+    });
+}

crates/services/src/state_manager/mod.rs

@@ -0,0 +1,66 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod ece;
+pub mod email_push;
+pub mod http;
+pub mod manager;
+pub mod push;
+
+use common::ipc::PushNotification;
+use email::push::PushSubscription;
+use reqwest::Client;
+use std::{
+    sync::Arc,
+    time::{Duration, Instant},
+};
+use tokio::sync::mpsc;
+use types::{id::Id, type_state::DataType};
+use utils::map::bitmap::Bitmap;
+
+const PURGE_EVERY: Duration = Duration::from_secs(3600);
+const SEND_TIMEOUT: Duration = Duration::from_millis(500);
+
+#[derive(Debug)]
+struct IpcSubscriber {
+    types: Bitmap<DataType>,
+    tx: mpsc::Sender<PushNotification>,
+}
+
+#[derive(Debug)]
+pub struct PushRegistration {
+    server: Arc<PushSubscription>,
+    member_account_ids: Vec<u32>,
+    num_attempts: u32,
+    last_request: Instant,
+    notifications: Vec<PushNotification>,
+    in_flight: bool,
+    client: Client,
+}
+
+#[derive(Debug)]
+pub enum Event {
+    Push {
+        notification: PushNotification,
+    },
+    Update {
+        account_id: u32,
+    },
+    DeliverySuccess {
+        id: Id,
+    },
+    DeliveryFailure {
+        id: Id,
+        notifications: Vec<PushNotification>,
+    },
+    Reset,
+}
+
+impl IpcSubscriber {
+    fn is_valid(&self) -> bool {
+        !self.tx.is_closed()
+    }
+}

crates/services/src/state_manager/push.rs

@@ -0,0 +1,521 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{
+    Event,
+    http::{build_push_client, http_request},
+};
+use crate::state_manager::PushRegistration;
+use common::{
+    BuildServer, IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, Server,
+    auth::BuildAccessToken,
+    ipc::{PushEvent, PushNotification},
+};
+use email::push::{PushSubscription, PushSubscriptions, Urgency};
+use std::{
+    collections::hash_map::Entry,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+use store::{
+    ValueKey,
+    ahash::{AHashMap, AHashSet},
+    write::{AlignedBytes, Archive, now},
+};
+use tokio::sync::mpsc;
+use trc::{AddContext, PushSubscriptionEvent, ServerEvent};
+use types::{collection::Collection, field::PrincipalField, id::Id};
+
+pub fn spawn_push_manager(inner: Arc<Inner>) -> mpsc::Sender<Event> {
+    let (push_tx_, mut push_rx) = mpsc::channel::<Event>(IPC_CHANNEL_BUFFER);
+    let push_tx = push_tx_.clone();
+
+    tokio::spawn(async move {
+        let mut push_servers: AHashMap<Id, PushRegistration> = AHashMap::default();
+        let mut account_push_ids: AHashMap<u32, AHashSet<Id>> = AHashMap::default();
+        let mut last_verify: AHashMap<u32, Instant> = AHashMap::default();
+        let mut last_retry = Instant::now();
+        let mut retry_timeout = LONG_1Y_SLUMBER;
+        let mut retry_ids = AHashSet::default();
+        let push_client = build_push_client();
+
+        // Load active subscriptions on startup
+        {
+            let server = inner.build_server();
+
+            if server.core.network.roles.push_notifications {
+                match server
+                    .document_ids(
+                        u32::MAX,
+                        Collection::Principal,
+                        PrincipalField::PushSubscriptions,
+                    )
+                    .await
+                {
+                    Ok(account_ids) => {
+                        for account_id in account_ids {
+                            if server.core.jmap.push_total_shards <= 1
+                                || account_id % server.core.jmap.push_total_shards
+                                    == server.registry().cluster_push_shard()
+                            {
+                                // Load push subscriptions for account
+                                let (subscriptions, member_account_ids) =
+                                    match load_push_subscriptions(&server, account_id).await {
+                                        Ok(subscriptions) => subscriptions,
+                                        Err(err) => {
+                                            trc::error!(err.caused_by(trc::location!()));
+                                            continue;
+                                        }
+                                    };
+                                let current_time = now();
+                                for subscription in subscriptions
+                                    .subscriptions
+                                    .into_iter()
+                                    .filter(|s| s.verified && s.expires > current_time)
+                                {
+                                    let id = Id::from_parts(subscription.id, account_id);
+                                    let subscription = Arc::new(subscription);
+
+                                    for account_id in &member_account_ids {
+                                        account_push_ids.entry(*account_id).or_default().insert(id);
+                                    }
+                                    push_servers.insert(
+                                        id,
+                                        PushRegistration {
+                                            member_account_ids: member_account_ids.clone(),
+                                            num_attempts: 0,
+                                            last_request: Instant::now()
+                                                - (server.core.jmap.push_throttle
+                                                    + Duration::from_millis(1)),
+                                            notifications: Vec::new(),
+                                            server: subscription.clone(),
+                                            in_flight: false,
+                                            client: push_client.clone(),
+                                        },
+                                    );
+                                }
+                            }
+                        }
+                    }
+                    Err(err) => {
+                        trc::error!(err.caused_by(trc::location!()));
+                    }
+                }
+
+                // Subscribe to push events
+                if !account_push_ids.is_empty()
+                    && server
+                        .inner
+                        .ipc
+                        .push_tx
+                        .clone()
+                        .send(PushEvent::PushServerRegister {
+                            activate: account_push_ids.keys().copied().collect(),
+                            expired: vec![],
+                        })
+                        .await
+                        .is_err()
+                {
+                    trc::event!(
+                        Server(ServerEvent::ThreadError),
+                        Details = "Error sending state change.",
+                        CausedBy = trc::location!()
+                    );
+                }
+            }
+        }
+
+        loop {
+            // Wait for the next event or timeout
+            let event_or_timeout = tokio::time::timeout(retry_timeout, push_rx.recv()).await;
+
+            // Load settings
+            let server = inner.build_server();
+            let push_attempt_interval = server.core.jmap.push_attempt_interval;
+            let push_attempts_max = server.core.jmap.push_attempts_max;
+            let push_retry_interval = server.core.jmap.push_retry_interval;
+            let push_timeout = server.core.jmap.push_timeout;
+            let push_verify_timeout = server.core.jmap.push_verify_timeout;
+            let push_throttle = server.core.jmap.push_throttle;
+
+            match event_or_timeout {
+                Ok(Some(event)) => match event {
+                    Event::Update { account_id } => {
+                        if server.core.jmap.push_total_shards > 1
+                            && account_id % server.core.jmap.push_total_shards
+                                != server.registry().cluster_push_shard()
+                        {
+                            continue;
+                        }
+
+                        // Load push subscriptions for account
+                        let (subscriptions, member_account_ids) =
+                            match load_push_subscriptions(&server, account_id).await {
+                                Ok(subscriptions) => subscriptions,
+                                Err(err) => {
+                                    trc::error!(err.caused_by(trc::location!()));
+                                    continue;
+                                }
+                            };
+                        let old_account_push_ids = account_push_ids
+                            .remove(&account_id)
+                            .filter(|v| !v.is_empty());
+
+                        // Process subscriptions
+                        let current_time = now();
+                        let mut newest_unverified: Option<Arc<PushSubscription>> = None;
+                        for subscription in subscriptions
+                            .subscriptions
+                            .into_iter()
+                            .filter(|s| s.expires > current_time)
+                        {
+                            let id = Id::from_parts(subscription.id, account_id);
+                            let subscription = Arc::new(subscription);
+
+                            if subscription.verified {
+                                for account_id in &member_account_ids {
+                                    account_push_ids.entry(*account_id).or_default().insert(id);
+                                }
+
+                                match push_servers.entry(id) {
+                                    Entry::Occupied(mut entry) => {
+                                        // Update existing subscription
+                                        let entry = entry.get_mut();
+                                        entry.server = subscription.clone();
+                                        entry.member_account_ids = member_account_ids.clone();
+                                    }
+                                    Entry::Vacant(entry) => {
+                                        entry.insert(PushRegistration {
+                                            member_account_ids: member_account_ids.clone(),
+                                            num_attempts: 0,
+                                            last_request: Instant::now()
+                                                - (push_throttle + Duration::from_millis(1)),
+                                            notifications: Vec::new(),
+                                            server: subscription.clone(),
+                                            in_flight: false,
+                                            client: push_client.clone(),
+                                        });
+                                    }
+                                }
+                            } else {
+                                match &newest_unverified {
+                                    Some(existing) if existing.id >= subscription.id => {}
+                                    _ => newest_unverified = Some(subscription),
+                                }
+                            }
+                        }
+
+                        if let Some(subscription) = newest_unverified {
+                            let current_time = Instant::now();
+
+                            #[cfg(feature = "test_mode")]
+                            if subscription.url.contains("skip_checks") {
+                                last_verify.insert(
+                                    account_id,
+                                    current_time - (push_verify_timeout + Duration::from_millis(1)),
+                                );
+                            }
+
+                            if last_verify
+                                .get(&account_id)
+                                .map(|last_verify| {
+                                    current_time - *last_verify > push_verify_timeout
+                                })
+                                .unwrap_or(true)
+                            {
+                                let core = server.core.clone();
+                                let push_client = push_client.clone();
+                                tokio::spawn(async move {
+                                    http_request(
+                                        &push_client,
+                                        &subscription,
+                                        format!(
+                                            concat!(
+                                                "{{\"@type\":\"PushVerification\",",
+                                                "\"pushSubscriptionId\":\"{}\",",
+                                                "\"verificationCode\":\"{}\"}}"
+                                            ),
+                                            Id::from(subscription.id),
+                                            subscription.verification_code
+                                        )
+                                        .into_bytes(),
+                                        push_timeout,
+                                        core.jmap.vapid.as_ref(),
+                                        Urgency::Normal,
+                                    )
+                                    .await;
+                                });
+
+                                last_verify.insert(account_id, current_time);
+                            } else {
+                                trc::event!(
+                                    PushSubscription(PushSubscriptionEvent::Error),
+                                    Details = "Failed to verify push subscription",
+                                    Url = subscription.url.clone(),
+                                    AccountId = account_id,
+                                    Reason = "Too many requests"
+                                );
+                            }
+                        }
+
+                        // Update subscriptions
+                        let mut remove_push_ids = AHashSet::new();
+                        let mut active_account_ids = Vec::new();
+                        let mut inactive_account_ids = Vec::new();
+                        match (old_account_push_ids, account_push_ids.get(&account_id)) {
+                            (Some(old), Some(current)) if &old != current => {
+                                for id in old.difference(current) {
+                                    remove_push_ids.insert(*id);
+                                }
+                                active_account_ids = member_account_ids;
+                            }
+                            (Some(old), None) => {
+                                remove_push_ids = old;
+                            }
+                            (None, Some(_)) => {
+                                active_account_ids = member_account_ids;
+                            }
+                            _ => {}
+                        }
+
+                        // Update push server registrations
+                        if !remove_push_ids.is_empty() {
+                            for id in remove_push_ids {
+                                if let Some(subscription) = push_servers.remove(&id) {
+                                    for account_id in &subscription.member_account_ids {
+                                        if let Some(ids) = account_push_ids.get_mut(account_id) {
+                                            ids.remove(&id);
+                                            if ids.is_empty() {
+                                                account_push_ids.remove(account_id);
+                                                inactive_account_ids.push(*account_id);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        if (!active_account_ids.is_empty() || !inactive_account_ids.is_empty())
+                            && server
+                                .inner
+                                .ipc
+                                .push_tx
+                                .clone()
+                                .send(PushEvent::PushServerRegister {
+                                    activate: active_account_ids,
+                                    expired: inactive_account_ids,
+                                })
+                                .await
+                                .is_err()
+                        {
+                            trc::event!(
+                                Server(ServerEvent::ThreadError),
+                                Details = "Error sending state change.",
+                                CausedBy = trc::location!()
+                            );
+                        }
+                    }
+                    Event::Push { notification } => {
+                        let account_id = notification.account_id();
+                        if let Some(ids) = account_push_ids.get_mut(&account_id) {
+                            let current_time = now();
+                            let mut remove_ids = Vec::new();
+
+                            for id in ids.iter() {
+                                if let Some(subscription) = push_servers.get_mut(id) {
+                                    if subscription.server.expires > current_time {
+                                        if let Some(mut notification) =
+                                            notification.filter_types(&subscription.server.types)
+                                        {
+                                            if let PushNotification::EmailPush(email_push) =
+                                                &notification
+                                                && !subscription
+                                                    .server
+                                                    .email_push
+                                                    .iter()
+                                                    .any(|ep| ep.account_id == account_id)
+                                            {
+                                                notification = PushNotification::StateChange(
+                                                    email_push.to_state_change(),
+                                                );
+                                            }
+
+                                            subscription.notifications.push(notification);
+                                            let last_request = subscription.last_request.elapsed();
+
+                                            if !subscription.in_flight
+                                                && ((subscription.num_attempts == 0
+                                                    && last_request > push_throttle)
+                                                    || ((1..push_attempts_max)
+                                                        .contains(&subscription.num_attempts)
+                                                        && last_request > push_attempt_interval))
+                                            {
+                                                subscription.send(
+                                                    *id,
+                                                    push_tx.clone(),
+                                                    push_timeout,
+                                                    server.clone(),
+                                                );
+                                                retry_ids.remove(id);
+                                            } else {
+                                                retry_ids.insert(*id);
+                                            }
+                                        }
+                                    } else {
+                                        push_servers.remove(id);
+                                    }
+                                } else {
+                                    remove_ids.push(*id);
+                                }
+                            }
+
+                            if !remove_ids.is_empty() {
+                                for remove_id in remove_ids {
+                                    ids.remove(&remove_id);
+                                }
+                                if ids.is_empty() {
+                                    account_push_ids.remove(&account_id);
+                                    if server
+                                        .inner
+                                        .ipc
+                                        .push_tx
+                                        .clone()
+                                        .send(PushEvent::PushServerRegister {
+                                            activate: vec![],
+                                            expired: vec![account_id],
+                                        })
+                                        .await
+                                        .is_err()
+                                    {
+                                        trc::event!(
+                                            Server(ServerEvent::ThreadError),
+                                            Details = "Error sending state change.",
+                                            CausedBy = trc::location!()
+                                        );
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Event::Reset => {
+                        push_servers.clear();
+                        account_push_ids.clear();
+                    }
+                    Event::DeliverySuccess { id } => {
+                        if let Some(subscription) = push_servers.get_mut(&id) {
+                            subscription.num_attempts = 0;
+                            subscription.in_flight = false;
+                            retry_ids.remove(&id);
+                        }
+                    }
+                    Event::DeliveryFailure { id, notifications } => {
+                        if let Some(subscription) = push_servers.get_mut(&id) {
+                            subscription.last_request = Instant::now();
+                            subscription.num_attempts += 1;
+                            subscription.notifications.extend(notifications);
+                            subscription.in_flight = false;
+                            retry_ids.insert(id);
+                        }
+                    }
+                },
+                Ok(None) => {
+                    break;
+                }
+                Err(_) => (),
+            }
+
+            retry_timeout = if !retry_ids.is_empty() {
+                let last_retry_elapsed = last_retry.elapsed();
+
+                if last_retry_elapsed >= push_retry_interval {
+                    let mut remove_ids = Vec::with_capacity(retry_ids.len());
+
+                    for retry_id in &retry_ids {
+                        if let Some(subscription) = push_servers.get_mut(retry_id) {
+                            let last_request = subscription.last_request.elapsed();
+
+                            if !subscription.in_flight
+                                && ((subscription.num_attempts == 0
+                                    && last_request >= push_throttle)
+                                    || (subscription.num_attempts > 0
+                                        && last_request >= push_attempt_interval))
+                            {
+                                if subscription.num_attempts < push_attempts_max {
+                                    subscription.send(
+                                        *retry_id,
+                                        push_tx.clone(),
+                                        push_timeout,
+                                        server.clone(),
+                                    );
+                                } else {
+                                    trc::event!(
+                                        PushSubscription(PushSubscriptionEvent::Error),
+                                        Details = "Failed to deliver push subscription",
+                                        Url = subscription.server.url.clone(),
+                                        Reason = "Too many failed attempts"
+                                    );
+
+                                    subscription.notifications.clear();
+                                    subscription.num_attempts = 0;
+                                }
+                                remove_ids.push(*retry_id);
+                            }
+                        } else {
+                            remove_ids.push(*retry_id);
+                        }
+                    }
+
+                    if remove_ids.len() < retry_ids.len() {
+                        for remove_id in remove_ids {
+                            retry_ids.remove(&remove_id);
+                        }
+                        last_retry = Instant::now();
+                        push_retry_interval
+                    } else {
+                        retry_ids.clear();
+                        LONG_1Y_SLUMBER
+                    }
+                } else {
+                    push_retry_interval - last_retry_elapsed
+                }
+            } else {
+                LONG_1Y_SLUMBER
+            };
+        }
+    });
+
+    push_tx_
+}
+
+async fn load_push_subscriptions(
+    server: &Server,
+    account_id: u32,
+) -> trc::Result<(PushSubscriptions, Vec<u32>)> {
+    let member_of = server
+        .access_token(account_id)
+        .await
+        .caused_by(trc::location!())?
+        .build()
+        .member_ids()
+        .collect::<Vec<_>>();
+
+    if let Some(push_subscriptions) = server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::property(
+            account_id,
+            Collection::Principal,
+            0,
+            PrincipalField::PushSubscriptions,
+        ))
+        .await?
+    {
+        push_subscriptions
+            .deserialize::<PushSubscriptions>()
+            .map(|push_subscriptions| (push_subscriptions, member_of))
+            .caused_by(trc::location!())
+    } else {
+        Ok((PushSubscriptions::default(), member_of))
+    }
+}

crates/services/src/task_manager/acme.rs

@@ -0,0 +1,113 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::{TaskFailureType, TaskResult};
+use common::{Server, network::acme::AcmeError};
+use registry::schema::structs::TaskDomainManagement;
+use std::time::Duration;
+use store::write::now;
+
+pub(crate) trait AcmeTask: Sync + Send {
+    fn acme_management(
+        &self,
+        task: &TaskDomainManagement,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl AcmeTask for Server {
+    async fn acme_management(&self, task: &TaskDomainManagement) -> TaskResult {
+        match acme_management(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.caused_by(trc::location!())
+                        .details("Failed to run ACME task")
+                );
+                result
+            }
+        }
+    }
+}
+
+#[cfg(not(feature = "test_mode"))]
+const MAX_RETRIES: u32 = 3;
+
+#[cfg(feature = "test_mode")]
+const MAX_RETRIES: u32 = 5;
+
+#[allow(unused_variables)]
+async fn acme_management(server: &Server, task: &TaskDomainManagement) -> trc::Result<TaskResult> {
+    let mut last_temporary_error = Ok(TaskResult::temporary(""));
+    for retry in 0..MAX_RETRIES {
+        last_temporary_error = match Box::pin(server.acme_renew(task.domain_id)).await {
+            Ok(tasks) => return Ok(TaskResult::Success(tasks)),
+            Err(err) => {
+                if !matches!(
+                    err,
+                    AcmeError::NotDue(_)
+                        | AcmeError::Internal(_)
+                        | AcmeError::AuthInvalid(_)
+                        | AcmeError::OrderInvalid(_)
+                        | AcmeError::AuthTimeout { .. }
+                        | AcmeError::Backoff { .. }
+                ) {
+                    trc::event!(
+                        Acme(trc::AcmeEvent::Error),
+                        Id = task.domain_id.to_string(),
+                        Total = retry as u64,
+                        Reason = err.to_string(),
+                    );
+                }
+
+                match err {
+                    AcmeError::Crypto(_)
+                    | AcmeError::Invalid(_)
+                    | AcmeError::NotDue(_)
+                    | AcmeError::ChallengeNotSupported { .. } => {
+                        return Ok(TaskResult::permanent(err.to_string()));
+                    }
+                    AcmeError::OrderInvalid(_) | AcmeError::Json(_) | AcmeError::Registry(_) => {
+                        return Ok(TaskResult::perpetual(err.to_string()));
+                    }
+                    AcmeError::Http(_)
+                    | AcmeError::HttpStatus(_)
+                    | AcmeError::Dns(_)
+                    | AcmeError::AuthInvalid(_) => Ok(TaskResult::temporary(err.to_string())),
+                    AcmeError::OrderTimeout { max_retries }
+                    | AcmeError::AuthTimeout { max_retries } => Ok(TaskResult::Failure {
+                        typ: TaskFailureType::Temporary,
+                        message: err.to_string(),
+                        max_attempts: (max_retries as u64).into(),
+                    }),
+                    AcmeError::Backoff { max_retries, wait } => {
+                        return if let Some(wait) = wait {
+                            Ok(TaskResult::Failure {
+                                typ: TaskFailureType::Retry(now() + wait.as_secs()),
+                                message: err.to_string(),
+                                max_attempts: (max_retries as u64).into(),
+                            })
+                        } else {
+                            Ok(TaskResult::Failure {
+                                typ: TaskFailureType::Temporary,
+                                message: err.to_string(),
+                                max_attempts: (max_retries as u64).into(),
+                            })
+                        };
+                    }
+                    AcmeError::Internal(error) => return Err(error),
+                }
+            }
+        };
+
+        #[cfg(not(feature = "test_mode"))]
+        tokio::time::sleep(Duration::from_secs(1 << (retry + 5))).await;
+
+        #[cfg(feature = "test_mode")]
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    }
+    last_temporary_error
+}

crates/services/src/task_manager/alarm.rs

@@ -0,0 +1,669 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use calcard::icalendar::{
+    ArchivedICalendarParameterName, ArchivedICalendarProperty, ICalendarProperty,
+};
+use common::{
+    DEFAULT_LOGO_BASE64, Server,
+    auth::{AccountInfo, BuildAccessToken},
+    config::groupware::CalendarTemplateVariable,
+    ipc::{CalendarAlert, PushNotification},
+    network::{ServerInstance, stream::NullIo},
+};
+use groupware::{
+    calendar::{ArchivedCalendarEvent, CalendarEvent},
+    scheduling::{
+        ItipTime, ItipValue,
+        format::{DateStyle, TextFormatter, hyperlink},
+    },
+    strip_mailto_scheme,
+};
+use mail_builder::{
+    MessageBuilder,
+    headers::{HeaderType, content_type::ContentType},
+    mime::{BodyPart, MimePart},
+};
+use mail_parser::decoders::html::html_to_text;
+use registry::{
+    schema::{
+        enums::Permission,
+        structs::{TaskCalendarAlarmEmail, TaskCalendarAlarmNotification},
+    },
+    types::EnumImpl,
+};
+use smtp::core::{Session, SessionData};
+use smtp_proto::{MailFrom, RcptTo};
+use std::{sync::Arc, time::Duration};
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive, now},
+};
+use trc::{AddContext, TaskManagerEvent};
+use types::collection::Collection;
+use utils::{sanitize_email, template::Variables};
+
+use crate::task_manager::TaskResult;
+
+pub(crate) trait SendAlarmTask: Sync + Send {
+    fn send_display_alarm(
+        &self,
+        task: &TaskCalendarAlarmNotification,
+    ) -> impl Future<Output = TaskResult> + Send;
+
+    fn send_email_alarm(
+        &self,
+        task: &TaskCalendarAlarmEmail,
+        server_instance: Arc<ServerInstance>,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl SendAlarmTask for Server {
+    async fn send_display_alarm(&self, task: &TaskCalendarAlarmNotification) -> TaskResult {
+        match send_display_alarm(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(task.account_id.document_id())
+                        .document_id(task.document_id.document_id())
+                        .caused_by(trc::location!())
+                        .details("Failed to process e-mail alarm")
+                );
+                result
+            }
+        }
+    }
+
+    async fn send_email_alarm(
+        &self,
+        task: &TaskCalendarAlarmEmail,
+        server_instance: Arc<ServerInstance>,
+    ) -> TaskResult {
+        match send_email_alarm(self, task, server_instance).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(task.account_id.document_id())
+                        .document_id(task.document_id.document_id())
+                        .caused_by(trc::location!())
+                        .details("Failed to process e-mail alarm")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn send_email_alarm(
+    server: &Server,
+    task: &TaskCalendarAlarmEmail,
+    server_instance: Arc<ServerInstance>,
+) -> trc::Result<TaskResult> {
+    // Obtain access token
+    let account_id = task.account_id.document_id();
+    let document_id = task.document_id.document_id();
+    let access_token = server
+        .access_token(account_id)
+        .await
+        .caused_by(trc::location!())?
+        .build();
+
+    if !access_token.has_permission(Permission::CalendarAlarmsSend) {
+        trc::event!(
+            Calendar(trc::CalendarEvent::AlarmSkipped),
+            Reason = "Account does not have permission to send calendar alarms",
+            AccountId = account_id,
+            DocumentId = document_id,
+        );
+        return Ok(TaskResult::Success(vec![]));
+    }
+    let account_info = server
+        .account_info(account_id)
+        .await
+        .caused_by(trc::location!())?;
+
+    if account_info.name().is_empty() {
+        trc::event!(
+            Calendar(trc::CalendarEvent::AlarmFailed),
+            Reason = "Account does not have any email addresses",
+            AccountId = account_id,
+            DocumentId = document_id,
+        );
+        return Ok(TaskResult::Success(vec![]));
+    }
+
+    // Fetch event
+    let Some(event_) = server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::CalendarEvent,
+            document_id,
+        ))
+        .await
+        .caused_by(trc::location!())?
+    else {
+        trc::event!(
+            TaskManager(TaskManagerEvent::MetadataNotFound),
+            Details = "Calendar Event metadata not found",
+            AccountId = account_id,
+            DocumentId = document_id,
+        );
+
+        return Ok(TaskResult::Success(vec![]));
+    };
+
+    // Unarchive event
+    let event = event_
+        .unarchive::<CalendarEvent>()
+        .caused_by(trc::location!())?;
+
+    // Build message body
+    let account_main_email = account_info.name();
+    let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost");
+    let logo_cid = format!("logo.{}@{account_main_domain}", now());
+    let Some(tpl) = build_template(server, &account_info, task, event, &logo_cid).await? else {
+        return Ok(TaskResult::Success(vec![]));
+    };
+    let txt_body = html_to_text(&tpl.body);
+
+    // Obtain logo image
+    let logo = match server.logo_resource(account_main_domain).await {
+        Ok(logo) => logo,
+        Err(err) => {
+            trc::error!(
+                err.caused_by(trc::location!())
+                    .details("Failed to fetch logo image")
+            );
+            None
+        }
+    };
+    let logo = if let Some(logo) = &logo {
+        MimePart::new(
+            ContentType::new(logo.content_type.as_ref()),
+            BodyPart::Binary(logo.contents.as_slice().into()),
+        )
+    } else {
+        MimePart::new(
+            ContentType::new("image/png"),
+            BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()),
+        )
+        .transfer_encoding("base64")
+    }
+    .inline()
+    .cid(&logo_cid);
+
+    // Build message
+    let mail_from = if let Some(from_email) = &server.core.groupware.alarms_from_email {
+        from_email.to_string()
+    } else {
+        format!("calendar-notification@{account_main_domain}")
+    };
+    let message = MessageBuilder::new()
+        .from((
+            server.core.groupware.alarms_from_name.as_str(),
+            mail_from.as_str(),
+        ))
+        .header("To", HeaderType::Text(tpl.to.as_str().into()))
+        .header("Auto-Submitted", HeaderType::Text("auto-generated".into()))
+        .header("Reply-To", HeaderType::Text(account_main_email.into()))
+        .message_id(server.core.network.message_id())
+        .subject(tpl.subject)
+        .body(MimePart::new(
+            ContentType::new("multipart/related"),
+            BodyPart::Multipart(vec![
+                MimePart::new(
+                    ContentType::new("multipart/alternative"),
+                    BodyPart::Multipart(vec![
+                        MimePart::new(
+                            ContentType::new("text/plain"),
+                            BodyPart::Text(txt_body.into()),
+                        ),
+                        MimePart::new(
+                            ContentType::new("text/html"),
+                            BodyPart::Text(tpl.body.into()),
+                        ),
+                    ]),
+                ),
+                logo,
+            ]),
+        ))
+        .write_to_vec()
+        .unwrap_or_default();
+
+    // Send message
+    let server_ = server.clone();
+    let mail_from = account_main_email.to_string();
+    let to = tpl.to;
+    let result = tokio::spawn(async move {
+        let mut session = Session::<NullIo>::local(
+            server_,
+            server_instance,
+            SessionData::local(account_info, None, vec![], vec![], 0),
+        );
+
+        // MAIL FROM
+        let _ = session
+            .handle_mail_from(MailFrom {
+                address: mail_from.into(),
+                ..Default::default()
+            })
+            .await;
+        if let Some(error) = session.has_failed() {
+            return Err(format!("Server rejected MAIL-FROM: {}", error.trim()));
+        }
+
+        // RCPT TO
+        session.params.rcpt_errors_wait = Duration::from_secs(0);
+        let _ = session
+            .handle_rcpt_to(RcptTo {
+                address: to.into(),
+                ..Default::default()
+            })
+            .await;
+        if let Some(error) = session.has_failed() {
+            return Err(format!("Server rejected RCPT-TO: {}", error.trim()));
+        }
+
+        // DATA
+        session.data.message = message;
+        let response = session.queue_message().await;
+        if let smtp::core::State::Accepted(queue_id) = session.state {
+            Ok(queue_id)
+        } else {
+            Err(format!(
+                "Server rejected DATA: {}",
+                std::str::from_utf8(&response).unwrap().trim()
+            ))
+        }
+    })
+    .await;
+
+    match result {
+        Ok(Ok(queue_id)) => {
+            trc::event!(
+                Calendar(trc::CalendarEvent::AlarmSent),
+                AccountId = account_id,
+                DocumentId = document_id,
+                QueueId = queue_id,
+            );
+        }
+        Ok(Err(err)) => {
+            trc::event!(
+                Calendar(trc::CalendarEvent::AlarmFailed),
+                AccountId = account_id,
+                DocumentId = document_id,
+                Reason = err,
+            );
+        }
+        Err(_) => {
+            trc::event!(
+                Server(trc::ServerEvent::ThreadError),
+                Details = "Join Error",
+                AccountId = account_id,
+                DocumentId = document_id,
+                CausedBy = trc::location!(),
+            );
+            return Ok(TaskResult::temporary("Thread join error"));
+        }
+    }
+
+    build_next_alarm(server, account_id, document_id, event)
+}
+
+async fn send_display_alarm(
+    server: &Server,
+    task: &TaskCalendarAlarmNotification,
+) -> trc::Result<TaskResult> {
+    // Fetch event
+    let account_id = task.account_id.document_id();
+    let document_id = task.document_id.document_id();
+    let Some(event_) = server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::CalendarEvent,
+            document_id,
+        ))
+        .await
+        .caused_by(trc::location!())?
+    else {
+        trc::event!(
+            TaskManager(TaskManagerEvent::MetadataNotFound),
+            Details = "Calendar Event metadata not found",
+            AccountId = account_id,
+            DocumentId = document_id,
+        );
+
+        return Ok(TaskResult::Success(vec![]));
+    };
+
+    // Unarchive event
+    let event = event_
+        .unarchive::<CalendarEvent>()
+        .caused_by(trc::location!())?;
+
+    let recurrence_id = task.recurrence_id;
+
+    let ical = &event.data.event;
+    server
+        .broadcast_push_notification(PushNotification::CalendarAlert(CalendarAlert {
+            account_id,
+            event_id: document_id,
+            recurrence_id,
+            uid: ical.uids().next().unwrap_or_default().to_string(),
+            alert_id: ical
+                .components
+                .get(task.alarm_id as usize)
+                .and_then(|c| c.property(&ICalendarProperty::Jsid))
+                .and_then(|v| v.values.first())
+                .and_then(|v| v.as_text())
+                .map(|v| v.to_string())
+                .unwrap_or_else(|| {
+                    format!(
+                        "k{}",
+                        ical.components
+                            .get(task.event_id as usize)
+                            .and_then(|c| c
+                                .component_ids
+                                .iter()
+                                .position(|id| id.to_native() == task.alarm_id as u32))
+                            .unwrap_or_default()
+                            + 1
+                    )
+                }),
+        }))
+        .await;
+
+    build_next_alarm(server, account_id, document_id, event)
+}
+
+fn build_next_alarm(
+    server: &Server,
+    account_id: u32,
+    document_id: u32,
+    event: &ArchivedCalendarEvent,
+) -> trc::Result<TaskResult> {
+    // Find next alarm time and write to task queue
+    let now = now() as i64;
+    if let Some(next_alarm) =
+        event
+            .data
+            .next_alarm(now, Default::default())
+            .and_then(|next_alarm| {
+                // Verify minimum interval
+                let max_next_alarm = now + server.core.groupware.alarms_minimum_interval;
+                if next_alarm.alarm_time < max_next_alarm {
+                    trc::event!(
+                        Calendar(trc::CalendarEvent::AlarmSkipped),
+                        Reason = "Next alarm skipped due to minimum interval",
+                        Details = next_alarm.alarm_time - now,
+                        AccountId = account_id,
+                        DocumentId = document_id,
+                    );
+                    event.data.next_alarm(max_next_alarm, Default::default())
+                } else {
+                    Some(next_alarm)
+                }
+            })
+    {
+        Ok(TaskResult::Update(
+            next_alarm.build_write_ops(account_id, document_id),
+        ))
+    } else {
+        Ok(TaskResult::Success(vec![]))
+    }
+}
+
+struct Details {
+    to: String,
+    subject: String,
+    body: String,
+}
+
+async fn build_template(
+    server: &Server,
+    account_info: &AccountInfo,
+    alarm: &TaskCalendarAlarmEmail,
+    event: &ArchivedCalendarEvent,
+    logo_cid: &str,
+) -> trc::Result<Option<Details>> {
+    let account_id = alarm.account_id.document_id();
+    let document_id = alarm.document_id.document_id();
+    let (Some(event_component), Some(alarm_component)) = (
+        event.data.event.components.get(alarm.event_id as usize),
+        event.data.event.components.get(alarm.alarm_id as usize),
+    ) else {
+        trc::event!(
+            TaskManager(TaskManagerEvent::MetadataNotFound),
+            Details = "Calendar Alarm component not found",
+            AccountId = account_id,
+            DocumentId = document_id,
+        );
+        return Ok(None);
+    };
+
+    // Build webcal URI
+    let webcal_uri = match event.webcal_uri(server, account_info).await {
+        Ok(uri) => uri,
+        Err(err) => {
+            trc::error!(
+                err.account_id(account_id)
+                    .document_id(document_id)
+                    .caused_by(trc::location!())
+                    .details("Failed to generate webcal URI")
+            );
+            String::from("#")
+        }
+    };
+
+    // Obtain alarm details
+    let mut summary = None;
+    let mut description = None;
+    let mut rcpt_to = None;
+    let mut location = None;
+    let mut conference = None;
+    let mut organizer = None;
+    let mut guests = vec![];
+
+    for entry in alarm_component.entries.iter() {
+        match &entry.name {
+            ArchivedICalendarProperty::Summary => {
+                summary = entry.values.first().and_then(|v| v.as_text());
+            }
+            ArchivedICalendarProperty::Description => {
+                description = entry.values.first().and_then(|v| v.as_text());
+            }
+            ArchivedICalendarProperty::Attendee => {
+                rcpt_to = entry
+                    .values
+                    .first()
+                    .and_then(|v| v.as_text())
+                    .map(strip_mailto_scheme)
+                    .and_then(sanitize_email);
+            }
+            _ => {}
+        }
+    }
+
+    for entry in event_component.entries.iter() {
+        match &entry.name {
+            ArchivedICalendarProperty::Summary if summary.is_none() => {
+                summary = entry.values.first().and_then(|v| v.as_text());
+            }
+            ArchivedICalendarProperty::Description if description.is_none() => {
+                description = entry.values.first().and_then(|v| v.as_text());
+            }
+            ArchivedICalendarProperty::Location => {
+                location = entry.values.first().and_then(|v| v.as_text());
+            }
+            ArchivedICalendarProperty::Conference if conference.is_none() => {
+                conference = entry.values.first().and_then(|v| v.as_text());
+            }
+            ArchivedICalendarProperty::Organizer | ArchivedICalendarProperty::Attendee => {
+                let email = entry
+                    .values
+                    .first()
+                    .and_then(|v| v.as_text())
+                    .map(strip_mailto_scheme);
+                let name = entry.params.iter().find_map(|param| {
+                    if let ArchivedICalendarParameterName::Cn = param.name {
+                        param.value.as_text()
+                    } else {
+                        None
+                    }
+                });
+
+                if email.is_some() || name.is_some() {
+                    if matches!(entry.name, ArchivedICalendarProperty::Organizer) {
+                        organizer = Some((email, name));
+                    } else {
+                        guests.push((email, name));
+                    }
+                }
+            }
+            _ => {}
+        }
+    }
+
+    // Validate recipient
+    let rcpt_to = if let Some(rcpt_to) = rcpt_to {
+        if server.core.groupware.alarms_allow_external_recipients
+            || account_info.addresses().contains(&rcpt_to)
+        {
+            rcpt_to
+        } else {
+            trc::event!(
+                Calendar(trc::CalendarEvent::AlarmRecipientOverride),
+                Reason = "External recipient not allowed for calendar alarms",
+                Details = rcpt_to,
+                AccountId = account_id,
+                DocumentId = document_id,
+            );
+
+            account_info.name().to_string()
+        }
+    } else {
+        account_info.name().to_string()
+    };
+
+
+    #[cfg(not(feature = "enterprise"))]
+    let template = &server.core.groupware.alarms_template;
+    let formatter = TextFormatter::new(account_info.locale().as_str())?;
+    let locale = formatter.locale;
+
+    let start = formatter.field_to_string(
+        &ItipValue::Time(ItipTime {
+            start: alarm.event_start.timestamp(),
+            tz_id: alarm.event_start_tz as u16,
+        }),
+        DateStyle::Short,
+    );
+    let end = formatter.field_to_string(
+        &ItipValue::Time(ItipTime {
+            start: alarm.event_end.timestamp(),
+            tz_id: alarm.event_end_tz as u16,
+        }),
+        DateStyle::Short,
+    );
+    let subject = format!(
+        "{}: {} @ {}",
+        locale.calendar_alarm_subject_prefix,
+        summary.or(description).unwrap_or("No Subject"),
+        start
+    );
+    let organizer = organizer
+        .map(|(email, name)| match (email, name) {
+            (Some(email), Some(name)) => format!("{} <{}>", name, email),
+            (Some(email), None) => email.to_string(),
+            (None, Some(name)) => name.to_string(),
+            _ => unreachable!(),
+        })
+        .unwrap_or_else(|| account_info.name().to_string());
+    let logo_cid = format!("cid:{logo_cid}");
+    let mut variables = Variables::new();
+    variables.insert_single(CalendarTemplateVariable::PageTitle, subject.as_str());
+    variables.insert_single(CalendarTemplateVariable::Lang, locale.name);
+    variables.insert_single(CalendarTemplateVariable::Dir, locale.direction);
+    variables.insert_single(
+        CalendarTemplateVariable::Header,
+        locale.calendar_alarm_header,
+    );
+    variables.insert_single(
+        CalendarTemplateVariable::Footer,
+        locale.calendar_alarm_footer,
+    );
+    variables.insert_single(
+        CalendarTemplateVariable::ActionName,
+        locale.calendar_alarm_open,
+    );
+    variables.insert_single(CalendarTemplateVariable::ActionUrl, webcal_uri.as_str());
+    variables.insert_single(
+        CalendarTemplateVariable::AttendeesTitle,
+        locale.calendar_attendees,
+    );
+    if let Some(summary) = summary.filter(|summary| !summary.is_empty()) {
+        variables.insert_single(CalendarTemplateVariable::EventTitle, summary);
+    }
+    variables.insert_single(CalendarTemplateVariable::LogoCid, logo_cid.as_str());
+    if let Some(description) = description {
+        variables.insert_single(CalendarTemplateVariable::EventDescription, description);
+    }
+    variables.insert_block(
+        CalendarTemplateVariable::EventDetails,
+        [
+            Some(vec![
+                (CalendarTemplateVariable::Key, locale.calendar_start),
+                (CalendarTemplateVariable::Value, start.as_str()),
+            ]),
+            Some(vec![
+                (CalendarTemplateVariable::Key, locale.calendar_end),
+                (CalendarTemplateVariable::Value, end.as_str()),
+            ]),
+            location.map(|location| {
+                vec![
+                    (CalendarTemplateVariable::Key, locale.calendar_location),
+                    (CalendarTemplateVariable::Value, location),
+                ]
+            }),
+            conference.map(|conference| {
+                let mut detail = vec![
+                    (CalendarTemplateVariable::Key, locale.calendar_conference),
+                    (CalendarTemplateVariable::Value, conference),
+                ];
+                if let Some(link) = hyperlink(conference) {
+                    detail.push((CalendarTemplateVariable::Link, link));
+                }
+                detail
+            }),
+            Some(vec![
+                (CalendarTemplateVariable::Key, locale.calendar_organizer),
+                (CalendarTemplateVariable::Value, organizer.as_str()),
+            ]),
+        ]
+        .into_iter()
+        .flatten(),
+    );
+    if !guests.is_empty() {
+        variables.insert_block(
+            CalendarTemplateVariable::Attendees,
+            guests.into_iter().map(|(email, name)| {
+                [
+                    (CalendarTemplateVariable::Key, name.unwrap_or_default()),
+                    (CalendarTemplateVariable::Value, email.unwrap_or_default()),
+                ]
+            }),
+        );
+    }
+    Ok(Some(Details {
+        to: rcpt_to,
+        body: template.eval(&variables),
+        subject,
+    }))
+}

crates/services/src/task_manager/destroy_account.rs

@@ -0,0 +1,236 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::TaskResult;
+use common::Server;
+use email::{message::metadata::MessageMetadata, sieve::SieveScript};
+use groupware::file::FileNode;
+use registry::{
+    schema::{
+        prelude::{ObjectType, Property},
+        structs::{ArchivedItem, TaskDestroyAccount},
+    },
+    types::EnumImpl,
+};
+use store::{
+    SerializeInfallible, ValueKey,
+    registry::RegistryQuery,
+    search::SearchQuery,
+    write::{BatchBuilder, BlobLink, BlobOp, RegistryClass, SearchIndex, ValueClass},
+};
+use trc::AddContext;
+use types::{
+    blob_hash::BlobHash,
+    collection::Collection,
+    field::{EmailField, Field},
+    id::Id,
+};
+
+pub(crate) trait DestroyAccountTask: Sync + Send {
+    fn destroy_account(&self, task: &TaskDestroyAccount)
+    -> impl Future<Output = TaskResult> + Send;
+}
+
+impl DestroyAccountTask for Server {
+    async fn destroy_account(&self, task: &TaskDestroyAccount) -> TaskResult {
+        match destroy_account(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(task.account_id.document_id())
+                        .details("Failed to destroy account")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Result<TaskResult> {
+    let account_id = task.account_id.document_id();
+
+    // Destroy public keys and masked emails
+    for object in [ObjectType::PublicKey, ObjectType::MaskedEmail] {
+        let mut batch = BatchBuilder::new();
+        let ids = server
+            .registry()
+            .query::<Vec<Id>>(RegistryQuery::new(object).with_account(account_id))
+            .await?;
+        let object_id = object.to_id();
+
+        for id in ids {
+            batch
+                .clear(ValueClass::Registry(RegistryClass::Item {
+                    object_id,
+                    item_id: id.id(),
+                }))
+                .clear(ValueClass::Registry(RegistryClass::IndexId {
+                    object_id,
+                    item_id: id.id(),
+                }))
+                .clear(ValueClass::Registry(RegistryClass::Index {
+                    index_id: Property::AccountId as u16,
+                    object_id,
+                    item_id: id.id(),
+                    key: (account_id as u64).serialize(),
+                }))
+                .clear(ValueClass::Registry(RegistryClass::Reference {
+                    to_object_id: ObjectType::Account as u16,
+                    to_item_id: account_id as u64,
+                    from_object_id: object_id,
+                    from_item_id: id.id(),
+                }));
+        }
+
+        if !batch.is_empty() {
+            server.store().write(batch.build_all()).await?;
+        }
+    }
+
+    // Remove archived items
+    let mut batch = BatchBuilder::new();
+    let ids = server
+        .registry()
+        .query::<Vec<Id>>(RegistryQuery::new(ObjectType::ArchivedItem).with_account(account_id))
+        .await?;
+    for id in ids {
+        let object_id = ObjectType::ArchivedItem.to_id();
+        let item_id = id.id();
+
+        if let Some(item) = server
+            .store()
+            .get_value::<ArchivedItem>(ValueKey::from(ValueClass::Registry(RegistryClass::Item {
+                object_id,
+                item_id,
+            })))
+            .await?
+        {
+            let until = item.archived_until().timestamp() as u64;
+            let blob_hash = item.into_blob_id().hash;
+
+            batch
+                .with_account_id(account_id)
+                .clear(BlobOp::Link {
+                    hash: blob_hash,
+                    to: BlobLink::Temporary { until },
+                })
+                .clear(ValueClass::Registry(RegistryClass::Index {
+                    index_id: Property::AccountId.to_id(),
+                    object_id,
+                    item_id,
+                    key: (account_id as u64).serialize(),
+                }))
+                .clear(ValueClass::Registry(RegistryClass::Item {
+                    object_id,
+                    item_id,
+                }));
+        }
+    }
+    if !batch.is_empty() {
+        server.store().write(batch.build_all()).await?;
+    }
+
+    // Remove search index
+    for index in [
+        SearchIndex::Email,
+        SearchIndex::Contacts,
+        SearchIndex::Calendar,
+    ] {
+        server
+            .search_store()
+            .unindex(SearchQuery::new(index).with_account_id(account_id))
+            .await?;
+    }
+
+    // Unlink all accounts's blobs
+    destroy_account_blobs(server, account_id).await?;
+
+    // Destroy account data
+    server
+        .store()
+        .danger_destroy_account(account_id)
+        .await
+        .caused_by(trc::location!())?;
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> {
+    let mut delete_keys = Vec::new();
+    for (collection, field) in [
+        (Collection::Email, u8::from(EmailField::Metadata)),
+        (Collection::FileNode, u8::from(Field::ARCHIVE)),
+        (Collection::SieveScript, u8::from(Field::ARCHIVE)),
+    ] {
+        server
+            .all_archives(account_id, collection, field, |document_id, archive| {
+                match collection {
+                    Collection::Email => {
+                        let message = archive.unarchive::<MessageMetadata>()?;
+                        delete_keys.push((
+                            collection,
+                            document_id,
+                            BlobHash::from(&message.blob_hash),
+                        ));
+                    }
+                    Collection::FileNode => {
+                        if let Some(file) = archive.unarchive::<FileNode>()?.file.as_ref() {
+                            delete_keys.push((
+                                collection,
+                                document_id,
+                                BlobHash::from(&file.blob_hash),
+                            ));
+                        }
+                    }
+                    Collection::SieveScript => {
+                        let sieve = archive.unarchive::<SieveScript>()?;
+                        delete_keys.push((
+                            collection,
+                            document_id,
+                            BlobHash::from(&sieve.blob_hash),
+                        ));
+                    }
+                    _ => {}
+                }
+                Ok(())
+            })
+            .await
+            .caused_by(trc::location!())?;
+    }
+
+    let mut batch = BatchBuilder::new();
+    batch.with_account_id(account_id);
+
+    for (collection, document_id, hash) in delete_keys {
+        if batch.is_large_batch() {
+            server
+                .store()
+                .write(batch.build_all())
+                .await
+                .caused_by(trc::location!())?;
+            batch = BatchBuilder::new();
+            batch.with_account_id(account_id);
+        }
+        batch
+            .with_collection(collection)
+            .with_document(document_id)
+            .clear(ValueClass::Blob(BlobOp::Link {
+                hash,
+                to: BlobLink::Document,
+            }));
+    }
+
+    if !batch.is_empty() {
+        server
+            .store()
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+    }
+
+    Ok(())
+}

crates/services/src/task_manager/dkim.rs

@@ -0,0 +1,633 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::TaskResult;
+use common::{
+    Server,
+    cache::invalidate::CacheInvalidationBuilder,
+    ipc::CacheInvalidation,
+    network::dkim::{
+        generate_dkim_dns_record, generate_dkim_dns_record_name, generate_dkim_private_key,
+        generate_dkim_selector,
+    },
+};
+use registry::{
+    schema::{
+        enums::{DkimRotationStage, DkimSignatureType, DnsRecordType},
+        prelude::{Object, ObjectType, Property},
+        structs::{
+            Dkim1Signature, Dkim2Signature, DkimManagement, DkimSignature, DnsManagement, Domain,
+            SecretText, SecretTextValue, Task, TaskDomainManagement, TaskStatus,
+        },
+    },
+    types::{datetime::UTCDateTime, id::ObjectId},
+};
+use std::fmt::Write;
+use store::{
+    registry::{
+        RegistryObject, RegistryQuery,
+        write::{RegistryWrite, RegistryWriteResult},
+    },
+    write::now,
+};
+use trc::{DkimEvent, DnsEvent};
+use types::id::Id;
+
+pub(crate) trait DkimManagementTask: Sync + Send {
+    fn dkim_management(
+        &self,
+        task: &TaskDomainManagement,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl DkimManagementTask for Server {
+    async fn dkim_management(&self, task: &TaskDomainManagement) -> TaskResult {
+        match dkim_management(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.caused_by(trc::location!())
+                        .details("Failed to run DKIM management task")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::Result<TaskResult> {
+    let Some(domain) = server.registry().object::<Domain>(task.domain_id).await? else {
+        return Ok(TaskResult::permanent("Domain not found".to_string()));
+    };
+    let DkimManagement::Automatic(dkim) = domain.dkim_management else {
+        return Ok(TaskResult::permanent(
+            "Domain is not set to automatic DKIM management".to_string(),
+        ));
+    };
+    let mut create_signatures = dkim.algorithms.into_inner();
+    if create_signatures.is_empty() {
+        return Ok(TaskResult::permanent(
+            "No DKIM algorithms configured for domain".to_string(),
+        ));
+    }
+
+    let dns_updater = match domain.dns_management {
+        DnsManagement::Automatic(props) if props.publish_records.contains(&DnsRecordType::Dkim) => {
+            match server.build_dns_updater(props.dns_server_id).await? {
+                Ok(updater) => Some((updater, props.origin.unwrap_or_else(|| domain.name.clone()))),
+                Err(err) => {
+                    return Ok(TaskResult::permanent(format!(
+                        "Failed to build DNS updater: {}",
+                        err
+                    )));
+                }
+            }
+        }
+        _ => None,
+    };
+
+    // Fetch existing DKIM keys
+    let mut publish_signatures = Vec::new();
+    let mut retire_signatures = Vec::new();
+    let mut retiring_signatures = Vec::new();
+    let mut delete_signatures = Vec::new();
+    let mut next_transition = None;
+
+    let signature_ids = server
+        .registry()
+        .query::<Vec<Id>>(
+            RegistryQuery::new(ObjectType::DkimSignature)
+                .equal(Property::DomainId, task.domain_id.document_id()),
+        )
+        .await?;
+
+    for id in signature_ids {
+        let id = ObjectId::new(ObjectType::DkimSignature, id);
+        let Some(key) = server.registry().get(id).await? else {
+            continue;
+        };
+        let key: RegistryObject<DkimSignature> = RegistryObject {
+            id,
+            revision: key.revision,
+            object: key.into(),
+        };
+
+        let key_algo = key.object.object_type();
+        if let Some(current_stage) = key.object.rotation_due() {
+            match current_stage {
+                DkimRotationStage::Pending => {
+                    create_signatures.retain(|algo| algo != &key_algo);
+                    publish_signatures.push(key)
+                }
+                DkimRotationStage::Active => retiring_signatures.push(key),
+                DkimRotationStage::Retiring => retire_signatures.push(key),
+                DkimRotationStage::Retired => delete_signatures.push(key),
+            }
+        } else {
+            if key.object.is_active() {
+                create_signatures.retain(|algo| algo != &key_algo);
+            }
+
+            if let Some(transition) = key.object.next_transition()
+                && next_transition.is_none_or(|next| transition < next)
+            {
+                next_transition = Some(transition);
+            }
+        }
+    }
+
+    let now = now();
+    let mut do_refresh = false;
+
+    for algorithm in create_signatures {
+        #[cfg(feature = "test_mode")]
+        let secret = {
+            if dkim.selector_template.contains("dummy") {
+                match algorithm {
+                    DkimSignatureType::Dkim1Ed25519Sha256
+                    | DkimSignatureType::Dkim2Ed25519Sha256 => TEST_ED25519_KEY.to_string(),
+                    DkimSignatureType::Dkim1RsaSha256 | DkimSignatureType::Dkim2RsaSha256 => {
+                        TEST_RSA_KEY.to_string()
+                    }
+                }
+            } else {
+                generate_dkim_private_key(algorithm).await.unwrap().unwrap()
+            }
+        };
+
+        // Generate new key and selector
+        #[cfg(not(feature = "test_mode"))]
+        let secret = match generate_dkim_private_key(algorithm).await? {
+            Ok(secret) => secret,
+            Err(err) => {
+                return Ok(TaskResult::permanent(err.to_string()));
+            }
+        };
+        let selector = match generate_dkim_selector(&dkim.selector_template, algorithm) {
+            Ok(selector) => selector,
+            Err(err) => {
+                return Ok(TaskResult::permanent(format!(
+                    "Failed to generate DKIM selector: {}",
+                    err
+                )));
+            }
+        };
+
+        // Build key
+        let private_key = SecretText::Text(SecretTextValue { secret });
+        let mut signature = match algorithm {
+            DkimSignatureType::Dkim1Ed25519Sha256 | DkimSignatureType::Dkim1RsaSha256 => {
+                let signature = Dkim1Signature {
+                    stage: DkimRotationStage::Active,
+                    domain_id: task.domain_id,
+                    member_tenant_id: domain.member_tenant_id,
+                    selector: selector.clone(),
+                    private_key,
+                    ..Default::default()
+                };
+                if algorithm == DkimSignatureType::Dkim1Ed25519Sha256 {
+                    DkimSignature::Dkim1Ed25519Sha256(signature)
+                } else {
+                    DkimSignature::Dkim1RsaSha256(signature)
+                }
+            }
+            DkimSignatureType::Dkim2Ed25519Sha256 | DkimSignatureType::Dkim2RsaSha256 => {
+                let signature = Dkim2Signature {
+                    stage: DkimRotationStage::Active,
+                    domain_id: task.domain_id,
+                    member_tenant_id: domain.member_tenant_id,
+                    selector: selector.clone(),
+                    private_key,
+                    ..Default::default()
+                };
+                if algorithm == DkimSignatureType::Dkim2Ed25519Sha256 {
+                    DkimSignature::Dkim2Ed25519Sha256(signature)
+                } else {
+                    DkimSignature::Dkim2RsaSha256(signature)
+                }
+            }
+        };
+
+        // Publish key
+        if let Some((updater, origin)) = &dns_updater {
+            let record = generate_dkim_dns_record(&signature, &domain.name).await?;
+            let dns_update::DnsRecord::TXT(txt_value) = &record.record else {
+                return Ok(TaskResult::permanent(
+                    "DKIM record must be a TXT record".to_string(),
+                ));
+            };
+            let propagation_target = txt_value.clone();
+            let published = updater
+                .set_rrset(
+                    origin,
+                    &record.name,
+                    dns_update::DnsRecordType::TXT,
+                    vec![record.record.clone()],
+                )
+                .await
+                .is_ok();
+            let signature_transition = if published
+                && updater
+                    .wait_for_txt_propagation(&record.name, origin, &propagation_target)
+                    .await
+            {
+                trc::event!(
+                    Dkim(DkimEvent::SignaturePublished),
+                    Id = selector.clone(),
+                    Details = domain.name.clone()
+                );
+
+                do_refresh = true;
+                UTCDateTime::from_timestamp((now + dkim.rotate_after.as_secs()) as i64)
+            } else {
+                // Something went wrong, reschedule.
+                signature.set_stage(DkimRotationStage::Pending);
+                UTCDateTime::from_timestamp((now + 60) as i64) // Retry after 1 minute
+            };
+
+            if next_transition.is_none_or(|next| signature_transition < next) {
+                next_transition = Some(signature_transition);
+            }
+
+            signature.set_next_transition(signature_transition);
+        }
+
+        // Write key
+        match server
+            .registry()
+            .write(RegistryWrite::insert(&signature.into()))
+            .await?
+        {
+            RegistryWriteResult::Success(_) => {
+                trc::event!(
+                    Dkim(DkimEvent::SignatureCreated),
+                    Id = selector,
+                    Details = domain.name.clone()
+                );
+            }
+            err => {
+                return Ok(TaskResult::permanent(format!(
+                    "Failed to write DKIM signature: {err}"
+                )));
+            }
+        }
+    }
+
+    // Publish signatures
+    let mut temporary_errors = String::new();
+    for signature in publish_signatures {
+        let record = generate_dkim_dns_record(&signature.object, &domain.name).await?;
+        if let Some((updater, origin)) = &dns_updater {
+            let dns_update::DnsRecord::TXT(txt_value) = &record.record else {
+                return Ok(TaskResult::permanent(
+                    "DKIM record must be a TXT record".to_string(),
+                ));
+            };
+            let propagation_target = txt_value.clone();
+            let publish_result = updater
+                .set_rrset(
+                    origin,
+                    &record.name,
+                    dns_update::DnsRecordType::TXT,
+                    vec![record.record.clone()],
+                )
+                .await;
+            let propagation_result = match &publish_result {
+                Ok(_) => Ok(updater
+                    .wait_for_txt_propagation(&record.name, origin, &propagation_target)
+                    .await),
+                Err(err) => Err(err.clone()),
+            };
+            match propagation_result {
+                Ok(true) => {
+                    let signature_transition =
+                        UTCDateTime::from_timestamp((now + dkim.rotate_after.as_secs()) as i64);
+
+                    if next_transition.is_none_or(|next| signature_transition < next) {
+                        next_transition = Some(signature_transition);
+                    }
+
+                    let mut new_signature = signature.object.clone();
+
+                    new_signature.set_next_transition(signature_transition);
+                    new_signature.set_stage(DkimRotationStage::Active);
+
+                    trc::event!(
+                        Dkim(DkimEvent::SignaturePublished),
+                        Id = new_signature.selector().to_string(),
+                        Details = domain.name.clone()
+                    );
+
+                    // Write key
+                    if let Some(task_result) = update_signature(
+                        server,
+                        signature,
+                        new_signature,
+                        &record.name,
+                        &mut temporary_errors,
+                    )
+                    .await?
+                    {
+                        return Ok(task_result);
+                    }
+                    do_refresh = true;
+                }
+                Ok(false) => {
+                    if !temporary_errors.is_empty() {
+                        temporary_errors.push_str("; ");
+                    }
+                    let _ = write!(
+                        &mut temporary_errors,
+                        "DKIM record {} did not propagate, will retry.",
+                        record.name
+                    );
+                }
+                Err(err) => {
+                    if !temporary_errors.is_empty() {
+                        temporary_errors.push_str("; ");
+                    }
+                    let _ = write!(
+                        &mut temporary_errors,
+                        "Failed to publish DKIM record {}: {err}.",
+                        record.name
+                    );
+                }
+            }
+        } else {
+            if !temporary_errors.is_empty() {
+                temporary_errors.push_str("; ");
+            }
+            let _ = write!(
+                &mut temporary_errors,
+                "No DNS server configured, cannot publish DKIM record {}.",
+                record.name
+            );
+        }
+    }
+
+    // Retiring signatures
+    for signature in retiring_signatures {
+        let record = generate_dkim_dns_record_name(&signature.object, &domain.name);
+        let signature_transition =
+            UTCDateTime::from_timestamp((now + dkim.retire_after.as_secs()) as i64);
+
+        if next_transition.is_none_or(|next| signature_transition < next) {
+            next_transition = Some(signature_transition);
+        }
+
+        let mut new_signature = signature.object.clone();
+
+        new_signature.set_next_transition(signature_transition);
+        new_signature.set_stage(DkimRotationStage::Retiring);
+
+        trc::event!(
+            Dkim(DkimEvent::SignatureRetiring),
+            Id = new_signature.selector().to_string(),
+            Details = domain.name.clone()
+        );
+
+        // Write key
+        if let Some(task_result) = update_signature(
+            server,
+            signature,
+            new_signature,
+            &record,
+            &mut temporary_errors,
+        )
+        .await?
+        {
+            return Ok(task_result);
+        }
+        do_refresh = true;
+    }
+
+    // Retire signatures
+    for signature in retire_signatures {
+        let record = generate_dkim_dns_record_name(&signature.object, &domain.name);
+        if let Some((updater, origin)) = &dns_updater {
+            match updater
+                .set_rrset(origin, &record, dns_update::DnsRecordType::TXT, Vec::new())
+                .await
+            {
+                Ok(_) => {
+                    let signature_transition =
+                        UTCDateTime::from_timestamp((now + dkim.delete_after.as_secs()) as i64);
+
+                    if next_transition.is_none_or(|next| signature_transition < next) {
+                        next_transition = Some(signature_transition);
+                    }
+
+                    let mut new_signature = signature.object.clone();
+
+                    new_signature.set_next_transition(signature_transition);
+                    new_signature.set_stage(DkimRotationStage::Retired);
+
+                    trc::event!(
+                        Dkim(DkimEvent::SignatureRetired),
+                        Id = new_signature.selector().to_string(),
+                        Details = domain.name.clone()
+                    );
+
+                    // Write key
+                    if let Some(task_result) = update_signature(
+                        server,
+                        signature,
+                        new_signature,
+                        &record,
+                        &mut temporary_errors,
+                    )
+                    .await?
+                    {
+                        return Ok(task_result);
+                    }
+
+                    do_refresh = true;
+                }
+                Err(err) => {
+                    if !temporary_errors.is_empty() {
+                        temporary_errors.push_str("; ");
+                    }
+                    let _ = write!(
+                        &mut temporary_errors,
+                        "Failed to remove DKIM record {}: {err}.",
+                        record
+                    );
+                }
+            }
+        } else {
+            if !temporary_errors.is_empty() {
+                temporary_errors.push_str("; ");
+            }
+            let _ = write!(
+                &mut temporary_errors,
+                "No DNS server configured, cannot retire DKIM record {}.",
+                record
+            );
+        }
+    }
+
+    // Delete signatures
+    for signature in delete_signatures {
+        let record = generate_dkim_dns_record_name(&signature.object, &domain.name);
+
+        if let Some((updater, origin)) = &dns_updater
+            && let Err(err) = updater
+                .set_rrset(origin, &record, dns_update::DnsRecordType::TXT, Vec::new())
+                .await
+        {
+            trc::event!(
+                Dns(DnsEvent::RecordDeletionFailed),
+                Hostname = record.clone(),
+                Details = origin.clone(),
+                Type = "TXT",
+                Reason = err.to_string(),
+            );
+        }
+
+        trc::event!(
+            Dkim(DkimEvent::SignatureDeleted),
+            Id = signature.object.selector().to_string(),
+            Details = domain.name.clone()
+        );
+
+        match server
+            .registry()
+            .write(RegistryWrite::delete_object(
+                signature.id,
+                &Object {
+                    inner: signature.object.into(),
+                    revision: signature.revision,
+                },
+            ))
+            .await
+        {
+            Ok(RegistryWriteResult::Success(_)) => {}
+            Ok(err) => {
+                return Ok(TaskResult::permanent(format!(
+                    "Failed to delete DKIM signature for record {record}: {err}"
+                )));
+            }
+            Err(err) => {
+                if err.is_assertion_failure() {
+                    if !temporary_errors.is_empty() {
+                        temporary_errors.push_str("; ");
+                    }
+                    let _ = write!(
+                        temporary_errors,
+                        "Failed to delete DKIM signature for record {record} due to concurrent modification, will retry.",
+                    );
+                } else {
+                    return Err(err);
+                }
+            }
+        }
+    }
+
+    if do_refresh
+        && let Err(err) = server
+            .invalidate_caches(CacheInvalidationBuilder::default().with_invalidation(
+                CacheInvalidation::DkimSignature(task.domain_id.document_id()),
+            ))
+            .await
+    {
+        trc::error!(
+            err.caused_by(trc::location!())
+                .details("Failed to invalidate caches after DKIM management task")
+        );
+    }
+
+    if !temporary_errors.is_empty() {
+        Ok(TaskResult::temporary(temporary_errors))
+    } else {
+        let tasks = if let Some(next_transition) = next_transition {
+            vec![Task::DkimManagement(TaskDomainManagement {
+                domain_id: task.domain_id,
+                status: TaskStatus::at(next_transition.timestamp()),
+            })]
+        } else {
+            vec![]
+        };
+
+        Ok(TaskResult::Success(tasks))
+    }
+}
+
+async fn update_signature(
+    server: &Server,
+    signature: RegistryObject<DkimSignature>,
+    new_signature: DkimSignature,
+    name: &str,
+    temporary_errors: &mut String,
+) -> trc::Result<Option<TaskResult>> {
+    match server
+        .registry()
+        .write(RegistryWrite::update(
+            signature.id.id(),
+            &new_signature.into(),
+            &Object {
+                inner: signature.object.into(),
+                revision: signature.revision,
+            },
+        ))
+        .await
+    {
+        Ok(RegistryWriteResult::Success(_)) => Ok(None),
+        Ok(err) => Ok(Some(TaskResult::permanent(format!(
+            "Failed to write DKIM signature for record {name}: {err}"
+        )))),
+        Err(err) => {
+            if err.is_assertion_failure() {
+                if !temporary_errors.is_empty() {
+                    temporary_errors.push_str("; ");
+                }
+                let _ = write!(
+                    temporary_errors,
+                    "Failed to write DKIM signature for record {name} due to concurrent modification, will retry.",
+                );
+                Ok(None)
+            } else {
+                Err(err)
+            }
+        }
+    }
+}
+
+#[cfg(feature = "test_mode")]
+const TEST_RSA_KEY: &str = r#"-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAv9XYXG3uK95115mB4nJ37nGeNe2CrARm1agrbcnSk5oIaEfM
+ZLUR/X8gPzoiNHZcfMZEVR6bAytxUhc5EvZIZrjSuEEeny+fFd/cTvcm3cOUUbIa
+UmSACj0dL2/KwW0LyUaza9z9zor7I5XdIl1M53qVd5GI62XBB76FH+Q0bWPZNkT4
+NclzTLspD/MTpNCCPhySM4Kdg5CuDczTH4aNzyS0TqgXdtw6A4Sdsp97VXT9fkPW
+9rso3lrkpsl/9EQ1mR/DWK6PBmRfIuSFuqnLKY6v/z2hXHxF7IoojfZLa2kZr9Ae
+d4l9WheQOTA19k5r2BmlRw/W9CrgCBo0Sdj+KQIDAQABAoIBAFPChEi/OvnulReB
+ECQWhOUYuNKlFKQU++2YEvZJ4+bMn5UgnE7wfJ1pj2Pr9xlfALz+OMHNrjMxGbaV
+KzdrT2uCkYcf78XjnhuH9gKIiXDUv4L4N+P3u6w8yOx4bFgOS9IjS53yDOPM7SC5
+g6dIg5aigHaHlffqIuFFv4yQMI/+Ai+zBKxS7wRhxK/7nnAuo28fe5MEdp57ho9/
+AGlDNsdg9zCgjwhokwFE3+AaD+bkUFm4gQ1XjkUFrlmnQn8vDQ0i9toEWhCj+UPY
+iOKL63MJnr90MXTXWLHoFj99wBp//mYygbF9Lj8fa28/oa8LWp3Jhb7QeMgH46iv
+3aLHbTECgYEA5M2dAw+nyMw9vYlkMejhwObKYP8Mr/6zcGMLCalYvRJM5iUAM0JI
+H6sM6pV9/nv167cbKocj3xYPdtE7FPOn4132MLM8Ne1f8nPE64Qrcbj5WBXvLnU8
+hpWbwe2Z8h7UUMKx6q4F1/TXYkc3ScxYwfjM4mP/pLsAOgVzRSEEgrUCgYEA1qNQ
+xaQHNWZ1O8WuTnqWd5JSsic6iURAmUcLeFDZY2PWhVoaQ8L/xMQhDYs1FIbLWArW
+4Qq3Ibu8AbSejAKuaJz7Uf26PX+PYVUwAOO0qamCJ8d/qd6So7qWMDyAY2yXI39Y
+1nMqRjr7bkEsggAZao7BKqA7ZtmogjOusBT38iUCgYEA06agJ8TDoKvOMRZ26PRU
+YO0dKLzGL8eclcoI29cbj0rud7aiiMg3j5PbTuUat95TjsjDCIQaWrM9etvxm2AJ
+Xfn9Uu96MyhyKQWOk46f4YMKpMElkARDCPw8KRhx39dE77AqhLyWCz8iPndCXbH6
+KPTOEl4OjYOuof2Is9nnIkECgYBh948RdsnXhNlzm8nwhiGRmBbou+EK8D0v+O5y
+Tyy6IcKzgSnFzgZh8EdJ4EUtBk1f9SqY8wQdgIvSl3daXorusuA/TzkngsaV3YUY
+ktZOLlF7CKLrjOyPkMWmZKcROmpNyH1q/IvKHHfQnizLdXIkYd4nL5WNX0F7lE1i
+j1+QhQKBgB2lviBK7rJFwlFYdQUP1NAN2dKxMZk8uJS8JglHrM0+8nRI83HbTdEQ
+vB0ManEKBkbS4T5n+gRtdEqKSDmWDTXDlrBfcdCHNQLwYtBpOotCqQn/AmfjcPBl
+byAbwh4+HiZ5JISoRZpiZqy67aJNVoXmdtb/E9mi7ozzytpxMNql
+-----END RSA PRIVATE KEY-----
+"#;
+
+#[cfg(feature = "test_mode")]
+const TEST_ED25519_KEY: &str = r#"-----BEGIN PRIVATE KEY-----
+MC4CAQAwBQYDK2VwBCIEIAO3hAf144lTAVjTkht3ZwBTK0CMCCd1bI0alggneN3B
+-----END PRIVATE KEY-----
+"#;

crates/services/src/task_manager/dns.rs

@@ -0,0 +1,205 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::TaskResult;
+use common::Server;
+use dns_update::{CAARecord, DnsRecord, DnsRecordType, Error as DnsUpdateError, KeyValue};
+use registry::schema::structs::{
+    DnsManagement, Domain, Task, TaskDnsManagement, TaskDomainManagement, TaskStatus,
+};
+use std::fmt::Write;
+use store::ahash::AHashMap;
+use trc::DnsEvent;
+
+pub(crate) trait DnsManagementTask: Sync + Send {
+    fn dns_management(&self, task: &TaskDnsManagement) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl DnsManagementTask for Server {
+    async fn dns_management(&self, task: &TaskDnsManagement) -> TaskResult {
+        match dns_management(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.caused_by(trc::location!())
+                        .details("Failed to run DNS management task")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Result<TaskResult> {
+    if task.update_records.is_empty() {
+        return Ok(TaskResult::permanent(
+            "No DNS records to update".to_string(),
+        ));
+    }
+    let Some(domain) = server.registry().object::<Domain>(task.domain_id).await? else {
+        return Ok(TaskResult::permanent("Domain not found".to_string()));
+    };
+    let DnsManagement::Automatic(props) = &domain.dns_management else {
+        return Ok(TaskResult::permanent(
+            "Domain is not set to automatic DNS management".to_string(),
+        ));
+    };
+    let dns_updater = match server.build_dns_updater(props.dns_server_id).await? {
+        Ok(updater) => updater,
+        Err(err) => {
+            return Ok(TaskResult::permanent(format!(
+                "Failed to build DNS updater: {}",
+                err
+            )));
+        }
+    };
+    let origin = props.origin.as_deref().unwrap_or(&domain.name);
+    let records = server
+        .build_dns_records(task.domain_id, &domain, task.update_records.as_slice())
+        .await?;
+
+    // Group records by (name, type) so each RRSet is published in one call.
+    let mut by_owner: AHashMap<(String, DnsRecordType), Vec<DnsRecord>> = AHashMap::new();
+    for record in records {
+        by_owner
+            .entry((record.name, record.record.as_type()))
+            .or_default()
+            .push(record.record);
+    }
+
+    let mut errors = String::new();
+    for ((name, record_type), mut recs) in by_owner {
+        let preserve_unrelated = match record_type {
+            DnsRecordType::TXT => !is_owned_txt_name(&name),
+            DnsRecordType::CAA => true,
+            _ => false,
+        };
+        if preserve_unrelated {
+            match dns_updater.list_rrset(origin, &name, record_type).await {
+                Ok(existing) => {
+                    for existing_rec in existing {
+                        if !recs.iter().any(|new| same_rrset_family(new, &existing_rec)) {
+                            recs.push(existing_rec);
+                        }
+                    }
+                }
+                Err(DnsUpdateError::Unsupported(reason)) => {
+                    trc::event!(
+                        Dns(DnsEvent::RecordLookupFailed),
+                        Hostname = name.clone(),
+                        Details = origin.to_string(),
+                        Type = record_type.as_str(),
+                        Reason = format!(
+                            "DNS provider cannot list RRSet, unrelated records at this name may be overwritten: {reason}"
+                        ),
+                    );
+                }
+                Err(err) => {
+                    trc::event!(
+                        Dns(DnsEvent::RecordLookupFailed),
+                        Hostname = name.clone(),
+                        Details = origin.to_string(),
+                        Type = record_type.as_str(),
+                        Reason = format!("DNS provider failed to list RRSet: {err}"),
+                    );
+                }
+            }
+        }
+
+        if let Err(err) = dns_updater
+            .set_rrset(origin, &name, record_type, recs)
+            .await
+        {
+            if !errors.is_empty() {
+                errors.push_str("; ");
+            }
+            let _ = write!(
+                &mut errors,
+                "Failed to set DNS RRSet for {}/{}: {}",
+                name,
+                record_type.as_str(),
+                err
+            );
+        }
+    }
+
+    if errors.is_empty() {
+        if task.on_success_renew_certificate {
+            Ok(TaskResult::Success(vec![Task::AcmeRenewal(
+                TaskDomainManagement {
+                    domain_id: task.domain_id,
+                    status: TaskStatus::now(),
+                },
+            )]))
+        } else {
+            Ok(TaskResult::Success(vec![]))
+        }
+    } else {
+        Ok(TaskResult::permanent(errors))
+    }
+}
+
+fn same_rrset_family(a: &DnsRecord, b: &DnsRecord) -> bool {
+    match (a, b) {
+        (DnsRecord::TXT(_), DnsRecord::TXT(_)) => same_txt_family(a, b),
+        (DnsRecord::CAA(_), DnsRecord::CAA(_)) => same_caa_family(a, b),
+        _ => false,
+    }
+}
+
+fn same_txt_family(a: &DnsRecord, b: &DnsRecord) -> bool {
+    match (a, b) {
+        (DnsRecord::TXT(va), DnsRecord::TXT(vb)) => match (txt_family(va), txt_family(vb)) {
+            (Some(fa), Some(fb)) => fa.eq_ignore_ascii_case(fb),
+            _ => false,
+        },
+        _ => false,
+    }
+}
+
+fn same_caa_family(a: &DnsRecord, b: &DnsRecord) -> bool {
+    match (a, b) {
+        (DnsRecord::CAA(ca), DnsRecord::CAA(cb)) => match (ca, cb) {
+            (CAARecord::Issue { options: oa, .. }, CAARecord::Issue { options: ob, .. })
+            | (
+                CAARecord::IssueWild { options: oa, .. },
+                CAARecord::IssueWild { options: ob, .. },
+            ) => match (caa_account_uri(oa), caa_account_uri(ob)) {
+                (Some(ua), Some(ub)) => ua.eq_ignore_ascii_case(ub),
+                _ => false,
+            },
+            (CAARecord::Iodef { url: ua, .. }, CAARecord::Iodef { url: ub, .. }) => {
+                ua.eq_ignore_ascii_case(ub)
+            }
+            _ => false,
+        },
+        _ => false,
+    }
+}
+
+fn caa_account_uri(options: &[KeyValue]) -> Option<&str> {
+    options
+        .iter()
+        .find(|kv| kv.key.eq_ignore_ascii_case("accounturi"))
+        .map(|kv| kv.value.as_str())
+}
+
+fn txt_family(value: &str) -> Option<&str> {
+    value.trim_start().strip_prefix("v=").map(|rest| {
+        rest.split_once([';', ' '])
+            .map_or(rest, |(family, _)| family)
+    })
+}
+
+fn is_owned_txt_name(name: &str) -> bool {
+    name.contains("_dmarc.")
+        || name.contains("_smtp._tls.")
+        || name.contains("_mta-sts.")
+        || name.contains("_ua-auto-config.")
+        || name.contains("_validation-persist.")
+        || name.contains("._domainkey.")
+}

crates/services/src/task_manager/imip.rs

@@ -0,0 +1,612 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::TaskResult;
+use calcard::icalendar::{ICalendarParticipationStatus, ICalendarProperty};
+use common::{
+    DEFAULT_LOGO_BASE64, Server,
+    auth::AccountInfo,
+    config::groupware::CalendarTemplateVariable,
+    network::{ServerInstance, stream::NullIo},
+};
+use groupware::{
+    calendar::itip::ItipIngest,
+    scheduling::{
+        ItipSummary, ItipValue,
+        format::{DateStyle, TextFormatter, hyperlink},
+    },
+};
+use mail_builder::{
+    MessageBuilder,
+    headers::{HeaderType, content_type::ContentType},
+    mime::{BodyPart, MimePart},
+};
+use mail_parser::decoders::html::html_to_text;
+use registry::{schema::structs::TaskCalendarItipMessage, types::EnumImpl};
+use smtp::core::{Session, SessionData};
+use smtp_proto::{MailFrom, RcptTo};
+use std::{sync::Arc, time::Duration};
+use store::{ahash::AHashMap, write::now};
+use trc::AddContext;
+use utils::template::{Variable, Variables};
+
+pub(crate) trait SendImipTask: Sync + Send {
+    fn send_imip(
+        &self,
+        task: &TaskCalendarItipMessage,
+        server_instance: Arc<ServerInstance>,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl SendImipTask for Server {
+    async fn send_imip(
+        &self,
+        task: &TaskCalendarItipMessage,
+        server_instance: Arc<ServerInstance>,
+    ) -> TaskResult {
+        match send_imip(self, task, server_instance).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(task.account_id.document_id())
+                        .document_id(task.document_id.document_id())
+                        .caused_by(trc::location!())
+                        .details("Failed to send iMIP message")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn send_imip(
+    server: &Server,
+    imip: &TaskCalendarItipMessage,
+    server_instance: Arc<ServerInstance>,
+) -> trc::Result<TaskResult> {
+    // Obtain iMIP payload
+    let account_id = imip.account_id.document_id();
+    let document_id = imip.document_id.document_id();
+
+    let sender_domain = imip
+        .messages
+        .iter()
+        .next()
+        .and_then(|msg| msg.from.rsplit('@').next())
+        .unwrap_or("localhost");
+
+    // Obtain logo image
+    let logo = match server.logo_resource(sender_domain).await {
+        Ok(logo) => logo,
+        Err(err) => {
+            trc::error!(
+                err.caused_by(trc::location!())
+                    .details("Failed to fetch logo image")
+            );
+            None
+        }
+    };
+    let logo_cid = format!("logo.{}@{sender_domain}", now());
+    let logo = if let Some(logo) = &logo {
+        MimePart::new(
+            ContentType::new(logo.content_type.as_ref()),
+            BodyPart::Binary(logo.contents.as_slice().into()),
+        )
+    } else {
+        MimePart::new(
+            ContentType::new("image/png"),
+            BodyPart::Binary(DEFAULT_LOGO_BASE64.as_bytes().into()),
+        )
+        .transfer_encoding("base64")
+    }
+    .inline()
+    .cid(&logo_cid);
+
+    let account_info = server
+        .account_info(account_id)
+        .await
+        .caused_by(trc::location!())?;
+
+    for itip_message in imip.messages.iter() {
+        let Ok(summary) = serde_json::from_str::<ItipSummary>(&itip_message.summary) else {
+            return Ok(TaskResult::permanent(
+                "Failed to parse iMIP message summary.",
+            ));
+        };
+
+        let organizer_info = match server
+            .account_id_from_email(itip_message.from.as_str(), true)
+            .await
+        {
+            Ok(Some(sender_id)) if sender_id != account_id => {
+                match server.account_info(sender_id).await {
+                    Ok(info) => Some(info),
+                    Err(err) => {
+                        trc::error!(
+                            err.account_id(account_id)
+                                .document_id(document_id)
+                                .caused_by(trc::location!())
+                                .details("Failed to load organizer account for iMIP sender")
+                        );
+                        None
+                    }
+                }
+            }
+            Ok(_) => None,
+            Err(err) => {
+                trc::error!(
+                    err.account_id(account_id)
+                        .document_id(document_id)
+                        .caused_by(trc::location!())
+                        .details("Failed to resolve organizer account for iMIP sender")
+                );
+                None
+            }
+        };
+        let sender_info = organizer_info.as_ref().unwrap_or(&account_info);
+
+        for recipient in itip_message.to.iter() {
+            // Build template
+            let tpl = build_itip_template(
+                server,
+                &account_info,
+                account_id,
+                document_id,
+                itip_message.from.as_str(),
+                recipient.as_str(),
+                &summary,
+                &logo_cid,
+            )
+            .await?;
+            let txt_body = html_to_text(&tpl.body);
+
+            // Build message
+            let message = MessageBuilder::new()
+                .from((
+                    sender_info.description().unwrap_or(sender_info.name()),
+                    itip_message.from.as_str(),
+                ))
+                .to(recipient.as_str())
+                .header("Auto-Submitted", HeaderType::Text("auto-generated".into()))
+                .header(
+                    "Reply-To",
+                    HeaderType::Text(itip_message.from.as_str().into()),
+                )
+                .message_id(server.core.network.message_id())
+                .subject(&tpl.subject)
+                .body(MimePart::new(
+                    ContentType::new("multipart/mixed"),
+                    BodyPart::Multipart(vec![
+                        MimePart::new(
+                            ContentType::new("multipart/related"),
+                            BodyPart::Multipart(vec![
+                                MimePart::new(
+                                    ContentType::new("multipart/alternative"),
+                                    BodyPart::Multipart(vec![
+                                        MimePart::new(
+                                            ContentType::new("text/plain"),
+                                            BodyPart::Text(txt_body.into()),
+                                        ),
+                                        MimePart::new(
+                                            ContentType::new("text/html"),
+                                            BodyPart::Text(tpl.body.as_str().into()),
+                                        ),
+                                        MimePart::new(
+                                            ContentType::new("text/calendar")
+                                                .attribute("method", summary.method())
+                                                .attribute("charset", "utf-8"),
+                                            BodyPart::Text(
+                                                itip_message.i_calendar_data.as_str().into(),
+                                            ),
+                                        ),
+                                    ]),
+                                ),
+                                logo.clone(),
+                            ]),
+                        ),
+                        MimePart::new(
+                            ContentType::new("application/ics").attribute("name", "event.ics"),
+                            BodyPart::Text(itip_message.i_calendar_data.as_str().into()),
+                        )
+                        .attachment("event.ics"),
+                    ]),
+                ))
+                .write_to_vec()
+                .unwrap_or_default();
+
+            // Send message
+            let server_ = server.clone();
+            let server_instance = server_instance.clone();
+            let sender_info = sender_info.clone();
+            let from = itip_message.from.to_string();
+            let to = recipient.to_string();
+            tokio::spawn(async move {
+                let mut session = Session::<NullIo>::local(
+                    server_,
+                    server_instance,
+                    SessionData::local(sender_info, None, vec![], vec![], 0),
+                );
+
+                // MAIL FROM
+                let _ = session
+                    .handle_mail_from(MailFrom {
+                        address: from.as_str().into(),
+                        ..Default::default()
+                    })
+                    .await;
+                if let Some(error) = session.has_failed() {
+                    trc::event!(
+                        Calendar(trc::CalendarEvent::ItipMessageError),
+                        AccountId = account_id,
+                        DocumentId = document_id,
+                        From = from,
+                        To = to,
+                        Reason = format!("Server rejected MAIL-FROM: {}", error.trim()),
+                    );
+                    return;
+                }
+
+                // RCPT TO
+                session.params.rcpt_errors_wait = Duration::from_secs(0);
+                let _ = session
+                    .handle_rcpt_to(RcptTo {
+                        address: to.as_str().into(),
+                        ..Default::default()
+                    })
+                    .await;
+                if let Some(error) = session.has_failed() {
+                    trc::event!(
+                        Calendar(trc::CalendarEvent::ItipMessageError),
+                        AccountId = account_id,
+                        DocumentId = document_id,
+                        From = from,
+                        To = to,
+                        Reason = format!("Server rejected RCPT-TO: {}", error.trim()),
+                    );
+                    return;
+                }
+
+                // DATA
+                session.data.message = message;
+                let response = session.queue_message().await;
+                if let smtp::core::State::Accepted(queue_id) = session.state {
+                    trc::event!(
+                        Calendar(trc::CalendarEvent::ItipMessageSent),
+                        From = from,
+                        To = to,
+                        AccountId = account_id,
+                        DocumentId = document_id,
+                        QueueId = queue_id,
+                    );
+                } else {
+                    trc::event!(
+                        Calendar(trc::CalendarEvent::ItipMessageError),
+                        From = from,
+                        To = to,
+                        AccountId = account_id,
+                        DocumentId = document_id,
+                        Reason = format!(
+                            "Server rejected DATA: {}",
+                            std::str::from_utf8(&response).unwrap().trim()
+                        ),
+                    );
+                }
+            })
+            .await
+            .map_err(|_| {
+                trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError))
+                    .caused_by(trc::location!())
+            })?;
+        }
+    }
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+pub struct Details {
+    pub subject: String,
+    pub body: String,
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn build_itip_template(
+    server: &Server,
+    account_info: &AccountInfo,
+    account_id: u32,
+    document_id: u32,
+    from: &str,
+    to: &str,
+    summary: &ItipSummary,
+    logo_cid: &str,
+) -> trc::Result<Details> {
+    #[cfg(not(feature = "enterprise"))]
+    let template = &server.core.groupware.itip_template;
+    let formatter = TextFormatter::new(account_info.locale().as_str())?;
+    let locale = formatter.locale;
+
+    let mut variables = Variables::new();
+    let mut subject;
+    let (fields, old_fields) = match summary {
+        ItipSummary::Invite(fields) => {
+            subject = format!("{}: ", locale.calendar_invitation);
+
+            (fields, None)
+        }
+        ItipSummary::Update {
+            current, previous, ..
+        } => {
+            subject = format!("{}: ", locale.calendar_updated_invitation);
+            variables.insert_single(
+                CalendarTemplateVariable::Header,
+                locale.calendar_event_updated.to_string(),
+            );
+            variables.insert_single(CalendarTemplateVariable::Color, "info".to_string());
+            (current, Some(previous))
+        }
+        ItipSummary::Cancel(fields) => {
+            subject = format!("{}: ", locale.calendar_cancelled);
+            variables.insert_single(
+                CalendarTemplateVariable::Header,
+                locale.calendar_event_cancelled.to_string(),
+            );
+            variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string());
+            (fields, None)
+        }
+        ItipSummary::Rsvp { part_stat, current } => {
+            let (color, value) = match part_stat {
+                ICalendarParticipationStatus::Accepted => {
+                    subject = format!("{}: ", locale.calendar_accepted);
+
+                    (
+                        "info",
+                        locale.calendar_participant_accepted.replace("$name", from),
+                    )
+                }
+                ICalendarParticipationStatus::Declined => {
+                    subject = format!("{}: ", locale.calendar_declined);
+                    (
+                        "danger",
+                        locale.calendar_participant_declined.replace("$name", from),
+                    )
+                }
+                ICalendarParticipationStatus::Tentative => {
+                    subject = format!("{}: ", locale.calendar_tentative);
+                    (
+                        "warning",
+                        locale.calendar_participant_tentative.replace("$name", from),
+                    )
+                }
+                ICalendarParticipationStatus::Delegated => {
+                    subject = format!("{}: ", locale.calendar_delegated);
+                    (
+                        "warning",
+                        locale.calendar_participant_delegated.replace("$name", from),
+                    )
+                }
+                _ => {
+                    subject = format!("{}: ", locale.calendar_reply);
+                    (
+                        "info",
+                        locale.calendar_participant_reply.replace("$name", from),
+                    )
+                }
+            };
+
+            variables.insert_single(CalendarTemplateVariable::Header, value);
+            variables.insert_single(CalendarTemplateVariable::Color, color.to_string());
+
+            (current, None)
+        }
+    };
+
+    let mut when_detail: Option<(usize, &ItipValue)> = None;
+    let mut details: Vec<AHashMap<CalendarTemplateVariable, String>> = Vec::with_capacity(4);
+    for field in [
+        ICalendarProperty::Summary,
+        ICalendarProperty::Description,
+        ICalendarProperty::Dtstart,
+        ICalendarProperty::Rrule,
+        ICalendarProperty::Location,
+        ICalendarProperty::Conference,
+    ] {
+        let mut old_entries = old_fields.into_iter().flatten().filter(|e| e.name == field);
+
+        for entry in fields.iter().filter(|e| e.name == field) {
+            let field_name = match &field {
+                ICalendarProperty::Summary => locale.calendar_summary,
+                ICalendarProperty::Description => locale.calendar_description,
+                ICalendarProperty::Dtstart | ICalendarProperty::Rrule => locale.calendar_when,
+                ICalendarProperty::Location => locale.calendar_location,
+                ICalendarProperty::Conference => locale.calendar_conference,
+                _ => continue,
+            };
+            let value = formatter.field_to_string(&entry.value, DateStyle::Long);
+
+            let old_entry = old_entries.next();
+
+            match &field {
+                ICalendarProperty::Summary => {
+                    subject.push_str(&value);
+                }
+                ICalendarProperty::Dtstart => {
+                    subject.push_str(" @ ");
+                    subject.push_str(&value);
+                }
+                ICalendarProperty::Rrule if when_detail.is_none() => {
+                    subject.push_str(" @ ");
+                    subject.push_str(&value);
+                }
+                _ => (),
+            }
+
+            if let ICalendarProperty::Summary | ICalendarProperty::Description = &field {
+                let variable = if matches!(field, ICalendarProperty::Summary) {
+                    CalendarTemplateVariable::EventTitle
+                } else {
+                    CalendarTemplateVariable::EventDescription
+                };
+
+                if old_entry.is_none() {
+                    variables.insert_single(variable, value);
+                    continue;
+                }
+                variables.insert_single(variable, value.clone());
+            }
+
+            if matches!(field, ICalendarProperty::Rrule)
+                && let Some((index, start_value)) = when_detail
+                && let Some(detail) = details.get_mut(index)
+            {
+                if let Some(when_value) = detail.get_mut(&CalendarTemplateVariable::Value) {
+                    when_value.push_str(", ");
+                    when_value.push_str(&value);
+                }
+
+                if let Some(old_entry) = old_entry {
+                    detail.insert(
+                        CalendarTemplateVariable::Changed,
+                        locale.calendar_changed.to_string(),
+                    );
+                    let old_value = detail
+                        .entry(CalendarTemplateVariable::OldValue)
+                        .or_insert_with(|| {
+                            formatter.field_to_string(start_value, DateStyle::Short)
+                        });
+                    old_value.push_str(", ");
+                    old_value
+                        .push_str(&formatter.field_to_string(&old_entry.value, DateStyle::Short));
+                }
+
+                continue;
+            }
+
+            let mut detail = AHashMap::with_capacity(4);
+            detail.insert(CalendarTemplateVariable::Key, field_name.to_string());
+            if matches!(field, ICalendarProperty::Conference)
+                && let Some(link) = hyperlink(&value)
+            {
+                detail.insert(CalendarTemplateVariable::Link, link.to_string());
+            }
+            detail.insert(CalendarTemplateVariable::Value, value);
+            if let Some(old_entry) = old_entry {
+                detail.insert(
+                    CalendarTemplateVariable::Changed,
+                    locale.calendar_changed.to_string(),
+                );
+                detail.insert(
+                    CalendarTemplateVariable::OldValue,
+                    formatter.field_to_string(&old_entry.value, DateStyle::Short),
+                );
+            }
+            if matches!(field, ICalendarProperty::Dtstart) && when_detail.is_none() {
+                when_detail = Some((details.len(), &entry.value));
+            }
+            details.push(detail);
+        }
+    }
+    if !details.is_empty() {
+        variables.items.insert(
+            CalendarTemplateVariable::EventDetails,
+            Variable::Block(details),
+        );
+    }
+    variables.insert_single(CalendarTemplateVariable::PageTitle, subject.clone());
+    variables.insert_single(CalendarTemplateVariable::Lang, locale.name.to_string());
+    variables.insert_single(CalendarTemplateVariable::Dir, locale.direction.to_string());
+    variables.insert_single(CalendarTemplateVariable::LogoCid, format!("cid:{logo_cid}"));
+
+    if let Some(guests) = fields
+        .iter()
+        .find(|e| e.name == ICalendarProperty::Attendee)
+        && let ItipValue::Participants(guests) = &guests.value
+    {
+        variables.insert_single(
+            CalendarTemplateVariable::AttendeesTitle,
+            locale.calendar_attendees.to_string(),
+        );
+        variables.insert_block(
+            CalendarTemplateVariable::Attendees,
+            guests.iter().map(|guest| {
+                [
+                    (
+                        CalendarTemplateVariable::Key,
+                        if guest.is_organizer {
+                            if let Some(name) = guest.name.as_ref() {
+                                format!("{name} - {}", locale.calendar_organizer)
+                            } else {
+                                locale.calendar_organizer.to_string()
+                            }
+                        } else {
+                            guest.name.as_deref().unwrap_or_default().to_string()
+                        },
+                    ),
+                    (CalendarTemplateVariable::Value, guest.email.to_string()),
+                ]
+            }),
+        );
+    }
+
+    // Add RSVP buttons
+    if matches!(summary, ItipSummary::Invite(_) | ItipSummary::Update { .. })
+        && let Some(rsvp_url) = server
+            .http_rsvp_url(account_id, account_info.name(), document_id, to)
+            .await
+    {
+        variables.insert_single(
+            CalendarTemplateVariable::Rsvp,
+            locale.calendar_reply_as.replace("$name", to),
+        );
+        variables.insert_block(
+            CalendarTemplateVariable::Actions,
+            [
+                (
+                    ICalendarParticipationStatus::Accepted,
+                    locale.calendar_yes.to_string(),
+                    "info",
+                ),
+                (
+                    ICalendarParticipationStatus::Declined,
+                    locale.calendar_no.to_string(),
+                    "danger",
+                ),
+                (
+                    ICalendarParticipationStatus::Tentative,
+                    locale.calendar_maybe.to_string(),
+                    "warning",
+                ),
+            ]
+            .into_iter()
+            .map(|(status, title, color)| {
+                [
+                    (CalendarTemplateVariable::ActionName, title.to_string()),
+                    (CalendarTemplateVariable::ActionUrl, rsvp_url.url(&status)),
+                    (CalendarTemplateVariable::Color, color.to_string()),
+                ]
+            }),
+        );
+    }
+
+    // Add footer
+    variables.insert_block(
+        CalendarTemplateVariable::Footer,
+        [
+            [(
+                CalendarTemplateVariable::Key,
+                locale.calendar_imip_footer_1.to_string(),
+            )],
+            [(
+                CalendarTemplateVariable::Key,
+                locale.calendar_imip_footer_2.to_string(),
+            )],
+        ],
+    );
+
+    Ok(Details {
+        subject,
+        body: template.eval(&variables),
+    })
+}

crates/services/src/task_manager/index.rs

@@ -0,0 +1,610 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::{Task, TaskDetails, TaskFailureType, TaskResult};
+use common::Server;
+use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata};
+use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard};
+use registry::{
+    schema::{
+        enums::IndexDocumentType,
+        prelude::{ObjectType, Property},
+        structs::{TaskIndexDocument, TaskIndexTrace, TaskStatus},
+    },
+    types::EnumImpl,
+};
+use std::cmp::Ordering;
+use store::{
+    IterateParams, ValueKey,
+    ahash::AHashMap,
+    rand::{self, RngExt},
+    search::{IndexDocument, SearchField, SearchFilter, SearchQuery},
+    write::{
+        AlignedBytes, Archive, BatchBuilder, SearchIndex, TelemetryClass, ValueClass,
+        key::DeserializeBigEndian, now,
+    },
+};
+use trc::{AddContext, TaskManagerEvent};
+use types::{
+    blob_hash::BlobHash,
+    collection::{Collection, SyncCollection},
+    field::EmailField,
+};
+
+pub(crate) trait SearchIndexTask: Sync + Send {
+    fn index(&self, tasks: &[TaskDetails]) -> impl Future<Output = Vec<IndexTaskResult>> + Send;
+}
+
+const NUM_INDEXES: usize = 5;
+const MISSING_DOCUMENT_MAX_ATTEMPTS: u64 = 3;
+const MISSING_DOCUMENT_RETRY_DELAY: u64 = 5;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum TaskType {
+    Insert,
+    Delete,
+}
+
+enum BuildResult {
+    Document(IndexDocument),
+    NotIndexed,
+    NotFound,
+}
+
+#[derive(Debug)]
+pub(crate) struct IndexTaskResult {
+    index: IndexDocumentType,
+    task_type: TaskType,
+    pub result: TaskResult,
+}
+
+impl SearchIndexTask for Server {
+    async fn index(&self, tasks: &[TaskDetails]) -> Vec<IndexTaskResult> {
+        let mut results: Vec<IndexTaskResult> = Vec::with_capacity(tasks.len());
+        let mut batch = BatchBuilder::new();
+        let mut document_insertions = Vec::new();
+        let mut document_deletions: [AHashMap<u32, Vec<u32>>; NUM_INDEXES] =
+            std::array::from_fn(|_| AHashMap::new());
+
+        for task in tasks {
+            match &task.task {
+                Task::IndexDocument(task) => {
+                    let account_id = task.account_id.document_id();
+                    let document_id = task.document_id.document_id();
+
+                    let document = match task.document_type {
+                        IndexDocumentType::Email => {
+                            build_email_document(self, account_id, document_id).await
+                        }
+                        IndexDocumentType::Calendar => {
+                            build_calendar_document(self, account_id, document_id).await
+                        }
+                        IndexDocumentType::Contacts => {
+                            build_contact_document(self, account_id, document_id).await
+                        }
+                        IndexDocumentType::File => {
+                            // File indexing not implemented yet
+                            continue;
+                        }
+                    };
+
+                    // Retry non found errors in case they are due to SQL read replication lag
+                    let result = match document {
+                        Ok(BuildResult::Document(doc)) if !doc.is_empty() => {
+                            document_insertions.push(doc);
+                            TaskResult::Success(vec![])
+                        }
+                        Err(err) => {
+                            let result = TaskResult::temporary(err.to_string());
+                            trc::error!(
+                                err.account_id(account_id)
+                                    .document_id(document_id)
+                                    .caused_by(trc::location!())
+                                    .ctx(trc::Key::Collection, task.document_type.as_str())
+                                    .details("Failed to build document for indexing")
+                            );
+                            result
+                        }
+                        Ok(BuildResult::NotFound)
+                            if attempt_number(&task.status) < MISSING_DOCUMENT_MAX_ATTEMPTS =>
+                        {
+                            TaskResult::Failure {
+                                typ: TaskFailureType::Retry(
+                                    now().saturating_add(MISSING_DOCUMENT_RETRY_DELAY),
+                                ),
+                                message: "Document not found in data store".into(),
+                                max_attempts: Some(MISSING_DOCUMENT_MAX_ATTEMPTS),
+                            }
+                        }
+                        Ok(BuildResult::NotFound) => {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskIgnored),
+                                Collection = task.document_type.as_str(),
+                                Reason = "Document no longer exists",
+                                AccountId = account_id,
+                                DocumentId = document_id,
+                            );
+                            TaskResult::Ignored
+                        }
+                        _ => {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskIgnored),
+                                Collection = task.document_type.as_str(),
+                                Reason = "Nothing to index",
+                                AccountId = account_id,
+                                DocumentId = document_id,
+                            );
+                            TaskResult::Ignored
+                        }
+                    };
+
+                    results.push(IndexTaskResult {
+                        task_type: TaskType::Insert,
+                        index: task.document_type,
+                        result,
+                    });
+                }
+                Task::IndexTrace(task) => {
+                    let result = match build_tracing_span_document(self, task.trace_id.id()).await {
+                        Ok(Some(doc)) if !doc.is_empty() => {
+                            document_insertions.push(doc);
+                            TaskResult::Success(vec![])
+                        }
+                        Err(err) => {
+                            let result = TaskResult::temporary(err.to_string());
+                            trc::error!(
+                                err.id(task.trace_id.id())
+                                    .caused_by(trc::location!())
+                                    .details("Failed to build document for indexing")
+                            );
+                            result
+                        }
+                        _ => {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskIgnored),
+                                Reason = "Nothing to index",
+                                Id = task.trace_id.id(),
+                            );
+                            TaskResult::Ignored
+                        }
+                    };
+
+                    results.push(IndexTaskResult {
+                        task_type: TaskType::Insert,
+                        index: IndexDocumentType::File, // use File index for tracing spans to avoid creating a new index type
+                        result,
+                    });
+                }
+                Task::UnindexDocument(task) => {
+                    let account_id = task.account_id.document_id();
+                    let document_id = task.document_id.document_id();
+                    let idx = match task.document_type {
+                        IndexDocumentType::Email => {
+                            if let Err(err) =
+                                delete_email_metadata(self, &mut batch, account_id, document_id)
+                                    .await
+                            {
+                                trc::error!(
+                                    err.account_id(account_id)
+                                        .document_id(document_id)
+                                        .caused_by(trc::location!())
+                                        .details("Failed to delete email metadata from index")
+                                );
+                                results.push(IndexTaskResult {
+                                    task_type: TaskType::Delete,
+                                    index: task.document_type,
+                                    result: TaskResult::temporary(
+                                        "Failed to delete email metadata from index",
+                                    ),
+                                });
+                                continue;
+                            }
+                            0
+                        }
+                        IndexDocumentType::Calendar => 1,
+                        IndexDocumentType::Contacts => 2,
+                        IndexDocumentType::File => 3,
+                    };
+
+                    document_deletions[idx]
+                        .entry(account_id)
+                        .or_default()
+                        .push(document_id);
+
+                    results.push(IndexTaskResult {
+                        task_type: TaskType::Delete,
+                        index: task.document_type,
+                        result: TaskResult::Success(vec![]),
+                    });
+                }
+                _ => unreachable!(),
+            }
+        }
+
+        // Commit deletion batch to data store
+        if !batch.is_empty()
+            && let Err(err) = self.store().write(batch.build_all()).await
+        {
+            trc::error!(
+                err.caused_by(trc::location!())
+                    .details("Failed to commit index deletions to data store")
+            );
+            for r in results.iter_mut() {
+                if r.task_type == TaskType::Delete
+                    && r.result.is_success()
+                    && r.index == IndexDocumentType::Email
+                {
+                    r.result =
+                        TaskResult::temporary("Failed to commit index deletions to data store");
+                }
+            }
+            return results;
+        }
+
+        // Index documents
+        if !document_insertions.is_empty()
+            && let Err(err) = self.search_store().index(document_insertions).await
+        {
+            let retry_at = deferred_retry_time(&err);
+            trc::error!(
+                err.caused_by(trc::location!())
+                    .details("Failed to index documents")
+            );
+            for r in results.iter_mut() {
+                if r.task_type == TaskType::Insert && r.result.is_success() {
+                    r.result = search_store_failure(retry_at, "Failed to index documents");
+                }
+            }
+            return results;
+        }
+
+        // Delete documents
+        for (accounts, index) in document_deletions.into_iter().zip([
+            SearchIndex::Email,
+            SearchIndex::Calendar,
+            SearchIndex::Contacts,
+        ]) {
+            let multi_account = match accounts.len().cmp(&1) {
+                Ordering::Greater => true,
+                Ordering::Equal => false,
+                Ordering::Less => continue,
+            };
+
+            let mut query = SearchQuery::new(index);
+            if multi_account {
+                query.add_filter(SearchFilter::Or);
+            }
+
+            for (account_id, document_ids) in accounts {
+                let multi_document = document_ids.len() > 1;
+                query
+                    .add_filter(SearchFilter::And)
+                    .add_filter(SearchFilter::eq(SearchField::AccountId, account_id));
+
+                if multi_document {
+                    query.add_filter(SearchFilter::Or);
+                }
+
+                for document_id in document_ids {
+                    query.add_filter(SearchFilter::eq(SearchField::DocumentId, document_id));
+                }
+
+                if multi_document {
+                    query.add_filter(SearchFilter::End);
+                }
+                query.add_filter(SearchFilter::End);
+            }
+
+            if multi_account {
+                query.add_filter(SearchFilter::End);
+            }
+
+            if let Err(err) = self.search_store().unindex(query).await {
+                let retry_at = deferred_retry_time(&err);
+                trc::error!(
+                    err.caused_by(trc::location!())
+                        .details("Failed to delete documents from index")
+                        .ctx(trc::Key::Collection, index.name())
+                );
+                for r in results.iter_mut() {
+                    if r.task_type == TaskType::Delete && r.result.is_success() {
+                        r.result =
+                            search_store_failure(retry_at, "Failed to delete documents from index");
+                    }
+                }
+                return results;
+            }
+        }
+
+        results
+    }
+}
+
+pub(crate) async fn reindex_telemetry(server: &Server) -> trc::Result<()> {
+    let mut spans = Vec::new();
+    server
+        .tracing_store()
+        .iterate(
+            IterateParams::new(
+                ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(0))),
+                ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(u64::MAX))),
+            )
+            .no_values(),
+            |key, _| {
+                spans.push(key.deserialize_be_u64(0)?);
+                Ok(true)
+            },
+        )
+        .await
+        .caused_by(trc::location!())?;
+
+    let mut batch = BatchBuilder::new();
+    let now = now() as i64;
+    for span_id in spans {
+        batch.schedule_task(Task::IndexTrace(TaskIndexTrace {
+            trace_id: span_id.into(),
+            status: TaskStatus::at(now + rand::rng().random_range(0..=300)),
+        }));
+        if batch.is_large_batch() {
+            server.core.storage.data.write(batch.build_all()).await?;
+            batch = BatchBuilder::new();
+        }
+    }
+
+    if !batch.is_empty() {
+        server.core.storage.data.write(batch.build_all()).await?;
+    }
+    Ok(())
+}
+
+pub(crate) async fn reindex_account(server: &Server, account_id: u32) -> trc::Result<()> {
+    let now = now() as i64;
+
+    let mut batch = BatchBuilder::new();
+
+    for document_id in server
+        .get_cached_messages(account_id)
+        .await
+        .caused_by(trc::location!())?
+        .emails
+        .items
+        .iter()
+        .map(|v| v.document_id)
+    {
+        batch.schedule_task(Task::IndexDocument(TaskIndexDocument {
+            account_id: account_id.into(),
+            document_id: document_id.into(),
+            document_type: IndexDocumentType::Email,
+            status: TaskStatus::at(now + rand::rng().random_range(0..=300)),
+        }));
+
+        if batch.is_large_batch() {
+            server.core.storage.data.write(batch.build_all()).await?;
+            batch = BatchBuilder::new();
+        }
+    }
+
+    for document_type in [IndexDocumentType::Calendar, IndexDocumentType::Contacts] {
+        let cache = server
+            .fetch_dav_resources(
+                account_id,
+                account_id,
+                if document_type == IndexDocumentType::Calendar {
+                    SyncCollection::Calendar
+                } else {
+                    SyncCollection::AddressBook
+                },
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        for document_id in cache.document_ids(false) {
+            batch.schedule_task(Task::IndexDocument(TaskIndexDocument {
+                account_id: account_id.into(),
+                document_id: document_id.into(),
+                document_type,
+                status: TaskStatus::at(now + rand::rng().random_range(0..=300)),
+            }));
+
+            if batch.is_large_batch() {
+                server.core.storage.data.write(batch.build_all()).await?;
+                batch = BatchBuilder::new();
+            }
+        }
+    }
+
+    if !batch.is_empty() {
+        server.core.storage.data.write(batch.build_all()).await?;
+    }
+
+    // Request indexing
+    server.notify_task_queue();
+
+    Ok(())
+}
+
+fn deferred_retry_time(err: &trc::Error) -> Option<u64> {
+    err.value(trc::Key::NextRetry)
+        .and_then(|value| value.to_uint())
+}
+
+fn search_store_failure(retry_at: Option<u64>, message: &'static str) -> TaskResult {
+    match retry_at {
+        Some(retry_at) => TaskResult::Failure {
+            typ: TaskFailureType::Retry(retry_at),
+            message: message.into(),
+            max_attempts: None,
+        },
+        None => TaskResult::temporary(message),
+    }
+}
+
+fn attempt_number(status: &TaskStatus) -> u64 {
+    match status {
+        TaskStatus::Pending(_) => 0,
+        TaskStatus::Retry(status) => status.attempt_number,
+        TaskStatus::Failed(status) => status.failed_attempt_number,
+    }
+}
+
+async fn build_email_document(
+    server: &Server,
+    account_id: u32,
+    document_id: u32,
+) -> trc::Result<BuildResult> {
+    let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Email) else {
+        return Ok(BuildResult::NotIndexed);
+    };
+
+    match server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::property(
+            account_id,
+            Collection::Email,
+            document_id,
+            EmailField::Metadata,
+        ))
+        .await?
+    {
+        Some(metadata_) => {
+            let metadata = metadata_
+                .unarchive::<MessageMetadata>()
+                .caused_by(trc::location!())?;
+
+            let raw_message = server
+                .blob_store()
+                .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
+                .await
+                .caused_by(trc::location!())?
+                .ok_or_else(|| {
+                    trc::StoreEvent::NotFound
+                        .into_err()
+                        .details("Blob not found")
+                })?;
+
+            Ok(BuildResult::Document(metadata.index_document(
+                account_id,
+                document_id,
+                &raw_message,
+                index_fields,
+                server.core.email.default_language,
+            )))
+        }
+        None => Ok(BuildResult::NotFound),
+    }
+}
+
+async fn build_calendar_document(
+    server: &Server,
+    account_id: u32,
+    document_id: u32,
+) -> trc::Result<BuildResult> {
+    let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Calendar) else {
+        return Ok(BuildResult::NotIndexed);
+    };
+
+    match server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::CalendarEvent,
+            document_id,
+        ))
+        .await?
+    {
+        Some(metadata_) => Ok(BuildResult::Document(
+            metadata_
+                .unarchive::<CalendarEvent>()
+                .caused_by(trc::location!())?
+                .index_document(
+                    account_id,
+                    document_id,
+                    index_fields,
+                    server.core.email.default_language,
+                ),
+        )),
+        None => Ok(BuildResult::NotFound),
+    }
+}
+
+async fn build_contact_document(
+    server: &Server,
+    account_id: u32,
+    document_id: u32,
+) -> trc::Result<BuildResult> {
+    let Some(index_fields) = server.core.email.index_fields.get(&SearchIndex::Contacts) else {
+        return Ok(BuildResult::NotIndexed);
+    };
+
+    match server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+            account_id,
+            Collection::ContactCard,
+            document_id,
+        ))
+        .await?
+    {
+        Some(metadata_) => Ok(BuildResult::Document(
+            metadata_
+                .unarchive::<ContactCard>()
+                .caused_by(trc::location!())?
+                .index_document(
+                    account_id,
+                    document_id,
+                    index_fields,
+                    server.core.email.default_language,
+                ),
+        )),
+        None => Ok(BuildResult::NotFound),
+    }
+}
+
+
+#[cfg(not(feature = "enterprise"))]
+async fn build_tracing_span_document(_: &Server, _: u64) -> trc::Result<Option<IndexDocument>> {
+    Ok(None)
+}
+
+async fn delete_email_metadata(
+    server: &Server,
+    batch: &mut BatchBuilder,
+    account_id: u32,
+    document_id: u32,
+) -> trc::Result<()> {
+    match server
+        .store()
+        .get_value::<Archive<AlignedBytes>>(ValueKey::property(
+            account_id,
+            Collection::Email,
+            document_id,
+            EmailField::Metadata,
+        ))
+        .await?
+    {
+        Some(metadata_) => {
+            batch
+                .with_account_id(account_id)
+                .with_collection(Collection::Email)
+                .with_document(document_id);
+            let metadata = metadata_
+                .unarchive::<MessageMetadata>()
+                .caused_by(trc::location!())?;
+            metadata.unindex(batch);
+
+        }
+        None => {
+            trc::event!(
+                TaskManager(TaskManagerEvent::MetadataNotFound),
+                Details = "E-mail metadata not found",
+                AccountId = account_id,
+                DocumentId = document_id,
+            );
+        }
+    }
+
+    Ok(())
+}

crates/services/src/task_manager/lock.rs

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::*;
+
+pub trait TaskLockManager: Sync + Send {
+    fn try_lock_task(&self, task: u64) -> impl Future<Output = bool> + Send;
+    fn remove_index_lock(&self, id: u64) -> impl Future<Output = ()> + Send;
+}
+
+impl TaskLockManager for Server {
+    async fn try_lock_task(&self, id: u64) -> bool {
+        match self
+            .in_memory_store()
+            .try_lock(KV_LOCK_TASK, &id.to_be_bytes(), DEFAULT_LOCK_EXPIRY)
+            .await
+        {
+            Ok(result) => {
+                if !result {
+                    trc::event!(
+                        TaskManager(TaskManagerEvent::TaskLocked),
+                        Id = id,
+                        Details = "Task details not available",
+                    );
+                }
+                result
+            }
+            Err(err) => {
+                trc::error!(err.id(id).details("Failed to lock task"));
+
+                false
+            }
+        }
+    }
+
+    async fn remove_index_lock(&self, id: u64) {
+        if let Err(err) = self
+            .in_memory_store()
+            .remove_lock(KV_LOCK_TASK, &id.to_be_bytes())
+            .await
+        {
+            trc::error!(
+                err.details("Failed to unlock task")
+                    .ctx(trc::Key::Id, id)
+                    .caused_by(trc::location!())
+            );
+        }
+    }
+}

crates/services/src/task_manager/maintenance.rs

@@ -0,0 +1,542 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::time::Instant;
+
+use crate::task_manager::{
+    TaskResult,
+    index::{reindex_account, reindex_telemetry},
+};
+use common::{
+    KV_ACME, KV_GREYLIST, KV_LOCK_DAV, KV_LOCK_QUEUE_MESSAGE, KV_LOCK_TASK, KV_OAUTH,
+    KV_QUOTA_BLOB, KV_RATE_LIMIT_AUTH, KV_RATE_LIMIT_CONTACT, KV_RATE_LIMIT_HTTP_ANONYMOUS,
+    KV_RATE_LIMIT_HTTP_AUTHENTICATED, KV_RATE_LIMIT_IMAP, KV_RATE_LIMIT_LOITER, KV_RATE_LIMIT_RCPT,
+    KV_RATE_LIMIT_SCAN, KV_RATE_LIMIT_SMTP, KV_SIEVE_ID, Server,
+    storage::index::ObjectIndexBuilder,
+};
+use email::{
+    cache::MessageCacheFetch,
+    message::{delete::EmailDeletion, ingest::EmailIngest, metadata::MessageData},
+    sieve::SieveScript,
+};
+use groupware::{
+    calendar::{Calendar, CalendarEvent, CalendarEventNotification},
+    contact::{AddressBook, ContactCard},
+    file::FileNode,
+};
+use registry::{
+    schema::{
+        enums::{TaskAccountMaintenanceType, TaskStoreMaintenanceType, TaskTenantMaintenanceType},
+        prelude::{Object, ObjectInner, ObjectType, Property},
+        structs::{
+            Task, TaskAccountMaintenance, TaskStatus, TaskStoreMaintenance, TaskTenantMaintenance,
+        },
+    },
+    types::EnumImpl,
+};
+use smtp::reporting::index::ExternalReportIndex;
+use store::{
+    Serialize, ValueKey,
+    rand::{self},
+    registry::{RegistryFilter, RegistryQuery},
+    roaring::RoaringBitmap,
+    write::{AlignedBytes, Archive, Archiver, BatchBuilder, RegistryClass, ValueClass, now},
+};
+use trc::{AddContext, StoreEvent};
+use types::{
+    collection::Collection,
+    field::{EmailField, MailboxField},
+    id::Id,
+};
+
+pub(crate) trait MaintenanceTask: Sync + Send {
+    fn store_maintenance(
+        &self,
+        task: &TaskStoreMaintenance,
+    ) -> impl Future<Output = TaskResult> + Send;
+    fn account_maintenance(
+        &self,
+        task: &TaskAccountMaintenance,
+    ) -> impl Future<Output = TaskResult> + Send;
+    fn tenant_maintenance(
+        &self,
+        task: &TaskTenantMaintenance,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl MaintenanceTask for Server {
+    async fn store_maintenance(&self, task: &TaskStoreMaintenance) -> TaskResult {
+        match store_maintenance(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(err.details("Failed to perform store maintenance task"));
+                result
+            }
+        }
+    }
+
+    async fn account_maintenance(&self, task: &TaskAccountMaintenance) -> TaskResult {
+        match account_maintenance(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(task.account_id.document_id())
+                        .details("Failed to perform account maintenance task")
+                );
+                result
+            }
+        }
+    }
+
+    async fn tenant_maintenance(&self, task: &TaskTenantMaintenance) -> TaskResult {
+        match tenant_maintenance(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(err.details("Failed to perform tenant maintenance task"));
+                result
+            }
+        }
+    }
+}
+
+async fn store_maintenance(
+    server: &Server,
+    task: &TaskStoreMaintenance,
+) -> trc::Result<TaskResult> {
+    match task.maintenance_type {
+        TaskStoreMaintenanceType::ReindexAccounts
+        | TaskStoreMaintenanceType::PurgeAccounts
+        | TaskStoreMaintenanceType::ResetUserQuotas => {
+            let mut batch = BatchBuilder::new();
+            let now = now() as i64;
+            let maintenance_type = match task.maintenance_type {
+                TaskStoreMaintenanceType::ReindexAccounts => TaskAccountMaintenanceType::Reindex,
+                TaskStoreMaintenanceType::PurgeAccounts => TaskAccountMaintenanceType::Purge,
+                TaskStoreMaintenanceType::ResetUserQuotas => {
+                    TaskAccountMaintenanceType::RecalculateQuota
+                }
+                _ => unreachable!(),
+            };
+            for account_id in server
+                .registry()
+                .query::<RoaringBitmap>(RegistryQuery::new(ObjectType::Account))
+                .await?
+            {
+                #[cfg(feature = "test_mode")]
+                let status = TaskStatus::at(now);
+
+                #[cfg(not(feature = "test_mode"))]
+                let status =
+                    TaskStatus::at(now + rand::RngExt::random_range(&mut rand::rng(), 0..=300));
+
+                batch.schedule_task(Task::AccountMaintenance(TaskAccountMaintenance {
+                    account_id: account_id.into(),
+                    maintenance_type,
+                    status,
+                }));
+
+                if batch.is_large_batch() {
+                    server.core.storage.data.write(batch.build_all()).await?;
+                    server.notify_task_queue();
+                    batch = BatchBuilder::new();
+                }
+            }
+
+            if !batch.is_empty() {
+                server.core.storage.data.write(batch.build_all()).await?;
+                server.notify_task_queue();
+            }
+        }
+        TaskStoreMaintenanceType::ReindexTelemetry => {
+            reindex_telemetry(server).await?;
+        }
+        TaskStoreMaintenanceType::PurgeData => {
+            // Delete expired external reports
+            let now = now();
+            let mut batch = BatchBuilder::new();
+            for object in [
+                ObjectType::DmarcExternalReport,
+                ObjectType::TlsExternalReport,
+                ObjectType::ArfExternalReport,
+            ] {
+                let ids = server
+                    .registry()
+                    .query::<Vec<Id>>(RegistryQuery::new(object).filter(RegistryFilter::less_than(
+                        Property::ExpiresAt,
+                        now,
+                        false,
+                    )))
+                    .await?;
+                let object_id = object.to_id();
+                for id in ids {
+                    let item_id = id.id();
+                    if let Some(report) = server
+                        .store()
+                        .get_value::<Object>(ValueKey::from(ValueClass::Registry(
+                            RegistryClass::Item { object_id, item_id },
+                        )))
+                        .await?
+                    {
+                        match &report.inner {
+                            ObjectInner::DmarcExternalReport(report) => {
+                                report.write_ops(&mut batch, item_id, false);
+                            }
+                            ObjectInner::TlsExternalReport(report) => {
+                                report.write_ops(&mut batch, item_id, false);
+                            }
+                            ObjectInner::ArfExternalReport(report) => {
+                                report.write_ops(&mut batch, item_id, false);
+                            }
+                            _ => {}
+                        }
+
+                        if batch.is_large_batch() {
+                            server.store().write(batch.build_all()).await?;
+                            batch = BatchBuilder::new();
+                        }
+                    }
+                }
+            }
+            if !batch.is_empty() {
+                server.store().write(batch.build_all()).await?;
+            }
+
+            let started = Instant::now();
+
+            server
+                .store()
+                .purge_store()
+                .await
+                .caused_by(trc::location!())?;
+
+            server
+                .in_memory_store()
+                .purge_in_memory_store()
+                .await
+                .caused_by(trc::location!())?;
+
+            server
+                .registry()
+                .purge_dead_nodes()
+                .await
+                .caused_by(trc::location!())?;
+
+
+            trc::event!(
+                Store(StoreEvent::DataStorePurged),
+                Elapsed = started.elapsed()
+            );
+        }
+        TaskStoreMaintenanceType::PurgeBlob => {
+            if let Some(shard_index) = task.shard_index {
+                server
+                    .store()
+                    .purge_blobs(server.blob_store().clone(), shard_index as u8)
+                    .await
+                    .caused_by(trc::location!())?;
+            } else {
+                let mut batch = BatchBuilder::new();
+                let now = now() as i64;
+                for shard_index in 0..=u8::MAX {
+                    batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance {
+                        maintenance_type: TaskStoreMaintenanceType::PurgeBlob,
+                        shard_index: Some(shard_index as u64),
+                        status: TaskStatus::at(now),
+                    }));
+
+                    if batch.is_large_batch() {
+                        server.core.storage.data.write(batch.build_all()).await?;
+                        server.notify_task_queue();
+                        batch = BatchBuilder::new();
+                    }
+                }
+
+                if !batch.is_empty() {
+                    server.core.storage.data.write(batch.build_all()).await?;
+                    server.notify_task_queue();
+                }
+            }
+        }
+        TaskStoreMaintenanceType::RemoveGreylist
+        | TaskStoreMaintenanceType::RemoveLockQueueMessage
+        | TaskStoreMaintenanceType::RemoveLockTask
+        | TaskStoreMaintenanceType::RemoveLockDav
+        | TaskStoreMaintenanceType::RemoveSieveId
+        | TaskStoreMaintenanceType::ResetRateLimiters
+        | TaskStoreMaintenanceType::ResetBlobQuotas
+        | TaskStoreMaintenanceType::RemoveAuthTokens => {
+            #[cfg(feature = "test_mode")]
+            if let Some(test_var) = task.shard_index {
+                use crate::task_manager::TaskFailureType;
+
+                // Simulate success for testing purposes
+                match test_var {
+                    0 => {
+                        return Ok(TaskResult::Success(vec![]));
+                    }
+                    1 => {
+                        return Ok(TaskResult::temporary(
+                            "Simulated temporary failure".to_string(),
+                        ));
+                    }
+                    2 => {
+                        return Ok(TaskResult::permanent("Simulated permanent failure"));
+                    }
+
+                    retry => {
+                        return Ok(TaskResult::Failure {
+                            typ: TaskFailureType::Retry(retry),
+                            message: "Simulated retry failure".to_string(),
+                            max_attempts: None,
+                        });
+                    }
+                }
+            }
+
+            let prefixes = match task.maintenance_type {
+                TaskStoreMaintenanceType::RemoveGreylist => &[KV_GREYLIST][..],
+                TaskStoreMaintenanceType::RemoveLockQueueMessage => &[KV_LOCK_QUEUE_MESSAGE][..],
+                TaskStoreMaintenanceType::RemoveLockTask => &[KV_LOCK_TASK][..],
+                TaskStoreMaintenanceType::RemoveLockDav => &[KV_LOCK_DAV][..],
+                TaskStoreMaintenanceType::RemoveSieveId => &[KV_SIEVE_ID][..],
+                TaskStoreMaintenanceType::ResetRateLimiters => &[
+                    KV_RATE_LIMIT_RCPT,
+                    KV_RATE_LIMIT_SCAN,
+                    KV_RATE_LIMIT_LOITER,
+                    KV_RATE_LIMIT_AUTH,
+                    KV_RATE_LIMIT_SMTP,
+                    KV_RATE_LIMIT_CONTACT,
+                    KV_RATE_LIMIT_HTTP_AUTHENTICATED,
+                    KV_RATE_LIMIT_HTTP_ANONYMOUS,
+                    KV_RATE_LIMIT_IMAP,
+                ][..],
+                TaskStoreMaintenanceType::ResetBlobQuotas => &[KV_QUOTA_BLOB][..],
+                TaskStoreMaintenanceType::RemoveAuthTokens => &[KV_ACME, KV_OAUTH][..],
+                _ => unreachable!(),
+            };
+
+            for &prefix in prefixes {
+                server
+                    .in_memory_store()
+                    .key_delete_prefix(&[prefix])
+                    .await?;
+            }
+        }
+        TaskStoreMaintenanceType::ResetTenantQuotas => {
+        }
+    }
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+async fn account_maintenance(
+    server: &Server,
+    task: &TaskAccountMaintenance,
+) -> trc::Result<TaskResult> {
+    match task.maintenance_type {
+        TaskAccountMaintenanceType::Purge => {
+            server.purge_account(task.account_id.document_id()).await?;
+        }
+        TaskAccountMaintenanceType::Reindex => {
+            reindex_account(server, task.account_id.document_id()).await?;
+        }
+        TaskAccountMaintenanceType::RecalculateImapUid => {
+            reset_imap_uids(server, task.account_id.document_id()).await?;
+        }
+        TaskAccountMaintenanceType::RecalculateQuota => {
+            recalculate_quota(server, task.account_id.document_id()).await?;
+        }
+    }
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+async fn tenant_maintenance(
+    server: &Server,
+    task: &TaskTenantMaintenance,
+) -> trc::Result<TaskResult> {
+    match task.maintenance_type {
+        TaskTenantMaintenanceType::RecalculateQuota => {
+            recalculate_tenant_quota(server, task.tenant_id.document_id()).await?;
+        }
+    }
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> {
+    let mut quota = 0;
+
+    for collection in [
+        Collection::Email,
+        Collection::Calendar,
+        Collection::CalendarEvent,
+        Collection::CalendarEventNotification,
+        Collection::AddressBook,
+        Collection::ContactCard,
+        Collection::FileNode,
+        Collection::SieveScript,
+    ] {
+        server
+            .archives(account_id, collection, &(), |_, archive| {
+                match collection {
+                    Collection::Email => {
+                        quota += archive.unarchive::<MessageData>()?.size.to_native() as i64;
+                    }
+                    Collection::Calendar => {
+                        quota += archive.unarchive::<Calendar>()?.size() as i64;
+                    }
+                    Collection::CalendarEvent => {
+                        quota += archive.unarchive::<CalendarEvent>()?.size() as i64;
+                    }
+                    Collection::CalendarEventNotification => {
+                        quota += archive.unarchive::<CalendarEventNotification>()?.size() as i64;
+                    }
+                    Collection::AddressBook => {
+                        quota += archive.unarchive::<AddressBook>()?.size() as i64;
+                    }
+                    Collection::ContactCard => {
+                        quota += archive.unarchive::<ContactCard>()?.size() as i64;
+                    }
+                    Collection::FileNode => {
+                        quota += archive.unarchive::<FileNode>()?.size() as i64;
+                    }
+                    Collection::SieveScript => {
+                        quota += u32::from(archive.unarchive::<SieveScript>()?.size) as i64;
+                    }
+                    _ => {}
+                }
+                Ok(true)
+            })
+            .await
+            .caused_by(trc::location!())?;
+    }
+
+    let mut batch = BatchBuilder::new();
+    batch
+        .with_account_id(account_id)
+        .clear(ValueClass::Quota)
+        .add(ValueClass::Quota, quota);
+    server
+        .store()
+        .write(batch.build_all())
+        .await
+        .caused_by(trc::location!())
+        .map(|_| ())
+}
+
+
+#[cfg(not(feature = "enterprise"))]
+async fn recalculate_tenant_quota(_server: &Server, _tenant_id: u32) -> trc::Result<()> {
+    Ok(())
+}
+
+async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> {
+    let mut mailbox_count = 0;
+    let mut email_count = 0;
+
+    let cache = server
+        .get_cached_messages(account_id)
+        .await
+        .caused_by(trc::location!())?;
+
+    for &mailbox_id in cache.mailboxes.index.keys() {
+        let mailbox = server
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                account_id,
+                Collection::Mailbox,
+                mailbox_id,
+            ))
+            .await
+            .caused_by(trc::location!())?
+            .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))?
+            .into_deserialized::<email::mailbox::Mailbox>()
+            .caused_by(trc::location!())?;
+        let mut new_mailbox = mailbox.inner.clone();
+        new_mailbox.uid_validity = rand::random::<u32>();
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::Mailbox)
+            .with_document(mailbox_id)
+            .custom(
+                ObjectIndexBuilder::new()
+                    .with_current(mailbox)
+                    .with_changes(new_mailbox),
+            )
+            .caused_by(trc::location!())?
+            .clear(MailboxField::UidCounter);
+        server
+            .store()
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+        mailbox_count += 1;
+    }
+
+    // Reset all UIDs
+    for message_id in cache.emails.items.iter().map(|i| i.document_id) {
+        let data = server
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                account_id,
+                Collection::Email,
+                message_id,
+            ))
+            .await
+            .caused_by(trc::location!())?;
+        let data_ = if let Some(data) = data {
+            data
+        } else {
+            continue;
+        };
+        let data = data_
+            .to_unarchived::<MessageData>()
+            .caused_by(trc::location!())?;
+        let mut new_data = data
+            .deserialize::<MessageData>()
+            .caused_by(trc::location!())?;
+
+        let ids = server
+            .assign_email_ids(
+                account_id,
+                new_data.mailboxes.iter().map(|m| m.mailbox_id),
+                false,
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        for (uid_mailbox, uid) in new_data.mailboxes.iter_mut().zip(ids) {
+            uid_mailbox.uid = uid;
+        }
+
+        // Prepare write batch
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::Email)
+            .with_document(message_id)
+            .assert_value(ValueClass::Property(EmailField::Archive.into()), &data)
+            .set(
+                EmailField::Archive,
+                Archiver::new(new_data)
+                    .serialize()
+                    .caused_by(trc::location!())?,
+            );
+        server
+            .store()
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+        email_count += 1;
+    }
+
+    Ok((mailbox_count, email_count))
+}

crates/services/src/task_manager/manager.rs

@@ -0,0 +1,687 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::acme::AcmeTask;
+use crate::task_manager::alarm::SendAlarmTask;
+use crate::task_manager::destroy_account::DestroyAccountTask;
+use crate::task_manager::dkim::DkimManagementTask;
+use crate::task_manager::dns::DnsManagementTask;
+use crate::task_manager::imip::SendImipTask;
+use crate::task_manager::index::SearchIndexTask;
+use crate::task_manager::lock::TaskLockManager;
+use crate::task_manager::maintenance::MaintenanceTask;
+use crate::task_manager::merge_threads::MergeThreadsTask;
+use crate::task_manager::report::{self, SubmitReportTask};
+use crate::task_manager::restore_item::RestoreItemTask;
+use crate::task_manager::spam_classifier::SpamFilterMaintenanceTask;
+use crate::task_manager::{
+    DEFAULT_LOCK_EXPIRY, Locked, QUEUE_REFRESH_INTERVAL, TaskDetails, TaskFailureType, TaskInfo,
+    TaskJob, TaskManagerIpc, TaskResult,
+};
+use common::BuildServer;
+use common::config::server::{DEFAULT_TLS_TIMEOUT, ServerProtocol};
+use common::network::limiter::ConcurrencyLimiter;
+use common::network::{ServerInstance, TcpAcceptor};
+use common::{Inner, Server};
+use registry::schema::enums::TaskType;
+use registry::schema::structs::{
+    Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry,
+};
+use registry::types::datetime::UTCDateTime;
+use registry::types::{EnumImpl, ObjectImpl};
+use std::collections::hash_map::Entry;
+use std::future::Future;
+use std::time::Duration;
+use std::{sync::Arc, time::Instant};
+use store::rand::seq::SliceRandom;
+use store::write::key::DeserializeBigEndian;
+use store::{
+    IterateParams, ValueKey,
+    write::{BatchBuilder, TaskQueueClass, ValueClass, assert::AssertValue, now},
+};
+use store::{SerializeInfallible, U64_LEN, rand};
+use tokio::sync::{mpsc, watch};
+use trc::TaskManagerEvent;
+use utils::snowflake::SnowflakeIdGenerator;
+
+const TASK_QUEUE_BUFFER: usize = 10;
+const PERPETUAL_RETRY_MIN_DELAY: u64 = 3600;
+const PERPETUAL_RETRY_MAX_DELAY: u64 = 21600;
+
+pub fn spawn_task_manager(inner: Arc<Inner>) {
+    let is_clustered = {
+        let server = inner.build_server();
+        let roles = &server.core.network.roles;
+
+        if !roles.account_maintenance
+            && !roles.store_maintenance
+            && !roles.search_indexing
+            && !roles.spam_training
+            && !roles.task_manager
+        {
+            return;
+        }
+
+        server.core.storage.coordinator.is_enabled()
+    };
+
+    trc::event!(TaskManager(TaskManagerEvent::ManagerStarted));
+
+    // Create dummy server instance for alarms
+    let server_instance = Arc::new(ServerInstance {
+        id: "_local".to_string(),
+        protocol: ServerProtocol::Smtp,
+        acceptor: TcpAcceptor::Plain,
+        limiter: ConcurrencyLimiter::new(100),
+        tls_timeout: DEFAULT_TLS_TIMEOUT,
+        shutdown_rx: watch::channel(false).1,
+        proxy_networks: vec![],
+        span_id_gen: Arc::new(SnowflakeIdGenerator::new()),
+    });
+
+    // Spawn workers for each task type
+    let mut txs = Vec::with_capacity(TaskType::COUNT);
+    for idx in 0..TaskType::COUNT {
+        let task_type = TaskType::from_id(idx as u16).unwrap();
+        let channel_capacity = match task_type {
+            TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace => {
+                std::cmp::max(
+                    inner.build_server().core.email.index_batch_size,
+                    TASK_QUEUE_BUFFER,
+                )
+            }
+            TaskType::DestroyAccount
+            | TaskType::AccountMaintenance
+            | TaskType::TenantMaintenance
+            | TaskType::StoreMaintenance => 1,
+            TaskType::SpamFilterMaintenance => 2,
+            TaskType::CalendarAlarmEmail
+            | TaskType::CalendarAlarmNotification
+            | TaskType::CalendarItipMessage
+            | TaskType::MergeThreads
+            | TaskType::DmarcReport
+            | TaskType::TlsReport
+            | TaskType::RestoreArchivedItem
+            | TaskType::AcmeRenewal
+            | TaskType::DkimManagement
+            | TaskType::DnsManagement => TASK_QUEUE_BUFFER,
+        };
+
+        let (tx, mut rx) = mpsc::channel::<TaskJob>(channel_capacity);
+        txs.push(tx);
+        let inner = inner.clone();
+        let server_instance = server_instance.clone();
+
+        if matches!(
+            task_type,
+            TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace,
+        ) {
+            tokio::spawn(async move {
+                while let Some(job) = rx.recv().await {
+                    let server = inner.build_server();
+                    let batch_size = server.core.email.index_batch_size;
+                    let mut batch = Vec::with_capacity(batch_size);
+                    match server
+                        .store()
+                        .get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
+                            TaskQueueClass::Task { id: job.id },
+                        )))
+                        .await
+                    {
+                        Ok(Some(task)) => {
+                            batch.push(TaskDetails { task, info: job });
+                        }
+                        Ok(None) => {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskIgnored),
+                                Id = job.id,
+                                Reason = "Task not found in store, likely already processed.",
+                            );
+                        }
+                        Err(err) => {
+                            trc::error!(
+                                err.id(job.id)
+                                    .details("Failed to retrieve task details.")
+                                    .caused_by(trc::location!())
+                            );
+                        }
+                    }
+
+                    while batch.len() < batch_size {
+                        match rx.try_recv() {
+                            Ok(job) => {
+                                match server
+                                    .store()
+                                    .get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
+                                        TaskQueueClass::Task { id: job.id },
+                                    )))
+                                    .await
+                                {
+                                    Ok(Some(task)) => {
+                                        batch.push(TaskDetails { task, info: job });
+                                    }
+                                    Ok(None) => {
+                                        trc::event!(
+                                            TaskManager(TaskManagerEvent::TaskIgnored),
+                                            Id = job.id,
+                                            Reason = "Task not found in store, likely already processed.",
+                                        );
+                                    }
+                                    Err(err) => {
+                                        trc::error!(
+                                            err.id(job.id)
+                                                .details("Failed to retrieve task details.")
+                                                .caused_by(trc::location!())
+                                        );
+                                    }
+                                }
+                            }
+                            Err(_) => break,
+                        }
+                    }
+
+                    // Dispatch
+                    let mut refresh_queue = false;
+                    let results = server.index(&batch).await.into_iter().map(|r| {
+                        refresh_queue |= r.result.is_retry();
+                        r.result
+                    });
+                    update_tasks(&server, &mut batch, results).await;
+
+                    if refresh_queue || rx.is_empty() {
+                        server.notify_task_queue();
+                    }
+                }
+            });
+        } else {
+            let server_instance = server_instance.clone();
+            tokio::spawn(async move {
+                while let Some(job) = rx.recv().await {
+                    let server = inner.build_server();
+                    let mut refresh_queue = false;
+
+                    match server
+                        .store()
+                        .get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
+                            TaskQueueClass::Task { id: job.id },
+                        )))
+                        .await
+                    {
+                        Ok(Some(task)) => {
+                            let result = match &task {
+                                Task::CalendarAlarmEmail(task) => {
+                                    server.send_email_alarm(task, server_instance.clone()).await
+                                }
+                                Task::CalendarAlarmNotification(task) => {
+                                    server.send_display_alarm(task).await
+                                }
+                                Task::CalendarItipMessage(task) => {
+                                    server.send_imip(task, server_instance.clone()).await
+                                }
+                                Task::MergeThreads(task) => server.merge_threads(task).await,
+                                Task::DmarcReport(task) => {
+                                    server
+                                        .submit_report(report::ReportId::Dmarc(task.report_id.id()))
+                                        .await
+                                }
+                                Task::TlsReport(task) => {
+                                    server
+                                        .submit_report(report::ReportId::Tls(task.report_id.id()))
+                                        .await
+                                }
+                                Task::RestoreArchivedItem(task) => server.restore_item(task).await,
+                                Task::DestroyAccount(task) => server.destroy_account(task).await,
+                                Task::AccountMaintenance(task) => {
+                                    server.account_maintenance(task).await
+                                }
+                                Task::TenantMaintenance(task) => {
+                                    server.tenant_maintenance(task).await
+                                }
+                                Task::StoreMaintenance(task) => {
+                                    server.store_maintenance(task).await
+                                }
+                                Task::SpamFilterMaintenance(task) => {
+                                    Box::pin(server.spam_filter_maintenance(task)).await
+                                }
+                                Task::AcmeRenewal(task) => server.acme_management(task).await,
+                                Task::DkimManagement(task_dkim_rotation) => {
+                                    server.dkim_management(task_dkim_rotation).await
+                                }
+                                Task::DnsManagement(task_dns_management) => {
+                                    server.dns_management(task_dns_management).await
+                                }
+                                Task::IndexDocument(_)
+                                | Task::UnindexDocument(_)
+                                | Task::IndexTrace(_) => unreachable!(),
+                            };
+
+                            refresh_queue = result.is_retry();
+
+                            update_tasks(
+                                &server,
+                                &mut [TaskDetails { task, info: job }],
+                                vec![result],
+                            )
+                            .await;
+                        }
+                        Ok(None) => {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskIgnored),
+                                Id = job.id,
+                                Reason = "Task not found in store, likely already processed.",
+                            );
+                        }
+                        Err(err) => {
+                            trc::error!(
+                                err.id(job.id)
+                                    .details("Failed to retrieve task details.")
+                                    .caused_by(trc::location!())
+                            );
+                        }
+                    }
+
+                    if refresh_queue || rx.is_empty() {
+                        server.notify_task_queue();
+                    }
+                }
+            });
+        }
+    }
+
+    const REFRESH_INTERVAL: Duration = Duration::from_secs(60);
+    tokio::spawn(async move {
+        let mut ipc = TaskManagerIpc {
+            txs: txs.try_into().expect("Incorrect number of task channels"),
+            locked: Default::default(),
+            revision: 0,
+        };
+        let rx = inner.ipc.task_tx.clone();
+        loop {
+            // Index any queued tasks
+            let mut sleep_for = inner.build_server().process_tasks(&mut ipc).await;
+            if is_clustered && sleep_for > REFRESH_INTERVAL {
+                sleep_for = REFRESH_INTERVAL;
+            }
+
+            // Wait for a signal or sleep until the next task is due
+            let _ = tokio::time::timeout(sleep_for, rx.notified()).await;
+        }
+    });
+}
+
+pub(crate) trait TaskQueueManager: Sync + Send {
+    fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> impl Future<Output = Duration> + Send;
+}
+
+impl TaskQueueManager for Server {
+    async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration {
+        let now_timestamp = now();
+        let from_key = ValueKey::<ValueClass> {
+            account_id: 0,
+            collection: 0,
+            document_id: 0,
+            class: ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 1 }),
+        };
+        let to_key = ValueKey::<ValueClass> {
+            account_id: u32::MAX,
+            collection: u8::MAX,
+            document_id: u32::MAX,
+            class: ValueClass::TaskQueue(TaskQueueClass::Due {
+                id: u64::MAX,
+                due: now_timestamp + QUEUE_REFRESH_INTERVAL,
+            }),
+        };
+
+        // Retrieve tasks pending to be processed
+        let mut tasks = Vec::new();
+        let now = Instant::now();
+        let mut next_event = None;
+        let roles = &self.core.network.roles;
+        ipc.revision += 1;
+        let _ = self
+            .store()
+            .iterate(
+                IterateParams::new(from_key, to_key).ascending(),
+                |key, value| {
+                    if key.len() == U64_LEN * 2 {
+                        let task_due = key.deserialize_be_u64(0)?;
+                        let task_id = key.deserialize_be_u64(U64_LEN)?;
+
+                        if task_due <= now_timestamp {
+                            let task_type_idx = value.deserialize_be_u16(0)?;
+                            let task_type = TaskType::from_id(task_type_idx).ok_or_else(|| {
+                                trc::StoreEvent::DataCorruption
+                                    .caused_by(trc::location!())
+                                    .ctx(trc::Key::Value, value)
+                            })?;
+                            let enabled = match task_type {
+                                TaskType::IndexDocument
+                                | TaskType::UnindexDocument
+                                | TaskType::IndexTrace => roles.search_indexing,
+                                TaskType::AccountMaintenance
+                                | TaskType::TenantMaintenance
+                                | TaskType::DestroyAccount => roles.account_maintenance,
+                                TaskType::StoreMaintenance => roles.store_maintenance,
+                                TaskType::SpamFilterMaintenance => roles.spam_training,
+                                TaskType::CalendarAlarmEmail
+                                | TaskType::CalendarAlarmNotification
+                                | TaskType::CalendarItipMessage
+                                | TaskType::MergeThreads
+                                | TaskType::DmarcReport
+                                | TaskType::TlsReport
+                                | TaskType::RestoreArchivedItem
+                                | TaskType::AcmeRenewal
+                                | TaskType::DkimManagement
+                                | TaskType::DnsManagement => true,
+                            };
+
+                            if !enabled {
+                                trc::event!(
+                                    TaskManager(TaskManagerEvent::TaskIgnored),
+                                    Id = task_id,
+                                    Details = task_type.as_str(),
+                                    Reason = "Task type is disabled by cluster roles.",
+                                );
+                                return Ok(true);
+                            }
+
+                            match ipc.locked.entry(task_id) {
+                                Entry::Occupied(mut entry) => {
+                                    let locked = entry.get_mut();
+                                    if locked.expires <= now || locked.due < task_due {
+                                        locked.expires = Instant::now()
+                                            + std::time::Duration::from_secs(
+                                                DEFAULT_LOCK_EXPIRY + 1,
+                                            );
+                                        locked.due = task_due;
+                                        tasks.push((
+                                            TaskJob {
+                                                id: task_id,
+                                                due: task_due,
+                                                typ: task_type,
+                                            },
+                                            task_type_idx,
+                                        ));
+                                    }
+                                    locked.revision = ipc.revision;
+                                }
+                                Entry::Vacant(entry) => {
+                                    entry.insert(Locked {
+                                        expires: Instant::now()
+                                            + std::time::Duration::from_secs(
+                                                DEFAULT_LOCK_EXPIRY + 1,
+                                            ),
+                                        due: task_due,
+                                        revision: ipc.revision,
+                                    });
+                                    tasks.push((
+                                        TaskJob {
+                                            id: task_id,
+                                            due: task_due,
+                                            typ: task_type,
+                                        },
+                                        task_type_idx,
+                                    ));
+                                }
+                            }
+
+                            Ok(true)
+                        } else {
+                            next_event = Some(task_due);
+                            Ok(false)
+                        }
+                    } else {
+                        Ok(true)
+                    }
+                },
+            )
+            .await
+            .map_err(|err| {
+                trc::error!(
+                    err.caused_by(trc::location!())
+                        .details("Failed to iterate over task queue.")
+                );
+            });
+
+        if !tasks.is_empty() {
+            trc::event!(
+                TaskManager(TaskManagerEvent::TaskAcquired),
+                Total = tasks.len(),
+                Details = ipc.locked.len(),
+            );
+        }
+
+        // Shuffle tasks
+        if tasks.len() > 1 {
+            tasks.shuffle(&mut rand::rng());
+        }
+
+        // Dispatch tasks
+        for (task_job, task_type_idx) in tasks {
+            let tx = &ipc.txs[task_type_idx as usize];
+
+            if tx.capacity() > 0 {
+                if self.try_lock_task(task_job.id).await && tx.send(task_job).await.is_err() {
+                    trc::event!(
+                        Server(trc::ServerEvent::ThreadError),
+                        Details = "Error sending task.",
+                        CausedBy = trc::location!()
+                    );
+                }
+            } else {
+                // If the channel is full, release the lock so it can be picked up in the next iteration
+                ipc.locked.remove(&task_job.id);
+            }
+        }
+
+        // Delete expired locks
+        let now = Instant::now();
+        ipc.locked
+            .retain(|_, locked| locked.expires > now && locked.revision == ipc.revision);
+        Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| {
+            timestamp.saturating_sub(store::write::now())
+        }))
+    }
+}
+
+async fn update_tasks(
+    server: &Server,
+    tasks: &mut [TaskDetails],
+    results: impl IntoIterator<Item = TaskResult>,
+) {
+    let mut batch = BatchBuilder::new();
+
+    for (task, result) in tasks.iter_mut().zip(results) {
+        let id = task.info.id;
+        batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
+            id,
+            due: task.info.due,
+        }));
+        match result {
+            TaskResult::Success(tasks) => {
+                for task in tasks {
+                    batch.schedule_task(task);
+                }
+                batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id }));
+            }
+            TaskResult::Ignored => {
+                batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id }));
+            }
+            TaskResult::Update(ops) => {
+                for op in ops {
+                    batch.any_op(op);
+                }
+            }
+            TaskResult::Failure {
+                typ,
+                message,
+                max_attempts,
+            } => {
+                let (attempt_number, retry_since) = match task.task.status() {
+                    TaskStatus::Pending(_) => (0, UTCDateTime::now()),
+                    TaskStatus::Retry(status) => (status.attempt_number, status.created_at),
+                    TaskStatus::Failed(status) => (status.failed_attempt_number, status.failed_at),
+                };
+                let retry_at = match typ {
+                    TaskFailureType::Retry(retry_at) => (attempt_number
+                        < max_attempts.unwrap_or(server.core.network.task_manager.max_attempts)
+                        && retry_at
+                            <= (retry_since.timestamp() as u64).saturating_add(
+                                server.core.network.task_manager.total_deadline.as_secs(),
+                            ))
+                    .then_some(retry_at)
+                    .or_else(|| perpetual_retry_time(task.info.typ, attempt_number)),
+                    TaskFailureType::Temporary => next_retry_time(
+                        &server.core.network.task_manager,
+                        max_attempts,
+                        retry_since.timestamp() as u64,
+                        attempt_number,
+                        now(),
+                    )
+                    .or_else(|| perpetual_retry_time(task.info.typ, attempt_number)),
+                    TaskFailureType::Perpetual => {
+                        perpetual_retry_time(task.info.typ, attempt_number)
+                    }
+                    TaskFailureType::Permanent => None,
+                };
+
+                let due = if let Some(retry_at) = retry_at {
+                    trc::event!(
+                        TaskManager(TaskManagerEvent::TaskRetry),
+                        Id = id,
+                        Details = task.task.name(),
+                        Reason = message.to_string(),
+                        NextRetry = trc::Value::Timestamp(retry_at),
+                    );
+
+                    task.task.set_status(TaskStatus::Retry(TaskStatusRetry {
+                        due: UTCDateTime::from_timestamp(retry_at as i64),
+                        attempt_number: attempt_number + 1,
+                        failure_reason: message,
+                        created_at: retry_since,
+                    }));
+
+                    retry_at
+                } else {
+                    trc::event!(
+                        TaskManager(TaskManagerEvent::TaskFailed),
+                        Id = id,
+                        Details = task.task.name(),
+                        Reason = message.to_string(),
+                    );
+
+                    task.task.set_status(TaskStatus::Failed(TaskStatusFailed {
+                        failed_at: UTCDateTime::now(),
+                        failed_attempt_number: attempt_number,
+                        failure_reason: message,
+                        created_at: retry_since,
+                    }));
+                    u64::MAX
+                };
+                batch
+                    .assert_value(
+                        ValueClass::TaskQueue(TaskQueueClass::Task { id }),
+                        AssertValue::Some,
+                    )
+                    .set(
+                        ValueClass::TaskQueue(TaskQueueClass::Due { id, due }),
+                        task.info.typ.to_id().serialize(),
+                    )
+                    .set(
+                        ValueClass::TaskQueue(TaskQueueClass::Task { id }),
+                        task.task.to_pickled_vec(),
+                    );
+            }
+        }
+    }
+
+    if let Err(err) = server.store().write(batch.build_all()).await {
+        if err.matches(trc::EventType::Store(trc::StoreEvent::AssertValueFailed)) {
+            trc::event!(
+                TaskManager(TaskManagerEvent::TaskIgnored),
+                Reason = "Task was deleted while being processed; skipping update.",
+            );
+        } else {
+            trc::error!(err.details("Failed to remove task(s) from queue."));
+        }
+    }
+
+    for task in tasks {
+        server.remove_index_lock(task.info.id).await;
+    }
+}
+
+pub fn perpetual_retry_time(typ: TaskType, attempt: u64) -> Option<u64> {
+    matches!(
+        typ,
+        TaskType::AcmeRenewal
+            | TaskType::DkimManagement
+            | TaskType::IndexDocument
+            | TaskType::UnindexDocument
+    )
+    .then(|| {
+        now().saturating_add(
+            PERPETUAL_RETRY_MIN_DELAY
+                .saturating_mul(1u64 << attempt.min(4))
+                .min(PERPETUAL_RETRY_MAX_DELAY),
+        )
+    })
+}
+
+pub fn next_retry_time(
+    manager: &TaskManager,
+    max_attempts_override: Option<u64>,
+    retry_since: u64,
+    attempt: u64,
+    now: u64,
+) -> Option<u64> {
+    if attempt >= max_attempts_override.unwrap_or(manager.max_attempts) {
+        return None;
+    }
+
+    let delay_secs: u64 = match &manager.strategy {
+        TaskRetryStrategy::FixedDelay(fixed) => fixed.delay.as_secs(),
+        TaskRetryStrategy::ExponentialBackoff(backoff) => {
+            let delay = (backoff.initial_delay.as_secs() as f64
+                * backoff.factor.into_inner().powi(attempt as i32))
+            .min(backoff.max_delay.as_secs() as f64) as u64;
+
+            if backoff.jitter {
+                let jitter_factor = rand::random::<f64>() + 0.5;
+                ((delay as f64 * jitter_factor) as u64).min(backoff.max_delay.as_secs())
+            } else {
+                delay
+            }
+        }
+    };
+
+    let next_time = now.saturating_add(delay_secs);
+    let deadline = retry_since.saturating_add(manager.total_deadline.as_secs());
+    if next_time > deadline {
+        return None;
+    }
+
+    Some(next_time)
+}
+
+impl TaskResult {
+    pub fn is_success(&self) -> bool {
+        matches!(self, TaskResult::Success(_))
+    }
+
+    pub fn is_retry(&self) -> bool {
+        matches!(
+            self,
+            TaskResult::Update(_)
+                | TaskResult::Failure {
+                    typ: TaskFailureType::Temporary
+                        | TaskFailureType::Retry(_)
+                        | TaskFailureType::Perpetual,
+                    ..
+                }
+        )
+    }
+}

crates/services/src/task_manager/merge_threads.rs

@@ -0,0 +1,253 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::TaskResult;
+use common::{Server, storage::index::ObjectIndexBuilder};
+use email::message::{
+    ingest::{ThreadMerge, has_message_id},
+    metadata::MessageData,
+};
+use registry::schema::structs::TaskMergeThreads;
+use std::{str::FromStr, time::Duration};
+use store::{
+    IterateParams, Key, U32_LEN, ValueKey,
+    ahash::AHashMap,
+    rand::RngExt,
+    write::{
+        AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, MergeResult, Params, ValueClass,
+        key::DeserializeBigEndian,
+    },
+};
+use trc::AddContext;
+use types::{
+    collection::{Collection, SyncCollection},
+    field::EmailField,
+};
+use utils::cheeky_hash::CheekyHash;
+
+const MAX_RETRIES: usize = 5;
+
+pub(crate) trait MergeThreadsTask: Sync + Send {
+    fn merge_threads(&self, threads: &TaskMergeThreads) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl MergeThreadsTask for Server {
+    async fn merge_threads(&self, threads: &TaskMergeThreads) -> TaskResult {
+        match merge_threads(self, threads).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(threads.account_id.document_id())
+                        .details("Failed to merge threads")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn merge_threads(
+    server: &Server,
+    task_merge_threads: &TaskMergeThreads,
+) -> trc::Result<TaskResult> {
+    let Ok(thread_hash) = CheekyHash::from_str(&task_merge_threads.thread_name) else {
+        return Ok(TaskResult::permanent("Invalid thread hash"));
+    };
+    let Ok(mut message_ids) = task_merge_threads
+        .message_ids
+        .iter()
+        .map(|id| CheekyHash::from_str(id))
+        .collect::<Result<Vec<_>, _>>()
+    else {
+        return Ok(TaskResult::permanent("Invalid message ids"));
+    };
+    message_ids.sort_unstable();
+
+    let account_id = task_merge_threads.account_id.document_id();
+    let mut try_count = 0;
+
+    let from_key = ValueKey {
+        account_id,
+        collection: Collection::Email.into(),
+        document_id: 0,
+        class: ValueClass::IndexProperty(IndexPropertyClass::Hash {
+            property: EmailField::Threading.into(),
+            hash: thread_hash,
+        }),
+    };
+    let to_key = ValueKey {
+        account_id,
+        collection: Collection::Email.into(),
+        document_id: u32::MAX,
+        class: ValueClass::IndexProperty(IndexPropertyClass::Hash {
+            property: EmailField::Threading.into(),
+            hash: thread_hash,
+        }),
+    };
+    let mut prefix = from_key.serialize(0);
+    let key_len = prefix.len();
+    let document_id_pos = key_len - U32_LEN;
+    prefix.truncate(document_id_pos);
+
+    'retry: loop {
+        // Merge threads
+        let mut thread_merge = ThreadMerge::new();
+        let mut same_subject_messages: AHashMap<u32, Vec<u32>> = AHashMap::new();
+
+        // Find thread ids
+        server
+            .store()
+            .iterate(
+                IterateParams::new(from_key.clone(), to_key.clone()).ascending(),
+                |key, value| {
+                    if key.len() == key_len && key.starts_with(&prefix) {
+                        // Find matching references
+                        let references = value.get(U32_LEN..).unwrap_or_default();
+                        let thread_id = value.deserialize_be_u32(0)?;
+                        let document_id = key.deserialize_be_u32(document_id_pos)?;
+
+                        if has_message_id(&message_ids, references) {
+                            thread_merge.add(thread_id, document_id);
+                        } else {
+                            // Keep track of messages with the same subject for potential future merges
+                            same_subject_messages
+                                .entry(thread_id)
+                                .or_default()
+                                .push(document_id);
+                        }
+                    }
+
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        if thread_merge.num_thread_ids() < 2 {
+            // Another process merged the threads already?
+            return Ok(TaskResult::Success(vec![]));
+        }
+
+        // Add other messages with the same subject to the merge if they share a
+        // thread id with a message that has a matching message id
+        for thread_id in thread_merge.thread_ids().copied().collect::<Vec<_>>() {
+            if let Some(document_ids) = same_subject_messages.get(&thread_id) {
+                for &document_id in document_ids {
+                    thread_merge.add(thread_id, document_id);
+                }
+            }
+        }
+
+        let thread_id = thread_merge.merge_thread_id();
+
+        // Delete all but the most common threadId
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::Thread);
+
+        for &delete_thread_id in thread_merge.thread_ids() {
+            if delete_thread_id != thread_id {
+                batch
+                    .with_document(delete_thread_id)
+                    .log_container_delete(SyncCollection::Thread);
+            }
+        }
+
+        // Move messages to the new threadId
+        batch.with_collection(Collection::Email);
+
+        for (&group_thread_id, document_ids) in thread_merge.thread_groups() {
+            if thread_id != group_thread_id {
+                for &document_id in document_ids {
+                    if let Some(data_) = server
+                        .store()
+                        .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                            account_id,
+                            Collection::Email,
+                            document_id,
+                        ))
+                        .await
+                        .caused_by(trc::location!())?
+                    {
+                        let data = data_
+                            .to_unarchived::<MessageData>()
+                            .caused_by(trc::location!())?;
+                        if data.inner.thread_id != group_thread_id {
+                            try_count += 1;
+                            continue 'retry;
+                        }
+
+                        // Update thread id
+                        let mut new_data = data
+                            .deserialize::<MessageData>()
+                            .caused_by(trc::location!())?;
+                        new_data.thread_id = thread_id;
+                        batch
+                            .with_document(document_id)
+                            .custom(
+                                ObjectIndexBuilder::new()
+                                    .with_current(data)
+                                    .with_changes(new_data),
+                            )
+                            .caused_by(trc::location!())?;
+
+                        // Update thread index property
+                        batch.merge_fnc(
+                            ValueClass::IndexProperty(IndexPropertyClass::Hash {
+                                property: EmailField::Threading.into(),
+                                hash: thread_hash,
+                            }),
+                            Params::with_capacity(3)
+                                .with_u64(thread_id as u64)
+                                .with_u64(group_thread_id as u64),
+                            |params, _, bytes| {
+                                let new_thread_id = params.u64(0) as u32;
+                                let old_thread_id = params.u64(1) as u32;
+
+                                let mut thread_index = bytes
+                                    .filter(|v| v.len() > U32_LEN)
+                                    .ok_or_else(|| {
+                                        trc::StoreEvent::AssertValueFailed
+                                            .into_err()
+                                            .details("Message no longer exists.")
+                                            .caused_by(trc::location!())
+                                    })?
+                                    .to_vec();
+
+                                if thread_index.as_slice().deserialize_be_u32(0)? != old_thread_id {
+                                    return Err(
+                                        trc::StoreEvent::AssertValueFailed
+                                            .into_err()
+                                            .details("Thread id mismatch, likely due to concurrent modification.")
+                                            .caused_by(trc::location!())
+                                    );
+                                }
+
+                                thread_index[0..U32_LEN].copy_from_slice(&new_thread_id.to_be_bytes());
+
+                                Ok(MergeResult::Update(thread_index))
+                            },
+                        );
+                    }
+                }
+            }
+        }
+
+        match server.commit_batch(batch).await {
+            Ok(_) => return Ok(TaskResult::Success(vec![])),
+            Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => {
+                let backoff = store::rand::rng().random_range(50..=300);
+                tokio::time::sleep(Duration::from_millis(backoff)).await;
+                try_count += 1;
+            }
+            Err(err) => {
+                return Err(err.caused_by(trc::location!()));
+            }
+        }
+    }
+}

crates/services/src/task_manager/mod.rs

@@ -0,0 +1,137 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::{KV_LOCK_TASK, Server};
+use registry::schema::enums::TaskType;
+use registry::schema::structs::Task;
+use registry::types::EnumImpl;
+use std::future::Future;
+use std::time::Instant;
+use store::ahash::AHashMap;
+use store::write::Operation;
+use tokio::sync::mpsc;
+use trc::TaskManagerEvent;
+
+pub mod acme;
+pub mod alarm;
+pub mod destroy_account;
+pub mod dkim;
+pub mod dns;
+pub mod imip;
+pub mod index;
+pub mod lock;
+pub mod maintenance;
+pub mod manager;
+pub mod merge_threads;
+pub mod report;
+pub mod restore_item;
+pub mod scheduler;
+pub mod spam_classifier;
+
+const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes
+const DEFAULT_LOCK_EXPIRY: u64 = 60 * 60; // 1 hour
+
+pub(crate) struct TaskManagerIpc {
+    txs: [mpsc::Sender<TaskJob>; TaskType::COUNT],
+    locked: AHashMap<u64, Locked>,
+    revision: u64,
+}
+
+#[derive(Debug)]
+pub(crate) struct Locked {
+    expires: Instant,
+    due: u64,
+    revision: u64,
+}
+
+#[derive(Debug)]
+pub(crate) struct TaskDetails {
+    task: Task,
+    info: TaskJob,
+}
+
+#[derive(Debug)]
+pub(crate) struct TaskJob {
+    id: u64,
+    due: u64,
+    typ: TaskType,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub(crate) enum TaskResult {
+    Success(Vec<Task>),
+    Update([Operation; 2]),
+    Failure {
+        typ: TaskFailureType,
+        message: String,
+        max_attempts: Option<u64>,
+    },
+    Ignored,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[allow(dead_code)]
+pub(crate) enum TaskFailureType {
+    Retry(u64),
+    Temporary,
+    Perpetual,
+    Permanent,
+}
+
+pub(crate) trait TaskInfo {
+    fn name(&self) -> &'static str;
+}
+
+impl TaskInfo for Task {
+    fn name(&self) -> &'static str {
+        match self {
+            Task::IndexDocument(_) => "IndexDocument",
+            Task::UnindexDocument(_) => "UnindexDocument",
+            Task::IndexTrace(_) => "IndexTrace",
+            Task::CalendarAlarmEmail(_) => "CalendarAlarmEmail",
+            Task::CalendarAlarmNotification(_) => "CalendarAlarmNotification",
+            Task::CalendarItipMessage(_) => "CalendarItipMessage",
+            Task::MergeThreads(_) => "MergeThreads",
+            Task::DmarcReport(_) => "DmarcReport",
+            Task::TlsReport(_) => "TlsReport",
+            Task::RestoreArchivedItem(_) => "RestoreArchivedItem",
+            Task::DestroyAccount(_) => "DestroyAccount",
+            Task::AccountMaintenance(_) => "AccountMaintenance",
+            Task::StoreMaintenance(_) => "StoreMaintenance",
+            Task::SpamFilterMaintenance(_) => "SpamFilterMaintenance",
+            Task::AcmeRenewal(_) => "AcmeRenewal",
+            Task::DkimManagement(_) => "DkimManagement",
+            Task::DnsManagement(_) => "DnsManagement",
+            Task::TenantMaintenance(_) => "TenantMaintenance",
+        }
+    }
+}
+
+impl TaskResult {
+    pub fn permanent(message: impl Into<String>) -> Self {
+        TaskResult::Failure {
+            typ: TaskFailureType::Permanent,
+            message: message.into(),
+            max_attempts: None,
+        }
+    }
+
+    pub fn temporary(message: impl Into<String>) -> Self {
+        TaskResult::Failure {
+            typ: TaskFailureType::Temporary,
+            message: message.into(),
+            max_attempts: None,
+        }
+    }
+
+    pub fn perpetual(message: impl Into<String>) -> Self {
+        TaskResult::Failure {
+            typ: TaskFailureType::Perpetual,
+            message: message.into(),
+            max_attempts: None,
+        }
+    }
+}

crates/services/src/task_manager/report.rs

@@ -0,0 +1,44 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::TaskResult;
+use common::Server;
+use smtp::reporting::{dmarc::DmarcReporting, tls::TlsReporting};
+
+pub enum ReportId {
+    Dmarc(u64),
+    Tls(u64),
+}
+
+pub(crate) trait SubmitReportTask: Sync + Send {
+    fn submit_report(&self, report_id: ReportId) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl SubmitReportTask for Server {
+    async fn submit_report(&self, report_id: ReportId) -> TaskResult {
+        match submit_report(self, report_id).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(err.details("Failed to submit report"));
+                result
+            }
+        }
+    }
+}
+
+async fn submit_report(server: &Server, report_id: ReportId) -> trc::Result<TaskResult> {
+    match report_id {
+        ReportId::Dmarc(item_id) => server
+            .send_dmarc_aggregate_report(item_id)
+            .await
+            .map(|_| TaskResult::Success(vec![])),
+        ReportId::Tls(item_id) => server
+            .send_tls_aggregate_report(item_id)
+            .await
+            .map(|_| TaskResult::Success(vec![])),
+    }
+}

crates/services/src/task_manager/restore_item.rs

@@ -0,0 +1,105 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::{Server, auth::BuildAccessToken};
+use email::{
+    mailbox::INBOX_ID,
+    message::ingest::{EmailIngest, IngestEmail, IngestSource},
+};
+use mail_parser::MessageParser;
+use registry::schema::{enums::ArchivedItemType, structs::TaskRestoreArchivedItem};
+use store::write::{BatchBuilder, BlobLink, BlobOp};
+use trc::AddContext;
+
+use crate::task_manager::TaskResult;
+
+pub(crate) trait RestoreItemTask: Sync + Send {
+    fn restore_item(
+        &self,
+        task: &TaskRestoreArchivedItem,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl RestoreItemTask for Server {
+    async fn restore_item(&self, task: &TaskRestoreArchivedItem) -> TaskResult {
+        match restore_item(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(
+                    err.account_id(task.account_id.document_id())
+                        .details("Failed to restore item")
+                );
+                result
+            }
+        }
+    }
+}
+
+async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::Result<TaskResult> {
+    match task.archived_item_type {
+        ArchivedItemType::Email => {
+            let account_id = task.account_id.document_id();
+            let access_token = server
+                .access_token(account_id)
+                .await
+                .caused_by(trc::location!())?;
+
+            let Some(bytes) = server
+                .blob_store()
+                .get_blob(task.blob_id.hash.as_slice(), 0..usize::MAX)
+                .await?
+            else {
+                return Ok(TaskResult::permanent("Blob not found"));
+            };
+
+            match server
+                .email_ingest(IngestEmail {
+                    raw_message: &bytes,
+                    message: MessageParser::new().parse(&bytes),
+                    blob_hash: Some(&task.blob_id.hash),
+                    access_token: &access_token.build(),
+                    mailbox_ids: vec![INBOX_ID],
+                    keywords: vec![],
+                    received_at: (task.created_at.timestamp() as u64).into(),
+                    source: IngestSource::Restore,
+                    session_id: 0,
+                })
+                .await
+            {
+                Ok(_) => {
+                    let mut batch = BatchBuilder::new();
+                    batch.with_account_id(account_id).clear(BlobOp::Link {
+                        hash: task.blob_id.hash.clone(),
+                        to: BlobLink::Temporary {
+                            until: task.archived_until.timestamp() as u64,
+                        },
+                    });
+                    server.store().write(batch.build_all()).await?;
+
+                    Ok(TaskResult::Success(vec![]))
+                }
+                Err(mut err)
+                    if err.matches(trc::EventType::MessageIngest(
+                        trc::MessageIngestEvent::Error,
+                    )) =>
+                {
+                    Ok(TaskResult::permanent(
+                        err.take_value(trc::Key::Reason)
+                            .and_then(|v| v.into_string())
+                            .unwrap()
+                            .to_string(),
+                    ))
+                }
+                Err(err) => Err(err.caused_by(trc::location!())),
+            }
+        }
+        ArchivedItemType::FileNode
+        | ArchivedItemType::CalendarEvent
+        | ArchivedItemType::ContactCard
+        | ArchivedItemType::SieveScript => Ok(TaskResult::permanent("Not implemented")),
+    }
+}

crates/services/src/task_manager/scheduler.rs

@@ -0,0 +1,399 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use std::time::Duration;
+use std::{
+    collections::BinaryHeap,
+    sync::Arc,
+    time::{Instant, SystemTime},
+};
+
+use common::{
+    BuildServer, Inner, LONG_1D_SLUMBER,
+    config::{mailstore::spamfilter, telemetry::OtelMetrics},
+};
+use registry::{
+    schema::{
+        enums::{TaskSpamFilterMaintenanceType, TaskStoreMaintenanceType, TaskType},
+        structs::{Task, TaskSpamFilterMaintenance, TaskStatus, TaskStoreMaintenance},
+    },
+    types::EnumImpl,
+};
+use store::write::{BatchBuilder, now};
+use trc::{ClusterEvent, Collector, MetricType, TaskManagerEvent, TelemetryEvent};
+
+#[derive(PartialEq, Eq)]
+struct Action {
+    due: Instant,
+    event: Event,
+}
+
+#[derive(PartialEq, Eq, Debug)]
+enum Event {
+    PurgeAccount,
+    PurgeDataStore,
+    PurgeBlobStore,
+    OtelMetrics,
+    CalculateMetrics,
+    TrainSpamClassifier,
+    RenewNodeIdLease,
+}
+
+#[derive(Default)]
+struct Queue {
+    heap: BinaryHeap<Action>,
+}
+
+
+pub fn spawn_task_scheduler(inner: Arc<Inner>) {
+    tokio::spawn(async move {
+        trc::event!(TaskManager(TaskManagerEvent::SchedulerStarted));
+        let start_time = SystemTime::now();
+
+        // Add all events to queue
+        let mut queue = Queue::default();
+        {
+            let server = inner.build_server();
+
+            // Account purge
+            queue.schedule(
+                Instant::now() + server.core.email.account_purge_frequency.time_to_next(),
+                Event::PurgeAccount,
+            );
+            queue.schedule(
+                Instant::now() + server.core.email.data_purge_frequency.time_to_next(),
+                Event::PurgeDataStore,
+            );
+            queue.schedule(
+                Instant::now() + server.core.email.blob_purge_frequency.time_to_next(),
+                Event::PurgeBlobStore,
+            );
+
+            // Node ID lease renewal
+            if server.core.storage.coordinator.is_enabled() {
+                queue.schedule(
+                    Instant::now() + server.registry().refresh_node_id_interval(),
+                    Event::RenewNodeIdLease,
+                );
+            }
+
+            // Spam classifier training
+            if let Some(train_frequency) = server
+                .core
+                .spam
+                .classifier
+                .as_ref()
+                .and_then(|c| c.train_frequency)
+            {
+                let next_train = match server.inner.data.spam_classifier.load().as_ref() {
+                    spamfilter::SpamClassifier::FhClassifier {
+                        last_trained_at, ..
+                    }
+                    | spamfilter::SpamClassifier::CcfhClassifier {
+                        last_trained_at, ..
+                    } => now().saturating_sub(*last_trained_at).min(train_frequency),
+                    spamfilter::SpamClassifier::Disabled => train_frequency,
+                };
+
+                queue.schedule(
+                    Instant::now() + Duration::from_secs(next_train),
+                    Event::TrainSpamClassifier,
+                );
+            }
+
+            // OTEL Push Metrics
+            if let Some(otel) = &server.core.metrics.otel {
+                OtelMetrics::enable_errors();
+                queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics);
+            }
+
+            // Calculate expensive metrics
+            queue.schedule(Instant::now(), Event::CalculateMetrics);
+
+        }
+
+
+        let mut next_metric_update = Instant::now();
+
+        loop {
+            tokio::time::sleep(queue.wake_up_time()).await;
+
+            let server = inner.build_server();
+            let roles = &server.core.network.roles;
+            let mut batch = (roles.task_scheduler).then(BatchBuilder::new);
+
+            while let Some(event) = queue.pop() {
+                match event.event {
+                    Event::PurgeAccount => {
+                        queue.schedule(
+                            Instant::now()
+                                + server.core.email.account_purge_frequency.time_to_next(),
+                            Event::PurgeAccount,
+                        );
+
+                        if let Some(batch) = batch.as_mut() {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskQueued),
+                                Type = TaskStoreMaintenanceType::PurgeAccounts.as_str()
+                            );
+
+                            batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance {
+                                maintenance_type: TaskStoreMaintenanceType::PurgeAccounts,
+                                status: TaskStatus::now(),
+                                shard_index: None,
+                            }));
+                        }
+                    }
+                    Event::PurgeDataStore => {
+                        queue.schedule(
+                            Instant::now() + server.core.email.data_purge_frequency.time_to_next(),
+                            Event::PurgeDataStore,
+                        );
+
+                        if let Some(batch) = batch.as_mut() {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskQueued),
+                                Type = TaskStoreMaintenanceType::PurgeData.as_str()
+                            );
+
+                            batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance {
+                                maintenance_type: TaskStoreMaintenanceType::PurgeData,
+                                status: TaskStatus::now(),
+                                shard_index: None,
+                            }));
+                        }
+                    }
+                    Event::PurgeBlobStore => {
+                        queue.schedule(
+                            Instant::now() + server.core.email.blob_purge_frequency.time_to_next(),
+                            Event::PurgeBlobStore,
+                        );
+
+                        if let Some(batch) = batch.as_mut() {
+                            trc::event!(
+                                TaskManager(TaskManagerEvent::TaskQueued),
+                                Type = TaskStoreMaintenanceType::PurgeBlob.as_str()
+                            );
+
+                            batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance {
+                                maintenance_type: TaskStoreMaintenanceType::PurgeBlob,
+                                status: TaskStatus::now(),
+                                shard_index: None,
+                            }));
+                        }
+                    }
+                    Event::RenewNodeIdLease => {
+                        queue.schedule(
+                            Instant::now() + server.registry().refresh_node_id_interval(),
+                            Event::RenewNodeIdLease,
+                        );
+
+                        trc::event!(
+                            Cluster(ClusterEvent::NodeIdRenewed),
+                            Id = server.registry().node_id()
+                        );
+
+                        let server = server.clone();
+                        tokio::spawn(async move {
+                            if let Err(err) = server.registry().refresh_node_id_lease().await {
+                                trc::error!(err.details("Failed to renew node ID lease"));
+                            }
+                        });
+                    }
+                    Event::OtelMetrics => {
+                        if let Some(otel) = &server.core.metrics.otel {
+                            queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics);
+
+                            if roles.metrics_push {
+                                let otel = otel.clone();
+
+
+                                #[cfg(not(feature = "enterprise"))]
+                                let is_enterprise = false;
+
+                                tokio::spawn(async move {
+                                    let elapsed = Instant::now();
+                                    otel.push_metrics(is_enterprise, start_time).await;
+
+                                    trc::event!(
+                                        Telemetry(TelemetryEvent::MetricsPushed),
+                                        Elapsed = elapsed.elapsed()
+                                    );
+                                });
+                            }
+                        }
+                    }
+                    Event::CalculateMetrics => {
+                        // Calculate expensive metrics every 5 minutes
+                        queue.schedule(
+                            Instant::now() + Duration::from_secs(5 * 60),
+                            Event::CalculateMetrics,
+                        );
+
+                        let update_other_metrics = if Instant::now() >= next_metric_update {
+                            next_metric_update = Instant::now() + Duration::from_secs(86400);
+                            true
+                        } else {
+                            false
+                        };
+
+                        let server = server.clone();
+                        tokio::spawn(async move {
+                            let elapsed = Instant::now();
+                            if server.core.network.roles.metrics_calculate {
+
+                                if update_other_metrics {
+                                    match server.total_accounts().await {
+                                        Ok(total) => {
+                                            Collector::update_gauge(
+                                                MetricType::UserCount,
+                                                total as u64,
+                                            );
+                                        }
+                                        Err(err) => {
+                                            trc::error!(
+                                                err.details("Failed to obtain account count")
+                                            );
+                                        }
+                                    }
+
+                                    match server.total_domains().await {
+                                        Ok(total) => {
+                                            Collector::update_gauge(
+                                                MetricType::DomainCount,
+                                                total as u64,
+                                            );
+                                        }
+                                        Err(err) => {
+                                            trc::error!(
+                                                err.details("Failed to obtain domain count")
+                                            );
+                                        }
+                                    }
+                                }
+                            }
+
+                            match tokio::task::spawn_blocking(memory_stats::memory_stats).await {
+                                Ok(Some(stats)) => {
+                                    Collector::update_gauge(
+                                        MetricType::ServerMemory,
+                                        stats.physical_mem as u64,
+                                    );
+                                }
+                                Ok(None) => {}
+                                Err(err) => {
+                                    trc::error!(
+                                        trc::EventType::Server(trc::ServerEvent::ThreadError,)
+                                            .reason(err)
+                                            .caused_by(trc::location!())
+                                            .details("Join Error")
+                                    );
+                                }
+                            }
+
+                            trc::event!(
+                                Telemetry(TelemetryEvent::MetricsCollected),
+                                Elapsed = elapsed.elapsed()
+                            );
+                        });
+                    }
+                    Event::TrainSpamClassifier => {
+                        if let Some(train_frequency) = server
+                            .core
+                            .spam
+                            .classifier
+                            .as_ref()
+                            .and_then(|c| c.train_frequency)
+                        {
+                            // Schedule next training
+                            queue.schedule(
+                                Instant::now() + Duration::from_secs(train_frequency),
+                                Event::TrainSpamClassifier,
+                            );
+
+                            if let Some(batch) = batch.as_mut() {
+                                trc::event!(
+                                    TaskManager(TaskManagerEvent::TaskQueued),
+                                    Type = TaskType::SpamFilterMaintenance.as_str()
+                                );
+
+                                batch.schedule_task(Task::SpamFilterMaintenance(
+                                    TaskSpamFilterMaintenance {
+                                        maintenance_type: TaskSpamFilterMaintenanceType::Train,
+                                        status: TaskStatus::now(),
+                                    },
+                                ));
+                            }
+                        }
+                    }
+
+                }
+            }
+
+            if let Some(mut batch) = batch
+                && !batch.is_empty()
+                && let Err(err) = server.store().write(batch.build_all()).await
+            {
+                trc::error!(err.details("Failed to write scheduled tasks"));
+            }
+        }
+    });
+}
+
+impl Queue {
+    pub fn schedule(&mut self, due: Instant, event: Event) {
+        trc::event!(
+            TaskManager(TaskManagerEvent::TaskScheduled),
+            Due = trc::Value::Timestamp(
+                now() + due.saturating_duration_since(Instant::now()).as_secs()
+            ),
+            Id = event.name()
+        );
+
+        self.heap.push(Action { due, event });
+    }
+
+    pub fn wake_up_time(&self) -> Duration {
+        self.heap
+            .peek()
+            .map(|e| e.due.saturating_duration_since(Instant::now()))
+            .unwrap_or(LONG_1D_SLUMBER)
+    }
+
+    pub fn pop(&mut self) -> Option<Action> {
+        if self.heap.peek()?.due <= Instant::now() {
+            self.heap.pop()
+        } else {
+            None
+        }
+    }
+}
+
+impl Ord for Action {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.due.cmp(&other.due).reverse()
+    }
+}
+
+impl PartialOrd for Action {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Event {
+    fn name(&self) -> &'static str {
+        match self {
+            Event::PurgeAccount => "purgeAccount",
+            Event::PurgeDataStore => "purgeDataStore",
+            Event::PurgeBlobStore => "purgeBlobStore",
+            Event::OtelMetrics => "otelMetrics",
+            Event::CalculateMetrics => "calculateMetrics",
+            Event::TrainSpamClassifier => "trainSpamClassifier",
+            Event::RenewNodeIdLease => "renewNodeIdLease",
+        }
+    }
+}

crates/services/src/task_manager/spam_classifier.rs

@@ -0,0 +1,402 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::task_manager::{TaskFailureType, TaskResult};
+use common::{
+    Server,
+    ipc::{BroadcastEvent, RegistryChange},
+    manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource},
+};
+use registry::{
+    schema::{
+        enums::TaskSpamFilterMaintenanceType,
+        prelude::ObjectType,
+        structs::{
+            HttpLookup, MemoryLookupKey, SpamDnsblServer, SpamFileExtension, SpamRule, SpamTag,
+            TaskSpamFilterMaintenance,
+        },
+    },
+    types::EnumImpl,
+};
+use spam_filter::modules::classifier::SpamClassifier;
+use std::time::{Duration, Instant};
+use store::{
+    ahash::AHashMap,
+    registry::write::{RegistryWrite, RegistryWriteResult},
+};
+use trc::{SpamEvent, Value};
+
+pub(crate) trait SpamFilterMaintenanceTask: Sync + Send {
+    fn spam_filter_maintenance(
+        &self,
+        task: &TaskSpamFilterMaintenance,
+    ) -> impl Future<Output = TaskResult> + Send;
+}
+
+impl SpamFilterMaintenanceTask for Server {
+    async fn spam_filter_maintenance(&self, task: &TaskSpamFilterMaintenance) -> TaskResult {
+        match spam_filter_maintenance(self, task).await {
+            Ok(result) => result,
+            Err(err) => {
+                let result = TaskResult::temporary(err.to_string());
+                trc::error!(err.details("Failed to perform spam filter maintenance task"));
+                result
+            }
+        }
+    }
+}
+
+async fn spam_filter_maintenance(
+    server: &Server,
+    task: &TaskSpamFilterMaintenance,
+) -> trc::Result<TaskResult> {
+    match task.maintenance_type {
+        TaskSpamFilterMaintenanceType::Train => {
+            if !server.inner.ipc.train_task_controller.is_running() {
+                Box::pin(server.spam_train(false)).await?;
+            }
+        }
+        TaskSpamFilterMaintenanceType::Retrain => {
+            if !server.inner.ipc.train_task_controller.is_running() {
+                Box::pin(server.spam_train(true)).await?;
+            }
+        }
+        TaskSpamFilterMaintenanceType::Reset => {
+            for key in [SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY] {
+                server.blob_store().delete_blob(key).await?;
+            }
+        }
+        TaskSpamFilterMaintenanceType::Abort => {
+            if server.inner.ipc.train_task_controller.is_running() {
+                server.inner.ipc.train_task_controller.stop();
+            }
+        }
+        TaskSpamFilterMaintenanceType::UpdateRules => {
+            return update_spam_rules(server).await;
+        }
+    }
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+struct RuleUpdateError {
+    typ: TaskFailureType,
+    reason: String,
+}
+
+#[derive(Default)]
+struct Rules {
+    rules: Vec<SpamRule>,
+    dnsbls: Vec<SpamDnsblServer>,
+    tags: Vec<SpamTag>,
+    http_lookups: Vec<HttpLookup>,
+    key_lookups: Vec<MemoryLookupKey>,
+    file_exts: Vec<SpamFileExtension>,
+}
+
+#[derive(Default)]
+struct RuleUpdateResult {
+    success: usize,
+    already_exists: usize,
+    failed: usize,
+}
+
+async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
+    let started = Instant::now();
+    let rules = match fetch_spam_rules(server).await {
+        Ok(rules) => rules,
+        Err(err) => {
+            return Ok(TaskResult::Failure {
+                typ: err.typ,
+                message: err.reason,
+                max_attempts: None,
+            });
+        }
+    };
+
+    let registry = server.registry();
+    let mut stats: AHashMap<ObjectType, RuleUpdateResult> = AHashMap::new();
+
+    let mut reload_settings = false;
+    let mut reload_lookups = false;
+
+    for rule in rules.rules {
+        match registry.write(RegistryWrite::insert(&rule.into())).await? {
+            RegistryWriteResult::Success(_) => {
+                stats.entry(ObjectType::SpamRule).or_default().success += 1;
+                reload_settings = true;
+            }
+            RegistryWriteResult::PrimaryKeyConflict { .. } => {
+                stats
+                    .entry(ObjectType::SpamRule)
+                    .or_default()
+                    .already_exists += 1;
+            }
+            _ => {
+                stats.entry(ObjectType::SpamRule).or_default().failed += 1;
+            }
+        }
+    }
+
+    for dnsbl in rules.dnsbls {
+        match registry.write(RegistryWrite::insert(&dnsbl.into())).await? {
+            RegistryWriteResult::Success(_) => {
+                stats
+                    .entry(ObjectType::SpamDnsblServer)
+                    .or_default()
+                    .success += 1;
+                reload_settings = true;
+            }
+            RegistryWriteResult::PrimaryKeyConflict { .. } => {
+                stats
+                    .entry(ObjectType::SpamDnsblServer)
+                    .or_default()
+                    .already_exists += 1;
+            }
+            _ => {
+                stats.entry(ObjectType::SpamDnsblServer).or_default().failed += 1;
+            }
+        }
+    }
+
+    for tag in rules.tags {
+        match registry.write(RegistryWrite::insert(&tag.into())).await? {
+            RegistryWriteResult::Success(_) => {
+                stats.entry(ObjectType::SpamTag).or_default().success += 1;
+                reload_settings = true;
+            }
+            RegistryWriteResult::PrimaryKeyConflict { .. } => {
+                stats.entry(ObjectType::SpamTag).or_default().already_exists += 1;
+            }
+            _ => {
+                stats.entry(ObjectType::SpamTag).or_default().failed += 1;
+            }
+        }
+    }
+
+    for lookup in rules.http_lookups {
+        match registry
+            .write(RegistryWrite::insert(&lookup.into()))
+            .await?
+        {
+            RegistryWriteResult::Success(_) => {
+                stats.entry(ObjectType::HttpLookup).or_default().success += 1;
+                reload_lookups = true;
+            }
+            RegistryWriteResult::PrimaryKeyConflict { .. } => {
+                stats
+                    .entry(ObjectType::HttpLookup)
+                    .or_default()
+                    .already_exists += 1;
+            }
+            _ => {
+                stats.entry(ObjectType::HttpLookup).or_default().failed += 1;
+            }
+        }
+    }
+
+    for key_lookup in rules.key_lookups {
+        match registry
+            .write(RegistryWrite::insert(&key_lookup.into()))
+            .await?
+        {
+            RegistryWriteResult::Success(_) => {
+                stats
+                    .entry(ObjectType::MemoryLookupKey)
+                    .or_default()
+                    .success += 1;
+                reload_lookups = true;
+            }
+            RegistryWriteResult::PrimaryKeyConflict { .. } => {
+                stats
+                    .entry(ObjectType::MemoryLookupKey)
+                    .or_default()
+                    .already_exists += 1;
+            }
+            _ => {
+                stats.entry(ObjectType::MemoryLookupKey).or_default().failed += 1;
+            }
+        }
+    }
+
+    for ext in rules.file_exts {
+        match registry.write(RegistryWrite::insert(&ext.into())).await? {
+            RegistryWriteResult::Success(_) => {
+                stats
+                    .entry(ObjectType::SpamFileExtension)
+                    .or_default()
+                    .success += 1;
+                reload_settings = true;
+            }
+            RegistryWriteResult::PrimaryKeyConflict { .. } => {
+                stats
+                    .entry(ObjectType::SpamFileExtension)
+                    .or_default()
+                    .already_exists += 1;
+            }
+            _ => {
+                stats
+                    .entry(ObjectType::SpamFileExtension)
+                    .or_default()
+                    .failed += 1;
+            }
+        }
+    }
+
+    if reload_settings {
+        if let Err(err) =
+            Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::SpamRule))).await
+        {
+            trc::error!(err.details("Failed to reload registry after updating spam rules"));
+        }
+        server
+            .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload(
+                ObjectType::SpamRule,
+            )))
+            .await;
+    }
+
+    if reload_lookups {
+        if let Err(err) =
+            Box::pin(server.reload_registry(RegistryChange::Reload(ObjectType::MemoryLookupKey)))
+                .await
+        {
+            trc::error!(err.details("Failed to reload registry after updating spam rules"));
+        }
+        server
+            .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload(
+                ObjectType::MemoryLookupKey,
+            )))
+            .await;
+    }
+
+    trc::event!(
+        Spam(SpamEvent::RulesUpdated),
+        Details = stats
+            .into_iter()
+            .map(|(object_type, result)| {
+                Value::Array(vec![
+                    Value::String(object_type.as_str().into()),
+                    Value::from(result.success),
+                    Value::from(result.already_exists),
+                    Value::from(result.failed),
+                ])
+            })
+            .collect::<Vec<_>>(),
+        Elapsed = started.elapsed(),
+    );
+
+    Ok(TaskResult::Success(vec![]))
+}
+
+async fn fetch_spam_rules(server: &Server) -> Result<Rules, RuleUpdateError> {
+    let Some(rules_url) = server.core.spam.spam_rules_url.as_ref() else {
+        return Err(RuleUpdateError {
+            typ: TaskFailureType::Permanent,
+            reason: "Spam rules resource URL not configured".to_string(),
+        });
+    };
+    let rules_json: AHashMap<String, Vec<serde_json::Value>> =
+        fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500)
+            .await
+            .map_err(|reason| RuleUpdateError {
+                typ: TaskFailureType::Temporary,
+                reason,
+            })
+            .and_then(|bytes| {
+                serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError {
+                    typ: TaskFailureType::Permanent,
+                    reason: format!("Failed to parse spam rules JSON: {err}"),
+                })
+            })?;
+
+    let mut rules = Rules::default();
+    for (object_type, values) in rules_json {
+        let Some(object_type) = ObjectType::parse(&object_type) else {
+            return Err(RuleUpdateError {
+                typ: TaskFailureType::Permanent,
+                reason: format!("Invalid object type in spam rules JSON: {object_type}"),
+            });
+        };
+
+        match object_type {
+            ObjectType::SpamRule => {
+                rules.rules = values
+                    .into_iter()
+                    .map(|value| {
+                        serde_json::from_value(value).map_err(|err| RuleUpdateError {
+                            typ: TaskFailureType::Permanent,
+                            reason: format!("Failed to parse spam rule: {err}"),
+                        })
+                    })
+                    .collect::<Result<Vec<SpamRule>, RuleUpdateError>>()?;
+            }
+            ObjectType::SpamDnsblServer => {
+                rules.dnsbls = values
+                    .into_iter()
+                    .map(|value| {
+                        serde_json::from_value(value).map_err(|err| RuleUpdateError {
+                            typ: TaskFailureType::Permanent,
+                            reason: format!("Failed to parse DNSBL server: {err}"),
+                        })
+                    })
+                    .collect::<Result<Vec<SpamDnsblServer>, RuleUpdateError>>()?;
+            }
+            ObjectType::SpamTag => {
+                rules.tags = values
+                    .into_iter()
+                    .map(|value| {
+                        serde_json::from_value(value).map_err(|err| RuleUpdateError {
+                            typ: TaskFailureType::Permanent,
+                            reason: format!("Failed to parse spam tag: {err}"),
+                        })
+                    })
+                    .collect::<Result<Vec<SpamTag>, RuleUpdateError>>()?;
+            }
+            ObjectType::HttpLookup => {
+                rules.http_lookups = values
+                    .into_iter()
+                    .map(|value| {
+                        serde_json::from_value(value).map_err(|err| RuleUpdateError {
+                            typ: TaskFailureType::Permanent,
+                            reason: format!("Failed to parse HTTP lookup: {err}"),
+                        })
+                    })
+                    .collect::<Result<Vec<HttpLookup>, RuleUpdateError>>()?;
+            }
+            ObjectType::MemoryLookupKey => {
+                rules.key_lookups = values
+                    .into_iter()
+                    .map(|value| {
+                        serde_json::from_value(value).map_err(|err| RuleUpdateError {
+                            typ: TaskFailureType::Permanent,
+                            reason: format!("Failed to parse memory lookup key: {err}"),
+                        })
+                    })
+                    .collect::<Result<Vec<MemoryLookupKey>, RuleUpdateError>>()?;
+            }
+            ObjectType::SpamFileExtension => {
+                rules.file_exts = values
+                    .into_iter()
+                    .map(|value| {
+                        serde_json::from_value(value).map_err(|err| RuleUpdateError {
+                            typ: TaskFailureType::Permanent,
+                            reason: format!("Failed to parse spam file extension: {err}"),
+                        })
+                    })
+                    .collect::<Result<Vec<SpamFileExtension>, RuleUpdateError>>()?;
+            }
+            _ => {
+                return Err(RuleUpdateError {
+                    typ: TaskFailureType::Permanent,
+                    reason: format!("Unsupported object type in spam rules: {object_type:?}"),
+                });
+            }
+        }
+    }
+
+    Ok(rules)
+}