Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

crates/email/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "email"
+version = "0.16.22"
+edition = "2024"
+
+[dependencies]
+utils = { path = "../utils" }
+nlp = { path = "../nlp" }
+store = { path = "../store" }
+trc = { path = "../trc" }
+types = { path = "../types" }
+jmap_proto = { path = "../jmap-proto" }
+jmap-tools = { version = "0.1" }
+common = { path =  "../common" }
+groupware = { path =  "../groupware" }
+registry = { path =  "../registry" }
+mail-parser = { version = "0.11", features = ["full_encoding"] } 
+mail-builder = { version = "1.0" }
+sieve-rs = { version = "0.7", features = ["rkyv"] } 
+tokio = { version = "1.53", features = ["net", "macros"] }
+aes = "0.9"
+aes-gcm = "0.11.1"
+chacha20poly1305 = "0.11"
+cbc = { version = "0.2", features = ["alloc"] }
+rasn = "0.28"
+rasn-cms = "0.28"
+rasn-pkix = "0.28"
+rsa = { version = "0.9.10", features = ["sha2"] }
+rand = "0.8"
+sequoia-openpgp = { version = "2.4", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] }
+hashify = "0.2"
+rkyv = { version = "0.8.18", features = ["little_endian"] }
+
+[features]
+test_mode = []
+enterprise = []
+
+[dev-dependencies]
+tokio = { version = "1.53", features = ["full"] }
+
+[lints]
+workspace = true

crates/email/src/cache/email.rs

@@ -0,0 +1,356 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::metadata::{ArchivedMessageData, MessageData};
+use common::{
+    MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken,
+    sharing::EffectiveAcl,
+};
+use store::write::{AlignedBytes, Archive};
+use store::{ValueKey, ahash::AHashMap, roaring::RoaringBitmap};
+use trc::AddContext;
+use types::{
+    acl::Acl,
+    collection::Collection,
+    keyword::{Keyword, OTHER},
+};
+use utils::map::bitmap::Bitmap;
+
+struct MessagesCacheBuilder {
+    pub change_id: u64,
+    pub items: Vec<MessageCache>,
+    pub index: AHashMap<u32, u32>,
+    pub keywords: Vec<Box<str>>,
+    pub size: u64,
+}
+
+pub(crate) async fn update_email_cache(
+    server: &Server,
+    account_id: u32,
+    changed_ids: &AHashMap<u32, bool>,
+    store_cache: &MessageStoreCache,
+) -> trc::Result<MessagesCache> {
+    let mut new_cache = MessagesCacheBuilder {
+        index: AHashMap::with_capacity(store_cache.emails.items.len()),
+        items: Vec::with_capacity(store_cache.emails.items.len()),
+        size: 0,
+        change_id: 0,
+        keywords: store_cache.emails.keywords.to_vec(),
+    };
+
+    for (document_id, is_update) in changed_ids {
+        if *is_update
+            && let Some(archive) = server
+                .store()
+                .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                    account_id,
+                    Collection::Email,
+                    *document_id,
+                ))
+                .await
+                .caused_by(trc::location!())?
+        {
+            insert_item(
+                &mut new_cache,
+                *document_id,
+                archive.to_unarchived::<MessageData>()?,
+            );
+        }
+    }
+
+    for item in &store_cache.emails.items {
+        if !changed_ids.contains_key(&item.document_id) {
+            email_insert(&mut new_cache, item.clone());
+        }
+    }
+
+    Ok(new_cache.build())
+}
+
+pub(crate) async fn full_email_cache_build(
+    server: &Server,
+    account_id: u32,
+) -> trc::Result<MessagesCache> {
+    // Build cache
+    let mut cache = MessagesCacheBuilder {
+        items: Vec::with_capacity(16),
+        index: AHashMap::with_capacity(16),
+        keywords: Vec::new(),
+        size: 0,
+        change_id: 0,
+    };
+
+    server
+        .archives(
+            account_id,
+            Collection::Email,
+            &(),
+            |document_id, archive| {
+                insert_item(
+                    &mut cache,
+                    document_id,
+                    archive.to_unarchived::<MessageData>()?,
+                );
+                Ok(true)
+            },
+        )
+        .await
+        .caused_by(trc::location!())?;
+
+    Ok(cache.build())
+}
+
+fn insert_item(
+    cache: &mut MessagesCacheBuilder,
+    document_id: u32,
+    archive: Archive<&ArchivedMessageData>,
+) {
+    let message = archive.inner;
+    let mut item = MessageCache {
+        mailboxes: message
+            .mailboxes
+            .iter()
+            .map(|m| MessageUidCache {
+                mailbox_id: m.mailbox_id.to_native(),
+                uid: m.uid.to_native(),
+            })
+            .collect(),
+        keywords: 0,
+        thread_id: message.thread_id.to_native(),
+        change_id: archive.version.change_id().unwrap_or_default(),
+        document_id,
+        size: message.size.to_native(),
+    };
+    for keyword in message.keywords.iter() {
+        match keyword.id() {
+            Ok(id) => {
+                item.keywords |= 1 << id;
+            }
+            Err(custom) => {
+                if let Some(idx) = cache.keywords.iter().position(|k| **k == *custom) {
+                    item.keywords |= 1 << (OTHER + idx);
+                } else if cache.keywords.len() < (128 - OTHER) {
+                    cache.keywords.push(custom.into());
+                    item.keywords |= 1 << (OTHER + cache.keywords.len() - 1);
+                }
+            }
+        }
+    }
+
+    email_insert(cache, item);
+}
+
+impl MessagesCacheBuilder {
+    pub fn build(mut self) -> MessagesCache {
+        self.index.shrink_to_fit();
+        MessagesCache {
+            change_id: self.change_id,
+            items: self.items.into_boxed_slice(),
+            index: self.index,
+            keywords: self.keywords.into_boxed_slice(),
+            size: self.size,
+        }
+    }
+}
+
+pub trait MessageCacheAccess {
+    fn email_by_id(&self, id: &u32) -> Option<&MessageCache>;
+
+    fn has_email_id(&self, id: &u32) -> bool;
+
+    fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator<Item = &MessageCache>;
+
+    fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator<Item = &MessageCache>;
+
+    fn in_thread(&self, thread_id: u32) -> impl Iterator<Item = &MessageCache>;
+
+    fn with_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache>;
+
+    fn without_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache>;
+
+    fn in_mailbox_with_keyword(
+        &self,
+        mailbox_id: u32,
+        keyword: &Keyword,
+    ) -> impl Iterator<Item = &MessageCache>;
+
+    fn in_mailbox_without_keyword(
+        &self,
+        mailbox_id: u32,
+        keyword: &Keyword,
+    ) -> impl Iterator<Item = &MessageCache>;
+
+    fn email_document_ids(&self) -> RoaringBitmap;
+
+    fn shared_messages(
+        &self,
+        access_token: &AccessToken,
+        check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
+    ) -> RoaringBitmap;
+
+    fn expand_keywords(&self, message: &MessageCache) -> impl Iterator<Item = Keyword>;
+
+    fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool;
+}
+
+impl MessageCacheAccess for MessageStoreCache {
+    fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator<Item = &MessageCache> {
+        self.emails
+            .items
+            .iter()
+            .filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id))
+    }
+
+    fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator<Item = &MessageCache> {
+        self.emails.items.iter().filter(move |m| {
+            m.mailboxes
+                .iter()
+                .any(|mb| mailbox_ids.contains(&mb.mailbox_id))
+        })
+    }
+
+    fn in_thread(&self, thread_id: u32) -> impl Iterator<Item = &MessageCache> {
+        self.emails
+            .items
+            .iter()
+            .filter(move |m| m.thread_id == thread_id)
+    }
+
+    fn with_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache> {
+        let keyword_id = keyword_to_id(self, keyword);
+        self.emails
+            .items
+            .iter()
+            .filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0))
+    }
+
+    fn without_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache> {
+        let keyword_id = keyword_to_id(self, keyword);
+        self.emails
+            .items
+            .iter()
+            .filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0))
+    }
+
+    fn in_mailbox_with_keyword(
+        &self,
+        mailbox_id: u32,
+        keyword: &Keyword,
+    ) -> impl Iterator<Item = &MessageCache> {
+        let keyword_id = keyword_to_id(self, keyword);
+        self.emails.items.iter().filter(move |m| {
+            m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)
+                && keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)
+        })
+    }
+
+    fn in_mailbox_without_keyword(
+        &self,
+        mailbox_id: u32,
+        keyword: &Keyword,
+    ) -> impl Iterator<Item = &MessageCache> {
+        let keyword_id = keyword_to_id(self, keyword);
+        self.emails.items.iter().filter(move |m| {
+            m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)
+                && keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)
+        })
+    }
+
+    fn shared_messages(
+        &self,
+        access_token: &AccessToken,
+        check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
+    ) -> RoaringBitmap {
+        let check_acls = check_acls.into();
+        let mut shared_messages = RoaringBitmap::new();
+        for mailbox in &self.mailboxes.items {
+            if mailbox
+                .acls
+                .as_slice()
+                .effective_acl(access_token)
+                .contains_all(check_acls)
+            {
+                shared_messages.extend(
+                    self.in_mailbox(mailbox.document_id)
+                        .map(|item| item.document_id),
+                );
+            }
+        }
+        shared_messages
+    }
+
+    fn email_document_ids(&self) -> RoaringBitmap {
+        RoaringBitmap::from_iter(self.emails.index.keys())
+    }
+
+    fn email_by_id(&self, id: &u32) -> Option<&MessageCache> {
+        self.emails
+            .index
+            .get(id)
+            .and_then(|idx| self.emails.items.get(*idx as usize))
+    }
+
+    fn has_email_id(&self, id: &u32) -> bool {
+        self.emails.index.contains_key(id)
+    }
+
+    fn expand_keywords(&self, message: &MessageCache) -> impl Iterator<Item = Keyword> {
+        KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) {
+            Ok(keyword) => keyword,
+            Err(id) => Keyword::Other(self.emails.keywords[id - OTHER].clone()),
+        })
+    }
+
+    fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool {
+        keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0)
+    }
+}
+
+fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) {
+    let id = item.document_id;
+    if let Some(idx) = cache.index.get(&id) {
+        cache.items[*idx as usize] = item;
+    } else {
+        cache.size += (std::mem::size_of::<MessageCache>()
+            + (std::mem::size_of::<u32>() * 2)
+            + (item.mailboxes.len() * std::mem::size_of::<MessageUidCache>()))
+            as u64;
+
+        let idx = cache.items.len() as u32;
+        cache.items.push(item);
+        cache.index.insert(id, idx);
+    }
+}
+
+#[inline]
+fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option<u32> {
+    match keyword.id() {
+        Ok(id) => Some(id),
+        Err(name) => cache
+            .emails
+            .keywords
+            .iter()
+            .position(|k| **k == *name)
+            .map(|idx| (OTHER + idx) as u32),
+    }
+}
+
+#[derive(Clone, Copy, Debug)]
+struct KeywordsIter(u128);
+
+impl Iterator for KeywordsIter {
+    type Item = usize;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.0 != 0 {
+            let item = 127 - self.0.leading_zeros();
+            self.0 ^= 1 << item;
+            Some(item as usize)
+        } else {
+            None
+        }
+    }
+}

crates/email/src/cache/mailbox.rs

@@ -0,0 +1,322 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::mailbox::{ArchivedMailbox, Mailbox, manage::MailboxFnc};
+use common::{
+    MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken,
+    sharing::EffectiveAcl,
+};
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use store::{ahash::AHashMap, roaring::RoaringBitmap};
+use trc::AddContext;
+use types::{
+    acl::{Acl, AclGrant},
+    collection::Collection,
+    special_use::SpecialUse,
+};
+use utils::{map::bitmap::Bitmap, topological::TopologicalSort};
+
+struct MailboxesCacheBuilder {
+    pub change_id: u64,
+    pub index: AHashMap<u32, u32>,
+    pub items: Vec<MailboxCache>,
+    pub size: u64,
+}
+
+pub(crate) async fn update_mailbox_cache(
+    server: &Server,
+    account_id: u32,
+    changed_ids: &AHashMap<u32, bool>,
+    store_cache: &MessageStoreCache,
+) -> trc::Result<MailboxesCache> {
+    let mut new_cache = MailboxesCacheBuilder {
+        items: Vec::with_capacity(store_cache.mailboxes.items.len()),
+        index: AHashMap::with_capacity(store_cache.mailboxes.items.len()),
+        size: 0,
+        change_id: 0,
+    };
+
+    for (document_id, is_update) in changed_ids {
+        if *is_update
+            && let Some(archive) = server
+                .store()
+                .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                    account_id,
+                    Collection::Mailbox,
+                    *document_id,
+                ))
+                .await
+                .caused_by(trc::location!())?
+        {
+            insert_item(
+                &mut new_cache,
+                *document_id,
+                archive.unarchive::<Mailbox>()?,
+            );
+        }
+    }
+
+    for item in store_cache.mailboxes.items.iter() {
+        if !changed_ids.contains_key(&item.document_id) {
+            mailbox_insert(&mut new_cache, item.clone());
+        }
+    }
+
+    build_tree(&mut new_cache);
+
+    Ok(new_cache.build())
+}
+
+pub(crate) async fn full_mailbox_cache_build(
+    server: &Server,
+    account_id: u32,
+) -> trc::Result<MailboxesCache> {
+    // Build cache
+    let mut cache = MailboxesCacheBuilder {
+        items: Default::default(),
+        index: Default::default(),
+        size: 0,
+        change_id: 0,
+    };
+
+    server
+        .archives(
+            account_id,
+            Collection::Mailbox,
+            &(),
+            |document_id, archive| {
+                insert_item(&mut cache, document_id, archive.unarchive::<Mailbox>()?);
+                Ok(true)
+            },
+        )
+        .await
+        .caused_by(trc::location!())?;
+
+    if cache.items.is_empty() {
+        server
+            .create_system_folders(account_id)
+            .await
+            .caused_by(trc::location!())?;
+        server
+            .archives(
+                account_id,
+                Collection::Mailbox,
+                &(),
+                |document_id, archive| {
+                    insert_item(&mut cache, document_id, archive.unarchive::<Mailbox>()?);
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())?;
+    }
+
+    build_tree(&mut cache);
+
+    Ok(cache.build())
+}
+
+fn insert_item(cache: &mut MailboxesCacheBuilder, document_id: u32, mailbox: &ArchivedMailbox) {
+    let parent_id = mailbox.parent_id.to_native();
+    let item = MailboxCache {
+        document_id,
+        name: mailbox.name.as_str().into(),
+        path: "".into(),
+        role: (&mailbox.role).into(),
+        parent_id: if parent_id > 0 {
+            parent_id - 1
+        } else {
+            u32::MAX
+        },
+        sort_order: mailbox
+            .sort_order
+            .as_ref()
+            .map(|s| s.to_native())
+            .unwrap_or(u32::MAX),
+        subscribers: mailbox.subscribers.iter().map(|s| s.to_native()).collect(),
+        uid_validity: mailbox.uid_validity.to_native(),
+        acls: mailbox
+            .acls
+            .iter()
+            .map(|acl| AclGrant {
+                account_id: acl.account_id.to_native(),
+                grants: Bitmap::from(&acl.grants),
+            })
+            .collect(),
+    };
+
+    mailbox_insert(cache, item);
+}
+
+fn build_tree(cache: &mut MailboxesCacheBuilder) {
+    cache.size = 0;
+    let mut topological_sort = TopologicalSort::with_capacity(cache.items.len());
+
+    for (idx, mailbox) in cache.items.iter_mut().enumerate() {
+        topological_sort.insert(
+            if mailbox.parent_id == u32::MAX {
+                0
+            } else {
+                mailbox.parent_id + 1
+            },
+            mailbox.document_id + 1,
+        );
+        mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) {
+            "INBOX".into()
+        } else if mailbox.is_root() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") {
+            format!("INBOX {}", idx + 1)
+        } else {
+            mailbox.name.clone()
+        };
+
+        cache.size += item_size(mailbox);
+    }
+
+    for folder_id in topological_sort.into_iterator() {
+        if folder_id != 0 {
+            let folder_id = folder_id - 1;
+            if let Some((path, parent_path)) = by_id(cache, &folder_id)
+                .and_then(|folder| {
+                    folder
+                        .parent_id()
+                        .map(|parent_id| (&folder.path, parent_id))
+                })
+                .and_then(|(path, parent_id)| {
+                    by_id(cache, &parent_id).map(|folder| (path, &folder.path))
+                })
+            {
+                let mut new_path = String::with_capacity(parent_path.len() + path.len() + 1);
+                new_path.push_str(parent_path.as_str());
+                new_path.push('/');
+                new_path.push_str(path.as_str());
+                let folder = by_id_mut(cache, &folder_id).unwrap();
+                folder.path = new_path;
+            }
+        }
+    }
+}
+
+impl MailboxesCacheBuilder {
+    fn build(mut self) -> MailboxesCache {
+        self.index.shrink_to_fit();
+        MailboxesCache {
+            change_id: self.change_id,
+            index: self.index,
+            items: self.items.into_boxed_slice(),
+            size: self.size,
+        }
+    }
+}
+
+pub trait MailboxCacheAccess {
+    fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache>;
+    fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache>;
+    fn mailbox_by_path(&self, name: &str) -> Option<&MailboxCache>;
+    fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>;
+    fn shared_mailboxes(
+        &self,
+        access_token: &AccessToken,
+        check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
+    ) -> RoaringBitmap;
+    fn has_mailbox_id(&self, id: &u32) -> bool;
+}
+
+impl MailboxCacheAccess for MessageStoreCache {
+    fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache> {
+        self.mailboxes
+            .items
+            .iter()
+            .find(|m| m.name.eq_ignore_ascii_case(name))
+    }
+
+    fn mailbox_by_path(&self, path: &str) -> Option<&MailboxCache> {
+        self.mailboxes
+            .items
+            .iter()
+            .find(|m| m.path.eq_ignore_ascii_case(path))
+    }
+
+    fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> {
+        self.mailboxes.items.iter().find(|m| &m.role == role)
+    }
+
+    fn shared_mailboxes(
+        &self,
+        access_token: &AccessToken,
+        check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
+    ) -> RoaringBitmap {
+        let check_acls = check_acls.into();
+
+        RoaringBitmap::from_iter(
+            self.mailboxes
+                .items
+                .iter()
+                .filter(|m| {
+                    m.acls
+                        .as_slice()
+                        .effective_acl(access_token)
+                        .contains_all(check_acls)
+                })
+                .map(|m| m.document_id),
+        )
+    }
+
+    fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache> {
+        self.mailboxes
+            .index
+            .get(id)
+            .and_then(|idx| self.mailboxes.items.get(*idx as usize))
+    }
+
+    fn has_mailbox_id(&self, id: &u32) -> bool {
+        self.mailboxes.index.contains_key(id)
+    }
+}
+
+#[inline(always)]
+fn by_id<'x>(cache: &'x MailboxesCacheBuilder, id: &u32) -> Option<&'x MailboxCache> {
+    cache
+        .index
+        .get(id)
+        .and_then(|idx| cache.items.get(*idx as usize))
+}
+
+#[inline(always)]
+fn by_id_mut<'x>(cache: &'x mut MailboxesCacheBuilder, id: &u32) -> Option<&'x mut MailboxCache> {
+    cache
+        .index
+        .get(id)
+        .and_then(|idx| cache.items.get_mut(*idx as usize))
+}
+
+fn mailbox_insert(cache: &mut MailboxesCacheBuilder, item: MailboxCache) {
+    let id = item.document_id;
+    if let Some(idx) = cache.index.get(&id) {
+        cache.items[*idx as usize] = item;
+    } else {
+        let idx = cache.items.len() as u32;
+        cache.items.push(item);
+        cache.index.insert(id, idx);
+    }
+}
+
+#[inline(always)]
+fn item_size(item: &MailboxCache) -> u64 {
+    (std::mem::size_of::<MailboxCache>()
+        + (if item.name.len() > std::mem::size_of::<String>() {
+            item.name.len()
+        } else {
+            0
+        })
+        + (if item.path.len() > std::mem::size_of::<String>() {
+            item.path.len()
+        } else {
+            0
+        })) as u64
+}

crates/email/src/cache/mod.rs

@@ -0,0 +1,248 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::{MessageStoreCache, Server, UpdateLock, cache::LockResult};
+use email::{full_email_cache_build, update_email_cache};
+use mailbox::{full_mailbox_cache_build, update_mailbox_cache};
+use std::{collections::hash_map::Entry, sync::Arc, time::Instant};
+use store::{
+    ahash::AHashMap,
+    query::log::{Change, Query},
+};
+use trc::{AddContext, StoreEvent};
+use types::collection::SyncCollection;
+use utils::cache::Cache;
+
+pub mod email;
+pub mod mailbox;
+
+pub trait MessageCacheFetch: Sync + Send {
+    fn get_cached_messages(
+        &self,
+        account_id: u32,
+    ) -> impl Future<Output = trc::Result<Arc<MessageStoreCache>>> + Send;
+}
+
+impl MessageCacheFetch for Server {
+    async fn get_cached_messages(&self, account_id: u32) -> trc::Result<Arc<MessageStoreCache>> {
+        let cache_store = &self.inner.cache.messages;
+        let mut cache = match cache_store.get_value_or_guard_async(&account_id).await {
+            Ok(cache) => cache,
+            Err(guard) => {
+                let start_time = Instant::now();
+                let cache = full_cache_build(self, account_id, Arc::new(UpdateLock::new())).await?;
+
+                if guard.insert(cache.clone()).is_err() {
+                    cache_store.update(account_id, cache.clone());
+                }
+                warn_if_uncacheable(cache_store, account_id, &cache);
+
+                trc::event!(
+                    Store(StoreEvent::CacheMiss),
+                    AccountId = account_id,
+                    Collection = SyncCollection::Email.as_str(),
+                    Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()],
+                    ChangeId = cache.last_change_id,
+                    Elapsed = start_time.elapsed(),
+                );
+
+                return Ok(cache);
+            }
+        };
+
+        // Obtain current state
+        let start_time = Instant::now();
+        let changes = self
+            .core
+            .storage
+            .data
+            .changes(
+                account_id,
+                SyncCollection::Email.into(),
+                Query::Since(cache.last_change_id),
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        // Regenerate cache if the change log has been truncated
+        if changes.is_truncated {
+            let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?;
+            cache_store.update(account_id, cache.clone());
+            warn_if_uncacheable(cache_store, account_id, &cache);
+
+            trc::event!(
+                Store(StoreEvent::CacheStale),
+                AccountId = account_id,
+                Collection = SyncCollection::Email.as_str(),
+                ChangeId = cache.last_change_id,
+                Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()],
+                Elapsed = start_time.elapsed(),
+            );
+
+            return Ok(cache);
+        }
+
+        // Verify changes
+        if changes.changes.is_empty() {
+            trc::event!(
+                Store(StoreEvent::CacheHit),
+                AccountId = account_id,
+                Collection = SyncCollection::Email.as_str(),
+                ChangeId = cache.last_change_id,
+                Elapsed = start_time.elapsed(),
+            );
+
+            return Ok(cache);
+        }
+
+        // Lock for updates
+        let lock = cache.update_lock.clone();
+        let _permit = match lock.acquire(cache.last_change_id).await? {
+            LockResult::Acquired(permit) => permit,
+            LockResult::Stale(permit) => {
+                cache = cache_store.peek(&account_id).unwrap_or(cache.clone());
+                if cache.last_change_id >= changes.to_change_id {
+                    trc::event!(
+                        Store(StoreEvent::CacheHit),
+                        AccountId = account_id,
+                        Collection = SyncCollection::Email.as_str(),
+                        ChangeId = cache.last_change_id,
+                        Elapsed = start_time.elapsed(),
+                    );
+                    return Ok(cache);
+                }
+
+                permit
+            }
+        };
+        let mut cache = cache.as_ref().clone();
+
+        let mut changed_items: AHashMap<u32, bool> = AHashMap::with_capacity(changes.changes.len());
+        let mut changed_containers: AHashMap<u32, bool> =
+            AHashMap::with_capacity(changes.changes.len());
+        let mut has_container_property_changes = false;
+
+        for change in changes.changes {
+            match change {
+                Change::InsertItem(id) => match changed_items.entry(id as u32) {
+                    Entry::Occupied(mut entry) => {
+                        *entry.get_mut() = true;
+                    }
+                    Entry::Vacant(entry) => {
+                        entry.insert(true);
+                    }
+                },
+                Change::UpdateItem(id) => {
+                    changed_items.insert(id as u32, true);
+                }
+                Change::DeleteItem(id) => {
+                    match changed_items.entry(id as u32) {
+                        Entry::Occupied(mut entry) => {
+                            // Thread reassignment
+                            *entry.get_mut() = true;
+                        }
+                        Entry::Vacant(entry) => {
+                            entry.insert(false);
+                        }
+                    }
+                }
+                Change::InsertContainer(id) | Change::UpdateContainer(id) => {
+                    changed_containers.insert(id as u32, true);
+                }
+                Change::DeleteContainer(id) => {
+                    changed_containers.insert(id as u32, false);
+                }
+                Change::UpdateContainerProperty(_) => {
+                    has_container_property_changes = true;
+                }
+            }
+        }
+
+        if !changed_items.is_empty() {
+            let mut email_cache =
+                update_email_cache(self, account_id, &changed_items, &cache).await?;
+            email_cache.change_id = changes.item_change_id.unwrap_or(changes.to_change_id);
+            cache.emails = Arc::new(email_cache);
+        }
+
+        if !changed_containers.is_empty() {
+            let mut mailbox_cache =
+                update_mailbox_cache(self, account_id, &changed_containers, &cache).await?;
+            mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id);
+            cache.mailboxes = Arc::new(mailbox_cache);
+        } else if has_container_property_changes {
+            let mut mailbox_cache = cache.mailboxes.as_ref().clone();
+            mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id);
+            cache.mailboxes = Arc::new(mailbox_cache);
+        }
+        cache.size = cache.emails.size + cache.mailboxes.size;
+        cache.last_change_id = changes.to_change_id;
+
+        cache.update_lock.set_revision(cache.last_change_id);
+        let cache = Arc::new(cache);
+        cache_store.update(account_id, cache.clone());
+        warn_if_uncacheable(cache_store, account_id, &cache);
+
+        trc::event!(
+            Store(StoreEvent::CacheUpdate),
+            AccountId = account_id,
+            Collection = SyncCollection::Email.as_str(),
+            ChangeId = cache.last_change_id,
+            Details = vec![changed_items.len(), changed_containers.len()],
+            Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()],
+            Elapsed = start_time.elapsed(),
+        );
+
+        Ok(cache)
+    }
+}
+
+#[inline(always)]
+fn warn_if_uncacheable(
+    cache_store: &Cache<u32, Arc<MessageStoreCache>>,
+    account_id: u32,
+    cache: &Arc<MessageStoreCache>,
+) {
+    let capacity = cache_store.weight_capacity();
+    if cache.size > capacity {
+        trc::event!(
+            Store(StoreEvent::CacheEntryTooLarge),
+            AccountId = account_id,
+            Collection = SyncCollection::Email.as_str(),
+            Size = cache.size,
+            Limit = capacity,
+        );
+    }
+}
+
+async fn full_cache_build(
+    server: &Server,
+    account_id: u32,
+    update_lock: Arc<UpdateLock>,
+) -> trc::Result<Arc<MessageStoreCache>> {
+    let last_change_id = server
+        .core
+        .storage
+        .data
+        .get_last_change_id(account_id, SyncCollection::Email.into())
+        .await
+        .caused_by(trc::location!())?
+        .unwrap_or_default();
+    let mut emails = full_email_cache_build(server, account_id).await?;
+    let mut mailboxes = full_mailbox_cache_build(server, account_id).await?;
+    let size = emails.size + mailboxes.size;
+    emails.change_id = last_change_id;
+    mailboxes.change_id = last_change_id;
+    update_lock.set_revision(last_change_id);
+
+    Ok(Arc::new(MessageStoreCache {
+        update_lock,
+        emails: Arc::new(emails),
+        mailboxes: Arc::new(mailboxes),
+        last_change_id,
+        size,
+    }))
+}

crates/email/src/identity/index.rs

@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ArchivedIdentity, Identity};
+use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
+use types::collection::SyncCollection;
+
+impl IndexableObject for Identity {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [IndexValue::LogItem {
+            sync_collection: SyncCollection::Identity,
+            prefix: None,
+        }]
+        .into_iter()
+    }
+}
+
+impl IndexableObject for &ArchivedIdentity {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [IndexValue::LogItem {
+            sync_collection: SyncCollection::Identity,
+            prefix: None,
+        }]
+        .into_iter()
+    }
+}
+
+impl IndexableAndSerializableObject for Identity {
+    fn is_versioned() -> bool {
+        false
+    }
+}

crates/email/src/identity/mod.rs

@@ -0,0 +1,25 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod index;
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub struct Identity {
+    pub name: String,
+    pub email: String,
+    pub reply_to: Option<Vec<EmailAddress>>,
+    pub bcc: Option<Vec<EmailAddress>>,
+    pub text_signature: String,
+    pub html_signature: String,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)]
+pub struct EmailAddress {
+    pub name: Option<String>,
+    pub email: String,
+}

crates/email/src/lib.rs

@@ -0,0 +1,15 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+#![warn(clippy::large_futures)]
+
+pub mod cache;
+pub mod identity;
+pub mod mailbox;
+pub mod message;
+pub mod push;
+pub mod sieve;
+pub mod submission;

crates/email/src/mailbox/destroy.rs

@@ -0,0 +1,237 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::*;
+use crate::{
+    cache::{MessageCacheFetch, email::MessageCacheAccess},
+    message::{delete::EmailDeletion, metadata::MessageData},
+};
+use common::{
+    Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder,
+};
+use registry::schema::{
+    enums::IndexDocumentType,
+    structs::{Task, TaskIndexDocument, TaskStatus},
+};
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use store::{roaring::RoaringBitmap, write::BatchBuilder};
+use trc::AddContext;
+use types::{
+    acl::Acl,
+    collection::{Collection, VanishedCollection},
+    field::MailboxField,
+};
+
+pub trait MailboxDestroy: Sync + Send {
+    fn mailbox_destroy(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        access_token: &AccessToken,
+        remove_emails: bool,
+    ) -> impl Future<Output = trc::Result<Result<Option<u64>, MailboxDestroyError>>> + Send;
+}
+
+pub enum MailboxDestroyError {
+    CannotDestroy,
+    Forbidden,
+    HasChildren,
+    HasEmails,
+    NotFound,
+    AssertionFailed,
+}
+
+impl MailboxDestroy for Server {
+    async fn mailbox_destroy(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        access_token: &AccessToken,
+        remove_emails: bool,
+    ) -> trc::Result<Result<Option<u64>, MailboxDestroyError>> {
+        // Internal folders cannot be deleted
+        #[cfg(not(feature = "test_mode"))]
+        if [INBOX_ID, TRASH_ID, JUNK_ID].contains(&document_id) {
+            return Ok(Err(MailboxDestroyError::CannotDestroy));
+        }
+
+        // Verify that this mailbox does not have sub-mailboxes
+        let cache = self
+            .get_cached_messages(account_id)
+            .await
+            .caused_by(trc::location!())?;
+        if cache
+            .mailboxes
+            .items
+            .iter()
+            .any(|item| item.parent_id == document_id)
+        {
+            return Ok(Err(MailboxDestroyError::HasChildren));
+        }
+
+        // Verify that the mailbox is empty
+        let mut batch = BatchBuilder::new();
+
+        batch.with_account_id(account_id);
+
+        let message_ids =
+            RoaringBitmap::from_iter(cache.in_mailbox(document_id).map(|m| m.document_id));
+
+        if !message_ids.is_empty() {
+            if remove_emails {
+                // If the message is in multiple mailboxes, untag it from the current mailbox,
+                // otherwise delete it.
+
+                let mut deleted_ids = RoaringBitmap::new();
+                let mut thread_ids = RoaringBitmap::new();
+                self.archives(
+                    account_id,
+                    Collection::Email,
+                    &message_ids,
+                    |message_id, message_data_| {
+                        // Remove mailbox from list
+                        let prev_message_data = message_data_
+                            .to_unarchived::<MessageData>()
+                            .caused_by(trc::location!())?;
+                        if !prev_message_data
+                            .inner
+                            .mailboxes
+                            .iter()
+                            .any(|id| id.mailbox_id == document_id)
+                        {
+                            return Ok(true);
+                        }
+
+                        if prev_message_data.inner.mailboxes.len() == 1 {
+                            // Delete message
+                            for mailbox in prev_message_data.inner.mailboxes.iter() {
+                                batch.log_vanished_item(
+                                    VanishedCollection::Email,
+                                    (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()),
+                                );
+                            }
+                            deleted_ids.insert(message_id);
+                            thread_ids.insert(prev_message_data.inner.thread_id.to_native());
+                            batch
+                                .with_collection(Collection::Email)
+                                .with_document(message_id)
+                                .custom(
+                                    ObjectIndexBuilder::<_, ()>::new()
+                                        .with_changed_by(access_token.account_tenant_ids())
+                                        .with_current(prev_message_data),
+                                )
+                                .caused_by(trc::location!())?
+                                .schedule_task(Task::UnindexDocument(TaskIndexDocument {
+                                    account_id: account_id.into(),
+                                    document_id: message_id.into(),
+                                    document_type: IndexDocumentType::Email,
+                                    status: TaskStatus::now(),
+                                }))
+                                .commit_point();
+                        } else {
+                            let new_message_data = MessageData {
+                                mailboxes: prev_message_data
+                                    .inner
+                                    .mailboxes
+                                    .iter()
+                                    .filter(|m| m.mailbox_id != document_id)
+                                    .map(|m| m.to_native())
+                                    .collect(),
+                                keywords: prev_message_data
+                                    .inner
+                                    .keywords
+                                    .iter()
+                                    .map(|k| k.to_native())
+                                    .collect(),
+                                thread_id: prev_message_data.inner.thread_id.to_native(),
+                                size: prev_message_data.inner.size.to_native(),
+                            };
+
+                            // Untag message from mailbox
+                            batch
+                                .with_collection(Collection::Email)
+                                .with_document(message_id)
+                                .custom(
+                                    ObjectIndexBuilder::new()
+                                        .with_changed_by(access_token.account_tenant_ids())
+                                        .with_changes(new_message_data)
+                                        .with_current(prev_message_data),
+                                )
+                                .caused_by(trc::location!())?
+                                .commit_point();
+                        }
+
+                        Ok(true)
+                    },
+                )
+                .await
+                .caused_by(trc::location!())?;
+
+                self.log_emptied_threads(account_id, &mut batch, thread_ids, &deleted_ids)
+                    .await
+                    .caused_by(trc::location!())?;
+            } else {
+                return Ok(Err(MailboxDestroyError::HasEmails));
+            }
+        }
+
+        // Obtain mailbox
+        if let Some(mailbox_) = self
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                account_id,
+                Collection::Mailbox,
+                document_id,
+            ))
+            .await
+            .caused_by(trc::location!())?
+        {
+            let mailbox = mailbox_
+                .to_unarchived::<Mailbox>()
+                .caused_by(trc::location!())?;
+            // Validate ACLs
+            if access_token.is_shared(account_id) {
+                let acl = mailbox.inner.acls.effective_acl(access_token);
+                if !acl.contains(Acl::Delete) || (remove_emails && !acl.contains(Acl::RemoveItems))
+                {
+                    return Ok(Err(MailboxDestroyError::Forbidden));
+                }
+            }
+            batch
+                .with_account_id(account_id)
+                .with_collection(Collection::Mailbox)
+                .with_document(document_id)
+                .clear(MailboxField::UidCounter)
+                .custom(ObjectIndexBuilder::<_, ()>::new().with_current(mailbox))
+                .caused_by(trc::location!())?;
+        } else {
+            return Ok(Err(MailboxDestroyError::NotFound));
+        };
+
+        if !batch.is_empty() {
+            match self
+                .commit_batch(batch)
+                .await
+                .and_then(|ids| ids.last_change_id(account_id))
+            {
+                Ok(change_id) => {
+                    self.notify_task_queue();
+
+                    Ok(Ok(Some(change_id)))
+                }
+                Err(err) if err.is_assertion_failure() => {
+                    Ok(Err(MailboxDestroyError::AssertionFailed))
+                }
+                Err(err) => Err(err.caused_by(trc::location!())),
+            }
+        } else {
+            Ok(Ok(None))
+        }
+    }
+}

crates/email/src/mailbox/index.rs

@@ -0,0 +1,48 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ArchivedMailbox, Mailbox};
+use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
+use types::{acl::AclGrant, collection::SyncCollection};
+
+impl IndexableObject for Mailbox {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::LogContainer {
+                sync_collection: SyncCollection::Email,
+            },
+            IndexValue::Acl {
+                value: (&self.acls).into(),
+            },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableObject for &ArchivedMailbox {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::LogContainer {
+                sync_collection: SyncCollection::Email,
+            },
+            IndexValue::Acl {
+                value: self
+                    .acls
+                    .iter()
+                    .map(AclGrant::from)
+                    .collect::<Vec<_>>()
+                    .into(),
+            },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableAndSerializableObject for Mailbox {
+    fn is_versioned() -> bool {
+        false
+    }
+}

crates/email/src/mailbox/manage.rs

@@ -0,0 +1,166 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::*;
+use crate::cache::MessageCacheFetch;
+use common::{Server, storage::index::ObjectIndexBuilder};
+use registry::schema::enums::StorageQuota;
+use std::future::Future;
+use store::write::BatchBuilder;
+use trc::AddContext;
+use types::collection::Collection;
+
+pub trait MailboxFnc: Sync + Send {
+    fn create_system_folders(
+        &self,
+        account_id: u32,
+    ) -> impl Future<Output = trc::Result<()>> + Send;
+
+    fn mailbox_create_path(
+        &self,
+        account_id: u32,
+        path: &str,
+    ) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
+}
+
+impl MailboxFnc for Server {
+    async fn create_system_folders(&self, account_id: u32) -> trc::Result<()> {
+        #[cfg(feature = "test_mode")]
+        if account_id == 0 {
+            return Ok(());
+        }
+
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::Mailbox);
+
+        // Create mailboxes
+        let mut last_document_id = ARCHIVE_ID;
+        for folder in &self.core.email.default_folders {
+            let document_id = match folder.special_use {
+                SpecialUse::Inbox => INBOX_ID,
+                SpecialUse::Trash => TRASH_ID,
+                SpecialUse::Junk => JUNK_ID,
+                SpecialUse::Drafts => DRAFTS_ID,
+                SpecialUse::Sent => SENT_ID,
+                SpecialUse::Archive => ARCHIVE_ID,
+                SpecialUse::None
+                | SpecialUse::Important
+                | SpecialUse::Memos
+                | SpecialUse::Scheduled
+                | SpecialUse::Snoozed => {
+                    last_document_id += 1;
+                    last_document_id
+                }
+                SpecialUse::Shared => unreachable!(),
+            };
+
+            let mut object = Mailbox::new(folder.name.clone()).with_role(folder.special_use);
+            if folder.subscribe {
+                object.add_subscriber(account_id);
+            }
+            batch
+                .with_document(document_id)
+                .custom(ObjectIndexBuilder::<(), _>::new().with_changes(object))
+                .caused_by(trc::location!())?;
+        }
+        self.store()
+            .assign_document_ids(account_id, Collection::Mailbox, (ARCHIVE_ID + 1) as u64)
+            .await
+            .caused_by(trc::location!())?;
+
+        self.core
+            .storage
+            .data
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+
+        Ok(())
+    }
+
+    async fn mailbox_create_path(&self, account_id: u32, path: &str) -> trc::Result<Option<u32>> {
+        let cache = self
+            .get_cached_messages(account_id)
+            .await
+            .caused_by(trc::location!())?;
+
+        let mut next_parent_id = 0;
+        let mut create_paths = Vec::with_capacity(2);
+
+        let mut path = path.split('/').map(|v| v.trim());
+        let mut found_path = String::with_capacity(16);
+        {
+            while let Some(name) = path.next() {
+                if !found_path.is_empty() {
+                    found_path.push('/');
+                }
+
+                for ch in name.chars() {
+                    for ch in ch.to_lowercase() {
+                        found_path.push(ch);
+                    }
+                }
+
+                if let Some(item) = cache
+                    .mailboxes
+                    .items
+                    .iter()
+                    .find(|item| item.path.to_lowercase() == found_path)
+                {
+                    next_parent_id = item.document_id + 1;
+                } else {
+                    create_paths.push(name.to_string());
+                    create_paths.extend(path.map(|v| v.to_string()));
+                    break;
+                }
+            }
+        }
+
+        // Create missing folders
+        if !create_paths.is_empty() {
+            if create_paths
+                .iter()
+                .any(|name| name.len() > self.core.email.mailbox_name_max_len)
+            {
+                return Ok(None);
+            }
+
+            let account = self.account(account_id).await.caused_by(trc::location!())?;
+            if cache.mailboxes.items.len() + create_paths.len()
+                > self.object_quota(account.object_quotas(), StorageQuota::MaxMailboxes) as usize
+            {
+                return Ok(None);
+            }
+
+            let mut next_document_id = self
+                .store()
+                .assign_document_ids(account_id, Collection::Mailbox, create_paths.len() as u64)
+                .await
+                .caused_by(trc::location!())?;
+            let mut batch = BatchBuilder::new();
+            for name in create_paths {
+                let document_id = next_document_id;
+                next_document_id -= 1;
+                batch
+                    .with_account_id(account_id)
+                    .with_collection(Collection::Mailbox)
+                    .with_document(document_id)
+                    .custom(
+                        ObjectIndexBuilder::<(), _>::new()
+                            .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)),
+                    )
+                    .caused_by(trc::location!())?;
+                next_parent_id = document_id + 1;
+            }
+
+            self.commit_batch(batch).await.caused_by(trc::location!())?;
+        }
+
+        Ok(Some(next_parent_id - 1))
+    }
+}

crates/email/src/mailbox/mod.rs

@@ -0,0 +1,112 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use types::{acl::AclGrant, special_use::SpecialUse};
+
+pub mod destroy;
+pub mod index;
+pub mod manage;
+
+pub const INBOX_ID: u32 = 0;
+pub const TRASH_ID: u32 = 1;
+pub const JUNK_ID: u32 = 2;
+pub const DRAFTS_ID: u32 = 3;
+pub const SENT_ID: u32 = 4;
+pub const ARCHIVE_ID: u32 = 5;
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)]
+#[rkyv(derive(Debug))]
+pub struct Mailbox {
+    pub name: String,
+    pub role: SpecialUse,
+    pub parent_id: u32,
+    pub sort_order: Option<u32>,
+    pub uid_validity: u32,
+    pub subscribers: Vec<u32>,
+    pub acls: Vec<AclGrant>,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy)]
+#[rkyv(derive(Debug, Clone, Copy))]
+pub struct UidMailbox {
+    pub mailbox_id: u32,
+    pub uid: u32,
+}
+
+impl Mailbox {
+    pub fn new(name: impl Into<String>) -> Self {
+        Mailbox {
+            name: name.into(),
+            role: SpecialUse::None,
+            parent_id: 0,
+            sort_order: None,
+            uid_validity: rand::random::<u32>(),
+            subscribers: vec![],
+            acls: vec![],
+        }
+    }
+
+    pub fn with_role(mut self, role: SpecialUse) -> Self {
+        self.role = role;
+        self
+    }
+
+    pub fn with_parent_id(mut self, parent_id: u32) -> Self {
+        self.parent_id = parent_id;
+        self
+    }
+
+    pub fn with_sort_order(mut self, sort_order: u32) -> Self {
+        self.sort_order = Some(sort_order);
+        self
+    }
+
+    pub fn with_subscriber(mut self, subscriber: u32) -> Self {
+        self.subscribers.push(subscriber);
+        self
+    }
+
+    pub fn add_subscriber(&mut self, subscriber: u32) -> bool {
+        if !self.subscribers.contains(&subscriber) {
+            self.subscribers.push(subscriber);
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn remove_subscriber(&mut self, subscriber: u32) {
+        self.subscribers.retain(|&x| x != subscriber);
+    }
+
+    pub fn is_subscribed(&self, subscriber: u32) -> bool {
+        self.subscribers.contains(&subscriber)
+    }
+}
+
+impl ArchivedMailbox {
+    pub fn is_subscribed(&self, subscriber: u32) -> bool {
+        self.subscribers.iter().any(|x| u32::from(x) == subscriber)
+    }
+}
+
+impl PartialEq for UidMailbox {
+    fn eq(&self, other: &Self) -> bool {
+        self.mailbox_id == other.mailbox_id
+    }
+}
+
+impl Eq for UidMailbox {}
+
+impl UidMailbox {
+    pub fn new(mailbox_id: u32, uid: u32) -> Self {
+        UidMailbox { mailbox_id, uid }
+    }
+
+    pub fn new_unassigned(mailbox_id: u32) -> Self {
+        UidMailbox { mailbox_id, uid: 0 }
+    }
+}

crates/email/src/message/body.rs

@@ -0,0 +1,395 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::metadata::{
+    ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, ArchivedMetadataPartType,
+    PART_ENCODING_BASE64, PART_ENCODING_QP, PART_SIZE_MASK,
+};
+use jmap_proto::object::email::{EmailProperty, EmailValue};
+use jmap_tools::{Map, Value};
+use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType};
+use types::blob::BlobId;
+use utils::chained_bytes::ChainedBytes;
+
+use super::headers::HeaderToValue;
+
+pub trait ToBodyPart {
+    fn to_body_part(
+        &self,
+        part_id: u32,
+        properties: &[EmailProperty],
+        raw_message: &ChainedBytes<'_>,
+        blob_id: &BlobId,
+        blob_body_offset: isize,
+    ) -> Value<'static, EmailProperty, EmailValue>;
+}
+
+impl ToBodyPart for Vec<MessagePart<'_>> {
+    fn to_body_part(
+        &self,
+        part_id: u32,
+        properties: &[EmailProperty],
+        raw_message: &ChainedBytes<'_>,
+        blob_id: &BlobId,
+        blob_body_offset: isize,
+    ) -> Value<'static, EmailProperty, EmailValue> {
+        let mut parts = vec![part_id].into_iter();
+        let mut parts_stack = Vec::new();
+        let mut subparts = Vec::with_capacity(1);
+
+        loop {
+            if let Some((part_id, part)) = parts
+                .next()
+                .map(|part_id| (part_id, &self[part_id as usize]))
+            {
+                let mut values = Map::with_capacity(properties.len());
+                let multipart = if let PartType::Multipart(parts) = &part.body {
+                    parts.into()
+                } else {
+                    None
+                };
+
+                for property in properties {
+                    let value = match property {
+                        EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(),
+                        EmailProperty::BlobId if multipart.is_none() => {
+                            let base_offset = blob_id.start_offset() as isize + blob_body_offset;
+                            BlobId::new_section(
+                                blob_id.hash.clone(),
+                                blob_id.class.clone(),
+                                (part.offset_body as isize + base_offset) as usize,
+                                (part.offset_end as isize + base_offset) as usize,
+                                part.encoding as u8,
+                            )
+                            .into()
+                        }
+                        EmailProperty::Size if multipart.is_none() => match &part.body {
+                            PartType::Text(text) | PartType::Html(text) => text.len(),
+                            PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(),
+                            PartType::Message(message) => message.root_part().raw_len() as usize,
+                            PartType::Multipart(_) => 0,
+                        }
+                        .into(),
+                        EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(),
+                        EmailProperty::Type => part
+                            .content_type()
+                            .map(|ct| {
+                                ct.subtype()
+                                    .map(|st| format!("{}/{}", ct.ctype(), st))
+                                    .unwrap_or_else(|| ct.ctype().to_string())
+                            })
+                            .or_else(|| match &part.body {
+                                PartType::Text(_) => Some("text/plain".to_string()),
+                                PartType::Html(_) => Some("text/html".to_string()),
+                                PartType::Message(_) => Some("message/rfc822".to_string()),
+                                _ => None,
+                            })
+                            .into(),
+                        EmailProperty::Charset => part
+                            .content_type()
+                            .and_then(|ct| ct.attribute("charset"))
+                            .or(match &part.body {
+                                PartType::Text(_) | PartType::Html(_) => Some("us-ascii"),
+                                _ => None,
+                            })
+                            .map(|v| v.to_string())
+                            .into(),
+                        EmailProperty::Disposition => part
+                            .content_disposition()
+                            .map(|cd| cd.ctype())
+                            .map(|v| v.to_string())
+                            .into(),
+                        EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(),
+                        EmailProperty::Language => match part.content_language() {
+                            HeaderValue::Text(text) => vec![text.to_string()].into(),
+                            HeaderValue::TextList(list) => list
+                                .iter()
+                                .map(|text| text.to_string().into())
+                                .collect::<Vec<Value<'static, EmailProperty, EmailValue>>>()
+                                .into(),
+                            _ => Value::Null,
+                        },
+                        EmailProperty::Location => {
+                            part.content_location().map(|v| v.to_string()).into()
+                        }
+                        EmailProperty::Header(_) => {
+                            part.headers.header_to_value(property, raw_message)
+                        }
+                        EmailProperty::Headers => part.headers.headers_to_value(raw_message),
+                        EmailProperty::SubParts => continue,
+                        _ => Value::Null,
+                    };
+                    values.insert_unchecked(property.clone(), value);
+                }
+
+                subparts.push(values);
+
+                if let Some(multipart) = multipart {
+                    if parts_stack.len() == 10_000 {
+                        debug_assert!(false, "Too much nesting in message metadata");
+                        return Value::Null;
+                    }
+                    let multipart = multipart.clone();
+                    parts_stack.push((
+                        parts,
+                        std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())),
+                    ));
+                    parts = multipart.into_iter();
+                }
+            } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() {
+                prev_subparts
+                    .last_mut()
+                    .unwrap()
+                    .insert_unchecked(EmailProperty::SubParts, subparts);
+                parts = prev_parts;
+                subparts = prev_subparts;
+            } else {
+                return subparts.pop().map(Into::into).unwrap_or_default();
+            }
+        }
+    }
+}
+
+impl ToBodyPart for ArchivedMessageMetadataContents {
+    fn to_body_part(
+        &self,
+        part_id: u32,
+        properties: &[EmailProperty],
+        raw_message: &ChainedBytes<'_>,
+        blob_id: &BlobId,
+        blob_body_offset: isize,
+    ) -> Value<'static, EmailProperty, EmailValue> {
+        let mut parts = vec![part_id].into_iter();
+        let mut parts_stack = Vec::new();
+        let mut subparts = Vec::with_capacity(1);
+
+        loop {
+            if let Some((part_id, part)) = parts
+                .next()
+                .map(|part_id| (part_id, &self.parts[part_id as usize]))
+            {
+                let mut values = Map::with_capacity(properties.len());
+                let multipart = if let ArchivedMetadataPartType::Multipart(parts) = &part.body {
+                    parts.into()
+                } else {
+                    None
+                };
+
+                for property in properties {
+                    let value = match property {
+                        EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(),
+                        EmailProperty::BlobId if multipart.is_none() => {
+                            let base_offset = blob_id.start_offset() as isize + blob_body_offset;
+                            let flags = part.flags.to_native();
+                            let encoding = if flags & PART_ENCODING_BASE64 != 0 {
+                                2
+                            } else if flags & PART_ENCODING_QP != 0 {
+                                1
+                            } else {
+                                0
+                            };
+                            BlobId::new_section(
+                                blob_id.hash.clone(),
+                                blob_id.class.clone(),
+                                (u32::from(part.offset_body) as isize + base_offset) as usize,
+                                (u32::from(part.offset_end) as isize + base_offset) as usize,
+                                encoding,
+                            )
+                            .into()
+                        }
+                        EmailProperty::Size if multipart.is_none() => {
+                            (part.flags.to_native() & PART_SIZE_MASK).into()
+                        }
+                        EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(),
+                        EmailProperty::Type => part
+                            .content_type()
+                            .map(|ct| {
+                                ct.subtype()
+                                    .map(|st| format!("{}/{}", ct.ctype(), st))
+                                    .unwrap_or_else(|| ct.ctype().to_string())
+                            })
+                            .or_else(|| match &part.body {
+                                ArchivedMetadataPartType::Text => Some("text/plain".to_string()),
+                                ArchivedMetadataPartType::Html => Some("text/html".to_string()),
+                                ArchivedMetadataPartType::Message(_) => {
+                                    Some("message/rfc822".to_string())
+                                }
+                                _ => None,
+                            })
+                            .into(),
+                        EmailProperty::Charset => {
+                            part.content_type()
+                                .and_then(|ct| ct.attribute("charset"))
+                                .or(match &part.body {
+                                    ArchivedMetadataPartType::Text
+                                    | ArchivedMetadataPartType::Html => Some("us-ascii"),
+                                    _ => None,
+                                })
+                                .map(|v| v.to_string())
+                                .into()
+                        }
+                        EmailProperty::Disposition => part
+                            .content_disposition()
+                            .map(|cd| cd.ctype())
+                            .map(|v| v.to_string())
+                            .into(),
+                        EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(),
+                        EmailProperty::Language => match part.content_language() {
+                            ArchivedMetadataHeaderValue::Text(text) => {
+                                vec![text.to_string()].into()
+                            }
+                            ArchivedMetadataHeaderValue::TextList(list) => list
+                                .iter()
+                                .map(|text| text.to_string().into())
+                                .collect::<Vec<Value<'static, EmailProperty, EmailValue>>>()
+                                .into(),
+                            _ => Value::Null,
+                        },
+                        EmailProperty::Location => {
+                            part.content_location().map(|v| v.to_string()).into()
+                        }
+                        EmailProperty::Header(_) => part.header_to_value(property, raw_message),
+                        EmailProperty::Headers => part.headers_to_value(raw_message),
+                        EmailProperty::SubParts => continue,
+                        _ => Value::Null,
+                    };
+                    values.insert_unchecked(property.clone(), value);
+                }
+
+                subparts.push(values);
+
+                if let Some(multipart) = multipart {
+                    if parts_stack.len() == 10_000 {
+                        debug_assert!(false, "Too much nesting in message metadata");
+                        return Value::Null;
+                    }
+                    let multipart = multipart
+                        .iter()
+                        .map(|id| u16::from(id) as u32)
+                        .collect::<Vec<_>>();
+                    parts_stack.push((
+                        parts,
+                        std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())),
+                    ));
+                    parts = multipart.into_iter();
+                }
+            } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() {
+                prev_subparts
+                    .last_mut()
+                    .unwrap()
+                    .insert_unchecked(EmailProperty::SubParts, subparts);
+                parts = prev_parts;
+                subparts = prev_subparts;
+            } else {
+                return subparts.pop().map(Into::into).unwrap_or_default();
+            }
+        }
+    }
+}
+
+pub trait TruncateBody {
+    fn truncate(&self, max_len: usize) -> (bool, String);
+}
+
+impl TruncateBody for PartType<'_> {
+    fn truncate(&self, max_len: usize) -> (bool, String) {
+        match self {
+            PartType::Text(text) => truncate_plain(text, max_len),
+            PartType::Html(html) => truncate_html(html, max_len),
+            PartType::Binary(bytes) | PartType::InlineBinary(bytes) => {
+                PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len)
+            }
+            _ => (false, "".into()),
+        }
+    }
+}
+
+pub fn truncate_plain(text: &str, mut max_len: usize) -> (bool, String) {
+    if max_len != 0 && text.len() > max_len {
+        let add_dots = max_len > 6;
+        if add_dots {
+            max_len -= 3;
+        }
+        let mut result = String::with_capacity(max_len);
+        for ch in text.chars() {
+            if ch != '\r' {
+                if ch.len_utf8() + result.len() > max_len {
+                    break;
+                }
+                result.push(ch);
+            }
+        }
+        if add_dots {
+            result.push_str("...");
+        }
+        (true, result)
+    } else {
+        (false, text.replace('\r', ""))
+    }
+}
+
+pub fn truncate_html(html: &str, mut max_len: usize) -> (bool, String) {
+    if max_len != 0 && html.len() > max_len {
+        let add_dots = max_len > 6;
+        if add_dots {
+            max_len -= 3;
+        }
+
+        let mut result = String::with_capacity(max_len);
+        let mut in_tag = false;
+        let mut in_comment = false;
+        let mut last_tag_end_pos = 0;
+        let mut cr_count = 0;
+        for (pos, ch) in html.char_indices() {
+            let mut set_last_tag = 0;
+            match ch {
+                '<' if !in_tag => {
+                    in_tag = true;
+                    if let Some("!--") = html.get(pos + 1..pos + 4) {
+                        in_comment = true;
+                    }
+                    set_last_tag = pos;
+                }
+                '>' if in_tag => {
+                    if in_comment {
+                        if let Some("--") = html.get(pos - 2..pos) {
+                            in_comment = false;
+                            in_tag = false;
+                            set_last_tag = pos + 1;
+                        }
+                    } else {
+                        in_tag = false;
+                        set_last_tag = pos + 1;
+                    }
+                }
+                '\r' => {
+                    cr_count += 1;
+                    continue;
+                }
+                _ => (),
+            }
+            if ch.len_utf8() + pos - cr_count > max_len {
+                result.push_str(
+                    &html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 {
+                        last_tag_end_pos
+                    } else {
+                        pos
+                    }]
+                        .replace('\r', ""),
+                );
+                if add_dots {
+                    result.push_str("...");
+                }
+                break;
+            } else if set_last_tag > 0 {
+                last_tag_end_pos = set_last_tag;
+            }
+        }
+        (true, result)
+    } else {
+        (false, html.replace('\r', ""))
+    }
+}

crates/email/src/message/copy.rs

@@ -0,0 +1,267 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{
+    ingest::{EmailIngest, IngestedEmail},
+    metadata::{MessageData, MessageMetadata},
+};
+use crate::{
+    mailbox::UidMailbox,
+    message::{
+        index::extractors::VisitTextArchived,
+        ingest::ThreadInfo,
+        metadata::{
+            MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MetadataHeaderName, MetadataHeaderValue,
+        },
+    },
+};
+use common::{Server, storage::index::ObjectIndexBuilder};
+use mail_parser::parsers::fields::thread::thread_name;
+use registry::{
+    schema::{
+        enums::IndexDocumentType,
+        structs::{Task, TaskIndexDocument, TaskMergeThreads, TaskStatus},
+    },
+    types::map::Map,
+};
+use store::write::{BatchBuilder, IndexPropertyClass, ValueClass};
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use trc::AddContext;
+use types::{
+    blob::{BlobClass, BlobId},
+    collection::{Collection, SyncCollection},
+    field::EmailField,
+    keyword::Keyword,
+};
+use utils::cheeky_hash::CheekyHash;
+
+pub enum CopyMessageError {
+    NotFound,
+    OverQuota,
+    AlreadyExists(u32),
+}
+
+pub trait EmailCopy: Sync + Send {
+    #[allow(clippy::too_many_arguments)]
+    fn copy_message(
+        &self,
+        from_account_id: u32,
+        from_message_id: u32,
+        to_account_id: u32,
+        mailboxes: Vec<u32>,
+        keywords: Vec<Keyword>,
+        received_at: Option<u64>,
+        session_id: u64,
+    ) -> impl Future<Output = trc::Result<Result<IngestedEmail, CopyMessageError>>> + Send;
+}
+
+impl EmailCopy for Server {
+    #[allow(clippy::too_many_arguments)]
+    async fn copy_message(
+        &self,
+        from_account_id: u32,
+        from_message_id: u32,
+        to_account_id: u32,
+        mailboxes: Vec<u32>,
+        keywords: Vec<Keyword>,
+        received_at: Option<u64>,
+        session_id: u64,
+    ) -> trc::Result<Result<IngestedEmail, CopyMessageError>> {
+        // Obtain metadata
+        let mut metadata = if let Some(metadata) = self
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::property(
+                from_account_id,
+                Collection::Email,
+                from_message_id,
+                EmailField::Metadata,
+            ))
+            .await?
+        {
+            metadata
+                .deserialize::<MessageMetadata>()
+                .caused_by(trc::location!())?
+        } else {
+            return Ok(Err(CopyMessageError::NotFound));
+        };
+
+        // Check quota
+        let size = metadata.root_part().offset_end;
+        let to_account = self.account(to_account_id).await?;
+        match self.has_available_quota(&to_account, size as u64).await {
+            Ok(_) => (),
+            Err(err) => {
+                if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota))
+                    || err.matches(trc::EventType::Limit(trc::LimitEvent::TenantQuota))
+                {
+                    trc::error!(err.account_id(to_account_id).span_id(session_id));
+                    return Ok(Err(CopyMessageError::OverQuota));
+                } else {
+                    return Err(err);
+                }
+            }
+        }
+
+        // Set receivedAt
+        if let Some(received_at) = received_at {
+            metadata.rcvd_attach = (metadata.rcvd_attach & MESSAGE_HAS_ATTACHMENT)
+                | (received_at & MESSAGE_RECEIVED_MASK);
+        }
+
+        // Obtain threadId
+        let mut message_ids = Vec::new();
+        let mut subject = "";
+        for header in &metadata.contents[0].parts[0].headers {
+            match &header.name {
+                MetadataHeaderName::MessageId => {
+                    header.value.visit_text(|id| {
+                        if !id.is_empty() {
+                            message_ids.push(CheekyHash::new(id.as_bytes()));
+                        }
+                    });
+                }
+                MetadataHeaderName::InReplyTo
+                | MetadataHeaderName::References
+                | MetadataHeaderName::ResentMessageId => {
+                    header.value.visit_text(|id| {
+                        if !id.is_empty() {
+                            message_ids.push(CheekyHash::new(id.as_bytes()));
+                        }
+                    });
+                }
+                MetadataHeaderName::Subject if subject.is_empty() => {
+                    subject = thread_name(match &header.value {
+                        MetadataHeaderValue::Text(text) => text.as_ref(),
+                        MetadataHeaderValue::TextList(list) if !list.is_empty() => {
+                            list.first().unwrap().as_ref()
+                        }
+                        _ => "",
+                    });
+                }
+                _ => (),
+            }
+        }
+
+        message_ids.sort_unstable();
+        message_ids.dedup();
+
+        // Obtain threadId
+        let thread_result = self
+            .find_thread_id(to_account_id, subject, &message_ids)
+            .await
+            .caused_by(trc::location!())?;
+
+        if let Some(&existing) = thread_result.duplicate_ids.first() {
+            return Ok(Err(CopyMessageError::AlreadyExists(existing)));
+        }
+
+        // Assign id
+        let mut email = IngestedEmail {
+            size: size as usize,
+            ..Default::default()
+        };
+        let blob_hash = metadata.blob_hash.clone();
+
+        // Assign IMAP UIDs
+        let mut mailbox_ids = Vec::with_capacity(mailboxes.len());
+        email.imap_uids = Vec::with_capacity(mailboxes.len());
+        let mut ids = self
+            .assign_email_ids(to_account_id, mailboxes.iter().copied(), true)
+            .await
+            .caused_by(trc::location!())?;
+        let document_id = ids.next().unwrap();
+        for (uid, mailbox_id) in ids.zip(mailboxes.iter().copied()) {
+            mailbox_ids.push(UidMailbox::new(mailbox_id, uid));
+            email.imap_uids.push(uid);
+        }
+
+        // Prepare batch
+        let mut batch = BatchBuilder::new();
+        batch.with_account_id(to_account_id);
+
+        // Determine thread id
+        let tenant_id = to_account.tenant_id();
+        let thread_id = if let Some(thread_id) = thread_result.thread_id {
+            thread_id
+        } else {
+            batch
+                .with_collection(Collection::Thread)
+                .with_document(document_id)
+                .log_container_insert(SyncCollection::Thread);
+            document_id
+        };
+        batch
+            .with_collection(Collection::Email)
+            .with_document(document_id)
+            .custom(
+                ObjectIndexBuilder::<(), _>::new()
+                    .with_tenant_id(tenant_id)
+                    .with_changes(MessageData {
+                        mailboxes: mailbox_ids.into_boxed_slice(),
+                        keywords: keywords.into_boxed_slice(),
+                        thread_id,
+                        size,
+                    }),
+            )
+            .caused_by(trc::location!())?
+            .set(
+                ValueClass::IndexProperty(IndexPropertyClass::Hash {
+                    property: EmailField::Threading.into(),
+                    hash: thread_result.thread_hash,
+                }),
+                ThreadInfo::serialize(thread_id, &message_ids),
+            )
+            .schedule_task(Task::IndexDocument(TaskIndexDocument {
+                account_id: to_account_id.into(),
+                document_id: document_id.into(),
+                document_type: IndexDocumentType::Email,
+                status: TaskStatus::now(),
+            }));
+
+        // Merge threads if necessary
+        if !thread_result.merge_ids.is_empty() {
+            batch.schedule_task(Task::MergeThreads(TaskMergeThreads {
+                account_id: to_account_id.into(),
+                status: TaskStatus::now(),
+                thread_name: thread_result.thread_hash.to_string(),
+                message_ids: Map::new(message_ids.into_iter().map(|id| id.to_string()).collect()),
+            }));
+        }
+
+        metadata
+            .index(&mut batch, true)
+            .caused_by(trc::location!())?;
+
+        // Insert and obtain ids
+        let change_id = self
+            .store()
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?
+            .last_change_id(to_account_id)?;
+
+        // Request indexing
+        self.notify_task_queue();
+
+        // Update response
+        email.document_id = document_id;
+        email.thread_id = thread_id;
+        email.change_id = change_id;
+        email.blob_id = BlobId::new(
+            blob_hash,
+            BlobClass::Linked {
+                account_id: to_account_id,
+                collection: Collection::Email.into(),
+                document_id,
+            },
+        );
+
+        Ok(Ok(email))
+    }
+}

crates/email/src/message/crypto.rs

@@ -0,0 +1,598 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use aes::cipher::{BlockModeEncrypt, KeyIvInit, block_padding::Pkcs7};
+use aes_gcm::{
+    Aes256Gcm,
+    aead::{AeadInOut, KeyInit},
+};
+use chacha20poly1305::ChaCha20Poly1305;
+use common::auth::{
+    ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, ACCOUNT_FLAG_ENCRYPT_ALGO_AES256_GCM,
+    ACCOUNT_FLAG_ENCRYPT_ALGO_CHACHA20_POLY1305, ACCOUNT_FLAG_ENCRYPT_APPEND,
+    ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, EncryptionKeys,
+};
+use mail_builder::{encoders::Base64Encoder, mime::make_boundary};
+use mail_parser::{Message, MimeHeaders, PartType};
+use openpgp::{
+    parse::Parse,
+    serialize::stream,
+    types::{KeyFlags, SymmetricAlgorithm},
+};
+use rand::{RngCore, SeedableRng, rngs::StdRng};
+use rasn::Encoder;
+use rasn::types::{OctetString, Oid, SetOf};
+use rasn_cms::{
+    AlgorithmIdentifier, AuthEnvelopedData, CONTENT_DATA, CONTENT_ENVELOPED_DATA, EncryptedContent,
+    EncryptedContentInfo, EncryptedKey, EnvelopedData, IssuerAndSerialNumber,
+    KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo,
+    algorithms::{AES128_CBC, AES256_CBC, RSA},
+    pkcs7_compat::EncapsulatedContentInfo,
+};
+use rsa::{Oaep, Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey, sha2::Sha256};
+use sequoia_openpgp as openpgp;
+use std::io::Cursor;
+
+const AES256_GCM: &Oid =
+    Oid::JOINT_ISO_ITU_T_COUNTRY_US_ORGANIZATION_GOV_CSOR_NIST_ALGORITHMS_AES256_GCM;
+const CHACHA20_POLY1305: &Oid = Oid::const_new(&[1, 2, 840, 113549, 1, 9, 16, 3, 18]);
+const CONTENT_AUTH_ENVELOPED_DATA: &Oid =
+    Oid::ISO_MEMBER_BODY_US_RSADSI_PKCS9_SMIME_CT_AUTH_ENVELOPED_DATA;
+const SHA256: &Oid =
+    Oid::JOINT_ISO_ITU_T_COUNTRY_US_ORGANIZATION_GOV_CSOR_NIST_ALGORITHMS_HASH_SHA256;
+const MGF1: &Oid = Oid::ISO_MEMBER_BODY_US_RSADSI_PKCS1_MGF1;
+const RSAES_OAEP: &Oid = Oid::ISO_MEMBER_BODY_US_RSADSI_PKCS1_RSAES_OAEP;
+
+#[derive(Debug)]
+pub enum EncryptMessageError {
+    AlreadyEncrypted,
+    Error(String),
+}
+
+#[allow(async_fn_in_trait)]
+pub trait EncryptMessage {
+    async fn encrypt(
+        &self,
+        keys: &EncryptionKeys,
+        flags: u64,
+    ) -> Result<Vec<u8>, EncryptMessageError>;
+    fn is_encrypted(&self) -> bool;
+}
+
+impl EncryptMessage for Message<'_> {
+    async fn encrypt(
+        &self,
+        keys: &EncryptionKeys,
+        flags: u64,
+    ) -> Result<Vec<u8>, EncryptMessageError> {
+        if flags & ACCOUNT_FLAG_ENCRYPT_METHOD_PGP != 0 && flags.cipher().is_aead() {
+            return Err(EncryptMessageError::Error(
+                "AES-256-GCM and ChaCha20-Poly1305 are only supported for S/MIME encryption."
+                    .into(),
+            ));
+        }
+
+        let root = self.root_part();
+        let raw_message = self.raw_message();
+        let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize);
+        let mut inner_message = Vec::with_capacity(raw_message.len());
+
+        // Move MIME headers and body to inner message
+        for header in root.headers() {
+            (if header.name.is_mime_header() {
+                &mut inner_message
+            } else {
+                &mut outer_message
+            })
+            .extend_from_slice(
+                &raw_message[header.offset_field() as usize..header.offset_end() as usize],
+            );
+        }
+        inner_message.extend_from_slice(b"\r\n");
+        inner_message.extend_from_slice(&raw_message[root.raw_body_offset() as usize..]);
+
+        // Encrypt inner message
+        if flags & ACCOUNT_FLAG_ENCRYPT_METHOD_PGP != 0 {
+            // Prepare encrypted message
+            let boundary = make_boundary("_");
+            outer_message.extend_from_slice(
+                concat!(
+                    "Content-Type: multipart/encrypted;\r\n\t",
+                    "protocol=\"application/pgp-encrypted\";\r\n\t",
+                    "boundary=\""
+                )
+                .as_bytes(),
+            );
+            outer_message.extend_from_slice(boundary.as_bytes());
+            outer_message.extend_from_slice(
+                concat!(
+                    "\"\r\n\r\n",
+                    "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n",
+                    "--"
+                )
+                .as_bytes(),
+            );
+            outer_message.extend_from_slice(boundary.as_bytes());
+            outer_message.extend_from_slice(
+                concat!(
+                    "\r\nContent-Type: application/pgp-encrypted\r\n\r\n",
+                    "Version: 1\r\n\r\n--"
+                )
+                .as_bytes(),
+            );
+            outer_message.extend_from_slice(boundary.as_bytes());
+            outer_message.extend_from_slice(
+                concat!(
+                    "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n",
+                    "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n"
+                )
+                .as_bytes(),
+            );
+
+            let certs = keys
+                .iter()
+                .map(openpgp::Cert::from_bytes)
+                .collect::<Result<Vec<_>, _>>()
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!(
+                        "Failed to parse OpenPGP public key: {}",
+                        err
+                    ))
+                })?;
+
+            // Encrypt contents (TODO: use rayon)
+            let encrypted_contents = tokio::task::spawn_blocking(move || {
+                // Parse public key
+                let mut keys = Vec::with_capacity(certs.len());
+                let policy = openpgp::policy::StandardPolicy::new();
+
+                for cert in &certs {
+                    for key in cert
+                        .keys()
+                        .with_policy(&policy, None)
+                        .supported()
+                        .alive()
+                        .revoked(false)
+                        .key_flags(KeyFlags::empty().set_transport_encryption())
+                    {
+                        keys.push(key);
+                    }
+                }
+
+                // Compose a writer stack corresponding to the output format and
+                // packet structure we want.
+                let mut sink = Vec::with_capacity(inner_message.len());
+
+                // Stream an OpenPGP message.
+                let message = stream::Armorer::new(stream::Message::new(&mut sink))
+                    .build()
+                    .map_err(|err| {
+                        EncryptMessageError::Error(format!("Failed to create armorer: {}", err))
+                    })?;
+                let message = stream::Encryptor::for_recipients(message, keys)
+                    .symmetric_algo(flags.algo())
+                    .build()
+                    .map_err(|err| {
+                        EncryptMessageError::Error(format!("Failed to build encryptor: {}", err))
+                    })?;
+                let mut message = stream::LiteralWriter::new(message).build().map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to create literal writer: {}", err))
+                })?;
+                std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
+                })?;
+                message.finalize().map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to finalize message: {}", err))
+                })?;
+
+                String::from_utf8(sink).map_err(|err| {
+                    EncryptMessageError::Error(format!(
+                        "Failed to convert encrypted message to UTF-8: {}",
+                        err
+                    ))
+                })
+            })
+            .await
+            .map_err(|err| {
+                EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
+            })??;
+            outer_message.extend_from_slice(encrypted_contents.as_bytes());
+            outer_message.extend_from_slice(b"\r\n--");
+            outer_message.extend_from_slice(boundary.as_bytes());
+            outer_message.extend_from_slice(b"--\r\n");
+        } else {
+            let cipher = flags.cipher();
+
+            // Generate random nonce
+            let mut rng = StdRng::from_entropy();
+            let mut nonce = vec![0u8; cipher.nonce_size()];
+            rng.fill_bytes(&mut nonce);
+
+            // Generate random key
+            let mut key = vec![0u8; cipher.key_size()];
+            rng.fill_bytes(&mut key);
+
+            // Encrypt contents (TODO: use rayon)
+            let (encrypted_contents, mac, key, nonce) = tokio::task::spawn_blocking(move || {
+                let (encrypted_contents, mac) = cipher.encrypt(&key, &nonce, &inner_message);
+                (encrypted_contents, mac, key, nonce)
+            })
+            .await
+            .map_err(|err| {
+                EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
+            })?;
+
+            // Encrypt key using public keys
+            let key_encryption_algorithm = cipher.key_encryption_algorithm()?;
+            let mut recipient_infos = SetOf::new();
+            for cert in keys.iter() {
+                let cert = rasn::der::decode::<rasn_pkix::Certificate>(cert).map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to parse certificate: {}", err))
+                })?;
+
+                let public_key = RsaPublicKey::from_pkcs1_der(
+                    cert.tbs_certificate
+                        .subject_public_key_info
+                        .subject_public_key
+                        .as_raw_slice(),
+                )
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to parse public key: {}", err))
+                })?;
+                let encrypted_key = if cipher.is_aead() {
+                    public_key.encrypt(&mut rng, Oaep::new::<Sha256>(), &key[..])
+                } else {
+                    public_key.encrypt(&mut rng, Pkcs1v15Encrypt, &key[..])
+                }
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to encrypt key: {}", err))
+                })?;
+
+                recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo(
+                    KeyTransRecipientInfo {
+                        version: 0.into(),
+                        rid: RecipientIdentifier::IssuerAndSerialNumber(IssuerAndSerialNumber {
+                            issuer: cert.tbs_certificate.issuer,
+                            serial_number: cert.tbs_certificate.serial_number,
+                        }),
+                        key_encryption_algorithm: key_encryption_algorithm.clone(),
+                        encrypted_key: EncryptedKey::from(encrypted_key),
+                    },
+                ));
+            }
+
+            let encrypted_content_info = EncryptedContentInfo {
+                content_type: CONTENT_DATA.into(),
+                content_encryption_algorithm: cipher.content_encryption_algorithm(&nonce)?,
+                encrypted_content: Some(EncryptedContent::from(encrypted_contents)),
+            };
+
+            let (content_type, content) = if let Some(mac) = mac {
+                (
+                    CONTENT_AUTH_ENVELOPED_DATA,
+                    rasn::der::encode(&AuthEnvelopedData {
+                        version: 0.into(),
+                        originator_info: None,
+                        recipient_infos,
+                        auth_encrypted_content_info: encrypted_content_info,
+                        auth_attrs: None,
+                        mac: OctetString::from(mac),
+                        unauth_attrs: None,
+                    })
+                    .map_err(|err| {
+                        EncryptMessageError::Error(format!(
+                            "Failed to encode AuthEnvelopedData: {}",
+                            err
+                        ))
+                    })?,
+                )
+            } else {
+                (
+                    CONTENT_ENVELOPED_DATA,
+                    rasn::der::encode(&EnvelopedData {
+                        version: 0.into(),
+                        originator_info: None,
+                        recipient_infos,
+                        encrypted_content_info,
+                        unprotected_attrs: None,
+                    })
+                    .map_err(|err| {
+                        EncryptMessageError::Error(format!(
+                            "Failed to encode EnvelopedData: {}",
+                            err
+                        ))
+                    })?,
+                )
+            };
+
+            let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo {
+                content_type: content_type.into(),
+                content: Some(content.into()),
+            })
+            .map_err(|err| {
+                EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err))
+            })?;
+
+            // Generate message
+            outer_message.extend_from_slice(b"Content-Type: application/pkcs7-mime;\r\n");
+            outer_message.extend_from_slice(b"\tname=\"smime.p7m\";\r\n\tsmime-type=");
+            outer_message.extend_from_slice(if cipher.is_aead() {
+                b"authenticated-enveloped-data\r\n"
+            } else {
+                b"enveloped-data\r\n"
+            });
+            outer_message.extend_from_slice(
+                concat!(
+                    "Content-Disposition: attachment;\r\n",
+                    "\tfilename=\"smime.p7m\"\r\n",
+                    "Content-Transfer-Encoding: base64\r\n\r\n"
+                )
+                .as_bytes(),
+            );
+            Base64Encoder::new()
+                .wrap_lines()
+                .encode_to_writer(&pkcs7, &mut outer_message)
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err))
+                })?;
+        }
+
+        Ok(outer_message)
+    }
+
+    fn is_encrypted(&self) -> bool {
+        if self.content_type().is_some_and(|ct| {
+            let main_type = ct.c_type.as_ref();
+            let sub_type = ct
+                .c_subtype
+                .as_ref()
+                .map(|s| s.as_ref())
+                .unwrap_or_default();
+
+            (main_type.eq_ignore_ascii_case("application")
+                && (sub_type.eq_ignore_ascii_case("pkcs7-mime")
+                    || sub_type.eq_ignore_ascii_case("pkcs7-signature")
+                    || (sub_type.eq_ignore_ascii_case("octet-stream")
+                        && self.attachment_name().is_some_and(|name| {
+                            name.rsplit_once('.')
+                                .is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext))
+                        }))))
+                || (main_type.eq_ignore_ascii_case("multipart")
+                    && sub_type.eq_ignore_ascii_case("encrypted"))
+        }) {
+            return true;
+        }
+
+        if self.parts.len() <= 2 {
+            let mut text_part = None;
+            let mut is_multipart = false;
+
+            for part in &self.parts {
+                match &part.body {
+                    PartType::Text(text) => {
+                        text_part = Some(text.as_ref());
+                    }
+                    PartType::Multipart(_) => {
+                        is_multipart = true;
+                    }
+                    _ => (),
+                }
+            }
+
+            match text_part {
+                Some(text)
+                    if (self.parts.len() == 1 || is_multipart)
+                        && text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") =>
+                {
+                    return true;
+                }
+                _ => (),
+            }
+        }
+
+        false
+    }
+}
+
+pub trait EncryptionFlags {
+    fn cipher(&self) -> SymmetricCipher;
+    fn can_train_spam_filter(&self) -> bool;
+    fn encrypt_on_append(&self) -> bool;
+    fn algo(&self) -> SymmetricAlgorithm;
+}
+
+impl EncryptionFlags for u64 {
+    fn cipher(&self) -> SymmetricCipher {
+        if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256_GCM != 0 {
+            SymmetricCipher::Aes256Gcm
+        } else if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_CHACHA20_POLY1305 != 0 {
+            SymmetricCipher::ChaCha20Poly1305
+        } else if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 {
+            SymmetricCipher::Aes256Cbc
+        } else {
+            SymmetricCipher::Aes128Cbc
+        }
+    }
+
+    fn can_train_spam_filter(&self) -> bool {
+        *self & ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER != 0
+    }
+
+    fn encrypt_on_append(&self) -> bool {
+        *self & ACCOUNT_FLAG_ENCRYPT_APPEND != 0
+    }
+
+    fn algo(&self) -> SymmetricAlgorithm {
+        if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 {
+            SymmetricAlgorithm::AES256
+        } else {
+            SymmetricAlgorithm::AES128
+        }
+    }
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum SymmetricCipher {
+    Aes128Cbc,
+    Aes256Cbc,
+    Aes256Gcm,
+    ChaCha20Poly1305,
+}
+
+impl SymmetricCipher {
+    fn key_size(self) -> usize {
+        match self {
+            SymmetricCipher::Aes128Cbc => 16,
+            SymmetricCipher::Aes256Cbc
+            | SymmetricCipher::Aes256Gcm
+            | SymmetricCipher::ChaCha20Poly1305 => 32,
+        }
+    }
+
+    fn nonce_size(self) -> usize {
+        match self {
+            SymmetricCipher::Aes128Cbc | SymmetricCipher::Aes256Cbc => 16,
+            SymmetricCipher::Aes256Gcm | SymmetricCipher::ChaCha20Poly1305 => 12,
+        }
+    }
+
+    fn is_aead(self) -> bool {
+        matches!(
+            self,
+            SymmetricCipher::Aes256Gcm | SymmetricCipher::ChaCha20Poly1305
+        )
+    }
+
+    fn encrypt(self, key: &[u8], nonce: &[u8], contents: &[u8]) -> (Vec<u8>, Option<Vec<u8>>) {
+        match self {
+            SymmetricCipher::Aes128Cbc => (
+                cbc::Encryptor::<aes::Aes128>::new_from_slices(key, nonce)
+                    .expect("invalid key or iv length")
+                    .encrypt_padded_vec::<Pkcs7>(contents),
+                None,
+            ),
+            SymmetricCipher::Aes256Cbc => (
+                cbc::Encryptor::<aes::Aes256>::new_from_slices(key, nonce)
+                    .expect("invalid key or iv length")
+                    .encrypt_padded_vec::<Pkcs7>(contents),
+                None,
+            ),
+            SymmetricCipher::Aes256Gcm => {
+                let cipher = Aes256Gcm::new_from_slice(key).expect("invalid key length");
+                let mut buffer = contents.to_vec();
+                let tag = cipher
+                    .encrypt_inout_detached(
+                        nonce.try_into().expect("invalid nonce length"),
+                        b"",
+                        buffer.as_mut_slice().into(),
+                    )
+                    .expect("AES-GCM encryption failed");
+                (buffer, Some(tag.to_vec()))
+            }
+            SymmetricCipher::ChaCha20Poly1305 => {
+                let cipher = ChaCha20Poly1305::new_from_slice(key).expect("invalid key length");
+                let mut buffer = contents.to_vec();
+                let tag = cipher
+                    .encrypt_inout_detached(
+                        nonce.try_into().expect("invalid nonce length"),
+                        b"",
+                        buffer.as_mut_slice().into(),
+                    )
+                    .expect("ChaCha20-Poly1305 encryption failed");
+                (buffer, Some(tag.to_vec()))
+            }
+        }
+    }
+
+    fn content_encryption_algorithm(
+        self,
+        nonce: &[u8],
+    ) -> Result<AlgorithmIdentifier, EncryptMessageError> {
+        let (algorithm, parameters) = match self {
+            SymmetricCipher::Aes128Cbc => (AES128_CBC, encode_octet_string(nonce)?),
+            SymmetricCipher::Aes256Cbc => (AES256_CBC, encode_octet_string(nonce)?),
+            SymmetricCipher::ChaCha20Poly1305 => (CHACHA20_POLY1305, encode_octet_string(nonce)?),
+            SymmetricCipher::Aes256Gcm => (
+                AES256_GCM,
+                rasn::der::encode(&GcmParameters {
+                    nonce: OctetString::from_slice(nonce),
+                    icv_len: 16,
+                })
+                .map_err(|err| {
+                    EncryptMessageError::Error(format!("Failed to encode GCM parameters: {}", err))
+                })?,
+            ),
+        };
+
+        Ok(AlgorithmIdentifier {
+            algorithm: algorithm.into(),
+            parameters: Some(parameters.into()),
+        })
+    }
+
+    fn key_encryption_algorithm(self) -> Result<AlgorithmIdentifier, EncryptMessageError> {
+        if self.is_aead() {
+            let sha256 = AlgorithmIdentifier {
+                algorithm: SHA256.into(),
+                parameters: Some(encode_null()?.into()),
+            };
+            let parameters = rasn::der::encode(&OaepParameters {
+                hash_algorithm: sha256.clone(),
+                mask_gen_algorithm: AlgorithmIdentifier {
+                    algorithm: MGF1.into(),
+                    parameters: Some(
+                        rasn::der::encode(&sha256)
+                            .map_err(|err| {
+                                EncryptMessageError::Error(format!(
+                                    "Failed to encode MGF1 parameters: {}",
+                                    err
+                                ))
+                            })?
+                            .into(),
+                    ),
+                },
+            })
+            .map_err(|err| {
+                EncryptMessageError::Error(format!("Failed to encode OAEP parameters: {}", err))
+            })?;
+
+            Ok(AlgorithmIdentifier {
+                algorithm: RSAES_OAEP.into(),
+                parameters: Some(parameters.into()),
+            })
+        } else {
+            Ok(AlgorithmIdentifier {
+                algorithm: RSA.into(),
+                parameters: Some(encode_null()?.into()),
+            })
+        }
+    }
+}
+
+#[derive(rasn::AsnType, rasn::Encode)]
+struct GcmParameters {
+    nonce: OctetString,
+    icv_len: u8,
+}
+
+#[derive(rasn::AsnType, rasn::Encode)]
+struct OaepParameters {
+    #[rasn(tag(explicit(0)))]
+    hash_algorithm: AlgorithmIdentifier,
+    #[rasn(tag(explicit(1)))]
+    mask_gen_algorithm: AlgorithmIdentifier,
+}
+
+fn encode_octet_string(value: &[u8]) -> Result<Vec<u8>, EncryptMessageError> {
+    rasn::der::encode(&OctetString::from_slice(value))
+        .map_err(|err| EncryptMessageError::Error(format!("Failed to encode nonce: {}", err)))
+}
+
+fn encode_null() -> Result<Vec<u8>, EncryptMessageError> {
+    rasn::der::encode(&()).map_err(|err| {
+        EncryptMessageError::Error(format!("Failed to encode NULL parameters: {}", err))
+    })
+}

crates/email/src/message/delete.rs

@@ -0,0 +1,315 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::metadata::MessageData;
+use crate::cache::{MessageCacheFetch, email::MessageCacheAccess};
+use common::{Server, storage::index::ObjectIndexBuilder};
+use groupware::calendar::storage::ItipAutoExpunge;
+use registry::schema::enums::IndexDocumentType;
+use registry::schema::structs::{Task, TaskIndexDocument, TaskStatus};
+use std::future::Future;
+use store::write::key::DeserializeBigEndian;
+use store::write::{IndexPropertyClass, now};
+use store::{IterateParams, U32_LEN, U64_LEN, ValueKey};
+use store::{
+    roaring::RoaringBitmap,
+    write::{BatchBuilder, ValueClass},
+};
+use trc::AddContext;
+use types::collection::{Collection, SyncCollection, VanishedCollection};
+use types::field::{EmailField, EmailSubmissionField};
+
+pub trait EmailDeletion: Sync + Send {
+    fn emails_delete(
+        &self,
+        account_id: u32,
+        tenant_id: Option<u32>,
+        batch: &mut BatchBuilder,
+        document_ids: RoaringBitmap,
+    ) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send;
+
+    fn purge_account(&self, account_id: u32) -> impl Future<Output = trc::Result<()>> + Send;
+
+    fn purge_email_submissions(
+        &self,
+        account_id: u32,
+        hold_period: u64,
+    ) -> impl Future<Output = trc::Result<()>> + Send;
+
+    fn emails_auto_expunge(
+        &self,
+        account_id: u32,
+        hold_period: u64,
+    ) -> impl Future<Output = trc::Result<()>> + Send;
+
+    fn log_emptied_threads(
+        &self,
+        account_id: u32,
+        batch: &mut BatchBuilder,
+        thread_ids: RoaringBitmap,
+        deleted_ids: &RoaringBitmap,
+    ) -> impl Future<Output = trc::Result<()>> + Send;
+}
+
+impl EmailDeletion for Server {
+    async fn emails_delete(
+        &self,
+        account_id: u32,
+        tenant_id: Option<u32>,
+        batch: &mut BatchBuilder,
+        document_ids: RoaringBitmap,
+    ) -> trc::Result<RoaringBitmap> {
+        let mut deleted_ids = RoaringBitmap::new();
+        let mut thread_ids = RoaringBitmap::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::Email);
+        self.archives(
+            account_id,
+            Collection::Email,
+            &document_ids,
+            |document_id, data_| {
+                // Add changes to batch
+                let metadata = data_
+                    .to_unarchived::<MessageData>()
+                    .caused_by(trc::location!())?;
+                for mailbox in metadata.inner.mailboxes.iter() {
+                    batch.log_vanished_item(
+                        VanishedCollection::Email,
+                        (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()),
+                    );
+                }
+                thread_ids.insert(metadata.inner.thread_id.to_native());
+                batch
+                    .with_document(document_id)
+                    .custom(
+                        ObjectIndexBuilder::<_, ()>::new()
+                            .with_tenant_id(tenant_id)
+                            .with_current(metadata),
+                    )
+                    .caused_by(trc::location!())?
+                    .schedule_task(Task::UnindexDocument(TaskIndexDocument {
+                        account_id: account_id.into(),
+                        document_id: document_id.into(),
+                        document_type: IndexDocumentType::Email,
+                        status: TaskStatus::now(),
+                    }))
+                    .commit_point();
+
+                deleted_ids.insert(document_id);
+
+                Ok(true)
+            },
+        )
+        .await?;
+
+        self.log_emptied_threads(account_id, batch, thread_ids, &deleted_ids)
+            .await?;
+
+        let not_destroyed = if document_ids.len() == deleted_ids.len() {
+            RoaringBitmap::new()
+        } else {
+            deleted_ids ^= document_ids;
+            deleted_ids
+        };
+
+        Ok(not_destroyed)
+    }
+
+    async fn log_emptied_threads(
+        &self,
+        account_id: u32,
+        batch: &mut BatchBuilder,
+        thread_ids: RoaringBitmap,
+        deleted_ids: &RoaringBitmap,
+    ) -> trc::Result<()> {
+        if !thread_ids.is_empty() {
+            let cache = self
+                .get_cached_messages(account_id)
+                .await
+                .caused_by(trc::location!())?;
+            for thread_id in &thread_ids {
+                if cache
+                    .in_thread(thread_id)
+                    .all(|message| deleted_ids.contains(message.document_id))
+                {
+                    batch
+                        .with_account_id(account_id)
+                        .with_collection(Collection::Thread)
+                        .with_document(thread_id)
+                        .log_container_delete(SyncCollection::Thread);
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn purge_account(&self, account_id: u32) -> trc::Result<()> {
+        // Auto-expunge deleted and junk messages
+        if let Some(hold_period) = self.core.email.mail_autoexpunge_after {
+            self.emails_auto_expunge(account_id, hold_period)
+                .await
+                .caused_by(trc::location!())?;
+        }
+
+        // Auto-expunge iMIP messages
+        if let Some(hold_period) = self.core.groupware.itip_inbox_auto_expunge {
+            self.itip_auto_expunge(account_id, hold_period)
+                .await
+                .caused_by(trc::location!())?;
+        }
+
+        // Delete old e-mail submissions
+        if let Some(hold_period) = self.core.email.email_submission_autoexpunge_after {
+            self.purge_email_submissions(account_id, hold_period)
+                .await
+                .caused_by(trc::location!())?;
+        }
+
+        // Purge changelogs
+        self.delete_changes(
+            account_id,
+            self.core.email.changes_max_history,
+            self.core.email.share_notification_max_history,
+        )
+        .await
+        .caused_by(trc::location!())?;
+
+        Ok(())
+    }
+
+    async fn emails_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
+        // Filter messages by received date
+        let mut destroy_ids = RoaringBitmap::new();
+        let cutoff = now().saturating_sub(hold_period);
+        self.store()
+            .iterate(
+                IterateParams::new(
+                    ValueKey {
+                        account_id,
+                        collection: Collection::Email.into(),
+                        document_id: 0,
+                        class: ValueClass::Property(EmailField::DeletedAt.into()),
+                    },
+                    ValueKey {
+                        account_id,
+                        collection: Collection::Email.into(),
+                        document_id: u32::MAX,
+                        class: ValueClass::Property(EmailField::DeletedAt.into()),
+                    },
+                )
+                .ascending(),
+                |key, value| {
+                    let deleted_at = value.deserialize_be_u64(0)?;
+                    if deleted_at <= cutoff {
+                        destroy_ids.insert(key.deserialize_be_u32(key.len() - U32_LEN)?);
+                    }
+
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        if destroy_ids.is_empty() {
+            return Ok(());
+        }
+
+        trc::event!(
+            Store(trc::StoreEvent::AutoExpunge),
+            Collection = Collection::Email.as_str(),
+            AccountId = account_id,
+            Total = destroy_ids.len(),
+        );
+
+        // Delete messages
+        let mut batch = BatchBuilder::new();
+        let tenant_id = self
+            .account(account_id)
+            .await
+            .caused_by(trc::location!())?
+            .tenant_id();
+        self.emails_delete(account_id, tenant_id, &mut batch, destroy_ids)
+            .await?;
+        self.commit_batch(batch).await?;
+        self.notify_task_queue();
+
+        Ok(())
+    }
+
+    async fn purge_email_submissions(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
+        // Filter messages by received date
+        let mut destroy_ids = Vec::new();
+        self.store()
+            .iterate(
+                IterateParams::new(
+                    ValueKey {
+                        account_id,
+                        collection: Collection::EmailSubmission.into(),
+                        document_id: 0,
+                        class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                            property: EmailSubmissionField::Metadata.into(),
+                            value: 0,
+                        }),
+                    },
+                    ValueKey {
+                        account_id,
+                        collection: Collection::Email.into(),
+                        document_id: u32::MAX,
+                        class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                            property: EmailSubmissionField::Metadata.into(),
+                            value: now().saturating_sub(hold_period),
+                        }),
+                    },
+                )
+                .ascending()
+                .no_values(),
+                |key, _| {
+                    destroy_ids.push((
+                        key.deserialize_be_u32(key.len() - U32_LEN)?,
+                        key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?,
+                    ));
+
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        if destroy_ids.is_empty() {
+            return Ok(());
+        }
+
+        trc::event!(
+            Store(trc::StoreEvent::AutoExpunge),
+            Collection = Collection::EmailSubmission.as_str(),
+            AccountId = account_id,
+            Total = destroy_ids.len(),
+        );
+
+        // Delete messages
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::EmailSubmission);
+
+        for (document_id, send_at) in destroy_ids {
+            batch
+                .with_document(document_id)
+                .clear(EmailSubmissionField::Metadata)
+                .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailSubmissionField::Metadata.into(),
+                    value: send_at,
+                }))
+                .commit_point();
+        }
+
+        self.commit_batch(batch).await?;
+
+        Ok(())
+    }
+}

crates/email/src/message/delivery.rs

@@ -0,0 +1,279 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::ingest::{EmailIngest, IngestEmail, IngestSource};
+use crate::{mailbox::INBOX_ID, sieve::ingest::SieveScriptIngest};
+use common::{
+    Server,
+    auth::BuildAccessToken,
+    ipc::{EmailPush, PushNotification},
+};
+use mail_parser::MessageParser;
+use registry::schema::enums::Permission;
+use std::{borrow::Cow, future::Future};
+use store::ahash::AHashMap;
+use types::blob_hash::BlobHash;
+
+#[derive(Debug)]
+pub struct IngestMessage {
+    pub sender_address: String,
+    pub sender_authenticated: bool,
+    pub recipients: Vec<IngestRecipient>,
+    pub message_blob: BlobHash,
+    pub message_size: u64,
+    pub session_id: u64,
+}
+
+#[derive(Debug)]
+pub struct IngestRecipient {
+    pub address: String,
+    pub orcpt: Option<String>,
+    pub spam_percentage: Option<u8>,
+}
+
+impl IngestRecipient {
+    pub fn is_spam(&self) -> bool {
+        self.spam_percentage
+            .is_some_and(|percentage| percentage >= 50)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum LocalDeliveryStatus {
+    Success,
+    TemporaryFailure {
+        reason: Cow<'static, str>,
+    },
+    PermanentFailure {
+        code: [u8; 3],
+        reason: Cow<'static, str>,
+    },
+}
+
+pub struct LocalDeliveryResult {
+    pub status: Vec<LocalDeliveryStatus>,
+    pub autogenerated: Vec<AutogeneratedMessage>,
+}
+
+pub struct AutogeneratedMessage {
+    pub sender_address: String,
+    pub recipients: Vec<String>,
+    pub message: Vec<u8>,
+}
+
+pub trait MailDelivery: Sync + Send {
+    fn deliver_message(
+        &self,
+        message: IngestMessage,
+    ) -> impl Future<Output = LocalDeliveryResult> + Send;
+}
+
+impl MailDelivery for Server {
+    async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult {
+        // Read message
+        let raw_message = match self
+            .core
+            .storage
+            .blob
+            .get_blob(message.message_blob.as_slice(), 0..usize::MAX)
+            .await
+        {
+            Ok(Some(raw_message)) => raw_message,
+            Ok(None) => {
+                trc::event!(
+                    MessageIngest(trc::MessageIngestEvent::Error),
+                    Reason = "Blob not found.",
+                    SpanId = message.session_id,
+                    CausedBy = trc::location!()
+                );
+
+                return LocalDeliveryResult {
+                    status: (0..message.recipients.len())
+                        .map(|_| LocalDeliveryStatus::TemporaryFailure {
+                            reason: "Blob not found.".into(),
+                        })
+                        .collect::<Vec<_>>(),
+                    autogenerated: vec![],
+                };
+            }
+            Err(err) => {
+                trc::error!(
+                    err.details("Failed to fetch message blob.")
+                        .span_id(message.session_id)
+                        .caused_by(trc::location!())
+                );
+
+                return LocalDeliveryResult {
+                    status: (0..message.recipients.len())
+                        .map(|_| LocalDeliveryStatus::TemporaryFailure {
+                            reason: "Temporary I/O error.".into(),
+                        })
+                        .collect::<Vec<_>>(),
+                    autogenerated: vec![],
+                };
+            }
+        };
+
+        // Obtain the account IDs for each recipient
+        let mut account_ids: AHashMap<u32, usize> =
+            AHashMap::with_capacity(message.recipients.len());
+        let mut result = LocalDeliveryResult {
+            status: Vec::with_capacity(message.recipients.len()),
+            autogenerated: Vec::new(),
+        };
+
+        for rcpt in message.recipients {
+            let account_id = match self.account_id_from_email(&rcpt.address, false).await {
+                Ok(Some(account_id)) => account_id,
+                Ok(None) => {
+                    // Something went wrong
+                    result.status.push(LocalDeliveryStatus::PermanentFailure {
+                        code: [5, 5, 0],
+                        reason: "Mailbox not found.".into(),
+                    });
+                    continue;
+                }
+                Err(err) => {
+                    trc::error!(
+                        err.details("Failed to lookup recipient.")
+                            .ctx(trc::Key::To, rcpt.address.to_string())
+                            .span_id(message.session_id)
+                            .caused_by(trc::location!())
+                    );
+                    result.status.push(LocalDeliveryStatus::TemporaryFailure {
+                        reason: "Address lookup failed.".into(),
+                    });
+                    continue;
+                }
+            };
+            if let Some(status) = account_ids
+                .get(&account_id)
+                .and_then(|pos| result.status.get(*pos))
+            {
+                result.status.push(status.clone());
+                continue;
+            }
+
+            // Obtain access token
+            let status = match self.access_token(account_id).await.and_then(|token| {
+                token
+                    .build()
+                    .assert_has_permission(Permission::EmailReceive)
+            }) {
+                Ok(access_token) => {
+                    // Check if there is an active sieve script
+                    match self.sieve_script_get_active(account_id).await {
+                        Ok(None) => {
+                            // Ingest message
+                            self.email_ingest(IngestEmail {
+                                raw_message: &raw_message,
+                                blob_hash: Some(&message.message_blob),
+                                message: MessageParser::new().parse(&raw_message),
+                                access_token: &access_token,
+                                mailbox_ids: vec![INBOX_ID],
+                                keywords: vec![],
+                                received_at: None,
+                                source: IngestSource::Smtp {
+                                    deliver_to: &rcpt.address,
+                                    is_sender_authenticated: message.sender_authenticated,
+                                    is_spam: rcpt.is_spam(),
+                                },
+                                session_id: message.session_id,
+                            })
+                            .await
+                        }
+                        Ok(Some(active_script)) => {
+                            self.sieve_script_ingest(
+                                &access_token,
+                                &message.message_blob,
+                                &raw_message,
+                                &message.sender_address,
+                                message.sender_authenticated,
+                                &rcpt,
+                                message.session_id,
+                                active_script,
+                                &mut result.autogenerated,
+                            )
+                            .await
+                        }
+                        Err(err) => Err(err),
+                    }
+                }
+
+                Err(err) => Err(err),
+            };
+
+            let status = match status {
+                Ok(ingested_message) => {
+                    // Notify state change
+                    if ingested_message.change_id != u64::MAX {
+                        self.broadcast_push_notification(PushNotification::EmailPush(EmailPush {
+                            account_id,
+                            email_id: ingested_message.document_id,
+                            change_id: ingested_message.change_id,
+                        }))
+                        .await;
+                    }
+
+                    LocalDeliveryStatus::Success
+                }
+                Err(err) => {
+                    let status = match err.as_ref() {
+                        trc::EventType::Limit(trc::LimitEvent::Quota) => {
+                            LocalDeliveryStatus::TemporaryFailure {
+                                reason: "Mailbox over quota.".into(),
+                            }
+                        }
+                        trc::EventType::Limit(trc::LimitEvent::TenantQuota) => {
+                            LocalDeliveryStatus::TemporaryFailure {
+                                reason: "Organization over quota.".into(),
+                            }
+                        }
+                        trc::EventType::Security(trc::SecurityEvent::Unauthorized) => {
+                            LocalDeliveryStatus::PermanentFailure {
+                                code: [5, 5, 0],
+                                reason: "This account is not authorized to receive email.".into(),
+                            }
+                        }
+                        trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => {
+                            LocalDeliveryStatus::PermanentFailure {
+                                code: err
+                                    .value(trc::Key::Code)
+                                    .and_then(|v| v.to_uint())
+                                    .map(|n| {
+                                        [(n / 100) as u8, ((n % 100) / 10) as u8, (n % 10) as u8]
+                                    })
+                                    .unwrap_or([5, 5, 0]),
+                                reason: err
+                                    .value_as_str(trc::Key::Reason)
+                                    .unwrap_or_default()
+                                    .to_string()
+                                    .into(),
+                            }
+                        }
+                        _ => LocalDeliveryStatus::TemporaryFailure {
+                            reason: "Transient server failure.".into(),
+                        },
+                    };
+
+                    trc::error!(
+                        err.ctx(trc::Key::To, rcpt.address.to_string())
+                            .span_id(message.session_id)
+                    );
+
+                    status
+                }
+            };
+
+            // Cache response for UID to avoid duplicate deliveries
+            account_ids.insert(account_id, result.status.len());
+
+            result.status.push(status);
+        }
+
+        result
+    }
+}

crates/email/src/message/headers.rs

@@ -0,0 +1,542 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::metadata::{ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue};
+use jmap_proto::{
+    object::email::{EmailProperty, EmailValue, HeaderForm, HeaderProperty},
+    types::date::UTCDate,
+};
+use jmap_tools::{Key, Map, Value};
+use mail_builder::{
+    MessageBuilder,
+    headers::{
+        address::{Address, EmailAddress, GroupedAddresses},
+        date::Date,
+        message_id::MessageId,
+        raw::Raw,
+        text::Text,
+        url::URL,
+    },
+};
+use mail_parser::{Addr, DateTime, Group, Header, HeaderName, HeaderValue, parsers::MessageStream};
+use utils::chained_bytes::ChainedBytes;
+
+pub trait HeaderToValue {
+    fn header_to_value(
+        &self,
+        property: &EmailProperty,
+        raw_message: &ChainedBytes<'_>,
+    ) -> Value<'static, EmailProperty, EmailValue>;
+    fn headers_to_value(
+        &self,
+        raw_message: &ChainedBytes<'_>,
+    ) -> Value<'static, EmailProperty, EmailValue>;
+}
+
+pub trait ValueToHeader<'x> {
+    fn try_into_grouped_addresses(self) -> Option<GroupedAddresses<'x>>;
+    fn try_into_address_list(self) -> Option<Vec<Address<'x>>>;
+    fn try_into_address(self) -> Option<EmailAddress<'x>>;
+}
+
+pub trait BuildHeader<'x>: Sized {
+    fn build_header(
+        self,
+        header: HeaderProperty,
+        value: Value<'x, EmailProperty, EmailValue>,
+    ) -> Result<Self, HeaderProperty>;
+}
+
+impl HeaderToValue for Vec<Header<'_>> {
+    fn header_to_value(
+        &self,
+        property: &EmailProperty,
+        raw_message: &ChainedBytes<'_>,
+    ) -> Value<'static, EmailProperty, EmailValue> {
+        let (header_name, form, all) = match property {
+            EmailProperty::Header(header) => (
+                HeaderName::parse(header.header.as_str())
+                    .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())),
+                header.form,
+                header.all,
+            ),
+            EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false),
+            EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false),
+            EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false),
+            EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false),
+            EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false),
+            EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false),
+            EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false),
+            EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false),
+            EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false),
+            EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false),
+            EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false),
+            _ => return Value::Null,
+        };
+
+        let is_raw = matches!(form, HeaderForm::Raw) || !header_name.is_structured();
+        let mut headers = Vec::new();
+        let header_name = header_name.as_str();
+        for header in self.iter().rev() {
+            if header.name.as_str().eq_ignore_ascii_case(header_name) {
+                let raw_header;
+                let header_value = if is_raw || matches!(header.value, HeaderValue::Empty) {
+                    raw_header =
+                        raw_message.get(header.offset_start as usize..header.offset_end as usize);
+
+                    if let Some(bytes) = &raw_header {
+                        let bytes = bytes.as_ref();
+                        match form {
+                            HeaderForm::Raw => {
+                                HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end()))
+                            }
+                            HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(),
+                            HeaderForm::Addresses
+                            | HeaderForm::GroupedAddresses
+                            | HeaderForm::URLs => MessageStream::new(bytes).parse_address(),
+                            HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(),
+                            HeaderForm::Date => MessageStream::new(bytes).parse_date(),
+                        }
+                    } else {
+                        HeaderValue::Empty
+                    }
+                } else {
+                    header.value.clone()
+                };
+                headers.push(header_value.into_form(&form));
+                if !all {
+                    break;
+                }
+            }
+        }
+
+        if !all {
+            headers.pop().unwrap_or_default()
+        } else {
+            if headers.len() > 1 {
+                headers.reverse();
+            }
+            Value::Array(headers)
+        }
+    }
+
+    fn headers_to_value(
+        &self,
+        raw_message: &ChainedBytes<'_>,
+    ) -> Value<'static, EmailProperty, EmailValue> {
+        let mut headers = Vec::with_capacity(self.len());
+        for header in self.iter() {
+            headers.push(Value::Object(
+                Map::with_capacity(2)
+                    .with_key_value(EmailProperty::Name, header.name().to_string())
+                    .with_key_value(
+                        EmailProperty::Value,
+                        String::from_utf8_lossy(
+                            raw_message
+                                .get(header.offset_start as usize..header.offset_end as usize)
+                                .unwrap_or_default()
+                                .as_ref()
+                                .trim_end(),
+                        )
+                        .into_owned(),
+                    ),
+            ));
+        }
+        headers.into()
+    }
+}
+
+impl<'x> ValueToHeader<'x> for Value<'x, EmailProperty, EmailValue> {
+    fn try_into_grouped_addresses(self) -> Option<GroupedAddresses<'x>> {
+        let mut obj = self.into_object()?;
+        Some(GroupedAddresses {
+            name: obj
+                .remove(&Key::Property(EmailProperty::Name))
+                .and_then(|n| n.into_string()),
+            addresses: obj
+                .remove(&Key::Property(EmailProperty::Addresses))?
+                .try_into_address_list()?,
+        })
+    }
+
+    fn try_into_address_list(self) -> Option<Vec<Address<'x>>> {
+        let list = self.into_array()?;
+        let mut addresses = Vec::with_capacity(list.len());
+        for value in list {
+            addresses.push(Address::Address(value.try_into_address()?));
+        }
+        Some(addresses)
+    }
+
+    fn try_into_address(self) -> Option<EmailAddress<'x>> {
+        let mut obj = self.into_object()?;
+        Some(EmailAddress {
+            name: obj
+                .remove(&Key::Property(EmailProperty::Name))
+                .and_then(|n| n.into_string()),
+            email: obj
+                .remove(&Key::Property(EmailProperty::Email))?
+                .into_string()?,
+        })
+    }
+}
+
+impl<'x> BuildHeader<'x> for MessageBuilder<'x> {
+    fn build_header(
+        self,
+        header: HeaderProperty,
+        value: Value<'x, EmailProperty, EmailValue>,
+    ) -> Result<Self, HeaderProperty> {
+        Ok(match (&header.form, header.all, value) {
+            (HeaderForm::Raw, false, Value::Str(value)) => {
+                self.header(header.header, Raw::from(value))
+            }
+            (HeaderForm::Raw, true, Value::Array(value)) => self.headers(
+                header.header,
+                value
+                    .into_iter()
+                    .filter_map(|v| Raw::from(v.into_string()?).into()),
+            ),
+            (HeaderForm::Date, false, Value::Element(EmailValue::Date(value))) => {
+                self.header(header.header, Date::new(value.timestamp()))
+            }
+            (HeaderForm::Date, true, Value::Array(value)) => self.headers(
+                header.header,
+                value
+                    .into_iter()
+                    .filter_map(|v| Date::new(unwrap_date(v)?.timestamp()).into()),
+            ),
+            (HeaderForm::Text, false, Value::Str(value)) => {
+                self.header(header.header, Text::from(value))
+            }
+            (HeaderForm::Text, true, Value::Array(value)) => self.headers(
+                header.header,
+                value
+                    .into_iter()
+                    .filter_map(|v| Text::from(v.into_string()?).into()),
+            ),
+            (HeaderForm::URLs, false, Value::Array(value)) => self.header(
+                header.header,
+                URL {
+                    url: value
+                        .into_iter()
+                        .filter_map(|v| v.into_string()?.into())
+                        .collect(),
+                },
+            ),
+            (HeaderForm::URLs, true, Value::Array(value)) => self.headers(
+                header.header,
+                value.into_iter().filter_map(|value| {
+                    URL {
+                        url: value
+                            .into_array()?
+                            .into_iter()
+                            .filter_map(|v| v.into_string()?.into())
+                            .collect(),
+                    }
+                    .into()
+                }),
+            ),
+            (HeaderForm::MessageIds, false, Value::Array(value)) => self.header(
+                header.header,
+                MessageId {
+                    id: value
+                        .into_iter()
+                        .filter_map(|v| v.into_string()?.into())
+                        .collect(),
+                },
+            ),
+            (HeaderForm::MessageIds, true, Value::Array(value)) => self.headers(
+                header.header,
+                value.into_iter().filter_map(|value| {
+                    MessageId {
+                        id: value
+                            .into_array()?
+                            .into_iter()
+                            .filter_map(|v| v.into_string()?.into())
+                            .collect(),
+                    }
+                    .into()
+                }),
+            ),
+            (HeaderForm::Addresses, false, Value::Array(value)) => self.header(
+                header.header,
+                Address::new_list(
+                    value
+                        .into_iter()
+                        .filter_map(|v| Address::Address(v.try_into_address()?).into())
+                        .collect(),
+                ),
+            ),
+            (HeaderForm::Addresses, true, Value::Array(value)) => self.headers(
+                header.header,
+                value
+                    .into_iter()
+                    .filter_map(|v| Address::new_list(v.try_into_address_list()?).into()),
+            ),
+            (HeaderForm::GroupedAddresses, false, Value::Array(value)) => self.header(
+                header.header,
+                Address::new_list(
+                    value
+                        .into_iter()
+                        .filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into())
+                        .collect(),
+                ),
+            ),
+            (HeaderForm::GroupedAddresses, true, Value::Array(value)) => self.headers(
+                header.header,
+                value.into_iter().filter_map(|v| {
+                    Address::new_list(
+                        v.into_array()?
+                            .into_iter()
+                            .filter_map(|v| Address::Group(v.try_into_grouped_addresses()?).into())
+                            .collect::<Vec<_>>(),
+                    )
+                    .into()
+                }),
+            ),
+            _ => {
+                return Err(header);
+            }
+        })
+    }
+}
+
+impl HeaderToValue for ArchivedMessageMetadataPart {
+    fn header_to_value(
+        &self,
+        property: &EmailProperty,
+        raw_message: &ChainedBytes<'_>,
+    ) -> Value<'static, EmailProperty, EmailValue> {
+        let (header_name, form, all) = match property {
+            EmailProperty::Header(header) => (
+                HeaderName::parse(header.header.as_str())
+                    .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())),
+                header.form,
+                header.all,
+            ),
+            EmailProperty::Sender => (HeaderName::Sender, HeaderForm::Addresses, false),
+            EmailProperty::From => (HeaderName::From, HeaderForm::Addresses, false),
+            EmailProperty::To => (HeaderName::To, HeaderForm::Addresses, false),
+            EmailProperty::Cc => (HeaderName::Cc, HeaderForm::Addresses, false),
+            EmailProperty::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false),
+            EmailProperty::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false),
+            EmailProperty::Subject => (HeaderName::Subject, HeaderForm::Text, false),
+            EmailProperty::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false),
+            EmailProperty::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false),
+            EmailProperty::References => (HeaderName::References, HeaderForm::MessageIds, false),
+            EmailProperty::SentAt => (HeaderName::Date, HeaderForm::Date, false),
+            _ => return Value::Null,
+        };
+
+        let is_raw = matches!(form, HeaderForm::Raw) || !header_name.is_structured();
+        let mut headers = Vec::new();
+        let header_name = header_name.as_str();
+        for header in self.headers.iter().rev() {
+            if header.name.as_str().eq_ignore_ascii_case(header_name) {
+                let raw_header;
+                let header_value =
+                    if is_raw || matches!(header.value, ArchivedMetadataHeaderValue::Empty) {
+                        raw_header = raw_message.get(header.value_range());
+
+                        if let Some(bytes) = &raw_header {
+                            let bytes = bytes.as_ref();
+                            match form {
+                                HeaderForm::Raw => {
+                                    HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end()))
+                                }
+                                HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(),
+                                HeaderForm::Addresses
+                                | HeaderForm::GroupedAddresses
+                                | HeaderForm::URLs => MessageStream::new(bytes).parse_address(),
+                                HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(),
+                                HeaderForm::Date => MessageStream::new(bytes).parse_date(),
+                            }
+                        } else {
+                            HeaderValue::Empty
+                        }
+                    } else {
+                        HeaderValue::from(&header.value)
+                    };
+                headers.push(header_value.into_form(&form));
+                if !all {
+                    break;
+                }
+            }
+        }
+
+        if !all {
+            headers.pop().unwrap_or_default()
+        } else {
+            if headers.len() > 1 {
+                headers.reverse();
+            }
+            Value::Array(headers)
+        }
+    }
+
+    fn headers_to_value(
+        &self,
+        raw_message: &ChainedBytes<'_>,
+    ) -> Value<'static, EmailProperty, EmailValue> {
+        let mut headers = Vec::with_capacity(self.headers.len());
+        for header in self.headers.iter() {
+            headers.push(Value::Object(
+                Map::with_capacity(2)
+                    .with_key_value(EmailProperty::Name, header.name.as_str().to_string())
+                    .with_key_value(
+                        EmailProperty::Value,
+                        String::from_utf8_lossy(
+                            raw_message
+                                .get(header.value_range())
+                                .unwrap_or_default()
+                                .as_ref()
+                                .trim_end(),
+                        )
+                        .into_owned(),
+                    ),
+            ));
+        }
+        headers.into()
+    }
+}
+
+trait ByteTrim {
+    fn trim_end(&self) -> Self;
+}
+
+impl ByteTrim for &[u8] {
+    fn trim_end(&self) -> Self {
+        let mut end = self.len();
+        while end > 0 && self[end - 1].is_ascii_whitespace() {
+            end -= 1;
+        }
+        &self[..end]
+    }
+}
+
+#[inline]
+pub(crate) fn unwrap_date(value: Value<'_, EmailProperty, EmailValue>) -> Option<UTCDate> {
+    match value {
+        Value::Element(EmailValue::Date(date)) => Some(date),
+        _ => None,
+    }
+}
+
+pub trait IntoForm {
+    fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue>;
+}
+
+impl IntoForm for HeaderValue<'_> {
+    fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue> {
+        match (self, form) {
+            (HeaderValue::Text(text), HeaderForm::Raw | HeaderForm::Text) => {
+                text.into_owned().into()
+            }
+            (HeaderValue::TextList(texts), HeaderForm::Raw | HeaderForm::Text) => {
+                texts.join(", ").into()
+            }
+            (HeaderValue::Text(text), HeaderForm::MessageIds) => {
+                Value::Array(vec![text.into_owned().into()])
+            }
+            (HeaderValue::TextList(texts), HeaderForm::MessageIds) => {
+                Value::Array(texts.into_iter().map(|t| t.into_owned().into()).collect())
+            }
+            (HeaderValue::DateTime(datetime), HeaderForm::Date) => from_mail_datetime(datetime),
+            (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::URLs) => {
+                Value::Array(
+                    addrlist
+                        .into_iter()
+                        .filter_map(|addr| match addr {
+                            Addr {
+                                address: Some(addr),
+                                ..
+                            } if addr.contains(':') => Some(addr.into_owned().into()),
+                            _ => None,
+                        })
+                        .collect(),
+                )
+            }
+            (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::Addresses) => {
+                from_mail_addrlist(addrlist)
+            }
+            (
+                HeaderValue::Address(mail_parser::Address::Group(grouplist)),
+                HeaderForm::Addresses,
+            ) => Value::Array(
+                grouplist
+                    .into_iter()
+                    .flat_map(|group| group.addresses.into_iter().map(from_mail_addr))
+                    .collect(),
+            ),
+            (
+                HeaderValue::Address(mail_parser::Address::List(addrlist)),
+                HeaderForm::GroupedAddresses,
+            ) => Value::Array(vec![
+                Map::with_capacity(2)
+                    .with_key_value(EmailProperty::Name, Value::Null)
+                    .with_key_value(EmailProperty::Addresses, from_mail_addrlist(addrlist))
+                    .into(),
+            ]),
+            (
+                HeaderValue::Address(mail_parser::Address::Group(grouplist)),
+                HeaderForm::GroupedAddresses,
+            ) => Value::Array(
+                grouplist
+                    .into_iter()
+                    .map(from_mail_group)
+                    .collect::<Vec<Value<'static, EmailProperty, EmailValue>>>(),
+            ),
+
+            _ => Value::Null,
+        }
+    }
+}
+
+fn from_mail_datetime(date: DateTime) -> Value<'static, EmailProperty, EmailValue> {
+    Value::Element(EmailValue::Date(UTCDate {
+        year: date.year,
+        month: date.month,
+        day: date.day,
+        hour: date.hour,
+        minute: date.minute,
+        second: date.second,
+        tz_before_gmt: date.tz_before_gmt,
+        tz_hour: date.tz_hour,
+        tz_minute: date.tz_minute,
+    }))
+}
+
+fn from_mail_addr(value: Addr<'_>) -> Value<'static, EmailProperty, EmailValue> {
+    Value::Object(
+        Map::with_capacity(2)
+            .with_key_value(EmailProperty::Name, value.name.map(|v| v.into_owned()))
+            .with_key_value(
+                EmailProperty::Email,
+                value.address.unwrap_or_default().into_owned(),
+            ),
+    )
+}
+
+fn from_mail_group(group: Group<'_>) -> Value<'static, EmailProperty, EmailValue> {
+    Value::Object(
+        Map::with_capacity(2)
+            .with_key_value(EmailProperty::Name, group.name.map(|v| v.into_owned()))
+            .with_key_value(
+                EmailProperty::Addresses,
+                from_mail_addrlist(group.addresses),
+            ),
+    )
+}
+
+fn from_mail_addrlist(addrlist: Vec<Addr<'_>>) -> Value<'static, EmailProperty, EmailValue> {
+    Value::Array(
+        addrlist
+            .into_iter()
+            .map(from_mail_addr)
+            .collect::<Vec<Value<'static, EmailProperty, EmailValue>>>(),
+    )
+}

crates/email/src/message/index/extractors.rs

@@ -0,0 +1,291 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::metadata::{
+    ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue,
+    MetadataHeaderName, MetadataHeaderValue,
+};
+use mail_parser::{Addr, Address, Group, HeaderValue};
+use nlp::language::Language;
+use rkyv::option::ArchivedOption;
+use std::borrow::Cow;
+
+impl ArchivedMessageMetadataContents {
+    pub fn is_html_part(&self, part_id: u16) -> bool {
+        self.html_body.iter().any(|&id| id == part_id)
+    }
+
+    pub fn is_text_part(&self, part_id: u16) -> bool {
+        self.text_body.iter().any(|&id| id == part_id)
+    }
+}
+
+impl ArchivedMessageMetadataPart {
+    pub fn language(&self) -> Option<Language> {
+        self.header_value(&MetadataHeaderName::ContentLanguage)
+            .and_then(|v| {
+                Language::from_iso_639(v.as_text()?)
+                    .unwrap_or(Language::Unknown)
+                    .into()
+            })
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum AddressElement {
+    Name,
+    Address,
+    GroupName,
+}
+
+pub trait VisitText {
+    fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str));
+    fn visit_text<'x>(&'x self, visitor: impl FnMut(&'x str));
+    fn into_visit_text(self, visitor: impl FnMut(String));
+}
+
+impl VisitText for HeaderValue<'_> {
+    fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
+        match self {
+            HeaderValue::Address(Address::List(addr_list)) => {
+                for addr in addr_list {
+                    if let Some(name) = &addr.name {
+                        visitor(AddressElement::Name, name);
+                    }
+                    if let Some(addr) = &addr.address {
+                        visitor(AddressElement::Address, addr);
+                    }
+                }
+            }
+            HeaderValue::Address(Address::Group(groups)) => {
+                for group in groups {
+                    if let Some(name) = &group.name {
+                        visitor(AddressElement::GroupName, name);
+                    }
+
+                    for addr in &group.addresses {
+                        if let Some(name) = &addr.name {
+                            visitor(AddressElement::Name, name);
+                        }
+                        if let Some(addr) = &addr.address {
+                            visitor(AddressElement::Address, addr);
+                        }
+                    }
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn visit_text<'x>(&'x self, mut visitor: impl FnMut(&'x str)) {
+        match &self {
+            HeaderValue::Text(text) => {
+                visitor(text.as_ref());
+            }
+            HeaderValue::TextList(texts) => {
+                for text in texts {
+                    visitor(text.as_ref());
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn into_visit_text(self, mut visitor: impl FnMut(String)) {
+        match self {
+            HeaderValue::Text(text) => {
+                visitor(text.into_owned());
+            }
+            HeaderValue::TextList(texts) => {
+                for text in texts {
+                    visitor(text.into_owned());
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub trait VisitTextArchived {
+    fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str));
+    fn visit_text(&self, visitor: impl FnMut(&str));
+}
+
+impl VisitTextArchived for MetadataHeaderValue {
+    fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
+        match self {
+            MetadataHeaderValue::AddressList(addr_list) => {
+                for addr in addr_list.iter() {
+                    if let Some(name) = &addr.name {
+                        visitor(AddressElement::Name, name);
+                    }
+                    if let Some(addr) = &addr.address {
+                        visitor(AddressElement::Address, addr);
+                    }
+                }
+            }
+            MetadataHeaderValue::AddressGroup(groups) => {
+                for group in groups.iter() {
+                    if let Some(name) = &group.name {
+                        visitor(AddressElement::GroupName, name);
+                    }
+
+                    for addr in group.addresses.iter() {
+                        if let Some(name) = &addr.name {
+                            visitor(AddressElement::Name, name);
+                        }
+                        if let Some(addr) = &addr.address {
+                            visitor(AddressElement::Address, addr);
+                        }
+                    }
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn visit_text(&self, mut visitor: impl FnMut(&str)) {
+        match &self {
+            MetadataHeaderValue::Text(text) => {
+                visitor(text.as_ref());
+            }
+            MetadataHeaderValue::TextList(texts) => {
+                for text in texts.iter() {
+                    visitor(text.as_ref());
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+impl VisitTextArchived for ArchivedMetadataHeaderValue {
+    fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
+        match self {
+            ArchivedMetadataHeaderValue::AddressList(addr_list) => {
+                for addr in addr_list.iter() {
+                    if let ArchivedOption::Some(name) = &addr.name {
+                        visitor(AddressElement::Name, name);
+                    }
+                    if let ArchivedOption::Some(addr) = &addr.address {
+                        visitor(AddressElement::Address, addr);
+                    }
+                }
+            }
+            ArchivedMetadataHeaderValue::AddressGroup(groups) => {
+                for group in groups.iter() {
+                    if let ArchivedOption::Some(name) = &group.name {
+                        visitor(AddressElement::GroupName, name);
+                    }
+
+                    for addr in group.addresses.iter() {
+                        if let ArchivedOption::Some(name) = &addr.name {
+                            visitor(AddressElement::Name, name);
+                        }
+                        if let ArchivedOption::Some(addr) = &addr.address {
+                            visitor(AddressElement::Address, addr);
+                        }
+                    }
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn visit_text(&self, mut visitor: impl FnMut(&str)) {
+        match &self {
+            ArchivedMetadataHeaderValue::Text(text) => {
+                visitor(text.as_ref());
+            }
+            ArchivedMetadataHeaderValue::TextList(texts) => {
+                for text in texts.iter() {
+                    visitor(text.as_ref());
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub trait TrimTextValue {
+    fn trim_text(self, length: usize) -> Self;
+}
+
+impl TrimTextValue for HeaderValue<'_> {
+    fn trim_text(self, length: usize) -> Self {
+        match self {
+            HeaderValue::Address(Address::List(v)) => {
+                HeaderValue::Address(Address::List(v.trim_text(length)))
+            }
+            HeaderValue::Address(Address::Group(v)) => {
+                HeaderValue::Address(Address::Group(v.trim_text(length)))
+            }
+            HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)),
+            HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)),
+            v => v,
+        }
+    }
+}
+
+impl TrimTextValue for Addr<'_> {
+    fn trim_text(self, length: usize) -> Self {
+        Self {
+            name: self.name.map(|v| v.trim_text(length)),
+            address: self.address.map(|v| v.trim_text(length)),
+        }
+    }
+}
+
+impl TrimTextValue for Group<'_> {
+    fn trim_text(self, length: usize) -> Self {
+        Self {
+            name: self.name.map(|v| v.trim_text(length)),
+            addresses: self.addresses.trim_text(length),
+        }
+    }
+}
+
+impl TrimTextValue for &str {
+    fn trim_text(self, length: usize) -> Self {
+        if self.len() < length {
+            self
+        } else {
+            let mut index = 0;
+
+            for (i, _) in self.char_indices() {
+                if i > length {
+                    break;
+                }
+                index = i;
+            }
+
+            &self[..index]
+        }
+    }
+}
+
+impl TrimTextValue for Cow<'_, str> {
+    fn trim_text(self, length: usize) -> Self {
+        if self.len() < length {
+            self
+        } else {
+            let mut result = String::with_capacity(length);
+            for (i, c) in self.char_indices() {
+                if i > length {
+                    break;
+                }
+                result.push(c);
+            }
+            result.into()
+        }
+    }
+}
+
+impl<T: TrimTextValue> TrimTextValue for Vec<T> {
+    fn trim_text(self, length: usize) -> Self {
+        self.into_iter().map(|v| v.trim_text(length)).collect()
+    }
+}

crates/email/src/message/index/metadata.rs

@@ -0,0 +1,251 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::{
+    index::{IndexMessage, MAX_MESSAGE_PARTS, PREVIEW_LENGTH},
+    metadata::{
+        ArchivedMessageMetadata, ArchivedMessageMetadataPart, ArchivedMetadataHeaderName,
+        MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata,
+        MessageMetadataPart, build_metadata_contents,
+    },
+};
+use common::storage::index::ObjectIndexBuilder;
+use mail_parser::{
+    PartType,
+    decoders::html::html_to_text,
+    parsers::{fields::thread::thread_name, preview::preview_text},
+};
+use store::{
+    Serialize,
+    write::{Archiver, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass},
+};
+use trc::AddContext;
+use types::{blob_hash::BlobHash, field::EmailField};
+use utils::cheeky_hash::CheekyHash;
+
+impl MessageMetadata {
+    #[inline(always)]
+    pub fn root_part(&self) -> &MessageMetadataPart {
+        &self.contents[0].parts[0]
+    }
+
+    pub fn index(self, batch: &mut BatchBuilder, set: bool) -> trc::Result<()> {
+        if set {
+            batch
+                .set(
+                    BlobOp::Link {
+                        hash: self.blob_hash.clone(),
+                        to: BlobLink::Document,
+                    },
+                    Vec::new(),
+                )
+                .set(EmailField::Metadata, Archiver::new(self).serialize()?);
+        } else {
+            batch
+                .clear(BlobOp::Link {
+                    hash: self.blob_hash.clone(),
+                    to: BlobLink::Document,
+                })
+                .clear(EmailField::Metadata);
+        }
+
+        Ok(())
+    }
+}
+
+impl ArchivedMessageMetadata {
+    #[inline(always)]
+    pub fn root_part(&self) -> &ArchivedMessageMetadataPart {
+        &self.contents[0].parts[0]
+    }
+
+    pub fn unindex(&self, batch: &mut BatchBuilder) {
+        // Delete metadata
+        let thread_name = self
+            .contents
+            .first()
+            .and_then(|c| c.parts.first())
+            .and_then(|p| {
+                p.headers.iter().rev().find_map(|h| {
+                    if let ArchivedMetadataHeaderName::Subject = &h.name {
+                        h.value.as_text()
+                    } else {
+                        None
+                    }
+                })
+            })
+            .map(thread_name)
+            .unwrap_or_default();
+
+        batch
+            .clear(EmailField::Metadata)
+            .clear(ValueClass::IndexProperty(IndexPropertyClass::Hash {
+                property: EmailField::Threading.into(),
+                hash: CheekyHash::new(if !thread_name.is_empty() {
+                    thread_name
+                } else {
+                    "!"
+                }),
+            }))
+            .clear(BlobOp::Link {
+                hash: BlobHash::from(&self.blob_hash),
+                to: BlobLink::Document,
+            });
+    }
+}
+
+impl IndexMessage for BatchBuilder {
+    fn index_message<'x>(
+        &mut self,
+        tenant_id: Option<u32>,
+        mut message: mail_parser::Message<'x>,
+        extra_headers: Vec<u8>,
+        mut extra_headers_parsed: Vec<mail_parser::Header<'x>>,
+        blob_hash: BlobHash,
+        data: MessageData,
+        received_at: u64,
+    ) -> trc::Result<&mut Self> {
+        let mut has_attachments = false;
+        let mut preview = None;
+        let preview_part_id = message
+            .text_body
+            .first()
+            .or_else(|| message.html_body.first())
+            .copied()
+            .unwrap_or(u32::MAX);
+
+        for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() {
+            let part_id = part_id as u32;
+            match &part.body {
+                mail_parser::PartType::Text(text) => {
+                    if part_id == preview_part_id {
+                        preview =
+                            preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
+                    }
+
+                    if !message.text_body.contains(&part_id)
+                        && !message.html_body.contains(&part_id)
+                    {
+                        has_attachments = true;
+                    }
+                }
+                mail_parser::PartType::Html(html) => {
+                    let text = html_to_text(html);
+                    if part_id == preview_part_id {
+                        preview =
+                            preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
+                    }
+
+                    if !message.text_body.contains(&part_id)
+                        && !message.html_body.contains(&part_id)
+                    {
+                        has_attachments = true;
+                    }
+                }
+                mail_parser::PartType::Binary(_) | mail_parser::PartType::Message(_)
+                    if !has_attachments =>
+                {
+                    has_attachments = true;
+                }
+                _ => {}
+            }
+        }
+
+        // Build raw headers
+        let root_part = message.root_part();
+        let mut raw_headers = Vec::with_capacity(
+            (root_part.offset_body - root_part.offset_header) as usize + extra_headers.len(),
+        );
+        raw_headers.extend_from_slice(&extra_headers);
+        raw_headers.extend_from_slice(
+            message
+                .raw_message
+                .as_ref()
+                .get(root_part.offset_header as usize..root_part.offset_body as usize)
+                .unwrap_or_default(),
+        );
+
+        // Add additional headers to message
+        let blob_body_offset = if !extra_headers.is_empty() {
+            // Add extra headers to root part
+            let offset_start = extra_headers.len() as u32;
+            let mut part_iter_stack = Vec::new();
+            let mut part_iter = message.parts.iter_mut();
+
+            loop {
+                if let Some(part) = part_iter.next() {
+                    // Increment header offsets
+                    for header in part.headers.iter_mut() {
+                        header.offset_field += offset_start;
+                        header.offset_start += offset_start;
+                        header.offset_end += offset_start;
+                    }
+
+                    // Adjust part offsets
+                    part.offset_body += offset_start;
+                    part.offset_end += offset_start;
+                    part.offset_header += offset_start;
+
+                    if let PartType::Message(sub_message) = &mut part.body
+                        && sub_message.root_part().offset_header != 0
+                    {
+                        part_iter_stack.push(part_iter);
+                        part_iter = sub_message.parts.iter_mut();
+                    }
+                } else if let Some(iter) = part_iter_stack.pop() {
+                    part_iter = iter;
+                } else {
+                    break;
+                }
+            }
+
+            // Add extra headers to root part
+            let root_part = &mut message.parts[0];
+            extra_headers_parsed.append(&mut root_part.headers);
+            root_part.offset_header = 0;
+            root_part.headers = extra_headers_parsed;
+            root_part.offset_body - offset_start
+        } else {
+            message.root_part().offset_body
+        };
+
+        // Build metadata
+        let metadata = MessageMetadata {
+            preview: preview.unwrap_or_default().into_owned().into_boxed_str(),
+            raw_headers: raw_headers.into_boxed_slice(),
+            contents: build_metadata_contents(message),
+            blob_hash,
+            blob_body_offset,
+            rcvd_attach: (if has_attachments {
+                MESSAGE_HAS_ATTACHMENT
+            } else {
+                0
+            }) | (received_at & MESSAGE_RECEIVED_MASK),
+        };
+
+        self.set(
+            BlobOp::Link {
+                hash: metadata.blob_hash.clone(),
+                to: BlobLink::Document,
+            },
+            Vec::new(),
+        )
+        .custom(
+            ObjectIndexBuilder::<(), _>::new()
+                .with_tenant_id(tenant_id)
+                .with_changes(data),
+        )
+        .caused_by(trc::location!())?
+        .set(
+            EmailField::Metadata,
+            Archiver::new(metadata)
+                .serialize()
+                .caused_by(trc::location!())?,
+        );
+
+        Ok(self)
+    }
+}

crates/email/src/message/index/mod.rs

@@ -0,0 +1,111 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    mailbox::{JUNK_ID, TRASH_ID},
+    message::metadata::{ArchivedMessageData, MessageData},
+};
+use common::storage::index::{IndexItem, IndexValue, IndexableObject};
+use store::write::now;
+use types::{blob_hash::BlobHash, collection::SyncCollection, field::EmailField};
+
+pub mod extractors;
+pub mod metadata;
+pub mod search;
+
+pub(super) const MAX_MESSAGE_PARTS: usize = 1000;
+pub const PREVIEW_LENGTH: usize = 256;
+
+impl IndexableObject for MessageData {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        let mut mailboxes = Vec::with_capacity(self.mailboxes.len());
+        let mut is_in_trash = false;
+
+        for mailbox in &self.mailboxes {
+            mailboxes.push(mailbox.mailbox_id);
+            is_in_trash |= mailbox.mailbox_id == TRASH_ID || mailbox.mailbox_id == JUNK_ID;
+        }
+
+        [
+            IndexValue::Property {
+                field: EmailField::DeletedAt.into(),
+                value: if is_in_trash {
+                    IndexItem::from(now())
+                } else {
+                    IndexItem::None
+                },
+            },
+            IndexValue::Quota { used: self.size },
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::Email,
+                prefix: self.thread_id.into(),
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Thread,
+                ids: vec![self.thread_id],
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Email,
+                ids: mailboxes,
+            },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableObject for &ArchivedMessageData {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        let mut mailboxes = Vec::with_capacity(self.mailboxes.len());
+        let mut is_in_trash = false;
+
+        for mailbox in self.mailboxes.iter() {
+            let mailbox_id = mailbox.mailbox_id.to_native();
+            mailboxes.push(mailbox_id);
+            is_in_trash |= mailbox_id == TRASH_ID || mailbox_id == JUNK_ID;
+        }
+
+        [
+            IndexValue::Property {
+                field: EmailField::DeletedAt.into(),
+                value: if is_in_trash {
+                    IndexItem::from(now())
+                } else {
+                    IndexItem::None
+                },
+            },
+            IndexValue::Quota {
+                used: self.size.to_native(),
+            },
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::Email,
+                prefix: self.thread_id.to_native().into(),
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Thread,
+                ids: vec![self.thread_id.to_native()],
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Email,
+                ids: mailboxes,
+            },
+        ]
+        .into_iter()
+    }
+}
+
+pub(super) trait IndexMessage {
+    #[allow(clippy::too_many_arguments)]
+    fn index_message<'x>(
+        &mut self,
+        tenant_id: Option<u32>,
+        message: mail_parser::Message<'x>,
+        extra_headers: Vec<u8>,
+        extra_headers_parsed: Vec<mail_parser::Header<'x>>,
+        blob_hash: BlobHash,
+        data: MessageData,
+        received_at: u64,
+    ) -> trc::Result<&mut Self>;
+}

crates/email/src/message/index/search.rs

@@ -0,0 +1,344 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::{
+    index::{MAX_MESSAGE_PARTS, extractors::VisitTextArchived},
+    metadata::{
+        ArchivedMessageMetadata, ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue,
+        ArchivedMetadataPartType, DecodedPartContent, MESSAGE_HAS_ATTACHMENT,
+        MESSAGE_RECEIVED_MASK, MetadataHeaderName,
+    },
+};
+use mail_parser::{DateTime, decoders::html::html_to_text, parsers::fields::thread::thread_name};
+use nlp::{
+    language::{
+        Language,
+        detect::{LanguageDetector, MIN_LANGUAGE_SCORE},
+    },
+    tokenizers::word::WordTokenizer,
+};
+use store::{
+    ahash::AHashSet,
+    backend::MAX_TOKEN_LENGTH,
+    search::{EmailSearchField, IndexDocument, SearchField},
+    write::SearchIndex,
+};
+use utils::chained_bytes::ChainedBytes;
+
+impl ArchivedMessageMetadata {
+    pub fn index_document(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        raw_message: &[u8],
+        index_fields: &AHashSet<SearchField>,
+        default_language: Language,
+    ) -> IndexDocument {
+        let mut detector = LanguageDetector::new();
+        let mut language = Language::Unknown;
+        let message_contents = &self.contents[0];
+        let mut document = IndexDocument::new(SearchIndex::Email)
+            .with_account_id(account_id)
+            .with_document_id(document_id);
+
+        let raw_message = ChainedBytes::new(self.raw_headers.as_ref()).with_last(
+            raw_message
+                .get(self.blob_body_offset.to_native() as usize..)
+                .unwrap_or_default(),
+        );
+
+        if index_fields.is_empty()
+            || index_fields.contains(&SearchField::Email(EmailSearchField::ReceivedAt))
+        {
+            document.index_unsigned(
+                SearchField::Email(EmailSearchField::ReceivedAt),
+                self.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK,
+            );
+        }
+        if index_fields.is_empty()
+            || index_fields.contains(&SearchField::Email(EmailSearchField::Size))
+        {
+            document.index_unsigned(
+                SearchField::Email(EmailSearchField::Size),
+                raw_message.len() as u32,
+            );
+        }
+
+        for (part_id, part) in message_contents
+            .parts
+            .iter()
+            .take(MAX_MESSAGE_PARTS)
+            .enumerate()
+        {
+            let part_language = part.language().unwrap_or(language);
+            if part_id == 0 {
+                language = part_language;
+
+                for header in part.headers.iter().rev() {
+                    match &header.name {
+                        ArchivedMetadataHeaderName::From => {
+                            if index_fields.is_empty()
+                                || index_fields
+                                    .contains(&SearchField::Email(EmailSearchField::From))
+                            {
+                                header.value.visit_addresses(|_, value| {
+                                    document.index_text(
+                                        SearchField::Email(EmailSearchField::From),
+                                        value,
+                                        Language::None,
+                                    );
+                                });
+                            }
+                        }
+                        ArchivedMetadataHeaderName::To => {
+                            if index_fields.is_empty()
+                                || index_fields.contains(&SearchField::Email(EmailSearchField::To))
+                            {
+                                header.value.visit_addresses(|_, value| {
+                                    document.index_text(
+                                        SearchField::Email(EmailSearchField::To),
+                                        value,
+                                        Language::None,
+                                    );
+                                });
+                            }
+                        }
+                        ArchivedMetadataHeaderName::Cc => {
+                            if index_fields.is_empty()
+                                || index_fields.contains(&SearchField::Email(EmailSearchField::Cc))
+                            {
+                                header.value.visit_addresses(|_, value| {
+                                    document.index_text(
+                                        SearchField::Email(EmailSearchField::Cc),
+                                        value,
+                                        Language::None,
+                                    );
+                                });
+                            }
+                        }
+                        ArchivedMetadataHeaderName::Bcc => {
+                            if index_fields.is_empty()
+                                || index_fields.contains(&SearchField::Email(EmailSearchField::Bcc))
+                            {
+                                header.value.visit_addresses(|_, value| {
+                                    document.index_text(
+                                        SearchField::Email(EmailSearchField::Bcc),
+                                        value,
+                                        Language::None,
+                                    );
+                                });
+                            }
+                        }
+                        ArchivedMetadataHeaderName::Subject => {
+                            if (index_fields.is_empty()
+                                || index_fields
+                                    .contains(&SearchField::Email(EmailSearchField::Subject)))
+                                && let Some(subject) = header.value.as_text()
+                            {
+                                let subject = thread_name(subject);
+
+                                if part_language.is_unknown() {
+                                    detector.detect(subject, MIN_LANGUAGE_SCORE);
+                                }
+
+                                document.index_text(
+                                    SearchField::Email(EmailSearchField::Subject),
+                                    subject,
+                                    part_language,
+                                );
+                            }
+                        }
+                        ArchivedMetadataHeaderName::Date => {
+                            if (index_fields.is_empty()
+                                || index_fields
+                                    .contains(&SearchField::Email(EmailSearchField::SentAt)))
+                                && let Some(date) = header.value.as_datetime()
+                            {
+                                document.index_integer(
+                                    SearchField::Email(EmailSearchField::SentAt),
+                                    DateTime::from(date).to_timestamp(),
+                                );
+                            }
+                        }
+                        _ => {
+                            #[cfg(not(feature = "test_mode"))]
+                            let index_headers = index_fields
+                                .contains(&SearchField::Email(EmailSearchField::Headers));
+
+                            #[cfg(feature = "test_mode")]
+                            let index_headers = true;
+
+                            if index_headers {
+                                let mut value = String::new();
+                                match &header.value {
+                                    ArchivedMetadataHeaderValue::AddressList(_)
+                                    | ArchivedMetadataHeaderValue::AddressGroup(_) => {
+                                        header.value.visit_addresses(|_, addr| {
+                                            if !value.is_empty() {
+                                                value.push(' ');
+                                            }
+                                            value.push_str(addr);
+                                        });
+                                    }
+                                    ArchivedMetadataHeaderValue::Text(_)
+                                    | ArchivedMetadataHeaderValue::TextList(_) => {
+                                        header.value.visit_text(|text| {
+                                            if !value.is_empty() {
+                                                value.push(' ');
+                                            }
+                                            value.push_str(text);
+                                        });
+                                    }
+                                    _ => {
+                                        if let Some(raw_value) =
+                                            raw_message.get(header.value_range())
+                                        {
+                                            let raw_value = std::str::from_utf8(raw_value.as_ref())
+                                                .unwrap_or_default();
+
+                                            for word in
+                                                WordTokenizer::new(raw_value, MAX_TOKEN_LENGTH)
+                                            {
+                                                if !value.is_empty() {
+                                                    value.push(' ');
+                                                }
+                                                value.push_str(word.word.as_ref());
+                                            }
+                                        }
+                                    }
+                                }
+
+                                document.insert_key_value(
+                                    EmailSearchField::Headers,
+                                    header.name.as_str(),
+                                    value,
+                                );
+                            }
+                        }
+                    }
+                }
+            }
+
+            let part_id = part_id as u16;
+            match &part.body {
+                ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
+                    let text = match (part.decode_contents(&raw_message), &part.body) {
+                        (DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text,
+                        (DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => {
+                            html_to_text(html.as_ref()).into()
+                        }
+                        _ => unreachable!(),
+                    };
+
+                    if message_contents.is_html_part(part_id)
+                        || message_contents.is_text_part(part_id)
+                    {
+                        if index_fields.is_empty()
+                            || index_fields.contains(&SearchField::Email(EmailSearchField::Body))
+                        {
+                            if part_language.is_unknown() {
+                                detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE);
+                            }
+
+                            document.index_text(
+                                SearchField::Email(EmailSearchField::Body),
+                                text.as_ref(),
+                                part_language,
+                            );
+                        }
+                    } else if index_fields.is_empty()
+                        || index_fields.contains(&SearchField::Email(EmailSearchField::Attachment))
+                    {
+                        if part_language.is_unknown() {
+                            detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE);
+                        }
+
+                        document.index_text(
+                            SearchField::Email(EmailSearchField::Attachment),
+                            text.as_ref(),
+                            part_language,
+                        );
+                    }
+                }
+                ArchivedMetadataPartType::Message(nested_message_id)
+                    if index_fields.is_empty()
+                        || index_fields
+                            .contains(&SearchField::Email(EmailSearchField::Attachment)) =>
+                {
+                    let nested_message = self.message_id(*nested_message_id);
+                    let nested_message_language = nested_message
+                        .root_part()
+                        .language()
+                        .unwrap_or(Language::Unknown);
+                    if let Some(ArchivedMetadataHeaderValue::Text(subject)) = nested_message
+                        .root_part()
+                        .header_value(&MetadataHeaderName::Subject)
+                    {
+                        if nested_message_language.is_unknown() {
+                            detector.detect(subject.as_ref(), MIN_LANGUAGE_SCORE);
+                        }
+
+                        document.index_text(
+                            SearchField::Email(EmailSearchField::Attachment),
+                            subject.as_ref(),
+                            nested_message_language,
+                        );
+                    }
+
+                    for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) {
+                        let language = sub_part.language().unwrap_or(nested_message_language);
+                        match &sub_part.body {
+                            ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
+                                let text = match (
+                                    sub_part.decode_contents(&raw_message),
+                                    &sub_part.body,
+                                ) {
+                                    (
+                                        DecodedPartContent::Text(text),
+                                        ArchivedMetadataPartType::Text,
+                                    ) => text,
+                                    (
+                                        DecodedPartContent::Text(html),
+                                        ArchivedMetadataPartType::Html,
+                                    ) => html_to_text(html.as_ref()).into(),
+                                    _ => unreachable!(),
+                                };
+
+                                if language.is_unknown() {
+                                    detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE);
+                                }
+
+                                document.index_text(
+                                    SearchField::Email(EmailSearchField::Attachment),
+                                    text.as_ref(),
+                                    language,
+                                );
+                            }
+                            _ => (),
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        #[cfg(not(feature = "test_mode"))]
+        document.set_unknown_language(
+            detector
+                .most_frequent_language()
+                .unwrap_or(default_language),
+        );
+
+        #[cfg(feature = "test_mode")]
+        document.set_unknown_language(default_language);
+
+        document.index_bool(
+            EmailSearchField::HasAttachment,
+            self.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT != 0,
+        );
+        document
+    }
+}

crates/email/src/message/mod.rs

@@ -0,0 +1,15 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+pub mod body;
+pub mod copy;
+pub mod crypto;
+pub mod delete;
+pub mod delivery;
+pub mod headers;
+pub mod index;
+pub mod ingest;
+pub mod metadata;

crates/email/src/push/mod.rs

@@ -0,0 +1,94 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use jmap_proto::method::query::Filter;
+use jmap_proto::object::email::EmailFilter;
+use jmap_proto::object::push_subscription::EmailPushProperty;
+use types::type_state::DataType;
+use utils::map::bitmap::Bitmap;
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq,
+)]
+pub struct PushSubscription {
+    pub id: u32,
+    pub url: String,
+    pub device_client_id: String,
+    pub expires: u64,
+    pub verification_code: String,
+    pub verified: bool,
+    pub types: Bitmap<DataType>,
+    pub keys: Option<Keys>,
+    pub email_push: Vec<EmailPush>,
+}
+
+#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)]
+pub struct Keys {
+    pub p256dh: Vec<u8>,
+    pub auth: Vec<u8>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq,
+)]
+pub struct PushSubscriptions {
+    pub subscriptions: Vec<PushSubscription>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq,
+)]
+pub struct EmailPush {
+    pub account_id: u32,
+    pub properties: Vec<EmailPushProperty>,
+    pub filter: Vec<Filter<EmailFilter>>,
+    pub urgency: Urgency,
+}
+
+#[derive(
+    rkyv::Archive,
+    rkyv::Deserialize,
+    rkyv::Serialize,
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Default,
+)]
+#[rkyv(compare(PartialEq), derive(Debug))]
+#[repr(u8)]
+pub enum Urgency {
+    VeryLow = 0,
+    Low = 1,
+    #[default]
+    Normal = 2,
+    High = 3,
+}
+
+impl From<&ArchivedUrgency> for Urgency {
+    fn from(value: &ArchivedUrgency) -> Self {
+        match value {
+            ArchivedUrgency::VeryLow => Urgency::VeryLow,
+            ArchivedUrgency::Low => Urgency::Low,
+            ArchivedUrgency::Normal => Urgency::Normal,
+            ArchivedUrgency::High => Urgency::High,
+        }
+    }
+}
+
+impl Urgency {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Urgency::VeryLow => "very-low",
+            Urgency::Low => "low",
+            Urgency::Normal => "normal",
+            Urgency::High => "high",
+        }
+    }
+}

crates/email/src/sieve/delete.rs

@@ -0,0 +1,67 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::SieveScript;
+use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder};
+use store::write::BatchBuilder;
+use store::{
+    ValueKey,
+    write::{AlignedBytes, Archive},
+};
+use trc::AddContext;
+use types::{collection::Collection, field::SieveField};
+
+pub trait SieveScriptDelete: Sync + Send {
+    fn sieve_script_delete(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        access_token: &AccessToken,
+        batch: &mut BatchBuilder,
+    ) -> impl Future<Output = trc::Result<bool>> + Send;
+}
+
+impl SieveScriptDelete for Server {
+    async fn sieve_script_delete(
+        &self,
+        account_id: u32,
+        document_id: u32,
+        access_token: &AccessToken,
+        batch: &mut BatchBuilder,
+    ) -> trc::Result<bool> {
+        // Fetch record
+        if let Some(obj_) = self
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                account_id,
+                Collection::SieveScript,
+                document_id,
+            ))
+            .await?
+        {
+            // Delete record
+            batch
+                .with_account_id(account_id)
+                .with_collection(Collection::SieveScript)
+                .with_document(document_id)
+                .clear(SieveField::Ids)
+                .custom(
+                    ObjectIndexBuilder::<_, ()>::new()
+                        .with_current(
+                            obj_.to_unarchived::<SieveScript>()
+                                .caused_by(trc::location!())?,
+                        )
+                        .with_changed_by(access_token.account_tenant_ids()),
+                )
+                .caused_by(trc::location!())?
+                .commit_point();
+
+            Ok(true)
+        } else {
+            Ok(false)
+        }
+    }
+}

crates/email/src/sieve/index.rs

@@ -0,0 +1,57 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ArchivedSieveScript, SieveScript};
+use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
+use types::{collection::SyncCollection, field::SieveField};
+
+impl IndexableObject for SieveScript {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::Index {
+                field: SieveField::Name.into(),
+                value: self.name.as_str().to_lowercase().into(),
+            },
+            IndexValue::Blob {
+                value: self.blob_hash.clone(),
+            },
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::SieveScript,
+                prefix: None,
+            },
+            IndexValue::Quota { used: self.size },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableAndSerializableObject for SieveScript {
+    fn is_versioned() -> bool {
+        false
+    }
+}
+
+impl IndexableObject for &ArchivedSieveScript {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::Index {
+                field: SieveField::Name.into(),
+                value: self.name.to_lowercase().into(),
+            },
+            IndexValue::Blob {
+                value: (&self.blob_hash).into(),
+            },
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::SieveScript,
+                prefix: None,
+            },
+            IndexValue::Quota {
+                used: u32::from(self.size),
+            },
+        ]
+        .into_iter()
+    }
+}

crates/email/src/sieve/ingest.rs

@@ -0,0 +1,793 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ActiveScript, SeenIdHash, SieveScript};
+use crate::{
+    cache::{MessageCacheFetch, mailbox::MailboxCacheAccess},
+    mailbox::{INBOX_ID, TRASH_ID, manage::MailboxFnc},
+    message::{
+        delivery::{AutogeneratedMessage, IngestRecipient},
+        ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail},
+    },
+};
+use common::{
+    Server, auth::AccessToken, config::mailstore::spamfilter::spam_status,
+    scripts::plugins::PluginContext,
+};
+use mail_builder::headers::date::Date;
+use mail_parser::{HeaderName, MessageParser};
+use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve};
+use std::{borrow::Cow, sync::Arc};
+use std::{future::Future, str::FromStr};
+use store::{
+    Deserialize, Serialize, ValueKey,
+    ahash::AHashMap,
+    dispatch::lookup::KeyValue,
+    write::{
+        AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobLink, BlobOp, ValueClass,
+    },
+};
+use trc::{AddContext, SieveEvent, SmtpEvent};
+use types::{
+    blob_hash::BlobHash,
+    collection::Collection,
+    field::{PrincipalField, SieveField},
+    id::Id,
+    keyword::Keyword,
+    special_use::SpecialUse,
+};
+
+struct SieveMessage<'x> {
+    pub raw_message: Cow<'x, [u8]>,
+    pub file_into: Vec<u32>,
+    pub did_file_into: bool,
+    pub flags: Vec<Keyword>,
+}
+
+pub trait SieveScriptIngest: Sync + Send {
+    #[allow(clippy::too_many_arguments)]
+    fn sieve_script_ingest(
+        &self,
+        access_token: &AccessToken,
+        blob_hash: &BlobHash,
+        raw_message: &[u8],
+        envelope_from: &str,
+        envelope_from_authenticated: bool,
+        envelope_to: &IngestRecipient,
+        session_id: u64,
+        active_script: ActiveScript,
+        autogenerated: &mut Vec<AutogeneratedMessage>,
+    ) -> impl Future<Output = trc::Result<IngestedEmail>> + Send;
+
+    fn sieve_script_get_active_id(
+        &self,
+        account_id: u32,
+    ) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
+
+    fn sieve_script_get_active(
+        &self,
+        account_id: u32,
+    ) -> impl Future<Output = trc::Result<Option<ActiveScript>>> + Send;
+
+    fn sieve_script_get_by_name(
+        &self,
+        account_id: u32,
+        name: &str,
+    ) -> impl Future<Output = trc::Result<Option<Sieve>>> + Send;
+
+    fn sieve_script_compile(
+        &self,
+        account_id: u32,
+        document_id: u32,
+    ) -> impl Future<Output = trc::Result<Option<CompiledScript>>> + Send;
+}
+
+impl SieveScriptIngest for Server {
+    #[allow(clippy::blocks_in_conditions)]
+    async fn sieve_script_ingest(
+        &self,
+        access_token: &AccessToken,
+        blob_hash: &BlobHash,
+        raw_message: &[u8],
+        envelope_from: &str,
+        envelope_from_authenticated: bool,
+        envelope_to: &IngestRecipient,
+        session_id: u64,
+        active_script: ActiveScript,
+        autogenerated: &mut Vec<AutogeneratedMessage>,
+    ) -> trc::Result<IngestedEmail> {
+        // Parse message
+        let message = if let Some(message) = MessageParser::new().parse(raw_message) {
+            message
+        } else {
+            return Err(
+                trc::EventType::MessageIngest(trc::MessageIngestEvent::Error)
+                    .ctx(trc::Key::Code, 550)
+                    .ctx(trc::Key::Reason, "Failed to parse e-mail message."),
+            );
+        };
+
+        let received_headers = message
+            .headers()
+            .iter()
+            .filter(|header| matches!(header.name, HeaderName::Received))
+            .count();
+
+        // Obtain mailboxIds
+        let account_id = access_token.account_id();
+        let mut cache = self
+            .get_cached_messages(account_id)
+            .await
+            .caused_by(trc::location!())?;
+
+        // Create Sieve instance
+        let mut instance = self.core.sieve.untrusted_runtime.filter_parsed(message);
+
+        // Set account name and email
+        let account_info = self.account(account_id).await.caused_by(trc::location!())?;
+        let mail_from = account_info.name().to_string();
+        instance.set_user_full_name(
+            account_info
+                .description()
+                .unwrap_or_else(|| account_info.name()),
+        );
+        instance.set_user_address(&mail_from);
+
+        // Set envelope
+        instance.set_envelope(Envelope::From, envelope_from);
+        instance.set_envelope(Envelope::To, envelope_to.address.as_str());
+        if let Some(orcpt) = &envelope_to.orcpt {
+            instance.set_envelope(Envelope::Orcpt, orcpt.as_str());
+        }
+        instance.set_spam_status(spam_status(envelope_to.spam_percentage));
+
+        let mut input = Input::script(
+            active_script.script_name.to_string(),
+            active_script.script.clone(),
+        );
+
+        let mut do_discard = false;
+        let mut do_deliver = false;
+        let mut do_redirect = false;
+
+        let mut reject_reason = None;
+        let mut messages: Vec<SieveMessage> = vec![SieveMessage {
+            raw_message: raw_message.into(),
+            file_into: Vec::new(),
+            flags: Vec::new(),
+            did_file_into: false,
+        }];
+        let mut ingested_message = IngestedEmail {
+            document_id: 0,
+            thread_id: 0,
+            change_id: u64::MAX,
+            blob_id: Default::default(),
+            size: raw_message.len(),
+            imap_uids: Vec::new(),
+        };
+        let mut checked_ids: AHashMap<SeenIdHash, bool> = AHashMap::new();
+
+        while let Some(event) = instance.run(input) {
+            match event {
+                Ok(event) => match event {
+                    Event::IncludeScript { name, .. } => match &name {
+                        sieve::Script::Personal(name_) => {
+                            if let Ok(Some(script)) =
+                                self.sieve_script_get_by_name(account_id, name_).await
+                            {
+                                input = Input::script(name, script);
+                            } else {
+                                input = false.into();
+                            }
+                        }
+                        sieve::Script::Global(name_) => {
+                            if let Some(script) = self.get_untrusted_sieve_script(name_, session_id)
+                            {
+                                input = Input::script(name, script.clone());
+                            } else {
+                                input = false.into();
+                            }
+                        }
+                    },
+                    Event::MailboxExists {
+                        mailboxes,
+                        special_use,
+                    } => {
+                        if !mailboxes.is_empty() {
+                            let mut special_use_ids = Vec::with_capacity(special_use.len());
+                            for role in special_use.iter().map(|v| SpecialUse::parse_use_attr(v)) {
+                                special_use_ids.push(match role {
+                                    Some(SpecialUse::Inbox) => INBOX_ID,
+                                    Some(SpecialUse::Trash) => TRASH_ID,
+                                    Some(role) => cache
+                                        .mailbox_by_role(&role)
+                                        .map(|m| m.document_id)
+                                        .unwrap_or(u32::MAX),
+                                    None => u32::MAX,
+                                });
+                            }
+
+                            let mut result = true;
+                            for mailbox in mailboxes {
+                                match mailbox {
+                                    Mailbox::Name(name) => {
+                                        if !matches!(
+                                            cache.mailbox_by_path(&name),
+                                            Some(item) if special_use_ids.is_empty() ||
+                                            special_use_ids.contains(&item.document_id)
+                                        ) {
+                                            result = false;
+                                            break;
+                                        }
+                                    }
+                                    Mailbox::Id(id) => {
+                                        if !matches!(Id::from_str(&id), Ok(id) if
+                                                        cache.has_mailbox_id(&id.document_id()) &&
+                                                        (special_use_ids.is_empty() ||
+                                                        special_use_ids.contains(&id.document_id())))
+                                        {
+                                            result = false;
+                                            break;
+                                        }
+                                    }
+                                }
+                            }
+                            input = result.into();
+                        } else if !special_use.is_empty() {
+                            let mut result = true;
+
+                            for role in special_use.iter().map(|v| SpecialUse::parse_use_attr(v)) {
+                                match role {
+                                    Some(SpecialUse::Inbox | SpecialUse::Trash) => {}
+                                    Some(other) if cache.mailbox_by_role(&other).is_some() => {}
+                                    _ => {
+                                        result = false;
+                                        break;
+                                    }
+                                }
+                            }
+                            input = result.into();
+                        } else {
+                            input = false.into();
+                        }
+                    }
+                    Event::DuplicateId { id, expiry, last } => {
+                        let id_hash = SeenIdHash::new(
+                            account_id,
+                            active_script.version.hash().unwrap_or_default(),
+                            &id,
+                        );
+                        if let Some(result) = checked_ids.get(&id_hash) {
+                            input = (*result).into();
+                        } else {
+                            let exists = self
+                                .in_memory_store()
+                                .key_exists(id_hash.key())
+                                .await
+                                .caused_by(trc::location!())?;
+
+                            if !exists || last {
+                                self.in_memory_store()
+                                    .key_set(KeyValue::new(id_hash.key(), vec![]).expires(expiry))
+                                    .await
+                                    .caused_by(trc::location!())?;
+                            }
+
+                            checked_ids.insert(id_hash, exists);
+                            input = exists.into();
+                        }
+                    }
+                    Event::Discard => {
+                        do_discard = true;
+                        input = true.into();
+                    }
+                    Event::Reject { reason, .. } => {
+                        reject_reason = reason.into();
+                        do_discard = true;
+                        input = true.into();
+                    }
+                    Event::Keep { flags, message_id } => {
+                        if let Some(message) = messages.get_mut(message_id) {
+                            message.flags = flags.into_iter().map(Keyword::from).collect();
+                            if !message.file_into.contains(&INBOX_ID) {
+                                message.file_into.push(INBOX_ID);
+                            }
+                            do_deliver = true;
+                        } else {
+                            trc::event!(
+                                Sieve(SieveEvent::UnexpectedError),
+                                Details = "Unknown message id.",
+                                MessageId = message_id,
+                                SpanId = session_id
+                            );
+                        }
+                        input = true.into();
+                    }
+                    Event::FileInto {
+                        folder,
+                        flags,
+                        mailbox_id,
+                        special_use,
+                        create,
+                        message_id,
+                    } => {
+                        let mut target_id = u32::MAX;
+
+                        // Find mailbox by Id
+                        if let Some(mailbox_id) = mailbox_id.and_then(|m| Id::from_str(&m).ok()) {
+                            let mailbox_id = mailbox_id.document_id();
+                            if cache.has_mailbox_id(&mailbox_id) {
+                                target_id = mailbox_id;
+                            }
+                        }
+
+                        // Find mailbox by role
+                        if target_id == u32::MAX
+                            && let Some(special_use) =
+                                special_use.as_deref().and_then(SpecialUse::parse_use_attr)
+                        {
+                            match special_use {
+                                SpecialUse::Inbox => {
+                                    target_id = INBOX_ID;
+                                }
+                                SpecialUse::Trash => {
+                                    target_id = TRASH_ID;
+                                }
+                                role => {
+                                    if let Some(item) = cache.mailbox_by_role(&role) {
+                                        target_id = item.document_id;
+                                    }
+                                }
+                            }
+                        }
+
+                        // Find mailbox by name
+                        if target_id == u32::MAX {
+                            if !create {
+                                if let Some(m) = cache.mailbox_by_path(&folder) {
+                                    target_id = m.document_id;
+                                }
+                            } else if let Some(document_id) = self
+                                .mailbox_create_path(account_id, &folder)
+                                .await
+                                .caused_by(trc::location!())?
+                            {
+                                cache = self
+                                    .get_cached_messages(account_id)
+                                    .await
+                                    .caused_by(trc::location!())?;
+                                target_id = document_id;
+                            }
+                        }
+
+                        // Default to Inbox
+                        if target_id == u32::MAX {
+                            target_id = INBOX_ID;
+                        }
+
+                        if let Some(message) = messages.get_mut(message_id) {
+                            message.flags = flags.into_iter().map(Keyword::from).collect();
+                            if !message.file_into.contains(&target_id) {
+                                message.file_into.push(target_id);
+                            }
+                            message.did_file_into = true;
+                            do_deliver = true;
+                        } else {
+                            trc::event!(
+                                Sieve(SieveEvent::UnexpectedError),
+                                Details = "Unknown message id.",
+                                MessageId = message_id,
+                                SpanId = session_id
+                            );
+                        }
+                        input = true.into();
+                    }
+                    Event::SendMessage {
+                        recipient,
+                        message_id,
+                        ..
+                    } => {
+                        input = true.into();
+                        if let Some(message) = messages.get(message_id) {
+                            if received_headers >= self.core.sieve.max_received_headers {
+                                trc::event!(
+                                    Smtp(SmtpEvent::LoopDetected),
+                                    From = mail_from.clone(),
+                                    Total = received_headers,
+                                    Limit = self.core.sieve.max_received_headers,
+                                    SpanId = session_id,
+                                );
+
+                                continue;
+                            }
+
+                            let recipients: Vec<String> = match recipient {
+                                Recipient::Address(rcpt) => vec![rcpt],
+                                Recipient::Group(rcpts) => rcpts,
+                                Recipient::List(_) => {
+                                    // Not yet implemented
+                                    continue;
+                                }
+                            };
+
+                            if message.raw_message.len() <= self.core.email.mail_max_size {
+                                trc::event!(
+                                    Sieve(SieveEvent::SendMessage),
+                                    From = mail_from.clone(),
+                                    To = recipients
+                                        .iter()
+                                        .map(|r| trc::Value::String(r.as_str().into()))
+                                        .collect::<Vec<_>>(),
+                                    Size = message.raw_message.len(),
+                                    SpanId = session_id
+                                );
+
+                                let mut raw_message =
+                                    Vec::with_capacity(160 + message.raw_message.len());
+                                write_received_header(
+                                    &mut raw_message,
+                                    &self.core.network.server_name,
+                                    session_id,
+                                );
+                                raw_message.extend_from_slice(message.raw_message.as_ref());
+
+                                autogenerated.push(AutogeneratedMessage {
+                                    sender_address: mail_from.clone(),
+                                    recipients,
+                                    message: raw_message,
+                                });
+                                do_redirect = true;
+                            } else {
+                                trc::event!(
+                                    Sieve(SieveEvent::MessageTooLarge),
+                                    From = mail_from.clone(),
+                                    To = recipients
+                                        .iter()
+                                        .map(|r| trc::Value::String(r.as_str().into()))
+                                        .collect::<Vec<_>>(),
+                                    Size = message.raw_message.len(),
+                                    Limit = self.core.email.mail_max_size,
+                                    SpanId = session_id,
+                                );
+                            }
+                        } else {
+                            trc::event!(
+                                Sieve(SieveEvent::UnexpectedError),
+                                Details = "Unknown message id.",
+                                MessageId = message_id,
+                                SpanId = session_id
+                            );
+
+                            continue;
+                        }
+                    }
+                    Event::ListContains { .. }
+                    | Event::Notify { .. }
+                    | Event::SetEnvelope { .. } => {
+                        // Not allowed
+                        input = false.into();
+                    }
+                    Event::Function { id, arguments } => {
+                        input = self
+                            .core
+                            .run_plugin(
+                                id,
+                                PluginContext {
+                                    session_id,
+                                    server: self,
+                                    message: instance.message(),
+                                    modifications: &mut Vec::new(),
+                                    access_token: access_token.into(),
+                                    arguments,
+                                },
+                            )
+                            .await;
+                    }
+                    Event::CreatedMessage { message, .. } => {
+                        messages.push(SieveMessage {
+                            raw_message: message.into(),
+                            file_into: Vec::new(),
+                            flags: Vec::new(),
+                            did_file_into: false,
+                        });
+                        input = true.into();
+                    }
+                },
+
+                #[cfg(feature = "test_mode")]
+                Err(sieve::runtime::RuntimeError::ScriptErrorMessage(err)) => {
+                    panic!("Sieve test failed: {}", err);
+                }
+
+                Err(err) => {
+                    trc::event!(
+                        Sieve(SieveEvent::RuntimeError),
+                        Reason = err.to_string(),
+                        SpanId = session_id
+                    );
+
+                    input = true.into();
+                }
+            }
+        }
+
+        // Fail-safe, no discard and no keep seen, assume that something went wrong and file anyway.
+        if !do_deliver && !do_discard && !do_redirect {
+            messages[0].file_into.push(INBOX_ID);
+        }
+
+        // Deliver messages
+        let mut last_temp_error = None;
+        let mut has_delivered = false;
+        for (message_id, sieve_message) in messages.into_iter().enumerate() {
+            if !sieve_message.file_into.is_empty() {
+                // Parse message if needed
+                let (blob_hash, message) = if message_id == 0 && !instance.has_message_changed() {
+                    (blob_hash.into(), instance.take_message())
+                } else if let Some(message) =
+                    MessageParser::new().parse(sieve_message.raw_message.as_ref())
+                {
+                    (None, message)
+                } else {
+                    trc::event!(
+                        Sieve(SieveEvent::UnexpectedError),
+                        Details = "Failed to parse Sieve generated message.",
+                        SpanId = session_id
+                    );
+
+                    continue;
+                };
+
+                // Deliver message
+                match self
+                    .email_ingest(IngestEmail {
+                        raw_message: &sieve_message.raw_message,
+                        blob_hash,
+                        message: message.into(),
+                        access_token,
+                        mailbox_ids: sieve_message.file_into,
+                        keywords: sieve_message.flags,
+                        received_at: None,
+                        source: IngestSource::Smtp {
+                            deliver_to: envelope_to.address.as_str(),
+                            is_sender_authenticated: envelope_from_authenticated,
+                            is_spam: envelope_to.is_spam() && !sieve_message.did_file_into,
+                        },
+                        session_id,
+                    })
+                    .await
+                {
+                    Ok(ingested_message_) => {
+                        has_delivered = true;
+                        ingested_message = ingested_message_;
+                    }
+                    Err(err) => {
+                        last_temp_error = err.into();
+                    }
+                }
+            }
+        }
+
+        if let Some(reject_reason) = reject_reason {
+            Err(
+                trc::EventType::MessageIngest(trc::MessageIngestEvent::Error)
+                    .ctx(trc::Key::Code, 571)
+                    .ctx(trc::Key::Reason, reject_reason),
+            )
+        } else if has_delivered || last_temp_error.is_none() {
+            Ok(ingested_message)
+        } else {
+            // There were problems during delivery
+            #[allow(clippy::unnecessary_unwrap)]
+            Err(last_temp_error.unwrap())
+        }
+    }
+
+    async fn sieve_script_get_active_id(&self, account_id: u32) -> trc::Result<Option<u32>> {
+        self.store()
+            .get_value::<u32>(ValueKey {
+                account_id,
+                collection: Collection::Principal.into(),
+                document_id: 0,
+                class: ValueClass::Property(PrincipalField::ActiveScriptId.into()),
+            })
+            .await
+            .caused_by(trc::location!())
+    }
+
+    async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result<Option<ActiveScript>> {
+        // Find the currently active script
+        if let Some(document_id) = self
+            .store()
+            .get_value::<u32>(ValueKey {
+                account_id,
+                collection: Collection::Principal.into(),
+                document_id: 0,
+                class: ValueClass::Property(PrincipalField::ActiveScriptId.into()),
+            })
+            .await
+            .caused_by(trc::location!())?
+        {
+            if let Some(script) = self.sieve_script_compile(account_id, document_id).await? {
+                Ok(Some(ActiveScript {
+                    document_id,
+                    script: Arc::new(script.script),
+                    script_name: script.name,
+                    version: script.version,
+                }))
+            } else {
+                Ok(None)
+            }
+        } else {
+            Ok(None)
+        }
+    }
+
+    async fn sieve_script_get_by_name(
+        &self,
+        account_id: u32,
+        name: &str,
+    ) -> trc::Result<Option<Sieve>> {
+        // Find the script by name
+        if let Some(document_id) = self
+            .document_ids_matching(
+                account_id,
+                Collection::SieveScript,
+                SieveField::Name,
+                name.to_lowercase().as_bytes(),
+            )
+            .await
+            .caused_by(trc::location!())?
+            .min()
+        {
+            self.sieve_script_compile(account_id, document_id)
+                .await
+                .map(|script| script.map(|s| s.script))
+        } else {
+            Ok(None)
+        }
+    }
+
+    #[allow(clippy::blocks_in_conditions)]
+    async fn sieve_script_compile(
+        &self,
+        account_id: u32,
+        document_id: u32,
+    ) -> trc::Result<Option<CompiledScript>> {
+        // Obtain script object
+        let Some(script_object) = self
+            .store()
+            .get_value::<Archive<AlignedBytes>>(ValueKey::archive(
+                account_id,
+                Collection::SieveScript,
+                document_id,
+            ))
+            .await?
+        else {
+            return Ok(None);
+        };
+
+        // Obtain the sieve script length
+        let version = script_object.version;
+        let unarchived_script = script_object
+            .unarchive::<SieveScript>()
+            .caused_by(trc::location!())?;
+        let script_offset = u32::from(unarchived_script.size) as usize;
+
+        // Obtain the sieve script blob
+        let script_bytes = self
+            .core
+            .storage
+            .blob
+            .get_blob(unarchived_script.blob_hash.0.as_ref(), 0..usize::MAX)
+            .await
+            .caused_by(trc::location!())?
+            .ok_or_else(|| {
+                trc::StoreEvent::NotFound
+                    .into_err()
+                    .caused_by(trc::location!())
+                    .document_id(document_id)
+            })?;
+
+        // Obtain the precompiled script
+        if let Some(script) = script_bytes.get(script_offset..).and_then(|bytes| {
+            <Archive<AlignedBytes> as Deserialize>::deserialize(bytes)
+                .ok()?
+                .deserialize::<Sieve>()
+                .ok()
+        }) {
+            Ok(Some(CompiledScript {
+                script,
+                name: unarchived_script.name.as_str().into(),
+                version,
+            }))
+        } else {
+            // Deserialization failed, probably because the script compiler version changed
+            match self.core.sieve.untrusted_compiler.compile(
+                script_bytes.get(0..script_offset).ok_or_else(|| {
+                    trc::StoreEvent::NotFound
+                        .into_err()
+                        .caused_by(trc::location!())
+                        .document_id(document_id)
+                })?,
+            ) {
+                Ok(sieve) => {
+                    // Store updated compiled sieve script
+                    let sieve = Archiver::new(sieve).untrusted();
+                    let compiled_bytes = sieve.serialize().caused_by(trc::location!())?;
+                    let mut updated_sieve_bytes =
+                        Vec::with_capacity(script_offset + compiled_bytes.len());
+                    updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]);
+                    updated_sieve_bytes.extend_from_slice(&compiled_bytes);
+
+                    // Store updated blob
+                    let (new_blob_hash, new_blob_hold) = self
+                        .put_temporary_blob(account_id, &updated_sieve_bytes, 60)
+                        .await?;
+                    let mut new_script_object =
+                        rkyv::deserialize(unarchived_script).caused_by(trc::location!())?;
+                    let blob_hash =
+                        std::mem::replace(&mut new_script_object.blob_hash, new_blob_hash.clone());
+                    let new_archive = Archiver::new(new_script_object);
+
+                    // Update script object
+                    let mut batch = BatchBuilder::new();
+                    batch
+                        .with_account_id(account_id)
+                        .with_collection(Collection::SieveScript)
+                        .with_document(document_id)
+                        .assert_value(SieveField::Archive, &script_object)
+                        .set(
+                            SieveField::Archive,
+                            new_archive.serialize().caused_by(trc::location!())?,
+                        )
+                        .clear(BlobOp::Link {
+                            hash: blob_hash,
+                            to: BlobLink::Document,
+                        })
+                        .set(
+                            BlobOp::Link {
+                                hash: new_blob_hash,
+                                to: BlobLink::Document,
+                            },
+                            Vec::new(),
+                        )
+                        .clear(new_blob_hold);
+                    self.store()
+                        .write(batch.build_all())
+                        .await
+                        .caused_by(trc::location!())?;
+
+                    Ok(Some(CompiledScript {
+                        script: sieve.into_inner(),
+                        name: new_archive.into_inner().name,
+                        version,
+                    }))
+                }
+                Err(error) => Err(trc::StoreEvent::UnexpectedError
+                    .caused_by(trc::location!())
+                    .reason(error)
+                    .details("Failed to compile Sieve script")),
+            }
+        }
+    }
+}
+
+fn write_received_header(buf: &mut Vec<u8>, hostname: &str, id: u64) {
+    buf.extend_from_slice(b"Received: from localhost (localhost [127.0.0.1])\r\n\tby ");
+    buf.extend_from_slice(hostname.as_bytes());
+    buf.extend_from_slice(b" (Stalwart SMTP) with LMTP id ");
+    buf.extend_from_slice(format!("{id:X}").as_bytes());
+    buf.extend_from_slice(b";\r\n\t");
+    buf.extend_from_slice(Date::now().to_rfc822().as_bytes());
+    buf.extend_from_slice(b"\r\n");
+}
+
+pub struct CompiledScript {
+    pub script: Sieve,
+    pub name: String,
+    pub version: ArchiveVersion,
+}

crates/email/src/sieve/mod.rs

@@ -0,0 +1,104 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use common::KV_SIEVE_ID;
+use sieve::Sieve;
+use std::sync::Arc;
+use store::{blake3, write::ArchiveVersion};
+use types::blob_hash::BlobHash;
+
+pub mod delete;
+pub mod index;
+pub mod ingest;
+
+#[derive(Debug, Clone)]
+pub struct ActiveScript {
+    pub document_id: u32,
+    pub version: ArchiveVersion,
+    pub script_name: String,
+    pub script: Arc<Sieve>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+#[rkyv(derive(Debug))]
+pub struct SieveScript {
+    pub name: String,
+    pub blob_hash: BlobHash,
+    pub size: u32,
+    pub vacation_response: Option<VacationResponse>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+#[rkyv(derive(Debug))]
+pub struct VacationResponse {
+    pub from_date: Option<u64>,
+    pub to_date: Option<u64>,
+    pub subject: Option<String>,
+    pub text_body: Option<String>,
+    pub html_body: Option<String>,
+}
+
+impl SieveScript {
+    pub fn new(name: impl Into<String>, blob_hash: BlobHash) -> Self {
+        SieveScript {
+            name: name.into(),
+            blob_hash,
+            vacation_response: None,
+            size: 0,
+        }
+    }
+
+    pub fn with_name(mut self, name: impl Into<String>) -> Self {
+        self.name = name.into();
+        self
+    }
+
+    pub fn with_blob_hash(mut self, blob_hash: BlobHash) -> Self {
+        self.blob_hash = blob_hash;
+        self
+    }
+
+    pub fn with_size(mut self, size: u32) -> Self {
+        self.size = size;
+        self
+    }
+
+    pub fn with_vacation_response(mut self, vacation_response: VacationResponse) -> Self {
+        self.vacation_response = Some(vacation_response);
+        self
+    }
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)]
+#[repr(transparent)]
+pub struct SeenIdHash(pub [u8; 32]);
+
+impl SeenIdHash {
+    pub fn new(account_id: u32, hash: u32, id: &str) -> Self {
+        let mut hasher = blake3::Hasher::new();
+        hasher.update(&account_id.to_be_bytes());
+        hasher.update(&hash.to_be_bytes());
+        hasher.update(id.as_bytes());
+        SeenIdHash(hasher.finalize().into())
+    }
+
+    pub fn key(&self) -> Vec<u8> {
+        let mut result = Vec::with_capacity(self.0.len() + 1);
+        result.push(KV_SIEVE_ID);
+        result.extend_from_slice(&self.0);
+        result
+    }
+}
+
+impl AsRef<[u8]> for SeenIdHash {
+    fn as_ref(&self) -> &[u8] {
+        &self.0
+    }
+}

crates/email/src/submission/index.rs

@@ -0,0 +1,71 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use super::{ArchivedEmailSubmission, EmailSubmission};
+use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
+use store::{
+    U32_LEN, U64_LEN,
+    write::{IndexPropertyClass, ValueClass, key::KeySerializer},
+};
+use types::{collection::SyncCollection, field::EmailSubmissionField};
+
+impl IndexableObject for EmailSubmission {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::Property {
+                field: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailSubmissionField::Metadata.into(),
+                    value: self.send_at,
+                }),
+                value: KeySerializer::new(U32_LEN * 3 + U64_LEN + 1)
+                    .write(self.email_id)
+                    .write(self.thread_id)
+                    .write(self.identity_id)
+                    .write(self.queue_id.unwrap_or_default())
+                    .write(self.undo_status.as_index())
+                    .finalize()
+                    .into(),
+            },
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::EmailSubmission,
+                prefix: None,
+            },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableObject for &ArchivedEmailSubmission {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::Property {
+                field: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailSubmissionField::Metadata.into(),
+                    value: self.send_at.to_native(),
+                }),
+                value: KeySerializer::new(U32_LEN * 3 + U64_LEN + 1)
+                    .write(self.email_id.to_native())
+                    .write(self.thread_id.to_native())
+                    .write(self.identity_id.to_native())
+                    .write(self.queue_id.as_ref().map(u64::from).unwrap_or_default())
+                    .write(self.undo_status.as_index())
+                    .finalize()
+                    .into(),
+            },
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::EmailSubmission,
+                prefix: None,
+            },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableAndSerializableObject for EmailSubmission {
+    fn is_versioned() -> bool {
+        false
+    }
+}

crates/email/src/submission/mod.rs

@@ -0,0 +1,151 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use utils::map::vec_map::VecMap;
+
+pub mod index;
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub struct EmailSubmission {
+    pub email_id: u32,
+    pub thread_id: u32,
+    pub identity_id: u32,
+    pub send_at: u64,
+    pub queue_id: Option<u64>,
+    pub undo_status: UndoStatus,
+    pub envelope: Envelope,
+    pub delivery_status: VecMap<String, DeliveryStatus>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub struct Envelope {
+    pub mail_from: Address,
+    pub rcpt_to: Vec<Address>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub struct Address {
+    pub email: String,
+    pub parameters: Option<VecMap<String, Option<String>>>,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub struct DeliveryStatus {
+    pub smtp_reply: String,
+    pub delivered: Delivered,
+    pub displayed: bool,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub enum Delivered {
+    Queued,
+    Yes,
+    No,
+    #[default]
+    Unknown,
+}
+
+#[derive(
+    rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq,
+)]
+pub enum UndoStatus {
+    #[default]
+    Pending,
+    Final,
+    Canceled,
+}
+
+impl UndoStatus {
+    pub fn parse(s: &str) -> Option<Self> {
+        hashify::tiny_map!(s.as_bytes(),
+            "pending" => UndoStatus::Pending,
+            "final" => UndoStatus::Final,
+            "canceled" => UndoStatus::Canceled,
+            "cancelled" => UndoStatus::Canceled,
+        )
+    }
+
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            UndoStatus::Pending => "pending",
+            UndoStatus::Final => "final",
+            UndoStatus::Canceled => "canceled",
+        }
+    }
+
+    pub fn as_index(&self) -> u8 {
+        match self {
+            UndoStatus::Pending => b'p',
+            UndoStatus::Final => b'f',
+            UndoStatus::Canceled => b'c',
+        }
+    }
+}
+
+impl ArchivedUndoStatus {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ArchivedUndoStatus::Pending => "pending",
+            ArchivedUndoStatus::Final => "final",
+            ArchivedUndoStatus::Canceled => "canceled",
+        }
+    }
+
+    pub fn as_index(&self) -> u8 {
+        match self {
+            ArchivedUndoStatus::Pending => b'p',
+            ArchivedUndoStatus::Final => b'f',
+            ArchivedUndoStatus::Canceled => b'c',
+        }
+    }
+}
+
+impl From<&ArchivedDeliveryStatus> for DeliveryStatus {
+    fn from(value: &ArchivedDeliveryStatus) -> Self {
+        DeliveryStatus {
+            smtp_reply: value.smtp_reply.to_string(),
+            delivered: match value.delivered {
+                ArchivedDelivered::Queued => Delivered::Queued,
+                ArchivedDelivered::Yes => Delivered::Yes,
+                ArchivedDelivered::No => Delivered::No,
+                ArchivedDelivered::Unknown => Delivered::Unknown,
+            },
+            displayed: value.displayed,
+        }
+    }
+}
+
+impl Delivered {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Delivered::Queued => "queued",
+            Delivered::Yes => "yes",
+            Delivered::No => "no",
+            Delivered::Unknown => "unknown",
+        }
+    }
+}
+
+impl ArchivedDelivered {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ArchivedDelivered::Queued => "queued",
+            ArchivedDelivered::Yes => "yes",
+            ArchivedDelivered::No => "no",
+            ArchivedDelivered::Unknown => "unknown",
+        }
+    }
+}