Import upstream v0.16.22, stripped

Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 117 in 50 files
Dangling module declarations removed: 5
Cargo edits turning enterprise off: 14
Verification: clean
Enterprise feature gates left for rebuilt features: 19 in 18 files

Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.

crates/common/src/storage/archive.rs

@@ -0,0 +1,111 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::Server;
+use store::{
+    Deserialize, IterateParams, U32_LEN, ValueKey,
+    dispatch::DocumentSet,
+    write::{AlignedBytes, Archive, ValueClass, key::DeserializeBigEndian},
+};
+use trc::AddContext;
+use types::{collection::Collection, field::Field};
+
+impl Server {
+    pub async fn archives<I, CB>(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        documents: &I,
+        mut cb: CB,
+    ) -> trc::Result<()>
+    where
+        I: DocumentSet + Send + Sync,
+        CB: FnMut(u32, Archive<AlignedBytes>) -> trc::Result<bool> + Send + Sync,
+    {
+        let collection: u8 = collection.into();
+
+        self.core
+            .storage
+            .data
+            .iterate(
+                IterateParams::new(
+                    ValueKey {
+                        account_id,
+                        collection,
+                        document_id: documents.min(),
+                        class: ValueClass::Property(Field::ARCHIVE.into()),
+                    },
+                    ValueKey {
+                        account_id,
+                        collection,
+                        document_id: documents.max(),
+                        class: ValueClass::Property(Field::ARCHIVE.into()),
+                    },
+                ),
+                |key, value| {
+                    let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
+                    if documents.contains(document_id) {
+                        <Archive<AlignedBytes> as Deserialize>::deserialize(value)
+                            .and_then(|archive| cb(document_id, archive))
+                    } else {
+                        Ok(true)
+                    }
+                },
+            )
+            .await
+            .add_context(|err| {
+                err.caused_by(trc::location!())
+                    .account_id(account_id)
+                    .collection(collection)
+            })
+    }
+
+    pub async fn all_archives<CB>(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        field: u8,
+        mut cb: CB,
+    ) -> trc::Result<()>
+    where
+        CB: FnMut(u32, Archive<AlignedBytes>) -> trc::Result<()> + Send + Sync,
+    {
+        let collection: u8 = collection.into();
+
+        self.core
+            .storage
+            .data
+            .iterate(
+                IterateParams::new(
+                    ValueKey {
+                        account_id,
+                        collection,
+                        document_id: 0,
+                        class: ValueClass::Property(field),
+                    },
+                    ValueKey {
+                        account_id,
+                        collection,
+                        document_id: u32::MAX,
+                        class: ValueClass::Property(field),
+                    },
+                ),
+                |key, value| {
+                    let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?;
+                    let archive = <Archive<AlignedBytes> as Deserialize>::deserialize(value)?;
+                    cb(document_id, archive)?;
+
+                    Ok(true)
+                },
+            )
+            .await
+            .add_context(|err| {
+                err.caused_by(trc::location!())
+                    .account_id(account_id)
+                    .collection(collection)
+            })
+    }
+}

crates/common/src/storage/blob.rs

@@ -0,0 +1,219 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{KV_QUOTA_BLOB, Server};
+use mail_parser::{
+    Encoding,
+    decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode},
+};
+use store::{
+    U32_LEN, U64_LEN,
+    dispatch::lookup::KeyValue,
+    write::{BatchBuilder, BlobLink, BlobOp, now},
+};
+use trc::AddContext;
+use types::{
+    blob::{BlobClass, BlobId, BlobSection},
+    blob_hash::BlobHash,
+};
+
+const COUNT_BYTES: u32 = 20;
+const COUNT_SHIFT: u32 = 64 - COUNT_BYTES;
+const SIZE_MASK: u64 = (1u64 << COUNT_SHIFT) - 1;
+
+pub struct BlobQuotaStatus {
+    pub allowed: bool,
+    pub expires_in: u64,
+}
+
+impl Server {
+    pub async fn blob_has_quota(
+        &self,
+        account_id: u32,
+        bytes: usize,
+    ) -> trc::Result<BlobQuotaStatus> {
+        if self.core.jmap.upload_tmp_quota_size > 0 || self.core.jmap.upload_tmp_quota_amount > 0 {
+            let now = now();
+            let range_start = now / self.core.jmap.upload_tmp_ttl;
+            let range_end =
+                (range_start * self.core.jmap.upload_tmp_ttl) + self.core.jmap.upload_tmp_ttl;
+            let expires_in = range_end - now;
+
+            let mut bucket = Vec::with_capacity(U32_LEN + U64_LEN + 1);
+            bucket.push(KV_QUOTA_BLOB);
+            bucket.extend_from_slice(account_id.to_be_bytes().as_slice());
+            bucket.extend_from_slice(range_start.to_be_bytes().as_slice());
+
+            self.in_memory_store()
+                .counter_incr(
+                    KeyValue::new(bucket, 1i64 << COUNT_SHIFT | bytes as i64).expires(expires_in),
+                    true,
+                )
+                .await
+                .caused_by(trc::location!())
+                .map(|v| {
+                    let v = v as u64;
+                    let count = v >> COUNT_SHIFT;
+                    let size = v & SIZE_MASK;
+
+                    let allowed = (self.core.jmap.upload_tmp_quota_amount == 0
+                        || count <= self.core.jmap.upload_tmp_quota_amount as u64)
+                        && (self.core.jmap.upload_tmp_quota_size == 0
+                            || size <= self.core.jmap.upload_tmp_quota_size as u64);
+
+                    BlobQuotaStatus {
+                        allowed,
+                        expires_in,
+                    }
+                })
+        } else {
+            Ok(BlobQuotaStatus {
+                allowed: true,
+                expires_in: 0,
+            })
+        }
+    }
+
+    #[allow(clippy::blocks_in_conditions)]
+    pub async fn put_jmap_blob(&self, account_id: u32, data: &[u8]) -> trc::Result<BlobId> {
+        // First reserve the hash
+        let hash = BlobHash::generate(data);
+        let mut batch = BatchBuilder::new();
+        let until = now() + self.core.jmap.upload_tmp_ttl;
+
+        batch.with_account_id(account_id).set(
+            BlobOp::Link {
+                hash: hash.clone(),
+                to: BlobLink::Temporary { until },
+            },
+            vec![],
+        );
+
+        self.core
+            .storage
+            .data
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+
+        if !self
+            .core
+            .storage
+            .data
+            .blob_exists(&hash)
+            .await
+            .caused_by(trc::location!())?
+        {
+            // Upload blob to store
+            self.core
+                .storage
+                .blob
+                .put_blob(hash.as_ref(), data, self.core.email.compression)
+                .await
+                .caused_by(trc::location!())?;
+
+            // Commit blob
+            let mut batch = BatchBuilder::new();
+            batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new());
+            self.core
+                .storage
+                .data
+                .write(batch.build_all())
+                .await
+                .caused_by(trc::location!())?;
+        }
+
+        Ok(BlobId {
+            hash,
+            class: BlobClass::Reserved {
+                account_id,
+                expires: until,
+            },
+            section: None,
+        })
+    }
+
+    pub async fn put_temporary_blob(
+        &self,
+        account_id: u32,
+        data: &[u8],
+        hold_for: u64,
+    ) -> trc::Result<(BlobHash, BlobOp)> {
+        // First reserve the hash
+        let hash = BlobHash::generate(data);
+        let mut batch = BatchBuilder::new();
+        let until = now() + hold_for;
+
+        batch.with_account_id(account_id).set(
+            BlobOp::Link {
+                hash: hash.clone(),
+                to: BlobLink::Temporary { until },
+            },
+            vec![],
+        );
+
+        self.core
+            .storage
+            .data
+            .write(batch.build_all())
+            .await
+            .caused_by(trc::location!())?;
+
+        if !self
+            .core
+            .storage
+            .data
+            .blob_exists(&hash)
+            .await
+            .caused_by(trc::location!())?
+        {
+            // Upload blob to store
+            self.core
+                .storage
+                .blob
+                .put_blob(hash.as_ref(), data, self.core.email.compression)
+                .await
+                .caused_by(trc::location!())?;
+
+            // Commit blob
+            let mut batch = BatchBuilder::new();
+            batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new());
+            self.core
+                .storage
+                .data
+                .write(batch.build_all())
+                .await
+                .caused_by(trc::location!())?;
+        }
+
+        Ok((
+            hash.clone(),
+            BlobOp::Link {
+                hash,
+                to: BlobLink::Temporary { until },
+            },
+        ))
+    }
+
+    pub async fn get_blob_section(
+        &self,
+        hash: &BlobHash,
+        section: &BlobSection,
+    ) -> trc::Result<Option<Vec<u8>>> {
+        Ok(self
+            .blob_store()
+            .get_blob(
+                hash.as_slice(),
+                (section.offset_start)..(section.offset_start.saturating_add(section.size)),
+            )
+            .await?
+            .and_then(|bytes| match Encoding::from(section.encoding) {
+                Encoding::None => Some(bytes),
+                Encoding::Base64 => base64_decode(&bytes),
+                Encoding::QuotedPrintable => quoted_printable_decode(&bytes),
+            }))
+    }
+}

crates/common/src/storage/dav.rs

@@ -0,0 +1,403 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    DavName, DavPath, DavResource, DavResourceMetadata, DavResourcePath, DavResources,
+    TinyCalendarPreferences,
+};
+use std::hash::{Hash, Hasher};
+use store::rand::{RngExt, distr::Alphanumeric};
+use types::acl::AclGrant;
+
+const SCHEDULE_INBOX_ID: u32 = u32::MAX - 1;
+
+impl DavResourcePath<'_> {
+    #[inline(always)]
+    pub fn document_id(&self) -> u32 {
+        self.resource.document_id
+    }
+
+    #[inline(always)]
+    pub fn parent_id(&self) -> Option<u32> {
+        self.path.parent_id
+    }
+
+    #[inline(always)]
+    pub fn path(&self) -> &str {
+        self.path.path.as_str()
+    }
+
+    #[inline(always)]
+    pub fn is_container(&self) -> bool {
+        self.resource.is_container()
+    }
+
+    #[inline(always)]
+    pub fn hierarchy_seq(&self) -> u32 {
+        self.path.hierarchy_seq
+    }
+
+    #[inline(always)]
+    pub fn size(&self) -> u32 {
+        self.resource.size().unwrap_or_default()
+    }
+}
+
+impl DavResources {
+    pub fn by_path(&self, name: &str) -> Option<DavResourcePath<'_>> {
+        self.paths.get(name).map(|path| DavResourcePath {
+            path,
+            resource: &self.resources[path.resource_idx],
+        })
+    }
+
+    pub fn container_resource_by_id(&self, id: u32) -> Option<&DavResource> {
+        self.resources
+            .iter()
+            .find(|res| res.document_id == id && res.is_container())
+    }
+
+    pub fn container_resource_path_by_id(&self, id: u32) -> Option<DavResourcePath<'_>> {
+        self.resources
+            .iter()
+            .enumerate()
+            .find(|(_, resource)| resource.document_id == id && resource.is_container())
+            .and_then(|(idx, resource)| {
+                self.paths
+                    .iter()
+                    .find(|path| path.resource_idx == idx)
+                    .map(|path| DavResourcePath { path, resource })
+            })
+    }
+
+    pub fn any_resource_path_by_id(&self, id: u32) -> Option<DavResourcePath<'_>> {
+        self.resources
+            .iter()
+            .enumerate()
+            .find(|(_, resource)| resource.document_id == id)
+            .and_then(|(idx, resource)| {
+                self.paths
+                    .iter()
+                    .find(|path| path.resource_idx == idx)
+                    .map(|path| DavResourcePath { path, resource })
+            })
+    }
+
+    pub fn subtree(&self, search_path: &str) -> impl Iterator<Item = DavResourcePath<'_>> {
+        let prefix = format!("{search_path}/");
+        self.paths.iter().filter_map(move |path| {
+            if path.path.starts_with(&prefix) || path.path == search_path {
+                Some(DavResourcePath {
+                    path,
+                    resource: &self.resources[path.resource_idx],
+                })
+            } else {
+                None
+            }
+        })
+    }
+
+    pub fn subtree_with_depth(
+        &self,
+        search_path: &str,
+        depth: usize,
+    ) -> impl Iterator<Item = DavResourcePath<'_>> {
+        let prefix = format!("{search_path}/");
+        self.paths.iter().filter_map(move |path| {
+            if path
+                .path
+                .strip_prefix(&prefix)
+                .is_some_and(|name| name.as_bytes().iter().filter(|&&c| c == b'/').count() < depth)
+                || path.path.as_str() == search_path
+            {
+                Some(DavResourcePath {
+                    path,
+                    resource: &self.resources[path.resource_idx],
+                })
+            } else {
+                None
+            }
+        })
+    }
+
+    pub fn tree_with_depth(&self, depth: usize) -> impl Iterator<Item = DavResourcePath<'_>> {
+        self.paths.iter().filter_map(move |path| {
+            if path.path.as_bytes().iter().filter(|&&c| c == b'/').count() <= depth {
+                Some(DavResourcePath {
+                    path,
+                    resource: &self.resources[path.resource_idx],
+                })
+            } else {
+                None
+            }
+        })
+    }
+
+    pub fn children(&self, parent_id: u32) -> impl Iterator<Item = DavResourcePath<'_>> {
+        self.paths
+            .iter()
+            .filter(move |item| item.parent_id.is_some_and(|id| id == parent_id))
+            .map(|path| DavResourcePath {
+                path,
+                resource: &self.resources[path.resource_idx],
+            })
+    }
+
+    pub fn children_ids(&self, parent_id: u32) -> impl Iterator<Item = u32> {
+        self.paths
+            .iter()
+            .filter(move |item| item.parent_id.is_some_and(|id| id == parent_id))
+            .map(|path| self.resources[path.resource_idx].document_id)
+    }
+
+    pub fn format_resource(&self, resource: DavResourcePath<'_>) -> String {
+        if resource.resource.is_container() {
+            format!("{}{}/", self.base_path, resource.path.path)
+        } else {
+            format!("{}{}", self.base_path, resource.path.path)
+        }
+    }
+
+    pub fn format_resource_paths_by_id(
+        &self,
+        document_id: u32,
+    ) -> impl Iterator<Item = String> + '_ {
+        self.paths
+            .iter()
+            .filter(move |path| self.resources[path.resource_idx].document_id == document_id)
+            .map(move |path| {
+                self.format_resource(DavResourcePath {
+                    path,
+                    resource: &self.resources[path.resource_idx],
+                })
+            })
+    }
+
+    pub fn format_resource_path_by_parent(
+        &self,
+        document_id: u32,
+        parent_id: u32,
+    ) -> Option<String> {
+        self.paths
+            .iter()
+            .find(|path| {
+                self.resources[path.resource_idx].document_id == document_id
+                    && path.parent_id == Some(parent_id)
+            })
+            .map(|path| {
+                self.format_resource(DavResourcePath {
+                    path,
+                    resource: &self.resources[path.resource_idx],
+                })
+            })
+    }
+
+    pub fn format_collection(&self, name: &str) -> String {
+        format!("{}{name}/", self.base_path)
+    }
+
+    pub fn format_item(&self, name: &str) -> String {
+        format!("{}{}", self.base_path, name)
+    }
+}
+
+impl DavResource {
+    pub fn is_child_of(&self, parent_id: u32) -> bool {
+        match &self.data {
+            DavResourceMetadata::File { parent_id: id, .. } => id.is_some_and(|id| id == parent_id),
+            DavResourceMetadata::CalendarEvent { names, .. } => {
+                names.iter().any(|name| name.parent_id == parent_id)
+            }
+            DavResourceMetadata::ContactCard { names } => {
+                names.iter().any(|name| name.parent_id == parent_id)
+            }
+            DavResourceMetadata::CalendarEventNotification { names } => {
+                names.is_empty() && parent_id == SCHEDULE_INBOX_ID
+            }
+            _ => false,
+        }
+    }
+
+    pub fn parent_id(&self) -> Option<u32> {
+        match &self.data {
+            DavResourceMetadata::File { parent_id, .. } => *parent_id,
+            DavResourceMetadata::CalendarEvent { names, .. } => {
+                names.first().map(|name| name.parent_id)
+            }
+            DavResourceMetadata::ContactCard { names } => names.first().map(|name| name.parent_id),
+            DavResourceMetadata::CalendarEventNotification { names } if names.is_empty() => {
+                Some(SCHEDULE_INBOX_ID)
+            }
+            _ => None,
+        }
+    }
+
+    pub fn child_names(&self) -> Option<&[DavName]> {
+        match &self.data {
+            DavResourceMetadata::CalendarEvent { names, .. } => Some(names.as_slice()),
+            DavResourceMetadata::ContactCard { names } => Some(names.as_slice()),
+            DavResourceMetadata::CalendarEventNotification { names } if !names.is_empty() => {
+                Some(names.as_slice())
+            }
+            _ => None,
+        }
+    }
+
+    pub fn container_name(&self) -> Option<&str> {
+        match &self.data {
+            DavResourceMetadata::File { name, .. } => Some(name.as_str()),
+            DavResourceMetadata::Calendar { name, .. } => Some(name.as_str()),
+            DavResourceMetadata::AddressBook { name, .. } => Some(name.as_str()),
+            DavResourceMetadata::CalendarEventNotification { names } if names.is_empty() => {
+                Some(if self.document_id == SCHEDULE_INBOX_ID {
+                    "inbox"
+                } else {
+                    "outbox"
+                })
+            }
+            _ => None,
+        }
+    }
+
+    pub fn has_hierarchy_changes(&self, other: &DavResource) -> bool {
+        match (&self.data, &other.data) {
+            (
+                DavResourceMetadata::File {
+                    name: a,
+                    parent_id: c,
+                    ..
+                },
+                DavResourceMetadata::File {
+                    name: b,
+                    parent_id: d,
+                    ..
+                },
+            ) => a != b || c != d,
+            (
+                DavResourceMetadata::Calendar { name: a, .. },
+                DavResourceMetadata::Calendar { name: b, .. },
+            ) => a != b,
+            (
+                DavResourceMetadata::AddressBook { name: a, .. },
+                DavResourceMetadata::AddressBook { name: b, .. },
+            ) => a != b,
+            (
+                DavResourceMetadata::CalendarEvent { names: a, .. },
+                DavResourceMetadata::CalendarEvent { names: b, .. },
+            ) => a != b,
+            (
+                DavResourceMetadata::ContactCard { names: a, .. },
+                DavResourceMetadata::ContactCard { names: b, .. },
+            ) => a != b,
+            (
+                DavResourceMetadata::CalendarEventNotification { names: a, .. },
+                DavResourceMetadata::CalendarEventNotification { names: b, .. },
+            ) => a != b,
+            _ => unreachable!(),
+        }
+    }
+
+    pub fn event_time_range(&self) -> Option<(i64, i64)> {
+        match &self.data {
+            DavResourceMetadata::CalendarEvent {
+                start, duration, ..
+            } => Some((*start, *start + *duration as i64)),
+            _ => None,
+        }
+    }
+
+    pub fn calendar_preferences(&self, account_id: u32) -> Option<&TinyCalendarPreferences> {
+        match &self.data {
+            DavResourceMetadata::Calendar { preferences, .. } => preferences
+                .iter()
+                .find(|pref| pref.account_id == account_id)
+                .or_else(|| preferences.first()),
+            _ => None,
+        }
+    }
+
+    pub fn is_container(&self) -> bool {
+        match &self.data {
+            DavResourceMetadata::File { size, .. } => size.is_none(),
+            DavResourceMetadata::Calendar { .. } | DavResourceMetadata::AddressBook { .. } => true,
+            DavResourceMetadata::CalendarEventNotification { names } => names.is_empty(),
+            _ => false,
+        }
+    }
+
+    pub fn size(&self) -> Option<u32> {
+        match &self.data {
+            DavResourceMetadata::File { size, .. } => *size,
+            _ => None,
+        }
+    }
+
+    pub fn acls(&self) -> Option<&[AclGrant]> {
+        match &self.data {
+            DavResourceMetadata::File { acls, .. } => Some(acls.as_slice()),
+            DavResourceMetadata::Calendar { acls, .. } => Some(acls.as_slice()),
+            DavResourceMetadata::AddressBook { acls, .. } => Some(acls.as_slice()),
+            _ => None,
+        }
+    }
+}
+
+impl Hash for DavPath {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.path.hash(state);
+    }
+}
+
+impl PartialEq for DavPath {
+    fn eq(&self, other: &Self) -> bool {
+        self.path == other.path
+    }
+}
+
+impl Eq for DavPath {}
+
+impl std::borrow::Borrow<str> for DavPath {
+    fn borrow(&self) -> &str {
+        &self.path
+    }
+}
+
+impl std::hash::Hash for DavResource {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.document_id.hash(state);
+    }
+}
+
+impl PartialEq for DavResource {
+    fn eq(&self, other: &Self) -> bool {
+        self.document_id == other.document_id
+    }
+}
+
+impl Eq for DavResource {}
+
+impl std::borrow::Borrow<u32> for DavResource {
+    fn borrow(&self) -> &u32 {
+        &self.document_id
+    }
+}
+
+impl DavName {
+    pub fn new(name: String, parent_id: u32) -> Self {
+        Self { name, parent_id }
+    }
+
+    pub fn new_with_rand_name(parent_id: u32) -> Self {
+        Self {
+            name: store::rand::rng()
+                .sample_iter(Alphanumeric)
+                .take(10)
+                .map(char::from)
+                .collect::<String>(),
+            parent_id,
+        }
+    }
+}

crates/common/src/storage/document.rs

@@ -0,0 +1,136 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use store::{
+    IndexKey, IndexKeyPrefix, IterateParams, U32_LEN, roaring::RoaringBitmap,
+    write::key::DeserializeBigEndian,
+};
+use trc::AddContext;
+use types::collection::Collection;
+
+use crate::Server;
+
+impl Server {
+    pub async fn document_ids(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        field: impl Into<u8>,
+    ) -> trc::Result<RoaringBitmap> {
+        let field = field.into();
+        let mut results = RoaringBitmap::new();
+        self.store()
+            .iterate(
+                IterateParams::new(
+                    IndexKeyPrefix {
+                        account_id,
+                        collection: collection.into(),
+                        field,
+                    },
+                    IndexKeyPrefix {
+                        account_id,
+                        collection: collection.into(),
+                        field: field + 1,
+                    },
+                )
+                .no_values(),
+                |key, _| {
+                    results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?);
+
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())
+            .map(|_| results)
+    }
+
+    pub async fn document_exists(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        field: impl Into<u8>,
+        filter: impl AsRef<[u8]>,
+    ) -> trc::Result<bool> {
+        let field = field.into();
+        let mut exists = false;
+        let filter = filter.as_ref();
+        let key_len = IndexKeyPrefix::len() + filter.len() + U32_LEN;
+
+        self.store()
+            .iterate(
+                IterateParams::new(
+                    IndexKey {
+                        account_id,
+                        collection: collection.into(),
+                        document_id: 0,
+                        field,
+                        key: filter,
+                    },
+                    IndexKey {
+                        account_id,
+                        collection: collection.into(),
+                        document_id: u32::MAX,
+                        field,
+                        key: filter,
+                    },
+                )
+                .no_values(),
+                |key, _| {
+                    exists = key.len() == key_len;
+
+                    Ok(!exists)
+                },
+            )
+            .await
+            .caused_by(trc::location!())
+            .map(|_| exists)
+    }
+
+    pub async fn document_ids_matching(
+        &self,
+        account_id: u32,
+        collection: Collection,
+        field: impl Into<u8>,
+        filter: impl AsRef<[u8]>,
+    ) -> trc::Result<RoaringBitmap> {
+        let field = field.into();
+        let filter = filter.as_ref();
+        let key_len = IndexKeyPrefix::len() + filter.len() + U32_LEN;
+        let mut results = RoaringBitmap::new();
+
+        self.store()
+            .iterate(
+                IterateParams::new(
+                    IndexKey {
+                        account_id,
+                        collection: collection.into(),
+                        document_id: 0,
+                        field,
+                        key: filter,
+                    },
+                    IndexKey {
+                        account_id,
+                        collection: collection.into(),
+                        document_id: u32::MAX,
+                        field,
+                        key: filter,
+                    },
+                )
+                .no_values(),
+                |key, _| {
+                    if key.len() == key_len {
+                        results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?);
+                    }
+
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())
+            .map(|_| results)
+    }
+}

crates/common/src/storage/encryption.rs

@@ -0,0 +1,163 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::auth::EncryptionKeys;
+use mail_parser::decoders::base64::base64_decode;
+use registry::schema::structs::PublicKey;
+use sequoia_openpgp::{Cert, parse::Parse, policy::StandardPolicy, types::KeyFlags};
+use std::borrow::Cow;
+
+const P: StandardPolicy<'static> = StandardPolicy::new();
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EncryptionMethod {
+    PGP,
+    SMIME,
+}
+
+pub struct EncryptionParams {
+    pub certs: EncryptionKeys,
+    pub method: EncryptionMethod,
+}
+
+#[allow(clippy::type_complexity)]
+pub fn parse_public_key(pk: &PublicKey) -> Result<Option<EncryptionParams>, Cow<'static, str>> {
+    let bytes_ = pk.key.as_bytes();
+    let mut bytes = bytes_.iter().enumerate();
+    let mut buf = vec![];
+    let mut method = None;
+    let mut certs: Vec<Box<[u8]>> = vec![];
+
+    loop {
+        // Find start of PEM block
+        let mut start_pos = 0;
+        for (pos, &ch) in bytes.by_ref() {
+            if ch.is_ascii_whitespace() {
+                continue;
+            } else if ch == b'-' {
+                start_pos = pos;
+                break;
+            } else {
+                return Ok(None);
+            }
+        }
+
+        // Find block type
+        for (_, &ch) in bytes.by_ref() {
+            match ch {
+                b'-' => (),
+                b'\n' => break,
+                _ => {
+                    if ch.is_ascii() {
+                        buf.push(ch.to_ascii_uppercase());
+                    } else {
+                        return Ok(None);
+                    }
+                }
+            }
+        }
+        if buf.is_empty() {
+            break;
+        }
+
+        // Find type
+        let tag = std::str::from_utf8(&buf).unwrap();
+        if tag.contains("CERTIFICATE") {
+            if method.is_some_and(|m| m == EncryptionMethod::PGP) {
+                return Err("Cannot mix OpenPGP and S/MIME certificates".into());
+            } else {
+                method = Some(EncryptionMethod::SMIME);
+            }
+        } else if tag.contains("PGP") {
+            if method.is_some_and(|m| m == EncryptionMethod::SMIME) {
+                return Err("Cannot mix OpenPGP and S/MIME certificates".into());
+            } else {
+                method = Some(EncryptionMethod::PGP);
+            }
+        } else {
+            // Ignore block
+            let mut found_end = false;
+            for (_, &ch) in bytes.by_ref() {
+                if ch == b'-' {
+                    found_end = true;
+                } else if ch == b'\n' && found_end {
+                    break;
+                }
+            }
+            buf.clear();
+            continue;
+        }
+
+        // Collect base64
+        buf.clear();
+        let mut found_end = false;
+        let mut end_pos = 0;
+        for (pos, &ch) in bytes.by_ref() {
+            match ch {
+                b'-' => {
+                    found_end = true;
+                }
+                b'\n' => {
+                    if found_end {
+                        end_pos = pos;
+                        break;
+                    }
+                }
+                _ => {
+                    if !ch.is_ascii_whitespace() {
+                        buf.push(ch);
+                    }
+                }
+            }
+        }
+
+        // Decode base64
+        let cert = base64_decode(&buf)
+            .ok_or_else(|| Cow::from("Failed to decode base64 certificate."))?
+            .into_boxed_slice();
+        match method.unwrap() {
+            EncryptionMethod::PGP => match Cert::from_bytes(bytes_) {
+                Ok(cert) => {
+                    if !has_pgp_keys(cert) {
+                        return Err("Could not find any suitable keys in OpenPGP public key".into());
+                    }
+                    certs.push(
+                        bytes_
+                            .get(start_pos..end_pos + 1)
+                            .unwrap_or_default()
+                            .into(),
+                    );
+                }
+                Err(err) => {
+                    return Err(format!("Failed to decode OpenPGP public key: {err}").into());
+                }
+            },
+            EncryptionMethod::SMIME => {
+                if let Err(err) = rasn::der::decode::<rasn_pkix::Certificate>(&cert) {
+                    return Err(format!("Failed to decode X509 certificate: {err}").into());
+                }
+                certs.push(cert);
+            }
+        }
+        buf.clear();
+    }
+
+    Ok(method.map(|method| EncryptionParams {
+        method,
+        certs: certs.into_boxed_slice(),
+    }))
+}
+
+fn has_pgp_keys(cert: Cert) -> bool {
+    cert.keys()
+        .with_policy(&P, None)
+        .supported()
+        .alive()
+        .revoked(false)
+        .key_flags(KeyFlags::empty().set_transport_encryption())
+        .next()
+        .is_some()
+}

crates/common/src/storage/index.rs

@@ -0,0 +1,784 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{auth::AccountTenantIds, sharing::notification::ShareNotification};
+use registry::schema::{
+    enums::IndexDocumentType,
+    structs::{Task, TaskIndexDocument, TaskStatus},
+};
+use rkyv::{
+    option::ArchivedOption,
+    primitive::{ArchivedU32, ArchivedU64},
+    string::ArchivedString,
+};
+use std::{borrow::Cow, fmt::Debug};
+use store::{
+    Serialize, SerializeInfallible,
+    write::{
+        Archive, Archiver, BatchBuilder, BlobLink, BlobOp, IntoOperations, Params, SearchIndex,
+        ValueClass,
+    },
+};
+use types::{
+    acl::AclGrant,
+    blob_hash::BlobHash,
+    collection::{Collection, SyncCollection},
+    field::Field,
+};
+use utils::{cheeky_hash::CheekyHash, map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum IndexValue<'x> {
+    Index {
+        field: Field,
+        value: IndexItem<'x>,
+    },
+    Property {
+        field: ValueClass,
+        value: IndexItem<'x>,
+    },
+    SearchIndex {
+        index: SearchIndex,
+        hash: u64,
+    },
+    Blob {
+        value: BlobHash,
+    },
+    Quota {
+        used: u32,
+    },
+    LogContainer {
+        sync_collection: SyncCollection,
+    },
+    LogContainerProperty {
+        sync_collection: SyncCollection,
+        ids: Vec<u32>,
+    },
+    LogItem {
+        sync_collection: SyncCollection,
+        prefix: Option<u32>,
+    },
+    Acl {
+        value: Cow<'x, [AclGrant]>,
+    },
+}
+
+#[derive(Debug, Clone)]
+pub enum IndexItem<'x> {
+    Vec(Vec<u8>),
+    Slice(&'x [u8]),
+    ShortInt([u8; std::mem::size_of::<u32>()]),
+    LongInt([u8; std::mem::size_of::<u64>()]),
+    Hash(CheekyHash),
+    None,
+}
+
+impl IndexItem<'_> {
+    pub fn as_slice(&self) -> &[u8] {
+        match self {
+            IndexItem::Vec(v) => v,
+            IndexItem::Slice(s) => s,
+            IndexItem::ShortInt(s) => s,
+            IndexItem::LongInt(s) => s,
+            IndexItem::Hash(h) => h.as_bytes(),
+            IndexItem::None => &[],
+        }
+    }
+
+    pub fn into_owned(self) -> Vec<u8> {
+        match self {
+            IndexItem::Vec(v) => v,
+            IndexItem::Slice(s) => s.to_vec(),
+            IndexItem::ShortInt(s) => s.to_vec(),
+            IndexItem::LongInt(s) => s.to_vec(),
+            IndexItem::Hash(h) => h.as_bytes().to_vec(),
+            IndexItem::None => vec![],
+        }
+    }
+
+    pub fn is_empty(&self) -> bool {
+        match self {
+            IndexItem::Vec(v) => v.is_empty(),
+            IndexItem::Slice(s) => s.is_empty(),
+            IndexItem::None => true,
+            _ => false,
+        }
+    }
+
+    pub fn is_none(&self) -> bool {
+        matches!(self, IndexItem::None)
+    }
+
+    pub fn is_some(&self) -> bool {
+        !self.is_none()
+    }
+}
+
+impl PartialEq for IndexItem<'_> {
+    fn eq(&self, other: &Self) -> bool {
+        self.as_slice() == other.as_slice()
+    }
+}
+
+impl Eq for IndexItem<'_> {}
+
+impl std::hash::Hash for IndexItem<'_> {
+    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+        match self {
+            IndexItem::Vec(v) => v.as_slice().hash(state),
+            IndexItem::Slice(s) => s.hash(state),
+            IndexItem::ShortInt(s) => s.as_slice().hash(state),
+            IndexItem::LongInt(s) => s.as_slice().hash(state),
+            IndexItem::Hash(h) => h.hash(state),
+            IndexItem::None => 0.hash(state),
+        }
+    }
+}
+
+impl From<u32> for IndexItem<'_> {
+    fn from(value: u32) -> Self {
+        IndexItem::ShortInt(value.to_be_bytes())
+    }
+}
+
+impl From<&u32> for IndexItem<'_> {
+    fn from(value: &u32) -> Self {
+        IndexItem::ShortInt(value.to_be_bytes())
+    }
+}
+
+impl From<u64> for IndexItem<'_> {
+    fn from(value: u64) -> Self {
+        IndexItem::LongInt(value.to_be_bytes())
+    }
+}
+
+impl From<i64> for IndexItem<'_> {
+    fn from(value: i64) -> Self {
+        IndexItem::LongInt(value.to_be_bytes())
+    }
+}
+
+impl<'x> From<&'x [u8]> for IndexItem<'x> {
+    fn from(value: &'x [u8]) -> Self {
+        IndexItem::Slice(value)
+    }
+}
+
+impl From<Vec<u8>> for IndexItem<'_> {
+    fn from(value: Vec<u8>) -> Self {
+        IndexItem::Vec(value)
+    }
+}
+
+impl<'x> From<&'x str> for IndexItem<'x> {
+    fn from(value: &'x str) -> Self {
+        IndexItem::Slice(value.as_bytes())
+    }
+}
+
+impl<'x> From<&'x String> for IndexItem<'x> {
+    fn from(value: &'x String) -> Self {
+        IndexItem::Slice(value.as_bytes())
+    }
+}
+
+impl From<String> for IndexItem<'_> {
+    fn from(value: String) -> Self {
+        IndexItem::Vec(value.into_bytes())
+    }
+}
+
+impl<'x> From<&'x ArchivedString> for IndexItem<'x> {
+    fn from(value: &'x ArchivedString) -> Self {
+        IndexItem::Slice(value.as_bytes())
+    }
+}
+
+impl From<ArchivedU32> for IndexItem<'_> {
+    fn from(value: ArchivedU32) -> Self {
+        IndexItem::ShortInt(value.to_native().to_be_bytes())
+    }
+}
+
+impl From<&ArchivedU32> for IndexItem<'_> {
+    fn from(value: &ArchivedU32) -> Self {
+        IndexItem::ShortInt(value.to_native().to_be_bytes())
+    }
+}
+
+impl From<ArchivedU64> for IndexItem<'_> {
+    fn from(value: ArchivedU64) -> Self {
+        IndexItem::LongInt(value.to_native().to_be_bytes())
+    }
+}
+
+impl<'x, T: Into<IndexItem<'x>>> From<Option<T>> for IndexItem<'x> {
+    fn from(value: Option<T>) -> Self {
+        match value {
+            Some(v) => v.into(),
+            None => IndexItem::None,
+        }
+    }
+}
+
+impl<'x, T: Into<IndexItem<'x>>> From<ArchivedOption<T>> for IndexItem<'x> {
+    fn from(value: ArchivedOption<T>) -> Self {
+        match value {
+            ArchivedOption::Some(v) => v.into(),
+            ArchivedOption::None => IndexItem::None,
+        }
+    }
+}
+
+pub trait IndexableObject: Sync + Send {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>>;
+}
+
+pub trait IndexableAndSerializableObject:
+    IndexableObject
+    + rkyv::Archive
+    + for<'a> rkyv::Serialize<
+        rkyv::api::high::HighSerializer<
+            rkyv::util::AlignedVec,
+            rkyv::ser::allocator::ArenaHandle<'a>,
+            rkyv::rancor::Error,
+        >,
+    >
+{
+    fn is_versioned() -> bool;
+}
+
+#[derive(Debug)]
+pub struct ObjectIndexBuilder<C: IndexableObject, N: IndexableAndSerializableObject> {
+    changed_by: u32,
+    tenant_id: Option<u32>,
+    current: Option<Archive<C>>,
+    changes: Option<N>,
+}
+
+impl<C: IndexableObject, N: IndexableAndSerializableObject> Default for ObjectIndexBuilder<C, N> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<C: IndexableObject, N: IndexableAndSerializableObject> ObjectIndexBuilder<C, N> {
+    pub fn new() -> Self {
+        Self {
+            current: None,
+            changes: None,
+            tenant_id: None,
+            changed_by: u32::MAX,
+        }
+    }
+
+    pub fn with_current(mut self, current: Archive<C>) -> Self {
+        self.current = Some(current);
+        self
+    }
+
+    pub fn with_changes(mut self, changes: N) -> Self {
+        self.changes = Some(changes);
+        self
+    }
+
+    pub fn with_current_opt(mut self, current: Option<Archive<C>>) -> Self {
+        self.current = current;
+        self
+    }
+
+    pub fn changes(&self) -> Option<&N> {
+        self.changes.as_ref()
+    }
+
+    pub fn changes_mut(&mut self) -> Option<&mut N> {
+        self.changes.as_mut()
+    }
+
+    pub fn current(&self) -> Option<&Archive<C>> {
+        self.current.as_ref()
+    }
+
+    pub fn with_changed_by(mut self, ids: AccountTenantIds) -> Self {
+        self.tenant_id = ids.tenant_id;
+        self.changed_by = ids.account_id;
+        self
+    }
+
+    pub fn with_tenant_id(mut self, tenant_id: Option<u32>) -> Self {
+        self.tenant_id = tenant_id;
+        self
+    }
+}
+
+impl<C: IndexableObject, N: IndexableAndSerializableObject> IntoOperations
+    for ObjectIndexBuilder<C, N>
+{
+    fn build(self, batch: &mut BatchBuilder) -> trc::Result<()> {
+        match (self.current, self.changes) {
+            (None, Some(changes)) => {
+                // Insertion
+                for item in changes.index_values() {
+                    build_index(batch, item, self.changed_by, self.tenant_id, true);
+                }
+                if N::is_versioned() {
+                    let (offset, bytes) = Archiver::new(changes).serialize_versioned()?;
+                    batch.set_fnc(
+                        Field::ARCHIVE,
+                        Params::with_capacity(2).with_bytes(bytes).with_u64(offset),
+                        |params, ids| {
+                            let change_id = ids.current_change_id()?;
+                            let archive = params.bytes(0);
+                            let offset = params.u64(1);
+
+                            let mut bytes = Vec::with_capacity(archive.len());
+                            bytes.extend_from_slice(&archive[..offset as usize]);
+                            bytes.extend_from_slice(&change_id.to_be_bytes()[..]);
+                            bytes.push(archive.last().copied().unwrap()); // Marker
+                            Ok(bytes)
+                        },
+                    );
+                } else {
+                    batch.set(Field::ARCHIVE, Archiver::new(changes).serialize()?);
+                }
+            }
+            (Some(current), Some(changes)) => {
+                // Update
+                batch.assert_value(Field::ARCHIVE, &current);
+                for (current, change) in current.inner.index_values().zip(changes.index_values()) {
+                    if current != change {
+                        merge_index(batch, current, change, self.changed_by, self.tenant_id)?;
+                    } else {
+                        match current {
+                            IndexValue::LogContainer { sync_collection } => {
+                                batch.log_container_update(sync_collection);
+                            }
+                            IndexValue::LogItem {
+                                sync_collection,
+                                prefix,
+                            } => {
+                                batch.log_item_update(sync_collection, prefix);
+                            }
+                            _ => (),
+                        }
+                    }
+                }
+                if N::is_versioned() {
+                    let (offset, bytes) = Archiver::new(changes).serialize_versioned()?;
+                    batch.set_fnc(
+                        Field::ARCHIVE,
+                        Params::with_capacity(2).with_bytes(bytes).with_u64(offset),
+                        |params, ids| {
+                            let change_id = ids.current_change_id()?;
+                            let archive = params.bytes(0);
+                            let offset = params.u64(1);
+
+                            let mut bytes = Vec::with_capacity(archive.len());
+                            bytes.extend_from_slice(&archive[..offset as usize]);
+                            bytes.extend_from_slice(&change_id.to_be_bytes()[..]);
+                            bytes.push(archive.last().copied().unwrap()); // Marker
+                            Ok(bytes)
+                        },
+                    );
+                } else {
+                    batch.set(Field::ARCHIVE, Archiver::new(changes).serialize()?);
+                }
+            }
+            (Some(current), None) => {
+                // Deletion
+                batch.assert_value(Field::ARCHIVE, &current);
+                for item in current.inner.index_values() {
+                    build_index(batch, item, self.changed_by, self.tenant_id, false);
+                }
+
+                batch.clear(Field::ARCHIVE);
+            }
+            (None, None) => unreachable!(),
+        }
+
+        Ok(())
+    }
+}
+
+fn build_index(
+    batch: &mut BatchBuilder,
+    item: IndexValue<'_>,
+    changed_by: u32,
+    tenant_id: Option<u32>,
+    set: bool,
+) {
+    match item {
+        IndexValue::Index { field, value } => {
+            if !value.is_empty() {
+                if set {
+                    batch.index(field, value.into_owned());
+                } else {
+                    batch.unindex(field, value.into_owned());
+                }
+            }
+        }
+        IndexValue::SearchIndex { index, .. } => {
+            let task = TaskIndexDocument {
+                account_id: batch.last_account_id().unwrap().into(),
+                document_id: batch.last_document_id().unwrap().into(),
+                document_type: match index {
+                    SearchIndex::Email => IndexDocumentType::Email,
+                    SearchIndex::Calendar => IndexDocumentType::Calendar,
+                    SearchIndex::Contacts => IndexDocumentType::Contacts,
+                    SearchIndex::File => IndexDocumentType::File,
+                    SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(),
+                },
+                status: TaskStatus::now(),
+            };
+            batch.schedule_task(if set {
+                Task::IndexDocument(task)
+            } else {
+                Task::UnindexDocument(task)
+            });
+        }
+        IndexValue::Property { field, value } => {
+            if !value.is_none() {
+                if set {
+                    batch.set(field, value.into_owned());
+                } else {
+                    batch.clear(field);
+                }
+            }
+        }
+        IndexValue::Blob { value } => {
+            if set {
+                batch.set(
+                    BlobOp::Link {
+                        hash: value,
+                        to: BlobLink::Document,
+                    },
+                    vec![],
+                );
+            } else {
+                batch.clear(BlobOp::Link {
+                    hash: value,
+                    to: BlobLink::Document,
+                });
+            }
+        }
+        IndexValue::Acl { value } => {
+            let object_account_id = batch.last_account_id().unwrap_or_default();
+            let object_type = batch.last_collection().unwrap_or(Collection::None);
+            let object_id = batch.last_document_id().unwrap_or_default();
+            let notification_id = SnowflakeIdGenerator::global_id().unwrap_or_default();
+
+            for item in value.as_ref() {
+                if set {
+                    batch.acl_grant(item.account_id, item.grants.bitmap.serialize());
+                    batch.log_share_notification(
+                        notification_id,
+                        item.account_id,
+                        ShareNotification {
+                            object_account_id,
+                            object_id,
+                            object_type,
+                            changed_by,
+                            old_rights: Default::default(),
+                            new_rights: item.grants,
+                            name: Default::default(),
+                        },
+                    );
+                } else {
+                    batch.acl_revoke(item.account_id);
+                    batch.log_share_notification(
+                        notification_id,
+                        item.account_id,
+                        ShareNotification {
+                            object_account_id,
+                            object_id,
+                            object_type,
+                            changed_by,
+                            old_rights: item.grants,
+                            new_rights: Default::default(),
+                            name: Default::default(),
+                        },
+                    );
+                }
+            }
+        }
+        IndexValue::Quota { used } => {
+            let value = if set { used as i64 } else { -(used as i64) };
+
+            batch.add(ValueClass::Quota, value);
+
+            if let Some(tenant_id) = tenant_id {
+                batch.add(ValueClass::TenantQuota(tenant_id), value);
+            }
+        }
+        IndexValue::LogItem {
+            sync_collection,
+            prefix,
+        } => {
+            if set {
+                batch.log_item_insert(sync_collection, prefix);
+            } else {
+                batch.log_item_delete(sync_collection, prefix);
+            }
+        }
+        IndexValue::LogContainer { sync_collection } => {
+            if set {
+                batch.log_container_insert(sync_collection);
+            } else {
+                batch.log_container_delete(sync_collection);
+            }
+        }
+        IndexValue::LogContainerProperty {
+            sync_collection,
+            ids,
+        } => {
+            for parent_id in ids {
+                batch.log_container_property_change(sync_collection, parent_id);
+            }
+        }
+    }
+}
+
+fn merge_index(
+    batch: &mut BatchBuilder,
+    current: IndexValue<'_>,
+    change: IndexValue<'_>,
+    changed_by: u32,
+    tenant_id: Option<u32>,
+) -> trc::Result<()> {
+    match (current, change) {
+        (
+            IndexValue::Index {
+                field,
+                value: old_value,
+            },
+            IndexValue::Index {
+                value: new_value, ..
+            },
+        ) => {
+            if !old_value.is_empty() {
+                batch.unindex(field, old_value.into_owned());
+            }
+
+            if !new_value.is_empty() {
+                batch.index(field, new_value.into_owned());
+            }
+        }
+        (IndexValue::SearchIndex { index, .. }, IndexValue::SearchIndex { .. }) => {
+            batch.schedule_task(Task::IndexDocument(TaskIndexDocument {
+                account_id: batch.last_account_id().unwrap().into(),
+                document_id: batch.last_document_id().unwrap().into(),
+                document_type: match index {
+                    SearchIndex::Email => IndexDocumentType::Email,
+                    SearchIndex::Calendar => IndexDocumentType::Calendar,
+                    SearchIndex::Contacts => IndexDocumentType::Contacts,
+                    SearchIndex::File => IndexDocumentType::File,
+                    SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(),
+                },
+                status: TaskStatus::now(),
+            }));
+        }
+        (
+            IndexValue::Property {
+                field: old_field,
+                value: old_value,
+            },
+            IndexValue::Property {
+                field: new_field,
+                value: new_value,
+                ..
+            },
+        ) => {
+            if old_field != new_field {
+                batch.clear(old_field);
+                batch.set(new_field, new_value.into_owned());
+            } else if new_value != old_value {
+                if new_value.is_some() {
+                    batch.set(old_field, new_value.into_owned());
+                } else {
+                    batch.clear(old_field);
+                }
+            }
+        }
+        (IndexValue::Blob { value: old_hash }, IndexValue::Blob { value: new_hash }) => {
+            batch.clear(BlobOp::Link {
+                hash: old_hash,
+                to: BlobLink::Document,
+            });
+            batch.set(
+                BlobOp::Link {
+                    hash: new_hash,
+                    to: BlobLink::Document,
+                },
+                vec![],
+            );
+        }
+        (IndexValue::Acl { value: old_acl }, IndexValue::Acl { value: new_acl }) => {
+            let has_old_acl = !old_acl.is_empty();
+            let has_new_acl = !new_acl.is_empty();
+
+            if !has_old_acl && !has_new_acl {
+                return Ok(());
+            }
+
+            let object_account_id = batch.last_account_id().unwrap_or_default();
+            let object_type = batch.last_collection().unwrap_or(Collection::None);
+            let object_id = batch.last_document_id().unwrap_or_default();
+            let notification_id = SnowflakeIdGenerator::global_id().unwrap_or_default();
+
+            match (has_old_acl, has_new_acl) {
+                (true, true) => {
+                    // Remove deleted ACLs
+                    for current_item in old_acl.as_ref() {
+                        if !new_acl
+                            .iter()
+                            .any(|item| item.account_id == current_item.account_id)
+                        {
+                            batch.acl_revoke(current_item.account_id);
+                            batch.log_share_notification(
+                                notification_id,
+                                current_item.account_id,
+                                ShareNotification {
+                                    object_account_id,
+                                    object_id,
+                                    object_type,
+                                    changed_by,
+                                    old_rights: current_item.grants,
+                                    new_rights: Default::default(),
+                                    name: Default::default(),
+                                },
+                            );
+                        }
+                    }
+
+                    // Update ACLs
+                    for item in new_acl.as_ref() {
+                        let mut add_item = true;
+                        let mut old_rights = Bitmap::default();
+                        for current_item in old_acl.as_ref() {
+                            if item.account_id == current_item.account_id {
+                                if item.grants == current_item.grants {
+                                    add_item = false;
+                                } else {
+                                    old_rights = current_item.grants;
+                                }
+                                break;
+                            }
+                        }
+                        if add_item {
+                            batch.acl_grant(item.account_id, item.grants.bitmap.serialize());
+                            batch.log_share_notification(
+                                notification_id,
+                                item.account_id,
+                                ShareNotification {
+                                    object_account_id,
+                                    object_id,
+                                    object_type,
+                                    changed_by,
+                                    old_rights,
+                                    new_rights: item.grants,
+                                    name: Default::default(),
+                                },
+                            );
+                        }
+                    }
+                }
+                (false, true) => {
+                    // Add all ACLs
+                    for item in new_acl.as_ref() {
+                        batch.acl_grant(item.account_id, item.grants.bitmap.serialize());
+                        batch.log_share_notification(
+                            notification_id,
+                            item.account_id,
+                            ShareNotification {
+                                object_account_id,
+                                object_id,
+                                object_type,
+                                changed_by,
+                                old_rights: Default::default(),
+                                new_rights: item.grants,
+                                name: Default::default(),
+                            },
+                        );
+                    }
+                }
+                (true, false) => {
+                    // Remove all ACLs
+                    for item in old_acl.as_ref() {
+                        batch.acl_revoke(item.account_id);
+                        batch.log_share_notification(
+                            notification_id,
+                            item.account_id,
+                            ShareNotification {
+                                object_account_id,
+                                object_id,
+                                object_type,
+                                changed_by,
+                                old_rights: item.grants,
+                                new_rights: Default::default(),
+                                name: Default::default(),
+                            },
+                        );
+                    }
+                }
+                _ => {}
+            }
+        }
+        (IndexValue::Quota { used: old_used }, IndexValue::Quota { used: new_used }) => {
+            let value = new_used as i64 - old_used as i64;
+            batch.add(ValueClass::Quota, value);
+
+            if let Some(tenant_id) = tenant_id {
+                batch.add(ValueClass::TenantQuota(tenant_id), value);
+            }
+        }
+        (
+            IndexValue::LogItem {
+                sync_collection,
+                prefix: old_prefix,
+            },
+            IndexValue::LogItem {
+                prefix: new_prefix, ..
+            },
+        ) => {
+            batch.log_item_delete(sync_collection, old_prefix);
+            batch.log_item_insert(sync_collection, new_prefix);
+        }
+        (
+            IndexValue::LogContainerProperty {
+                sync_collection,
+                ids: old_ids,
+            },
+            IndexValue::LogContainerProperty { ids: new_ids, .. },
+        ) => {
+            for parent_id in &old_ids {
+                if !new_ids.contains(parent_id) {
+                    batch.log_container_property_change(sync_collection, *parent_id);
+                }
+            }
+            for parent_id in new_ids {
+                if !old_ids.contains(&parent_id) {
+                    batch.log_container_property_change(sync_collection, parent_id);
+                }
+            }
+        }
+        _ => unreachable!(),
+    }
+
+    Ok(())
+}
+
+impl IndexableObject for () {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        std::iter::empty()
+    }
+}
+
+impl IndexableAndSerializableObject for () {
+    fn is_versioned() -> bool {
+        false
+    }
+}

crates/common/src/storage/mod.rs

@@ -0,0 +1,105 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::Server;
+use directory::Directory;
+use registry::{
+    schema::{
+        enums::{StorageQuota, TenantStorageQuota},
+        prelude::ObjectType,
+    },
+    types::EnumImpl,
+};
+use std::sync::Arc;
+use store::{BlobStore, InMemoryStore, RegistryStore, SearchStore, Store};
+
+pub mod archive;
+pub mod blob;
+pub mod dav;
+pub mod document;
+pub mod encryption;
+pub mod index;
+pub mod quota;
+pub mod state;
+pub mod transaction;
+
+#[derive(Debug, Clone)]
+pub struct ObjectQuota([u32; StorageQuota::COUNT - 1]);
+
+#[derive(Debug, Clone)]
+pub struct TenantQuota([u32; TenantStorageQuota::COUNT - 1]);
+
+impl Server {
+    #[inline(always)]
+    pub fn registry(&self) -> &RegistryStore {
+        &self.core.storage.registry
+    }
+
+    #[inline(always)]
+    pub fn store(&self) -> &Store {
+        &self.core.storage.data
+    }
+
+    #[inline(always)]
+    pub fn blob_store(&self) -> &BlobStore {
+        &self.core.storage.blob
+    }
+
+    #[inline(always)]
+    pub fn search_store(&self) -> &SearchStore {
+        &self.core.storage.search
+    }
+
+    #[inline(always)]
+    pub fn in_memory_store(&self) -> &InMemoryStore {
+        &self.core.storage.memory
+    }
+
+    #[inline(always)]
+    pub fn tracing_store(&self) -> &Store {
+        &self.core.storage.tracing
+    }
+
+    #[inline(always)]
+    pub fn metrics_store(&self) -> &Store {
+        &self.core.storage.metrics
+    }
+
+    #[inline(always)]
+    pub fn get_directory(&self, id: &u32) -> Option<&Arc<Directory>> {
+        self.core.storage.directories.get(id)
+    }
+
+    #[inline(always)]
+    pub fn get_default_directory(&self) -> Option<&Arc<Directory>> {
+        self.core.storage.directory.as_ref()
+    }
+
+    #[inline(always)]
+    pub fn get_lookup_store(&self, name: &str) -> Option<InMemoryStore> {
+        if !name.is_empty() && name != "*" {
+            self.inner.data.lookup_stores.load().get(name).cloned()
+        } else {
+            self.in_memory_store().clone().into()
+        }
+    }
+
+    pub async fn total_accounts(&self) -> trc::Result<usize> {
+        self.registry().count_object(ObjectType::Account).await
+    }
+
+    pub async fn total_domains(&self) -> trc::Result<usize> {
+        self.registry().count_object(ObjectType::Domain).await
+    }
+
+    #[cfg(not(feature = "enterprise"))]
+    pub async fn logo_resource(
+        &self,
+        _: &str,
+    ) -> trc::Result<Option<crate::manager::application::Resource<Vec<u8>>>> {
+        Ok(None)
+    }
+}

crates/common/src/storage/quota.rs

@@ -0,0 +1,99 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    Server,
+    auth::AccountCache,
+    storage::{ObjectQuota, TenantQuota},
+};
+use registry::{
+    schema::enums::{StorageQuota, TenantStorageQuota},
+    types::EnumImpl,
+};
+use store::{ValueKey, write::ValueClass};
+use trc::AddContext;
+
+impl Server {
+    pub async fn get_used_quota_account(&self, account_id: u32) -> trc::Result<i64> {
+        self.core
+            .storage
+            .data
+            .get_counter(ValueKey {
+                account_id,
+                collection: 0,
+                document_id: 0,
+                class: ValueClass::Quota,
+            })
+            .await
+            .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))
+    }
+
+    #[cfg(not(feature = "enterprise"))]
+    pub async fn get_used_quota_tenant(&self, _tenant_id: u32) -> trc::Result<i64> {
+        Ok(0)
+    }
+
+    pub async fn has_available_quota(
+        &self,
+        account: &AccountCache,
+        item_size: u64,
+    ) -> trc::Result<()> {
+        if account.quota_disk != 0 {
+            let used_quota = self.get_used_quota_account(account.id).await?.max(0) as u64;
+
+            if used_quota + item_size > account.quota_disk {
+                return Err(trc::LimitEvent::Quota
+                    .into_err()
+                    .ctx(trc::Key::Limit, account.quota_disk)
+                    .ctx(trc::Key::Size, used_quota));
+            }
+        }
+
+
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub fn object_quota(&self, user_quotas: Option<&ObjectQuota>, object: StorageQuota) -> u32 {
+        user_quotas.unwrap_or(&self.core.email.max_objects).0[object as usize]
+    }
+}
+
+impl ObjectQuota {
+    #[inline(always)]
+    pub fn set(&mut self, item: StorageQuota, max: u32) {
+        self.0[item as usize] = max;
+    }
+
+    #[inline(always)]
+    pub fn get(&self, item: StorageQuota) -> u32 {
+        self.0[item as usize]
+    }
+}
+
+impl TenantQuota {
+    #[inline(always)]
+    pub fn set(&mut self, item: TenantStorageQuota, max: u32) {
+        self.0[item as usize] = max;
+    }
+
+    #[inline(always)]
+    pub fn get(&self, item: TenantStorageQuota) -> u32 {
+        self.0[item as usize]
+    }
+}
+
+impl Default for ObjectQuota {
+    fn default() -> Self {
+        Self([u32::MAX; StorageQuota::COUNT - 1])
+    }
+}
+
+impl Default for TenantQuota {
+    fn default() -> Self {
+        Self([u32::MAX; TenantStorageQuota::COUNT - 1])
+    }
+}

crates/common/src/storage/state.rs

@@ -0,0 +1,82 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    IPC_CHANNEL_BUFFER, Server,
+    auth::AccessToken,
+    ipc::{BroadcastEvent, PushEvent, PushNotification},
+};
+use tokio::sync::mpsc;
+use types::type_state::DataType;
+use utils::map::bitmap::Bitmap;
+
+impl Server {
+    pub async fn subscribe_push_manager(
+        &self,
+        access_token: &AccessToken,
+        types: Bitmap<DataType>,
+    ) -> trc::Result<mpsc::Receiver<PushNotification>> {
+        let (tx, rx) = mpsc::channel::<PushNotification>(IPC_CHANNEL_BUFFER);
+        let push_tx = self.inner.ipc.push_tx.clone();
+
+        push_tx
+            .send(PushEvent::Subscribe {
+                account_ids: access_token.member_ids().collect(),
+                types,
+                tx,
+            })
+            .await
+            .map_err(|err| {
+                trc::EventType::Server(trc::ServerEvent::ThreadError)
+                    .reason(err)
+                    .caused_by(trc::location!())
+            })?;
+
+        Ok(rx)
+    }
+
+    #[inline(always)]
+    pub fn notify_task_queue(&self) {
+        self.inner.ipc.task_tx.notify_one();
+    }
+
+    pub async fn broadcast_push_notification(&self, notification: PushNotification) -> bool {
+        match self
+            .inner
+            .ipc
+            .push_tx
+            .clone()
+            .send(PushEvent::Publish {
+                notification,
+                broadcast: true,
+            })
+            .await
+        {
+            Ok(_) => true,
+            Err(_) => {
+                trc::event!(
+                    Server(trc::ServerEvent::ThreadError),
+                    Details = "Error sending state change.",
+                    CausedBy = trc::location!()
+                );
+
+                false
+            }
+        }
+    }
+
+    pub async fn cluster_broadcast(&self, event: BroadcastEvent) {
+        if let Some(broadcast_tx) = &self.inner.ipc.broadcast_tx.clone()
+            && broadcast_tx.send(event).await.is_err()
+        {
+            trc::event!(
+                Server(trc::ServerEvent::ThreadError),
+                Details = "Error sending broadcast event.",
+                CausedBy = trc::location!()
+            );
+        }
+    }
+}

crates/common/src/storage/transaction.rs

@@ -0,0 +1,198 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{Server, ipc::PushNotification};
+use std::time::Duration;
+use store::{
+    IterateParams, Key, LogKey, SUBSPACE_LOGS, U64_LEN,
+    write::{AnyClass, AssignedIds, BatchBuilder, ValueClass, key::DeserializeBigEndian},
+};
+use trc::AddContext;
+use types::{
+    collection::SyncCollection,
+    type_state::{DataType, StateChange},
+};
+use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator};
+
+impl Server {
+    pub async fn commit_batch(&self, mut builder: BatchBuilder) -> trc::Result<AssignedIds> {
+        let mut assigned_ids = AssignedIds::default();
+        let mut commit_points = builder.commit_points();
+
+        for commit_point in commit_points.iter() {
+            let batch = builder.build_one(commit_point);
+            assigned_ids
+                .ids
+                .extend(self.store().write(batch).await?.ids);
+        }
+
+        if let Some(changes) = builder.changes() {
+            for (account_id, changed_collections) in changes {
+                let mut state_change = StateChange::new(account_id);
+                for changed_collection in changed_collections.changed_containers {
+                    if let Some(data_type) = DataType::try_from_sync(changed_collection, true) {
+                        state_change.set_change(data_type);
+                    }
+                }
+                for changed_collection in changed_collections.changed_items {
+                    if let Some(data_type) = DataType::try_from_sync(changed_collection, false) {
+                        state_change.set_change(data_type);
+                    }
+                }
+                if state_change.has_changes() {
+                    self.broadcast_push_notification(PushNotification::StateChange(
+                        state_change.with_change_id(assigned_ids.last_change_id(account_id)?),
+                    ))
+                    .await;
+                }
+                if let Some(change_id) = changed_collections.share_notification_id {
+                    self.broadcast_push_notification(PushNotification::StateChange(StateChange {
+                        account_id,
+                        change_id,
+                        types: Bitmap::from_iter([DataType::ShareNotification]),
+                    }))
+                    .await;
+                }
+            }
+        }
+
+        Ok(assigned_ids)
+    }
+
+    pub async fn delete_changes(
+        &self,
+        account_id: u32,
+        max_entries: Option<usize>,
+        max_duration: Option<Duration>,
+    ) -> trc::Result<()> {
+        if let Some(max_entries) = max_entries {
+            for sync_collection in [
+                SyncCollection::Email,
+                SyncCollection::Thread,
+                SyncCollection::Identity,
+                SyncCollection::EmailSubmission,
+                SyncCollection::SieveScript,
+                SyncCollection::FileNode,
+                SyncCollection::AddressBook,
+                SyncCollection::Calendar,
+                SyncCollection::CalendarEventNotification,
+            ] {
+                let collection = sync_collection.into();
+                let from_key = LogKey {
+                    account_id,
+                    collection,
+                    change_id: 0,
+                };
+                let to_key = LogKey {
+                    account_id,
+                    collection,
+                    change_id: u64::MAX,
+                };
+
+                let mut first_change_id = 0;
+                let mut num_changes = 0;
+
+                self.store()
+                    .iterate(
+                        IterateParams::new(from_key, to_key)
+                            .descending()
+                            .no_values(),
+                        |key, _| {
+                            first_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?;
+                            num_changes += 1;
+
+                            Ok(num_changes <= max_entries)
+                        },
+                    )
+                    .await
+                    .caused_by(trc::location!())?;
+
+                if num_changes > max_entries {
+                    self.store()
+                        .delete_range(
+                            LogKey {
+                                account_id,
+                                collection,
+                                change_id: 0,
+                            },
+                            LogKey {
+                                account_id,
+                                collection,
+                                change_id: first_change_id,
+                            },
+                        )
+                        .await
+                        .caused_by(trc::location!())?;
+
+                    // Delete vanished items
+                    if let Some(vanished_collection) =
+                        sync_collection.vanished_collection().map(u8::from)
+                    {
+                        self.store()
+                            .delete_range(
+                                LogKey {
+                                    account_id,
+                                    collection: vanished_collection,
+                                    change_id: 0,
+                                },
+                                LogKey {
+                                    account_id,
+                                    collection: vanished_collection,
+                                    change_id: first_change_id,
+                                },
+                            )
+                            .await
+                            .caused_by(trc::location!())?;
+                    }
+
+                    // Write truncation entry for cache
+                    let mut batch = BatchBuilder::new();
+                    batch.with_account_id(account_id).set(
+                        ValueClass::Any(AnyClass {
+                            subspace: SUBSPACE_LOGS,
+                            key: LogKey {
+                                account_id,
+                                collection,
+                                change_id: first_change_id,
+                            }
+                            .serialize(0),
+                        }),
+                        Vec::new(),
+                    );
+                    self.store()
+                        .write(batch.build_all())
+                        .await
+                        .caused_by(trc::location!())?;
+                }
+            }
+        }
+
+        if let Some(max_duration) = max_duration {
+            self.store()
+                .delete_range(
+                    LogKey {
+                        account_id,
+                        collection: SyncCollection::ShareNotification.into(),
+                        change_id: 0,
+                    },
+                    LogKey {
+                        account_id,
+                        collection: SyncCollection::ShareNotification.into(),
+                        change_id: SnowflakeIdGenerator::from_duration(max_duration)
+                            .unwrap_or_default(),
+                    },
+                )
+                .await
+                .caused_by(trc::location!())?;
+        }
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub fn generate_snowflake_id(&self) -> u64 {
+        self.inner.data.jmap_id_gen.generate()
+    }
+}