Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
Vendored
+356
@@ -0,0 +1,356 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::message::metadata::{ArchivedMessageData, MessageData};
|
||||
use common::{
|
||||
MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken,
|
||||
sharing::EffectiveAcl,
|
||||
};
|
||||
use store::write::{AlignedBytes, Archive};
|
||||
use store::{ValueKey, ahash::AHashMap, roaring::RoaringBitmap};
|
||||
use trc::AddContext;
|
||||
use types::{
|
||||
acl::Acl,
|
||||
collection::Collection,
|
||||
keyword::{Keyword, OTHER},
|
||||
};
|
||||
use utils::map::bitmap::Bitmap;
|
||||
|
||||
struct MessagesCacheBuilder {
|
||||
pub change_id: u64,
|
||||
pub items: Vec<MessageCache>,
|
||||
pub index: AHashMap<u32, u32>,
|
||||
pub keywords: Vec<Box<str>>,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
pub(crate) async fn update_email_cache(
|
||||
server: &Server,
|
||||
account_id: u32,
|
||||
changed_ids: &AHashMap<u32, bool>,
|
||||
store_cache: &MessageStoreCache,
|
||||
) -> trc::Result<MessagesCache> {
|
||||
let mut new_cache = MessagesCacheBuilder {
|
||||
index: AHashMap::with_capacity(store_cache.emails.items.len()),
|
||||
items: Vec::with_capacity(store_cache.emails.items.len()),
|
||||
size: 0,
|
||||
change_id: 0,
|
||||
keywords: store_cache.emails.keywords.to_vec(),
|
||||
};
|
||||
|
||||
for (document_id, is_update) in changed_ids {
|
||||
if *is_update
|
||||
&& let Some(archive) = server
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::archive(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
*document_id,
|
||||
))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
insert_item(
|
||||
&mut new_cache,
|
||||
*document_id,
|
||||
archive.to_unarchived::<MessageData>()?,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for item in &store_cache.emails.items {
|
||||
if !changed_ids.contains_key(&item.document_id) {
|
||||
email_insert(&mut new_cache, item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(new_cache.build())
|
||||
}
|
||||
|
||||
pub(crate) async fn full_email_cache_build(
|
||||
server: &Server,
|
||||
account_id: u32,
|
||||
) -> trc::Result<MessagesCache> {
|
||||
// Build cache
|
||||
let mut cache = MessagesCacheBuilder {
|
||||
items: Vec::with_capacity(16),
|
||||
index: AHashMap::with_capacity(16),
|
||||
keywords: Vec::new(),
|
||||
size: 0,
|
||||
change_id: 0,
|
||||
};
|
||||
|
||||
server
|
||||
.archives(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
&(),
|
||||
|document_id, archive| {
|
||||
insert_item(
|
||||
&mut cache,
|
||||
document_id,
|
||||
archive.to_unarchived::<MessageData>()?,
|
||||
);
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
Ok(cache.build())
|
||||
}
|
||||
|
||||
fn insert_item(
|
||||
cache: &mut MessagesCacheBuilder,
|
||||
document_id: u32,
|
||||
archive: Archive<&ArchivedMessageData>,
|
||||
) {
|
||||
let message = archive.inner;
|
||||
let mut item = MessageCache {
|
||||
mailboxes: message
|
||||
.mailboxes
|
||||
.iter()
|
||||
.map(|m| MessageUidCache {
|
||||
mailbox_id: m.mailbox_id.to_native(),
|
||||
uid: m.uid.to_native(),
|
||||
})
|
||||
.collect(),
|
||||
keywords: 0,
|
||||
thread_id: message.thread_id.to_native(),
|
||||
change_id: archive.version.change_id().unwrap_or_default(),
|
||||
document_id,
|
||||
size: message.size.to_native(),
|
||||
};
|
||||
for keyword in message.keywords.iter() {
|
||||
match keyword.id() {
|
||||
Ok(id) => {
|
||||
item.keywords |= 1 << id;
|
||||
}
|
||||
Err(custom) => {
|
||||
if let Some(idx) = cache.keywords.iter().position(|k| **k == *custom) {
|
||||
item.keywords |= 1 << (OTHER + idx);
|
||||
} else if cache.keywords.len() < (128 - OTHER) {
|
||||
cache.keywords.push(custom.into());
|
||||
item.keywords |= 1 << (OTHER + cache.keywords.len() - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
email_insert(cache, item);
|
||||
}
|
||||
|
||||
impl MessagesCacheBuilder {
|
||||
pub fn build(mut self) -> MessagesCache {
|
||||
self.index.shrink_to_fit();
|
||||
MessagesCache {
|
||||
change_id: self.change_id,
|
||||
items: self.items.into_boxed_slice(),
|
||||
index: self.index,
|
||||
keywords: self.keywords.into_boxed_slice(),
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait MessageCacheAccess {
|
||||
fn email_by_id(&self, id: &u32) -> Option<&MessageCache>;
|
||||
|
||||
fn has_email_id(&self, id: &u32) -> bool;
|
||||
|
||||
fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn in_thread(&self, thread_id: u32) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn with_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn without_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn in_mailbox_with_keyword(
|
||||
&self,
|
||||
mailbox_id: u32,
|
||||
keyword: &Keyword,
|
||||
) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn in_mailbox_without_keyword(
|
||||
&self,
|
||||
mailbox_id: u32,
|
||||
keyword: &Keyword,
|
||||
) -> impl Iterator<Item = &MessageCache>;
|
||||
|
||||
fn email_document_ids(&self) -> RoaringBitmap;
|
||||
|
||||
fn shared_messages(
|
||||
&self,
|
||||
access_token: &AccessToken,
|
||||
check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
|
||||
) -> RoaringBitmap;
|
||||
|
||||
fn expand_keywords(&self, message: &MessageCache) -> impl Iterator<Item = Keyword>;
|
||||
|
||||
fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool;
|
||||
}
|
||||
|
||||
impl MessageCacheAccess for MessageStoreCache {
|
||||
fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator<Item = &MessageCache> {
|
||||
self.emails
|
||||
.items
|
||||
.iter()
|
||||
.filter(move |m| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id))
|
||||
}
|
||||
|
||||
fn in_mailboxes(&self, mailbox_ids: &[u32]) -> impl Iterator<Item = &MessageCache> {
|
||||
self.emails.items.iter().filter(move |m| {
|
||||
m.mailboxes
|
||||
.iter()
|
||||
.any(|mb| mailbox_ids.contains(&mb.mailbox_id))
|
||||
})
|
||||
}
|
||||
|
||||
fn in_thread(&self, thread_id: u32) -> impl Iterator<Item = &MessageCache> {
|
||||
self.emails
|
||||
.items
|
||||
.iter()
|
||||
.filter(move |m| m.thread_id == thread_id)
|
||||
}
|
||||
|
||||
fn with_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache> {
|
||||
let keyword_id = keyword_to_id(self, keyword);
|
||||
self.emails
|
||||
.items
|
||||
.iter()
|
||||
.filter(move |m| keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0))
|
||||
}
|
||||
|
||||
fn without_keyword(&self, keyword: &Keyword) -> impl Iterator<Item = &MessageCache> {
|
||||
let keyword_id = keyword_to_id(self, keyword);
|
||||
self.emails
|
||||
.items
|
||||
.iter()
|
||||
.filter(move |m| keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0))
|
||||
}
|
||||
|
||||
fn in_mailbox_with_keyword(
|
||||
&self,
|
||||
mailbox_id: u32,
|
||||
keyword: &Keyword,
|
||||
) -> impl Iterator<Item = &MessageCache> {
|
||||
let keyword_id = keyword_to_id(self, keyword);
|
||||
self.emails.items.iter().filter(move |m| {
|
||||
m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)
|
||||
&& keyword_id.is_some_and(|id| m.keywords & (1 << id) != 0)
|
||||
})
|
||||
}
|
||||
|
||||
fn in_mailbox_without_keyword(
|
||||
&self,
|
||||
mailbox_id: u32,
|
||||
keyword: &Keyword,
|
||||
) -> impl Iterator<Item = &MessageCache> {
|
||||
let keyword_id = keyword_to_id(self, keyword);
|
||||
self.emails.items.iter().filter(move |m| {
|
||||
m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)
|
||||
&& keyword_id.is_none_or(|id| m.keywords & (1 << id) == 0)
|
||||
})
|
||||
}
|
||||
|
||||
fn shared_messages(
|
||||
&self,
|
||||
access_token: &AccessToken,
|
||||
check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
|
||||
) -> RoaringBitmap {
|
||||
let check_acls = check_acls.into();
|
||||
let mut shared_messages = RoaringBitmap::new();
|
||||
for mailbox in &self.mailboxes.items {
|
||||
if mailbox
|
||||
.acls
|
||||
.as_slice()
|
||||
.effective_acl(access_token)
|
||||
.contains_all(check_acls)
|
||||
{
|
||||
shared_messages.extend(
|
||||
self.in_mailbox(mailbox.document_id)
|
||||
.map(|item| item.document_id),
|
||||
);
|
||||
}
|
||||
}
|
||||
shared_messages
|
||||
}
|
||||
|
||||
fn email_document_ids(&self) -> RoaringBitmap {
|
||||
RoaringBitmap::from_iter(self.emails.index.keys())
|
||||
}
|
||||
|
||||
fn email_by_id(&self, id: &u32) -> Option<&MessageCache> {
|
||||
self.emails
|
||||
.index
|
||||
.get(id)
|
||||
.and_then(|idx| self.emails.items.get(*idx as usize))
|
||||
}
|
||||
|
||||
fn has_email_id(&self, id: &u32) -> bool {
|
||||
self.emails.index.contains_key(id)
|
||||
}
|
||||
|
||||
fn expand_keywords(&self, message: &MessageCache) -> impl Iterator<Item = Keyword> {
|
||||
KeywordsIter(message.keywords).map(move |id| match Keyword::try_from_id(id) {
|
||||
Ok(keyword) => keyword,
|
||||
Err(id) => Keyword::Other(self.emails.keywords[id - OTHER].clone()),
|
||||
})
|
||||
}
|
||||
|
||||
fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool {
|
||||
keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0)
|
||||
}
|
||||
}
|
||||
|
||||
fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) {
|
||||
let id = item.document_id;
|
||||
if let Some(idx) = cache.index.get(&id) {
|
||||
cache.items[*idx as usize] = item;
|
||||
} else {
|
||||
cache.size += (std::mem::size_of::<MessageCache>()
|
||||
+ (std::mem::size_of::<u32>() * 2)
|
||||
+ (item.mailboxes.len() * std::mem::size_of::<MessageUidCache>()))
|
||||
as u64;
|
||||
|
||||
let idx = cache.items.len() as u32;
|
||||
cache.items.push(item);
|
||||
cache.index.insert(id, idx);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option<u32> {
|
||||
match keyword.id() {
|
||||
Ok(id) => Some(id),
|
||||
Err(name) => cache
|
||||
.emails
|
||||
.keywords
|
||||
.iter()
|
||||
.position(|k| **k == *name)
|
||||
.map(|idx| (OTHER + idx) as u32),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct KeywordsIter(u128);
|
||||
|
||||
impl Iterator for KeywordsIter {
|
||||
type Item = usize;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.0 != 0 {
|
||||
let item = 127 - self.0.leading_zeros();
|
||||
self.0 ^= 1 << item;
|
||||
Some(item as usize)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+322
@@ -0,0 +1,322 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::mailbox::{ArchivedMailbox, Mailbox, manage::MailboxFnc};
|
||||
use common::{
|
||||
MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken,
|
||||
sharing::EffectiveAcl,
|
||||
};
|
||||
use store::{
|
||||
ValueKey,
|
||||
write::{AlignedBytes, Archive},
|
||||
};
|
||||
use store::{ahash::AHashMap, roaring::RoaringBitmap};
|
||||
use trc::AddContext;
|
||||
use types::{
|
||||
acl::{Acl, AclGrant},
|
||||
collection::Collection,
|
||||
special_use::SpecialUse,
|
||||
};
|
||||
use utils::{map::bitmap::Bitmap, topological::TopologicalSort};
|
||||
|
||||
struct MailboxesCacheBuilder {
|
||||
pub change_id: u64,
|
||||
pub index: AHashMap<u32, u32>,
|
||||
pub items: Vec<MailboxCache>,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
pub(crate) async fn update_mailbox_cache(
|
||||
server: &Server,
|
||||
account_id: u32,
|
||||
changed_ids: &AHashMap<u32, bool>,
|
||||
store_cache: &MessageStoreCache,
|
||||
) -> trc::Result<MailboxesCache> {
|
||||
let mut new_cache = MailboxesCacheBuilder {
|
||||
items: Vec::with_capacity(store_cache.mailboxes.items.len()),
|
||||
index: AHashMap::with_capacity(store_cache.mailboxes.items.len()),
|
||||
size: 0,
|
||||
change_id: 0,
|
||||
};
|
||||
|
||||
for (document_id, is_update) in changed_ids {
|
||||
if *is_update
|
||||
&& let Some(archive) = server
|
||||
.store()
|
||||
.get_value::<Archive<AlignedBytes>>(ValueKey::archive(
|
||||
account_id,
|
||||
Collection::Mailbox,
|
||||
*document_id,
|
||||
))
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
insert_item(
|
||||
&mut new_cache,
|
||||
*document_id,
|
||||
archive.unarchive::<Mailbox>()?,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for item in store_cache.mailboxes.items.iter() {
|
||||
if !changed_ids.contains_key(&item.document_id) {
|
||||
mailbox_insert(&mut new_cache, item.clone());
|
||||
}
|
||||
}
|
||||
|
||||
build_tree(&mut new_cache);
|
||||
|
||||
Ok(new_cache.build())
|
||||
}
|
||||
|
||||
pub(crate) async fn full_mailbox_cache_build(
|
||||
server: &Server,
|
||||
account_id: u32,
|
||||
) -> trc::Result<MailboxesCache> {
|
||||
// Build cache
|
||||
let mut cache = MailboxesCacheBuilder {
|
||||
items: Default::default(),
|
||||
index: Default::default(),
|
||||
size: 0,
|
||||
change_id: 0,
|
||||
};
|
||||
|
||||
server
|
||||
.archives(
|
||||
account_id,
|
||||
Collection::Mailbox,
|
||||
&(),
|
||||
|document_id, archive| {
|
||||
insert_item(&mut cache, document_id, archive.unarchive::<Mailbox>()?);
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
if cache.items.is_empty() {
|
||||
server
|
||||
.create_system_folders(account_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
server
|
||||
.archives(
|
||||
account_id,
|
||||
Collection::Mailbox,
|
||||
&(),
|
||||
|document_id, archive| {
|
||||
insert_item(&mut cache, document_id, archive.unarchive::<Mailbox>()?);
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
}
|
||||
|
||||
build_tree(&mut cache);
|
||||
|
||||
Ok(cache.build())
|
||||
}
|
||||
|
||||
fn insert_item(cache: &mut MailboxesCacheBuilder, document_id: u32, mailbox: &ArchivedMailbox) {
|
||||
let parent_id = mailbox.parent_id.to_native();
|
||||
let item = MailboxCache {
|
||||
document_id,
|
||||
name: mailbox.name.as_str().into(),
|
||||
path: "".into(),
|
||||
role: (&mailbox.role).into(),
|
||||
parent_id: if parent_id > 0 {
|
||||
parent_id - 1
|
||||
} else {
|
||||
u32::MAX
|
||||
},
|
||||
sort_order: mailbox
|
||||
.sort_order
|
||||
.as_ref()
|
||||
.map(|s| s.to_native())
|
||||
.unwrap_or(u32::MAX),
|
||||
subscribers: mailbox.subscribers.iter().map(|s| s.to_native()).collect(),
|
||||
uid_validity: mailbox.uid_validity.to_native(),
|
||||
acls: mailbox
|
||||
.acls
|
||||
.iter()
|
||||
.map(|acl| AclGrant {
|
||||
account_id: acl.account_id.to_native(),
|
||||
grants: Bitmap::from(&acl.grants),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
|
||||
mailbox_insert(cache, item);
|
||||
}
|
||||
|
||||
fn build_tree(cache: &mut MailboxesCacheBuilder) {
|
||||
cache.size = 0;
|
||||
let mut topological_sort = TopologicalSort::with_capacity(cache.items.len());
|
||||
|
||||
for (idx, mailbox) in cache.items.iter_mut().enumerate() {
|
||||
topological_sort.insert(
|
||||
if mailbox.parent_id == u32::MAX {
|
||||
0
|
||||
} else {
|
||||
mailbox.parent_id + 1
|
||||
},
|
||||
mailbox.document_id + 1,
|
||||
);
|
||||
mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) {
|
||||
"INBOX".into()
|
||||
} else if mailbox.is_root() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") {
|
||||
format!("INBOX {}", idx + 1)
|
||||
} else {
|
||||
mailbox.name.clone()
|
||||
};
|
||||
|
||||
cache.size += item_size(mailbox);
|
||||
}
|
||||
|
||||
for folder_id in topological_sort.into_iterator() {
|
||||
if folder_id != 0 {
|
||||
let folder_id = folder_id - 1;
|
||||
if let Some((path, parent_path)) = by_id(cache, &folder_id)
|
||||
.and_then(|folder| {
|
||||
folder
|
||||
.parent_id()
|
||||
.map(|parent_id| (&folder.path, parent_id))
|
||||
})
|
||||
.and_then(|(path, parent_id)| {
|
||||
by_id(cache, &parent_id).map(|folder| (path, &folder.path))
|
||||
})
|
||||
{
|
||||
let mut new_path = String::with_capacity(parent_path.len() + path.len() + 1);
|
||||
new_path.push_str(parent_path.as_str());
|
||||
new_path.push('/');
|
||||
new_path.push_str(path.as_str());
|
||||
let folder = by_id_mut(cache, &folder_id).unwrap();
|
||||
folder.path = new_path;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MailboxesCacheBuilder {
|
||||
fn build(mut self) -> MailboxesCache {
|
||||
self.index.shrink_to_fit();
|
||||
MailboxesCache {
|
||||
change_id: self.change_id,
|
||||
index: self.index,
|
||||
items: self.items.into_boxed_slice(),
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait MailboxCacheAccess {
|
||||
fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache>;
|
||||
fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache>;
|
||||
fn mailbox_by_path(&self, name: &str) -> Option<&MailboxCache>;
|
||||
fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache>;
|
||||
fn shared_mailboxes(
|
||||
&self,
|
||||
access_token: &AccessToken,
|
||||
check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
|
||||
) -> RoaringBitmap;
|
||||
fn has_mailbox_id(&self, id: &u32) -> bool;
|
||||
}
|
||||
|
||||
impl MailboxCacheAccess for MessageStoreCache {
|
||||
fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache> {
|
||||
self.mailboxes
|
||||
.items
|
||||
.iter()
|
||||
.find(|m| m.name.eq_ignore_ascii_case(name))
|
||||
}
|
||||
|
||||
fn mailbox_by_path(&self, path: &str) -> Option<&MailboxCache> {
|
||||
self.mailboxes
|
||||
.items
|
||||
.iter()
|
||||
.find(|m| m.path.eq_ignore_ascii_case(path))
|
||||
}
|
||||
|
||||
fn mailbox_by_role(&self, role: &SpecialUse) -> Option<&MailboxCache> {
|
||||
self.mailboxes.items.iter().find(|m| &m.role == role)
|
||||
}
|
||||
|
||||
fn shared_mailboxes(
|
||||
&self,
|
||||
access_token: &AccessToken,
|
||||
check_acls: impl Into<Bitmap<Acl>> + Sync + Send,
|
||||
) -> RoaringBitmap {
|
||||
let check_acls = check_acls.into();
|
||||
|
||||
RoaringBitmap::from_iter(
|
||||
self.mailboxes
|
||||
.items
|
||||
.iter()
|
||||
.filter(|m| {
|
||||
m.acls
|
||||
.as_slice()
|
||||
.effective_acl(access_token)
|
||||
.contains_all(check_acls)
|
||||
})
|
||||
.map(|m| m.document_id),
|
||||
)
|
||||
}
|
||||
|
||||
fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache> {
|
||||
self.mailboxes
|
||||
.index
|
||||
.get(id)
|
||||
.and_then(|idx| self.mailboxes.items.get(*idx as usize))
|
||||
}
|
||||
|
||||
fn has_mailbox_id(&self, id: &u32) -> bool {
|
||||
self.mailboxes.index.contains_key(id)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn by_id<'x>(cache: &'x MailboxesCacheBuilder, id: &u32) -> Option<&'x MailboxCache> {
|
||||
cache
|
||||
.index
|
||||
.get(id)
|
||||
.and_then(|idx| cache.items.get(*idx as usize))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn by_id_mut<'x>(cache: &'x mut MailboxesCacheBuilder, id: &u32) -> Option<&'x mut MailboxCache> {
|
||||
cache
|
||||
.index
|
||||
.get(id)
|
||||
.and_then(|idx| cache.items.get_mut(*idx as usize))
|
||||
}
|
||||
|
||||
fn mailbox_insert(cache: &mut MailboxesCacheBuilder, item: MailboxCache) {
|
||||
let id = item.document_id;
|
||||
if let Some(idx) = cache.index.get(&id) {
|
||||
cache.items[*idx as usize] = item;
|
||||
} else {
|
||||
let idx = cache.items.len() as u32;
|
||||
cache.items.push(item);
|
||||
cache.index.insert(id, idx);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn item_size(item: &MailboxCache) -> u64 {
|
||||
(std::mem::size_of::<MailboxCache>()
|
||||
+ (if item.name.len() > std::mem::size_of::<String>() {
|
||||
item.name.len()
|
||||
} else {
|
||||
0
|
||||
})
|
||||
+ (if item.path.len() > std::mem::size_of::<String>() {
|
||||
item.path.len()
|
||||
} else {
|
||||
0
|
||||
})) as u64
|
||||
}
|
||||
Vendored
+248
@@ -0,0 +1,248 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use common::{MessageStoreCache, Server, UpdateLock, cache::LockResult};
|
||||
use email::{full_email_cache_build, update_email_cache};
|
||||
use mailbox::{full_mailbox_cache_build, update_mailbox_cache};
|
||||
use std::{collections::hash_map::Entry, sync::Arc, time::Instant};
|
||||
use store::{
|
||||
ahash::AHashMap,
|
||||
query::log::{Change, Query},
|
||||
};
|
||||
use trc::{AddContext, StoreEvent};
|
||||
use types::collection::SyncCollection;
|
||||
use utils::cache::Cache;
|
||||
|
||||
pub mod email;
|
||||
pub mod mailbox;
|
||||
|
||||
pub trait MessageCacheFetch: Sync + Send {
|
||||
fn get_cached_messages(
|
||||
&self,
|
||||
account_id: u32,
|
||||
) -> impl Future<Output = trc::Result<Arc<MessageStoreCache>>> + Send;
|
||||
}
|
||||
|
||||
impl MessageCacheFetch for Server {
|
||||
async fn get_cached_messages(&self, account_id: u32) -> trc::Result<Arc<MessageStoreCache>> {
|
||||
let cache_store = &self.inner.cache.messages;
|
||||
let mut cache = match cache_store.get_value_or_guard_async(&account_id).await {
|
||||
Ok(cache) => cache,
|
||||
Err(guard) => {
|
||||
let start_time = Instant::now();
|
||||
let cache = full_cache_build(self, account_id, Arc::new(UpdateLock::new())).await?;
|
||||
|
||||
if guard.insert(cache.clone()).is_err() {
|
||||
cache_store.update(account_id, cache.clone());
|
||||
}
|
||||
warn_if_uncacheable(cache_store, account_id, &cache);
|
||||
|
||||
trc::event!(
|
||||
Store(StoreEvent::CacheMiss),
|
||||
AccountId = account_id,
|
||||
Collection = SyncCollection::Email.as_str(),
|
||||
Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()],
|
||||
ChangeId = cache.last_change_id,
|
||||
Elapsed = start_time.elapsed(),
|
||||
);
|
||||
|
||||
return Ok(cache);
|
||||
}
|
||||
};
|
||||
|
||||
// Obtain current state
|
||||
let start_time = Instant::now();
|
||||
let changes = self
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.changes(
|
||||
account_id,
|
||||
SyncCollection::Email.into(),
|
||||
Query::Since(cache.last_change_id),
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Regenerate cache if the change log has been truncated
|
||||
if changes.is_truncated {
|
||||
let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?;
|
||||
cache_store.update(account_id, cache.clone());
|
||||
warn_if_uncacheable(cache_store, account_id, &cache);
|
||||
|
||||
trc::event!(
|
||||
Store(StoreEvent::CacheStale),
|
||||
AccountId = account_id,
|
||||
Collection = SyncCollection::Email.as_str(),
|
||||
ChangeId = cache.last_change_id,
|
||||
Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()],
|
||||
Elapsed = start_time.elapsed(),
|
||||
);
|
||||
|
||||
return Ok(cache);
|
||||
}
|
||||
|
||||
// Verify changes
|
||||
if changes.changes.is_empty() {
|
||||
trc::event!(
|
||||
Store(StoreEvent::CacheHit),
|
||||
AccountId = account_id,
|
||||
Collection = SyncCollection::Email.as_str(),
|
||||
ChangeId = cache.last_change_id,
|
||||
Elapsed = start_time.elapsed(),
|
||||
);
|
||||
|
||||
return Ok(cache);
|
||||
}
|
||||
|
||||
// Lock for updates
|
||||
let lock = cache.update_lock.clone();
|
||||
let _permit = match lock.acquire(cache.last_change_id).await? {
|
||||
LockResult::Acquired(permit) => permit,
|
||||
LockResult::Stale(permit) => {
|
||||
cache = cache_store.peek(&account_id).unwrap_or(cache.clone());
|
||||
if cache.last_change_id >= changes.to_change_id {
|
||||
trc::event!(
|
||||
Store(StoreEvent::CacheHit),
|
||||
AccountId = account_id,
|
||||
Collection = SyncCollection::Email.as_str(),
|
||||
ChangeId = cache.last_change_id,
|
||||
Elapsed = start_time.elapsed(),
|
||||
);
|
||||
return Ok(cache);
|
||||
}
|
||||
|
||||
permit
|
||||
}
|
||||
};
|
||||
let mut cache = cache.as_ref().clone();
|
||||
|
||||
let mut changed_items: AHashMap<u32, bool> = AHashMap::with_capacity(changes.changes.len());
|
||||
let mut changed_containers: AHashMap<u32, bool> =
|
||||
AHashMap::with_capacity(changes.changes.len());
|
||||
let mut has_container_property_changes = false;
|
||||
|
||||
for change in changes.changes {
|
||||
match change {
|
||||
Change::InsertItem(id) => match changed_items.entry(id as u32) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
*entry.get_mut() = true;
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(true);
|
||||
}
|
||||
},
|
||||
Change::UpdateItem(id) => {
|
||||
changed_items.insert(id as u32, true);
|
||||
}
|
||||
Change::DeleteItem(id) => {
|
||||
match changed_items.entry(id as u32) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
// Thread reassignment
|
||||
*entry.get_mut() = true;
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Change::InsertContainer(id) | Change::UpdateContainer(id) => {
|
||||
changed_containers.insert(id as u32, true);
|
||||
}
|
||||
Change::DeleteContainer(id) => {
|
||||
changed_containers.insert(id as u32, false);
|
||||
}
|
||||
Change::UpdateContainerProperty(_) => {
|
||||
has_container_property_changes = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !changed_items.is_empty() {
|
||||
let mut email_cache =
|
||||
update_email_cache(self, account_id, &changed_items, &cache).await?;
|
||||
email_cache.change_id = changes.item_change_id.unwrap_or(changes.to_change_id);
|
||||
cache.emails = Arc::new(email_cache);
|
||||
}
|
||||
|
||||
if !changed_containers.is_empty() {
|
||||
let mut mailbox_cache =
|
||||
update_mailbox_cache(self, account_id, &changed_containers, &cache).await?;
|
||||
mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id);
|
||||
cache.mailboxes = Arc::new(mailbox_cache);
|
||||
} else if has_container_property_changes {
|
||||
let mut mailbox_cache = cache.mailboxes.as_ref().clone();
|
||||
mailbox_cache.change_id = changes.container_change_id.unwrap_or(changes.to_change_id);
|
||||
cache.mailboxes = Arc::new(mailbox_cache);
|
||||
}
|
||||
cache.size = cache.emails.size + cache.mailboxes.size;
|
||||
cache.last_change_id = changes.to_change_id;
|
||||
|
||||
cache.update_lock.set_revision(cache.last_change_id);
|
||||
let cache = Arc::new(cache);
|
||||
cache_store.update(account_id, cache.clone());
|
||||
warn_if_uncacheable(cache_store, account_id, &cache);
|
||||
|
||||
trc::event!(
|
||||
Store(StoreEvent::CacheUpdate),
|
||||
AccountId = account_id,
|
||||
Collection = SyncCollection::Email.as_str(),
|
||||
ChangeId = cache.last_change_id,
|
||||
Details = vec![changed_items.len(), changed_containers.len()],
|
||||
Total = vec![cache.emails.items.len(), cache.mailboxes.items.len()],
|
||||
Elapsed = start_time.elapsed(),
|
||||
);
|
||||
|
||||
Ok(cache)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn warn_if_uncacheable(
|
||||
cache_store: &Cache<u32, Arc<MessageStoreCache>>,
|
||||
account_id: u32,
|
||||
cache: &Arc<MessageStoreCache>,
|
||||
) {
|
||||
let capacity = cache_store.weight_capacity();
|
||||
if cache.size > capacity {
|
||||
trc::event!(
|
||||
Store(StoreEvent::CacheEntryTooLarge),
|
||||
AccountId = account_id,
|
||||
Collection = SyncCollection::Email.as_str(),
|
||||
Size = cache.size,
|
||||
Limit = capacity,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn full_cache_build(
|
||||
server: &Server,
|
||||
account_id: u32,
|
||||
update_lock: Arc<UpdateLock>,
|
||||
) -> trc::Result<Arc<MessageStoreCache>> {
|
||||
let last_change_id = server
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.get_last_change_id(account_id, SyncCollection::Email.into())
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.unwrap_or_default();
|
||||
let mut emails = full_email_cache_build(server, account_id).await?;
|
||||
let mut mailboxes = full_mailbox_cache_build(server, account_id).await?;
|
||||
let size = emails.size + mailboxes.size;
|
||||
emails.change_id = last_change_id;
|
||||
mailboxes.change_id = last_change_id;
|
||||
update_lock.set_revision(last_change_id);
|
||||
|
||||
Ok(Arc::new(MessageStoreCache {
|
||||
update_lock,
|
||||
emails: Arc::new(emails),
|
||||
mailboxes: Arc::new(mailboxes),
|
||||
last_change_id,
|
||||
size,
|
||||
}))
|
||||
}
|
||||
Reference in New Issue
Block a user