Sort and filter by received date on the message cache #172

Merged
jcoffey-dev merged 2 commits from perf/local-received-at into main 2026-10-08 20:02:22 +00:00
9 changed files with 396 additions and 53 deletions

No files matched your search

+11 -8
View File
@@ -9,7 +9,6 @@
// inbuxa: composite stores (sharded members, read replicas) nest store
// futures deeply enough to pass rustc's default query depth
#![recursion_limit = "512"]
#![warn(clippy::large_futures)]
use crate::auth::{AccessTokenInner, EmailAddress};
@@ -65,25 +64,24 @@ use utils::{
snowflake::SnowflakeIdGenerator,
};
pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU)
pub mod auth;
pub mod cache;
pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU)
pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH)
pub mod privacy; // inbuxa: the personal-data catalog, evaluated
pub mod reachability; // inbuxa: whether the outside world reaches each node's ports
pub mod config;
pub mod enterprise; // inbuxa: rebuilt features (AI spam classification)
pub mod expr;
pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH)
pub mod i18n;
pub mod ipc;
pub mod manager;
pub mod network;
pub mod enterprise; // inbuxa: rebuilt features (AI spam classification)
pub mod privacy; // inbuxa: the personal-data catalog, evaluated
pub mod reachability; // inbuxa: whether the outside world reaches each node's ports
pub mod scripts;
pub mod sharing;
pub mod storage;
pub mod telemetry;
pub use psl;
pub static VERSION_PRIVATE: &str = env!("CARGO_PKG_VERSION");
@@ -254,6 +252,9 @@ pub struct MessagesCache {
pub index: AHashMap<u32, u32>,
pub keywords: Box<[Box<str>]>,
pub size: u64,
/// inbuxa: every item's `received_at` is filled in, so receivedAt
/// sorts and date filters run on the cache instead of the search store
pub has_received_at: bool,
}
#[derive(Debug, Clone)]
@@ -264,6 +265,9 @@ pub struct MessageCache {
pub thread_id: u32,
pub change_id: u64,
pub size: u32,
/// inbuxa: the message's received date (IMAP INTERNALDATE) in seconds,
/// valid when the owning `MessagesCache::has_received_at` is set
pub received_at: u64,
}
#[derive(Debug, Default, Clone, Copy)]
@@ -410,7 +414,6 @@ pub struct Core {
pub spam: SpamFilterConfig,
pub groupware: GroupwareConfig,
pub metrics: Metrics,
}
pub trait BuildServer {
+151 -3
View File
@@ -6,17 +6,25 @@
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::message::metadata::{ArchivedMessageData, MessageData};
use crate::message::metadata::{
ArchivedMessageData, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata,
};
use common::{
MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken,
sharing::EffectiveAcl,
};
use store::write::{AlignedBytes, Archive};
use store::{READ_BATCH_SIZE, ValueKey, ahash::AHashMap, roaring::RoaringBitmap};
use store::write::{AlignedBytes, Archive, SearchIndex};
use store::{
READ_BATCH_SIZE, ValueKey,
ahash::AHashMap,
roaring::RoaringBitmap,
search::{EmailSearchField, SearchField},
};
use trc::AddContext;
use types::{
acl::Acl,
collection::Collection,
field::EmailField,
keyword::{Keyword, OTHER},
};
use utils::map::bitmap::Bitmap;
@@ -27,6 +35,7 @@ struct MessagesCacheBuilder {
pub index: AHashMap<u32, u32>,
pub keywords: Vec<Box<str>>,
pub size: u64,
pub has_received_at: bool,
}
pub(crate) async fn update_email_cache(
@@ -41,6 +50,7 @@ pub(crate) async fn update_email_cache(
size: 0,
change_id: 0,
keywords: store_cache.emails.keywords.to_vec(),
has_received_at: store_cache.emails.has_received_at,
};
// inbuxa: one round trip per batch of changed messages, not one per
@@ -72,6 +82,22 @@ pub(crate) async fn update_email_cache(
}
}
// inbuxa: a message's received date never changes, so an updated
// message keeps the date the old cache had; a new one reads its metadata
if new_cache.has_received_at {
let mut new_ids = Vec::new();
for document_id in &updated {
let Some(idx) = new_cache.index.get(document_id) else {
continue;
};
match store_cache.email_by_id(document_id) {
Some(old) => new_cache.items[*idx as usize].received_at = old.received_at,
None => new_ids.push(*document_id),
}
}
fill_received_dates(server, account_id, &mut new_cache, new_ids).await?;
}
for item in &store_cache.emails.items {
if !changed_ids.contains_key(&item.document_id) {
email_insert(&mut new_cache, item.clone());
@@ -92,6 +118,7 @@ pub(crate) async fn full_email_cache_build(
keywords: Vec::new(),
size: 0,
change_id: 0,
has_received_at: false,
};
server
@@ -111,9 +138,82 @@ pub(crate) async fn full_email_cache_build(
.await
.caused_by(trc::location!())?;
// inbuxa: the received dates come from the search index in one query
// (an SQL search store), and from the metadata of the messages the
// index task has not reached yet. With another search store the dates
// stay unknown and sorts keep going through the store.
if let Some(values) = server
.search_store()
.unsigned_values(
SearchIndex::Email,
SearchField::Email(EmailSearchField::ReceivedAt),
account_id,
)
.await
.caused_by(trc::location!())?
{
let mut missing = cache.index.clone();
for (document_id, received_at) in values {
if let Some(idx) = missing.remove(&document_id) {
cache.items[idx as usize].received_at = received_at;
}
}
cache.has_received_at = true;
fill_received_dates(
server,
account_id,
&mut cache,
missing.into_keys().collect(),
)
.await?;
}
Ok(cache.build())
}
/// inbuxa: sets the received date of `document_ids` from their metadata,
/// a batch at a time
async fn fill_received_dates(
server: &Server,
account_id: u32,
cache: &mut MessagesCacheBuilder,
mut document_ids: Vec<u32>,
) -> trc::Result<()> {
document_ids.sort_unstable();
for document_ids in document_ids.chunks(READ_BATCH_SIZE) {
let keys = document_ids
.iter()
.map(|document_id| {
ValueKey::property(
account_id,
Collection::Email,
*document_id,
EmailField::Metadata,
)
})
.collect::<Vec<_>>();
for (document_id, metadata) in document_ids.iter().zip(
server
.store()
.get_values::<Archive<AlignedBytes>>(&keys)
.await
.caused_by(trc::location!())?,
) {
if let (Some(metadata), Some(idx)) = (metadata, cache.index.get(document_id)) {
let received_at = metadata
.unarchive::<MessageMetadata>()
.caused_by(trc::location!())?
.rcvd_attach
.to_native()
& MESSAGE_RECEIVED_MASK;
cache.items[*idx as usize].received_at = received_at;
} else {
}
}
}
Ok(())
}
fn insert_item(
cache: &mut MessagesCacheBuilder,
document_id: u32,
@@ -134,6 +234,7 @@ fn insert_item(
change_id: archive.version.change_id().unwrap_or_default(),
document_id,
size: message.size.to_native(),
received_at: 0,
};
for keyword in message.keywords.iter() {
match keyword.id() {
@@ -163,6 +264,7 @@ impl MessagesCacheBuilder {
index: self.index,
keywords: self.keywords.into_boxed_slice(),
size: self.size,
has_received_at: self.has_received_at,
}
}
}
@@ -205,6 +307,15 @@ pub trait MessageCacheAccess {
fn expand_keywords(&self, message: &MessageCache) -> impl Iterator<Item = Keyword>;
fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool;
/// inbuxa: every message's rank by received date, for a local
/// receivedAt sort; `None` when the cache has no dates (see
/// `MessagesCache`)
fn received_at_order(&self) -> Option<AHashMap<u32, u32>>;
/// inbuxa: the messages received in `[from, to)` seconds, for a local
/// date filter; `None` when the cache has no dates
fn received_between(&self, from: u64, to: u64) -> Option<RoaringBitmap>;
}
impl MessageCacheAccess for MessageStoreCache {
@@ -318,6 +429,43 @@ impl MessageCacheAccess for MessageStoreCache {
fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool {
keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0)
}
fn received_at_order(&self) -> Option<AHashMap<u32, u32>> {
// Ranks rather than the dates themselves: a sorted set holds u32
// positions, and dates past 2106 would otherwise tie
self.emails.has_received_at.then(|| {
let mut order = self
.emails
.items
.iter()
.map(|m| (m.received_at, m.document_id))
.collect::<Vec<_>>();
order.sort_unstable();
let mut rank = 0;
let mut last = None;
order
.into_iter()
.map(|(received_at, document_id)| {
if last != Some(received_at) {
last = Some(received_at);
rank += 1;
}
(document_id, rank)
})
.collect()
})
}
fn received_between(&self, from: u64, to: u64) -> Option<RoaringBitmap> {
self.emails.has_received_at.then(|| {
self.emails
.items
.iter()
.filter(|m| (from..to).contains(&m.received_at))
.map(|m| m.document_id)
.collect()
})
}
}
fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) {
+42 -17
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use super::{FromModSeq, ToModSeq};
@@ -331,8 +333,13 @@ impl<T: SessionStream> SessionData<T> {
.map(|m| m.document_id),
)));
}
// inbuxa: date filters run on the cache when it carries the
// received dates (see MessagesCache)
Filter::Before(date) => {
filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date));
filters.push(match cache.received_between(0, date.max(0) as u64) {
Some(set) => SearchFilter::is_in_set(set),
None => SearchFilter::lt(EmailSearchField::ReceivedAt, date),
});
}
Filter::Deleted => {
filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter(
@@ -360,10 +367,16 @@ impl<T: SessionStream> SessionData<T> {
filters.push(SearchFilter::gt(EmailSearchField::Size, size));
}
Filter::On(date) => {
filters.push(SearchFilter::And);
filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date));
filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400));
filters.push(SearchFilter::End);
if let Some(set) =
cache.received_between(date.max(0) as u64, date.max(0) as u64 + 86400)
{
filters.push(SearchFilter::is_in_set(set));
} else {
filters.push(SearchFilter::And);
filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date));
filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400));
filters.push(SearchFilter::End);
}
}
Filter::Seen => {
filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter(
@@ -383,7 +396,10 @@ impl<T: SessionStream> SessionData<T> {
filters.push(SearchFilter::ge(EmailSearchField::SentAt, date));
}
Filter::Since(date) => {
filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date));
filters.push(match cache.received_between(date.max(0) as u64, u64::MAX) {
Some(set) => SearchFilter::is_in_set(set),
None => SearchFilter::ge(EmailSearchField::ReceivedAt, date),
});
}
Filter::Smaller(size) => {
filters.push(SearchFilter::lt(EmailSearchField::Size, size));
@@ -448,16 +464,18 @@ impl<T: SessionStream> SessionData<T> {
filters.push(SearchFilter::End);*/
}
Filter::Older(secs) => {
filters.push(SearchFilter::le(
EmailSearchField::ReceivedAt,
now().saturating_sub(secs as u64),
));
let date = now().saturating_sub(secs as u64);
filters.push(match cache.received_between(0, date + 1) {
Some(set) => SearchFilter::is_in_set(set),
None => SearchFilter::le(EmailSearchField::ReceivedAt, date),
});
}
Filter::Younger(secs) => {
filters.push(SearchFilter::ge(
EmailSearchField::ReceivedAt,
now().saturating_sub(secs as u64),
));
let date = now().saturating_sub(secs as u64);
filters.push(match cache.received_between(date, u64::MAX) {
Some(set) => SearchFilter::is_in_set(set),
None => SearchFilter::ge(EmailSearchField::ReceivedAt, date),
});
}
Filter::ModSeq((modseq, _)) => {
let mut set = RoaringBitmap::new();
@@ -659,9 +677,16 @@ impl<T: SessionStream> SessionData<T> {
let mut comparators = Vec::with_capacity(imap_comparator.len());
for comparator in imap_comparator {
comparators.push(match comparator.sort {
search::Sort::Arrival => {
SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending)
}
// inbuxa: sorted on the cache when it carries the dates
search::Sort::Arrival => match cache.received_at_order() {
Some(set) => SearchComparator::SortedSet {
set,
ascending: comparator.ascending,
},
None => {
SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending)
}
},
search::Sort::Cc => {
return Err(trc::ImapEvent::Error
.into_err()
+31 -11
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState};
@@ -181,14 +183,22 @@ impl EmailQuery for Server {
}),
)));
}
EmailFilter::Before(date) => filters.push(SearchFilter::lt(
EmailSearchField::ReceivedAt,
date.timestamp(),
)),
EmailFilter::After(date) => filters.push(SearchFilter::gt(
EmailSearchField::ReceivedAt,
date.timestamp(),
)),
// inbuxa: received dates come from the cache when it has
// them, so these filters cost no query (see MessagesCache)
EmailFilter::Before(date) => {
let date = date.timestamp().max(0) as u64;
filters.push(match cached_messages.received_between(0, date) {
Some(set) => SearchFilter::is_in_set(set),
None => SearchFilter::lt(EmailSearchField::ReceivedAt, date),
})
}
EmailFilter::After(date) => {
let date = date.timestamp().max(0) as u64;
filters.push(match cached_messages.received_between(date + 1, u64::MAX) {
Some(set) => SearchFilter::is_in_set(set),
None => SearchFilter::gt(EmailSearchField::ReceivedAt, date),
})
}
EmailFilter::MinSize(size) => {
filters.push(SearchFilter::ge(EmailSearchField::Size, size))
}
@@ -282,9 +292,19 @@ impl EmailQuery for Server {
.unwrap_or_default()
{
comparators.push(match comparator.property {
EmailComparator::ReceivedAt => {
SearchComparator::field(EmailSearchField::ReceivedAt, comparator.is_ascending)
}
// inbuxa: a local sort when the cache carries received dates;
// upstream pulled every message id of the account, ordered,
// from the search store on each query
EmailComparator::ReceivedAt => match cached_messages.received_at_order() {
Some(set) => SearchComparator::SortedSet {
set,
ascending: comparator.is_ascending,
},
None => SearchComparator::field(
EmailSearchField::ReceivedAt,
comparator.is_ascending,
),
},
EmailComparator::Size => {
SearchComparator::field(EmailSearchField::Size, comparator.is_ascending)
}
+12 -3
View File
@@ -2,6 +2,8 @@
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*
* Modified by Coffey Labs in 2026 for INBUXA.
*/
use crate::changes::state::StateManager;
@@ -95,9 +97,16 @@ impl ThreadGet for Server {
SearchQuery::new(SearchIndex::Email)
.with_account_id(account_id)
.with_mask(all_ids)
.with_comparator(SearchComparator::Field {
field: SearchField::Email(EmailSearchField::ReceivedAt),
ascending: true,
// inbuxa: sorted on the cache when it has the dates
.with_comparator(match cache.received_at_order() {
Some(set) => SearchComparator::SortedSet {
set,
ascending: true,
},
None => SearchComparator::Field {
field: SearchField::Email(EmailSearchField::ReceivedAt),
ascending: true,
},
}),
)
.await?,
+34 -2
View File
@@ -15,8 +15,8 @@ use crate::{
},
},
search::{
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
SearchQuery, SearchValue,
IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter,
SearchOperator, SearchQuery, SearchValue,
},
write::SearchIndex,
};
@@ -115,6 +115,38 @@ impl MysqlStore {
bounded(conn, result, limit)
}
/// inbuxa: every document's value of one unsigned field in an account,
/// in one round trip (see the PostgreSQL backend).
pub async fn unsigned_values(
&self,
index: SearchIndex,
field: SearchField,
account_id: u32,
) -> trc::Result<Vec<(u32, u64)>> {
let query = format!(
"SELECT {}, {} FROM {} WHERE {} = ?",
SearchField::DocumentId.column(),
field.column(),
index.mysql_table(),
SearchField::AccountId.column()
);
let mut conn = self.conn().await?;
let limit = self.timeouts.query;
let result = tokio::time::timeout(limit, async {
let s = conn.prep(query).await.map_err(into_error)?;
conn.exec::<(i32, i64), _, _>(s, (account_id as i32,))
.await
.map(|rows| {
rows.into_iter()
.map(|(id, value)| (id as u32, value as u64))
.collect()
})
.map_err(into_error)
})
.await;
bounded(conn, result, limit)
}
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
let table = filter.index.mysql_table();
let mut query = format!("DELETE FROM {table} ");
+11 -1
View File
@@ -14,7 +14,7 @@ use crate::{
tls::MakeRustlsConnect,
},
search::{
CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField,
CalendarSearchField, ContactSearchField, EmailSearchField, SearchField, SearchableField,
TracingSearchField,
},
*,
@@ -304,6 +304,16 @@ async fn create_search_tables<T: SearchableField + PsqlSearchField + 'static>(
conn.execute(&create_index_query, &[])
.await
.map_err(into_error)?;
// inbuxa: every sorted query is for one account, so an index led
// by the account column answers "this account, in this order"
// with an ordered scan instead of sorting the account's rows
let account_column = SearchField::AccountId.column();
let create_index_query = format!(
"CREATE INDEX IF NOT EXISTS idx_{table_name}_{account_column}_{column_name} ON {table_name}({account_column}, {column_name})",
);
conn.execute(&create_index_query, &[])
.await
.map_err(into_error)?;
}
}
+40 -2
View File
@@ -15,8 +15,8 @@ use crate::{
},
},
search::{
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
SearchQuery, SearchValue,
IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter,
SearchOperator, SearchQuery, SearchValue,
},
write::SearchIndex,
};
@@ -197,6 +197,44 @@ impl PostgresStore {
bounded(conn, result, limit)
}
/// inbuxa: every document's value of one unsigned field in an account,
/// in one round trip. Feeds the message cache's received dates so
/// sorting by receivedAt needs no query at all.
pub async fn unsigned_values(
&self,
index: SearchIndex,
field: SearchField,
account_id: u32,
) -> trc::Result<Vec<(u32, u64)>> {
let query = format!(
"SELECT {}, {} FROM {} WHERE {} = $1",
SearchField::DocumentId.column(),
field.column(),
index.psql_table(),
SearchField::AccountId.column()
);
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
let limit = self.timeouts.query;
let result = tokio::time::timeout(limit, async {
let s = conn.prepare_cached(&query).await.map_err(into_error)?;
conn.query(&s, &[&(account_id as i32)])
.await
.and_then(|rows| {
rows.into_iter()
.map(|row| {
Ok((
row.try_get::<_, i32>(0)? as u32,
row.try_get::<_, i64>(1)? as u64,
))
})
.collect::<Result<Vec<_>, _>>()
})
.map_err(into_error)
})
.await;
bounded(conn, result, limit)
}
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
debug_assert!(!filter.filters.is_empty());
let table = filter.index.psql_table();
+64 -6
View File
@@ -9,8 +9,8 @@
use crate::{
SearchStore, Store,
search::{
IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery,
SearchValue,
IndexDocument, QueryResults, SearchComparator, SearchField, SearchFilter, SearchOperator,
SearchQuery, SearchValue,
split::{SplitFilter, split_filters},
},
write::SearchIndex,
@@ -113,10 +113,17 @@ impl SearchStore {
};
// Merge results locally
let results = SearchQuery::new(query.index)
.with_filters(filters)
.with_mask(query.mask)
.filter();
let results = if has_local_filters || has_external_filters {
SearchQuery::new(query.index)
.with_filters(filters)
.with_mask(query.mask)
.filter()
} else {
// inbuxa: only the account filter, which the local filter skips,
// and it would answer with nothing: the result is the mask,
// sorted by the local comparators below
QueryResults::new(query.mask, Vec::new())
};
let total_results = results.results().len();
match total_results.cmp(&1) {
@@ -308,6 +315,57 @@ impl SearchStore {
}
}
/// inbuxa: every document's value of one unsigned field in an account
/// (one round trip), or `None` when the search store is not SQL and
/// callers must keep sorting through the store. ST-6, ST-12: a replica
/// answers in a read scope.
pub async fn unsigned_values(
&self,
index: SearchIndex,
field: SearchField,
account_id: u32,
) -> trc::Result<Option<Vec<(u32, u64)>>> {
if let SearchStore::Store(Store::Replicated(store)) = self {
if !store.primary.is_pg_or_mysql() {
return Ok(None);
}
return match store.read_target(crate::SUBSPACE_SEARCH_INDEX).await {
Some(replica) => match crate::sql_backend!(
&store.replicas[replica].store,
db => db.unsigned_values(index, field.clone(), account_id).await
) {
Ok(values) => {
store.served(replica);
Ok(Some(values))
}
Err(err) => {
store.failed(replica, err);
crate::sql_backend!(&store.primary, db => db.unsigned_values(index, field, account_id).await)
.map(Some)
}
},
None => {
crate::sql_backend!(&store.primary, db => db.unsigned_values(index, field, account_id).await)
.map(Some)
}
};
}
match self {
#[cfg(feature = "postgres")]
SearchStore::Store(Store::PostgreSQL(store)) => store
.unsigned_values(index, field, account_id)
.await
.map(Some),
#[cfg(feature = "mysql")]
SearchStore::Store(Store::MySQL(store)) => store
.unsigned_values(index, field, account_id)
.await
.map(Some),
_ => Ok(None),
}
.caused_by(trc::location!())
}
pub fn internal_fts(&self) -> Option<&Store> {
match self {
SearchStore::Store(store) => match store {