From 37da91c0a68f5e7dbd4383221284e4685c21378e Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 8 Oct 2026 12:25:31 -0700 Subject: [PATCH 1/2] Sort and filter by received date on the message cache The message cache now carries each message's received date. On an SQL search store a full cache build reads the account's dates from the search index in one query and the metadata of the messages the index task has not reached yet; a refresh keeps the dates of the messages it had and reads the metadata of the new ones, a batch at a time. With any other search store the dates stay unknown and nothing changes. With the dates in the cache, JMAP Email/query and IMAP SORT by received date, Thread/get's ordering, and the before/after/since/on/older/younger filters run on the cache. Upstream answered each of those with a query that pulled every message id of the account from the search store in order, with no limit, on every call. The PostgreSQL search tables also get an (account, column) index for each sorted column, so the sorts that still go to the database walk an ordered index instead of sorting the account's rows. --- crates/common/src/lib.rs | 19 +-- crates/email/src/cache/email.rs | 136 +++++++++++++++++++- crates/imap/src/op/search.rs | 59 ++++++--- crates/jmap/src/email/query.rs | 42 ++++-- crates/jmap/src/thread/get.rs | 15 ++- crates/store/src/backend/mysql/search.rs | 36 +++++- crates/store/src/backend/postgres/main.rs | 12 +- crates/store/src/backend/postgres/search.rs | 42 +++++- crates/store/src/dispatch/search.rs | 51 ++++++++ 9 files changed, 365 insertions(+), 47 deletions(-) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index f51ebca8..1a40f35b 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -9,7 +9,6 @@ // inbuxa: composite stores (sharded members, read replicas) nest store // futures deeply enough to pass rustc's default query depth #![recursion_limit = "512"] - #![warn(clippy::large_futures)] use crate::auth::{AccessTokenInner, EmailAddress}; @@ -65,25 +64,24 @@ use utils::{ snowflake::SnowflakeIdGenerator, }; +pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU) pub mod auth; pub mod cache; -pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU) -pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH) -pub mod privacy; // inbuxa: the personal-data catalog, evaluated -pub mod reachability; // inbuxa: whether the outside world reaches each node's ports pub mod config; +pub mod enterprise; // inbuxa: rebuilt features (AI spam classification) pub mod expr; +pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH) pub mod i18n; pub mod ipc; pub mod manager; pub mod network; -pub mod enterprise; // inbuxa: rebuilt features (AI spam classification) +pub mod privacy; // inbuxa: the personal-data catalog, evaluated +pub mod reachability; // inbuxa: whether the outside world reaches each node's ports pub mod scripts; pub mod sharing; pub mod storage; pub mod telemetry; - pub use psl; pub static VERSION_PRIVATE: &str = env!("CARGO_PKG_VERSION"); @@ -254,6 +252,9 @@ pub struct MessagesCache { pub index: AHashMap, pub keywords: Box<[Box]>, pub size: u64, + /// inbuxa: every item's `received_at` is filled in, so receivedAt + /// sorts and date filters run on the cache instead of the search store + pub has_received_at: bool, } #[derive(Debug, Clone)] @@ -264,6 +265,9 @@ pub struct MessageCache { pub thread_id: u32, pub change_id: u64, pub size: u32, + /// inbuxa: the message's received date (IMAP INTERNALDATE) in seconds, + /// valid when the owning `MessagesCache::has_received_at` is set + pub received_at: u32, } #[derive(Debug, Default, Clone, Copy)] @@ -410,7 +414,6 @@ pub struct Core { pub spam: SpamFilterConfig, pub groupware: GroupwareConfig, pub metrics: Metrics, - } pub trait BuildServer { diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index 6607ebf3..133d61aa 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -6,17 +6,25 @@ * Modified by Coffey Labs in 2026 for INBUXA. */ -use crate::message::metadata::{ArchivedMessageData, MessageData}; +use crate::message::metadata::{ + ArchivedMessageData, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata, +}; use common::{ MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; -use store::write::{AlignedBytes, Archive}; -use store::{READ_BATCH_SIZE, ValueKey, ahash::AHashMap, roaring::RoaringBitmap}; +use store::write::{AlignedBytes, Archive, SearchIndex}; +use store::{ + READ_BATCH_SIZE, ValueKey, + ahash::AHashMap, + roaring::RoaringBitmap, + search::{EmailSearchField, SearchField}, +}; use trc::AddContext; use types::{ acl::Acl, collection::Collection, + field::EmailField, keyword::{Keyword, OTHER}, }; use utils::map::bitmap::Bitmap; @@ -27,6 +35,7 @@ struct MessagesCacheBuilder { pub index: AHashMap, pub keywords: Vec>, pub size: u64, + pub has_received_at: bool, } pub(crate) async fn update_email_cache( @@ -41,6 +50,7 @@ pub(crate) async fn update_email_cache( size: 0, change_id: 0, keywords: store_cache.emails.keywords.to_vec(), + has_received_at: store_cache.emails.has_received_at, }; // inbuxa: one round trip per batch of changed messages, not one per @@ -72,6 +82,22 @@ pub(crate) async fn update_email_cache( } } + // inbuxa: a message's received date never changes, so an updated + // message keeps the date the old cache had; a new one reads its metadata + if new_cache.has_received_at { + let mut new_ids = Vec::new(); + for document_id in &updated { + let Some(idx) = new_cache.index.get(document_id) else { + continue; + }; + match store_cache.email_by_id(document_id) { + Some(old) => new_cache.items[*idx as usize].received_at = old.received_at, + None => new_ids.push(*document_id), + } + } + fill_received_dates(server, account_id, &mut new_cache, new_ids).await?; + } + for item in &store_cache.emails.items { if !changed_ids.contains_key(&item.document_id) { email_insert(&mut new_cache, item.clone()); @@ -92,6 +118,7 @@ pub(crate) async fn full_email_cache_build( keywords: Vec::new(), size: 0, change_id: 0, + has_received_at: false, }; server @@ -111,9 +138,81 @@ pub(crate) async fn full_email_cache_build( .await .caused_by(trc::location!())?; + // inbuxa: the received dates come from the search index in one query + // (an SQL search store), and from the metadata of the messages the + // index task has not reached yet. With another search store the dates + // stay unknown and sorts keep going through the store. + if let Some(values) = server + .search_store() + .unsigned_values( + SearchIndex::Email, + SearchField::Email(EmailSearchField::ReceivedAt), + account_id, + ) + .await + .caused_by(trc::location!())? + { + let mut missing = cache.index.clone(); + for (document_id, received_at) in values { + if let Some(idx) = missing.remove(&document_id) { + cache.items[idx as usize].received_at = received_at.min(u32::MAX as u64) as u32; + } + } + cache.has_received_at = true; + fill_received_dates( + server, + account_id, + &mut cache, + missing.into_keys().collect(), + ) + .await?; + } + Ok(cache.build()) } +/// inbuxa: sets the received date of `document_ids` from their metadata, +/// a batch at a time +async fn fill_received_dates( + server: &Server, + account_id: u32, + cache: &mut MessagesCacheBuilder, + mut document_ids: Vec, +) -> trc::Result<()> { + document_ids.sort_unstable(); + for document_ids in document_ids.chunks(READ_BATCH_SIZE) { + let keys = document_ids + .iter() + .map(|document_id| { + ValueKey::property( + account_id, + Collection::Email, + *document_id, + EmailField::Metadata, + ) + }) + .collect::>(); + for (document_id, metadata) in document_ids.iter().zip( + server + .store() + .get_values::>(&keys) + .await + .caused_by(trc::location!())?, + ) { + if let (Some(metadata), Some(idx)) = (metadata, cache.index.get(document_id)) { + let received_at = metadata + .unarchive::() + .caused_by(trc::location!())? + .rcvd_attach + .to_native() + & MESSAGE_RECEIVED_MASK; + cache.items[*idx as usize].received_at = received_at.min(u32::MAX as u64) as u32; + } + } + } + Ok(()) +} + fn insert_item( cache: &mut MessagesCacheBuilder, document_id: u32, @@ -134,6 +233,7 @@ fn insert_item( change_id: archive.version.change_id().unwrap_or_default(), document_id, size: message.size.to_native(), + received_at: 0, }; for keyword in message.keywords.iter() { match keyword.id() { @@ -163,6 +263,7 @@ impl MessagesCacheBuilder { index: self.index, keywords: self.keywords.into_boxed_slice(), size: self.size, + has_received_at: self.has_received_at, } } } @@ -205,6 +306,14 @@ pub trait MessageCacheAccess { fn expand_keywords(&self, message: &MessageCache) -> impl Iterator; fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; + + /// inbuxa: every message's received date, for a local receivedAt sort; + /// `None` when the cache has no dates (see `MessagesCache`) + fn received_at_order(&self) -> Option>; + + /// inbuxa: the messages received in `[from, to)` seconds, for a local + /// date filter; `None` when the cache has no dates + fn received_between(&self, from: u64, to: u64) -> Option; } impl MessageCacheAccess for MessageStoreCache { @@ -318,6 +427,27 @@ impl MessageCacheAccess for MessageStoreCache { fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool { keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0) } + + fn received_at_order(&self) -> Option> { + self.emails.has_received_at.then(|| { + self.emails + .items + .iter() + .map(|m| (m.document_id, m.received_at)) + .collect() + }) + } + + fn received_between(&self, from: u64, to: u64) -> Option { + self.emails.has_received_at.then(|| { + self.emails + .items + .iter() + .filter(|m| (from..to).contains(&(m.received_at as u64))) + .map(|m| m.document_id) + .collect() + }) + } } fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) { diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 0e2fb57c..4139f885 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use super::{FromModSeq, ToModSeq}; @@ -331,8 +333,13 @@ impl SessionData { .map(|m| m.document_id), ))); } + // inbuxa: date filters run on the cache when it carries the + // received dates (see MessagesCache) Filter::Before(date) => { - filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date)); + filters.push(match cache.received_between(0, date.max(0) as u64) { + Some(set) => SearchFilter::is_in_set(set), + None => SearchFilter::lt(EmailSearchField::ReceivedAt, date), + }); } Filter::Deleted => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( @@ -360,10 +367,16 @@ impl SessionData { filters.push(SearchFilter::gt(EmailSearchField::Size, size)); } Filter::On(date) => { - filters.push(SearchFilter::And); - filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); - filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400)); - filters.push(SearchFilter::End); + if let Some(set) = + cache.received_between(date.max(0) as u64, date.max(0) as u64 + 86400) + { + filters.push(SearchFilter::is_in_set(set)); + } else { + filters.push(SearchFilter::And); + filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); + filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400)); + filters.push(SearchFilter::End); + } } Filter::Seen => { filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( @@ -383,7 +396,10 @@ impl SessionData { filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); } Filter::Since(date) => { - filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); + filters.push(match cache.received_between(date.max(0) as u64, u64::MAX) { + Some(set) => SearchFilter::is_in_set(set), + None => SearchFilter::ge(EmailSearchField::ReceivedAt, date), + }); } Filter::Smaller(size) => { filters.push(SearchFilter::lt(EmailSearchField::Size, size)); @@ -448,16 +464,18 @@ impl SessionData { filters.push(SearchFilter::End);*/ } Filter::Older(secs) => { - filters.push(SearchFilter::le( - EmailSearchField::ReceivedAt, - now().saturating_sub(secs as u64), - )); + let date = now().saturating_sub(secs as u64); + filters.push(match cache.received_between(0, date + 1) { + Some(set) => SearchFilter::is_in_set(set), + None => SearchFilter::le(EmailSearchField::ReceivedAt, date), + }); } Filter::Younger(secs) => { - filters.push(SearchFilter::ge( - EmailSearchField::ReceivedAt, - now().saturating_sub(secs as u64), - )); + let date = now().saturating_sub(secs as u64); + filters.push(match cache.received_between(date, u64::MAX) { + Some(set) => SearchFilter::is_in_set(set), + None => SearchFilter::ge(EmailSearchField::ReceivedAt, date), + }); } Filter::ModSeq((modseq, _)) => { let mut set = RoaringBitmap::new(); @@ -659,9 +677,16 @@ impl SessionData { let mut comparators = Vec::with_capacity(imap_comparator.len()); for comparator in imap_comparator { comparators.push(match comparator.sort { - search::Sort::Arrival => { - SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending) - } + // inbuxa: sorted on the cache when it carries the dates + search::Sort::Arrival => match cache.received_at_order() { + Some(set) => SearchComparator::SortedSet { + set, + ascending: comparator.ascending, + }, + None => { + SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending) + } + }, search::Sort::Cc => { return Err(trc::ImapEvent::Error .into_err() diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index a5cedbb1..fb200ed4 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; @@ -181,14 +183,22 @@ impl EmailQuery for Server { }), ))); } - EmailFilter::Before(date) => filters.push(SearchFilter::lt( - EmailSearchField::ReceivedAt, - date.timestamp(), - )), - EmailFilter::After(date) => filters.push(SearchFilter::gt( - EmailSearchField::ReceivedAt, - date.timestamp(), - )), + // inbuxa: received dates come from the cache when it has + // them, so these filters cost no query (see MessagesCache) + EmailFilter::Before(date) => { + let date = date.timestamp().max(0) as u64; + filters.push(match cached_messages.received_between(0, date) { + Some(set) => SearchFilter::is_in_set(set), + None => SearchFilter::lt(EmailSearchField::ReceivedAt, date), + }) + } + EmailFilter::After(date) => { + let date = date.timestamp().max(0) as u64; + filters.push(match cached_messages.received_between(date + 1, u64::MAX) { + Some(set) => SearchFilter::is_in_set(set), + None => SearchFilter::gt(EmailSearchField::ReceivedAt, date), + }) + } EmailFilter::MinSize(size) => { filters.push(SearchFilter::ge(EmailSearchField::Size, size)) } @@ -282,9 +292,19 @@ impl EmailQuery for Server { .unwrap_or_default() { comparators.push(match comparator.property { - EmailComparator::ReceivedAt => { - SearchComparator::field(EmailSearchField::ReceivedAt, comparator.is_ascending) - } + // inbuxa: a local sort when the cache carries received dates; + // upstream pulled every message id of the account, ordered, + // from the search store on each query + EmailComparator::ReceivedAt => match cached_messages.received_at_order() { + Some(set) => SearchComparator::SortedSet { + set, + ascending: comparator.is_ascending, + }, + None => SearchComparator::field( + EmailSearchField::ReceivedAt, + comparator.is_ascending, + ), + }, EmailComparator::Size => { SearchComparator::field(EmailSearchField::Size, comparator.is_ascending) } diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 654b1b1a..995d0b16 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -2,6 +2,8 @@ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + * + * Modified by Coffey Labs in 2026 for INBUXA. */ use crate::changes::state::StateManager; @@ -95,9 +97,16 @@ impl ThreadGet for Server { SearchQuery::new(SearchIndex::Email) .with_account_id(account_id) .with_mask(all_ids) - .with_comparator(SearchComparator::Field { - field: SearchField::Email(EmailSearchField::ReceivedAt), - ascending: true, + // inbuxa: sorted on the cache when it has the dates + .with_comparator(match cache.received_at_order() { + Some(set) => SearchComparator::SortedSet { + set, + ascending: true, + }, + None => SearchComparator::Field { + field: SearchField::Email(EmailSearchField::ReceivedAt), + ascending: true, + }, }), ) .await?, diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index 7b596ba7..083aab13 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -15,8 +15,8 @@ use crate::{ }, }, search::{ - IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, - SearchQuery, SearchValue, + IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter, + SearchOperator, SearchQuery, SearchValue, }, write::SearchIndex, }; @@ -115,6 +115,38 @@ impl MysqlStore { bounded(conn, result, limit) } + /// inbuxa: every document's value of one unsigned field in an account, + /// in one round trip (see the PostgreSQL backend). + pub async fn unsigned_values( + &self, + index: SearchIndex, + field: SearchField, + account_id: u32, + ) -> trc::Result> { + let query = format!( + "SELECT {}, {} FROM {} WHERE {} = ?", + SearchField::DocumentId.column(), + field.column(), + index.mysql_table(), + SearchField::AccountId.column() + ); + let mut conn = self.conn().await?; + let limit = self.timeouts.query; + let result = tokio::time::timeout(limit, async { + let s = conn.prep(query).await.map_err(into_error)?; + conn.exec::<(i32, i64), _, _>(s, (account_id as i32,)) + .await + .map(|rows| { + rows.into_iter() + .map(|(id, value)| (id as u32, value as u64)) + .collect() + }) + .map_err(into_error) + }) + .await; + bounded(conn, result, limit) + } + pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { let table = filter.index.mysql_table(); let mut query = format!("DELETE FROM {table} "); diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 743d2d5c..9854871d 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -14,7 +14,7 @@ use crate::{ tls::MakeRustlsConnect, }, search::{ - CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField, + CalendarSearchField, ContactSearchField, EmailSearchField, SearchField, SearchableField, TracingSearchField, }, *, @@ -304,6 +304,16 @@ async fn create_search_tables( conn.execute(&create_index_query, &[]) .await .map_err(into_error)?; + // inbuxa: every sorted query is for one account, so an index led + // by the account column answers "this account, in this order" + // with an ordered scan instead of sorting the account's rows + let account_column = SearchField::AccountId.column(); + let create_index_query = format!( + "CREATE INDEX IF NOT EXISTS idx_{table_name}_{account_column}_{column_name} ON {table_name}({account_column}, {column_name})", + ); + conn.execute(&create_index_query, &[]) + .await + .map_err(into_error)?; } } diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index 8a27ef4c..56161e5d 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -15,8 +15,8 @@ use crate::{ }, }, search::{ - IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, - SearchQuery, SearchValue, + IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter, + SearchOperator, SearchQuery, SearchValue, }, write::SearchIndex, }; @@ -197,6 +197,44 @@ impl PostgresStore { bounded(conn, result, limit) } + /// inbuxa: every document's value of one unsigned field in an account, + /// in one round trip. Feeds the message cache's received dates so + /// sorting by receivedAt needs no query at all. + pub async fn unsigned_values( + &self, + index: SearchIndex, + field: SearchField, + account_id: u32, + ) -> trc::Result> { + let query = format!( + "SELECT {}, {} FROM {} WHERE {} = $1", + SearchField::DocumentId.column(), + field.column(), + index.psql_table(), + SearchField::AccountId.column() + ); + let conn = self.conn_pool.get().await.map_err(into_pool_error)?; + let limit = self.timeouts.query; + let result = tokio::time::timeout(limit, async { + let s = conn.prepare_cached(&query).await.map_err(into_error)?; + conn.query(&s, &[&(account_id as i32)]) + .await + .and_then(|rows| { + rows.into_iter() + .map(|row| { + Ok(( + row.try_get::<_, i32>(0)? as u32, + row.try_get::<_, i64>(1)? as u64, + )) + }) + .collect::, _>>() + }) + .map_err(into_error) + }) + .await; + bounded(conn, result, limit) + } + pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { debug_assert!(!filter.filters.is_empty()); let table = filter.index.psql_table(); diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs index b8afd7e3..084c6e72 100644 --- a/crates/store/src/dispatch/search.rs +++ b/crates/store/src/dispatch/search.rs @@ -308,6 +308,57 @@ impl SearchStore { } } + /// inbuxa: every document's value of one unsigned field in an account + /// (one round trip), or `None` when the search store is not SQL and + /// callers must keep sorting through the store. ST-6, ST-12: a replica + /// answers in a read scope. + pub async fn unsigned_values( + &self, + index: SearchIndex, + field: SearchField, + account_id: u32, + ) -> trc::Result>> { + if let SearchStore::Store(Store::Replicated(store)) = self { + if !store.primary.is_pg_or_mysql() { + return Ok(None); + } + return match store.read_target(crate::SUBSPACE_SEARCH_INDEX).await { + Some(replica) => match crate::sql_backend!( + &store.replicas[replica].store, + db => db.unsigned_values(index, field.clone(), account_id).await + ) { + Ok(values) => { + store.served(replica); + Ok(Some(values)) + } + Err(err) => { + store.failed(replica, err); + crate::sql_backend!(&store.primary, db => db.unsigned_values(index, field, account_id).await) + .map(Some) + } + }, + None => { + crate::sql_backend!(&store.primary, db => db.unsigned_values(index, field, account_id).await) + .map(Some) + } + }; + } + match self { + #[cfg(feature = "postgres")] + SearchStore::Store(Store::PostgreSQL(store)) => store + .unsigned_values(index, field, account_id) + .await + .map(Some), + #[cfg(feature = "mysql")] + SearchStore::Store(Store::MySQL(store)) => store + .unsigned_values(index, field, account_id) + .await + .map(Some), + _ => Ok(None), + } + .caused_by(trc::location!()) + } + pub fn internal_fts(&self) -> Option<&Store> { match self { SearchStore::Store(store) => match store { -- 2.54.0 From 68a67be170713030c4b72437bba3d9ff36e14cf7 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Thu, 8 Oct 2026 12:41:40 -0700 Subject: [PATCH 2/2] Keep the received date whole and answer a mask-only query The cache holds the date as u64 and the local sort ranks the messages, so dates past 2106 no longer tie. A query with no filter beyond the account used to fall into the local filter, which skips operators and answered with nothing; it now starts from the mask. --- crates/common/src/lib.rs | 2 +- crates/email/src/cache/email.rs | 32 ++++++++++++++++++++++------- crates/store/src/dispatch/search.rs | 19 +++++++++++------ 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 1a40f35b..70be56e7 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -267,7 +267,7 @@ pub struct MessageCache { pub size: u32, /// inbuxa: the message's received date (IMAP INTERNALDATE) in seconds, /// valid when the owning `MessagesCache::has_received_at` is set - pub received_at: u32, + pub received_at: u64, } #[derive(Debug, Default, Clone, Copy)] diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index 133d61aa..23a454e7 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -155,7 +155,7 @@ pub(crate) async fn full_email_cache_build( let mut missing = cache.index.clone(); for (document_id, received_at) in values { if let Some(idx) = missing.remove(&document_id) { - cache.items[idx as usize].received_at = received_at.min(u32::MAX as u64) as u32; + cache.items[idx as usize].received_at = received_at; } } cache.has_received_at = true; @@ -206,7 +206,8 @@ async fn fill_received_dates( .rcvd_attach .to_native() & MESSAGE_RECEIVED_MASK; - cache.items[*idx as usize].received_at = received_at.min(u32::MAX as u64) as u32; + cache.items[*idx as usize].received_at = received_at; + } else { } } } @@ -307,8 +308,9 @@ pub trait MessageCacheAccess { fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool; - /// inbuxa: every message's received date, for a local receivedAt sort; - /// `None` when the cache has no dates (see `MessagesCache`) + /// inbuxa: every message's rank by received date, for a local + /// receivedAt sort; `None` when the cache has no dates (see + /// `MessagesCache`) fn received_at_order(&self) -> Option>; /// inbuxa: the messages received in `[from, to)` seconds, for a local @@ -429,11 +431,27 @@ impl MessageCacheAccess for MessageStoreCache { } fn received_at_order(&self) -> Option> { + // Ranks rather than the dates themselves: a sorted set holds u32 + // positions, and dates past 2106 would otherwise tie self.emails.has_received_at.then(|| { - self.emails + let mut order = self + .emails .items .iter() - .map(|m| (m.document_id, m.received_at)) + .map(|m| (m.received_at, m.document_id)) + .collect::>(); + order.sort_unstable(); + let mut rank = 0; + let mut last = None; + order + .into_iter() + .map(|(received_at, document_id)| { + if last != Some(received_at) { + last = Some(received_at); + rank += 1; + } + (document_id, rank) + }) .collect() }) } @@ -443,7 +461,7 @@ impl MessageCacheAccess for MessageStoreCache { self.emails .items .iter() - .filter(|m| (from..to).contains(&(m.received_at as u64))) + .filter(|m| (from..to).contains(&m.received_at)) .map(|m| m.document_id) .collect() }) diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs index 084c6e72..93d9e20d 100644 --- a/crates/store/src/dispatch/search.rs +++ b/crates/store/src/dispatch/search.rs @@ -9,8 +9,8 @@ use crate::{ SearchStore, Store, search::{ - IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, - SearchValue, + IndexDocument, QueryResults, SearchComparator, SearchField, SearchFilter, SearchOperator, + SearchQuery, SearchValue, split::{SplitFilter, split_filters}, }, write::SearchIndex, @@ -113,10 +113,17 @@ impl SearchStore { }; // Merge results locally - let results = SearchQuery::new(query.index) - .with_filters(filters) - .with_mask(query.mask) - .filter(); + let results = if has_local_filters || has_external_filters { + SearchQuery::new(query.index) + .with_filters(filters) + .with_mask(query.mask) + .filter() + } else { + // inbuxa: only the account filter, which the local filter skips, + // and it would answer with nothing: the result is the mask, + // sorted by the local comparators below + QueryResults::new(query.mask, Vec::new()) + }; let total_results = results.results().len(); match total_results.cmp(&1) { -- 2.54.0