Sort and filter by received date on the message cache #172
No files matched your search
@@ -9,7 +9,6 @@
|
||||
// inbuxa: composite stores (sharded members, read replicas) nest store
|
||||
// futures deeply enough to pass rustc's default query depth
|
||||
#![recursion_limit = "512"]
|
||||
|
||||
#![warn(clippy::large_futures)]
|
||||
|
||||
use crate::auth::{AccessTokenInner, EmailAddress};
|
||||
@@ -65,25 +64,24 @@ use utils::{
|
||||
snowflake::SnowflakeIdGenerator,
|
||||
};
|
||||
|
||||
pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU)
|
||||
pub mod auth;
|
||||
pub mod cache;
|
||||
pub mod audit; // inbuxa: the audit log (audit-hold-lock spec, AU)
|
||||
pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH)
|
||||
pub mod privacy; // inbuxa: the personal-data catalog, evaluated
|
||||
pub mod reachability; // inbuxa: whether the outside world reaches each node's ports
|
||||
pub mod config;
|
||||
pub mod enterprise; // inbuxa: rebuilt features (AI spam classification)
|
||||
pub mod expr;
|
||||
pub mod hold; // inbuxa: legal holds (audit-hold-lock spec, LH)
|
||||
pub mod i18n;
|
||||
pub mod ipc;
|
||||
pub mod manager;
|
||||
pub mod network;
|
||||
pub mod enterprise; // inbuxa: rebuilt features (AI spam classification)
|
||||
pub mod privacy; // inbuxa: the personal-data catalog, evaluated
|
||||
pub mod reachability; // inbuxa: whether the outside world reaches each node's ports
|
||||
pub mod scripts;
|
||||
pub mod sharing;
|
||||
pub mod storage;
|
||||
pub mod telemetry;
|
||||
|
||||
|
||||
pub use psl;
|
||||
|
||||
pub static VERSION_PRIVATE: &str = env!("CARGO_PKG_VERSION");
|
||||
@@ -254,6 +252,9 @@ pub struct MessagesCache {
|
||||
pub index: AHashMap<u32, u32>,
|
||||
pub keywords: Box<[Box<str>]>,
|
||||
pub size: u64,
|
||||
/// inbuxa: every item's `received_at` is filled in, so receivedAt
|
||||
/// sorts and date filters run on the cache instead of the search store
|
||||
pub has_received_at: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -264,6 +265,9 @@ pub struct MessageCache {
|
||||
pub thread_id: u32,
|
||||
pub change_id: u64,
|
||||
pub size: u32,
|
||||
/// inbuxa: the message's received date (IMAP INTERNALDATE) in seconds,
|
||||
/// valid when the owning `MessagesCache::has_received_at` is set
|
||||
pub received_at: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
@@ -410,7 +414,6 @@ pub struct Core {
|
||||
pub spam: SpamFilterConfig,
|
||||
pub groupware: GroupwareConfig,
|
||||
pub metrics: Metrics,
|
||||
|
||||
}
|
||||
|
||||
pub trait BuildServer {
|
||||
|
||||
Vendored
+151
-3
@@ -6,17 +6,25 @@
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::message::metadata::{ArchivedMessageData, MessageData};
|
||||
use crate::message::metadata::{
|
||||
ArchivedMessageData, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata,
|
||||
};
|
||||
use common::{
|
||||
MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken,
|
||||
sharing::EffectiveAcl,
|
||||
};
|
||||
use store::write::{AlignedBytes, Archive};
|
||||
use store::{READ_BATCH_SIZE, ValueKey, ahash::AHashMap, roaring::RoaringBitmap};
|
||||
use store::write::{AlignedBytes, Archive, SearchIndex};
|
||||
use store::{
|
||||
READ_BATCH_SIZE, ValueKey,
|
||||
ahash::AHashMap,
|
||||
roaring::RoaringBitmap,
|
||||
search::{EmailSearchField, SearchField},
|
||||
};
|
||||
use trc::AddContext;
|
||||
use types::{
|
||||
acl::Acl,
|
||||
collection::Collection,
|
||||
field::EmailField,
|
||||
keyword::{Keyword, OTHER},
|
||||
};
|
||||
use utils::map::bitmap::Bitmap;
|
||||
@@ -27,6 +35,7 @@ struct MessagesCacheBuilder {
|
||||
pub index: AHashMap<u32, u32>,
|
||||
pub keywords: Vec<Box<str>>,
|
||||
pub size: u64,
|
||||
pub has_received_at: bool,
|
||||
}
|
||||
|
||||
pub(crate) async fn update_email_cache(
|
||||
@@ -41,6 +50,7 @@ pub(crate) async fn update_email_cache(
|
||||
size: 0,
|
||||
change_id: 0,
|
||||
keywords: store_cache.emails.keywords.to_vec(),
|
||||
has_received_at: store_cache.emails.has_received_at,
|
||||
};
|
||||
|
||||
// inbuxa: one round trip per batch of changed messages, not one per
|
||||
@@ -72,6 +82,22 @@ pub(crate) async fn update_email_cache(
|
||||
}
|
||||
}
|
||||
|
||||
// inbuxa: a message's received date never changes, so an updated
|
||||
// message keeps the date the old cache had; a new one reads its metadata
|
||||
if new_cache.has_received_at {
|
||||
let mut new_ids = Vec::new();
|
||||
for document_id in &updated {
|
||||
let Some(idx) = new_cache.index.get(document_id) else {
|
||||
continue;
|
||||
};
|
||||
match store_cache.email_by_id(document_id) {
|
||||
Some(old) => new_cache.items[*idx as usize].received_at = old.received_at,
|
||||
None => new_ids.push(*document_id),
|
||||
}
|
||||
}
|
||||
fill_received_dates(server, account_id, &mut new_cache, new_ids).await?;
|
||||
}
|
||||
|
||||
for item in &store_cache.emails.items {
|
||||
if !changed_ids.contains_key(&item.document_id) {
|
||||
email_insert(&mut new_cache, item.clone());
|
||||
@@ -92,6 +118,7 @@ pub(crate) async fn full_email_cache_build(
|
||||
keywords: Vec::new(),
|
||||
size: 0,
|
||||
change_id: 0,
|
||||
has_received_at: false,
|
||||
};
|
||||
|
||||
server
|
||||
@@ -111,9 +138,82 @@ pub(crate) async fn full_email_cache_build(
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// inbuxa: the received dates come from the search index in one query
|
||||
// (an SQL search store), and from the metadata of the messages the
|
||||
// index task has not reached yet. With another search store the dates
|
||||
// stay unknown and sorts keep going through the store.
|
||||
if let Some(values) = server
|
||||
.search_store()
|
||||
.unsigned_values(
|
||||
SearchIndex::Email,
|
||||
SearchField::Email(EmailSearchField::ReceivedAt),
|
||||
account_id,
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
let mut missing = cache.index.clone();
|
||||
for (document_id, received_at) in values {
|
||||
if let Some(idx) = missing.remove(&document_id) {
|
||||
cache.items[idx as usize].received_at = received_at;
|
||||
}
|
||||
}
|
||||
cache.has_received_at = true;
|
||||
fill_received_dates(
|
||||
server,
|
||||
account_id,
|
||||
&mut cache,
|
||||
missing.into_keys().collect(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(cache.build())
|
||||
}
|
||||
|
||||
/// inbuxa: sets the received date of `document_ids` from their metadata,
|
||||
/// a batch at a time
|
||||
async fn fill_received_dates(
|
||||
server: &Server,
|
||||
account_id: u32,
|
||||
cache: &mut MessagesCacheBuilder,
|
||||
mut document_ids: Vec<u32>,
|
||||
) -> trc::Result<()> {
|
||||
document_ids.sort_unstable();
|
||||
for document_ids in document_ids.chunks(READ_BATCH_SIZE) {
|
||||
let keys = document_ids
|
||||
.iter()
|
||||
.map(|document_id| {
|
||||
ValueKey::property(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
*document_id,
|
||||
EmailField::Metadata,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (document_id, metadata) in document_ids.iter().zip(
|
||||
server
|
||||
.store()
|
||||
.get_values::<Archive<AlignedBytes>>(&keys)
|
||||
.await
|
||||
.caused_by(trc::location!())?,
|
||||
) {
|
||||
if let (Some(metadata), Some(idx)) = (metadata, cache.index.get(document_id)) {
|
||||
let received_at = metadata
|
||||
.unarchive::<MessageMetadata>()
|
||||
.caused_by(trc::location!())?
|
||||
.rcvd_attach
|
||||
.to_native()
|
||||
& MESSAGE_RECEIVED_MASK;
|
||||
cache.items[*idx as usize].received_at = received_at;
|
||||
} else {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_item(
|
||||
cache: &mut MessagesCacheBuilder,
|
||||
document_id: u32,
|
||||
@@ -134,6 +234,7 @@ fn insert_item(
|
||||
change_id: archive.version.change_id().unwrap_or_default(),
|
||||
document_id,
|
||||
size: message.size.to_native(),
|
||||
received_at: 0,
|
||||
};
|
||||
for keyword in message.keywords.iter() {
|
||||
match keyword.id() {
|
||||
@@ -163,6 +264,7 @@ impl MessagesCacheBuilder {
|
||||
index: self.index,
|
||||
keywords: self.keywords.into_boxed_slice(),
|
||||
size: self.size,
|
||||
has_received_at: self.has_received_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,6 +307,15 @@ pub trait MessageCacheAccess {
|
||||
fn expand_keywords(&self, message: &MessageCache) -> impl Iterator<Item = Keyword>;
|
||||
|
||||
fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool;
|
||||
|
||||
/// inbuxa: every message's rank by received date, for a local
|
||||
/// receivedAt sort; `None` when the cache has no dates (see
|
||||
/// `MessagesCache`)
|
||||
fn received_at_order(&self) -> Option<AHashMap<u32, u32>>;
|
||||
|
||||
/// inbuxa: the messages received in `[from, to)` seconds, for a local
|
||||
/// date filter; `None` when the cache has no dates
|
||||
fn received_between(&self, from: u64, to: u64) -> Option<RoaringBitmap>;
|
||||
}
|
||||
|
||||
impl MessageCacheAccess for MessageStoreCache {
|
||||
@@ -318,6 +429,43 @@ impl MessageCacheAccess for MessageStoreCache {
|
||||
fn has_keyword(&self, message: &MessageCache, keyword: &Keyword) -> bool {
|
||||
keyword_to_id(self, keyword).is_some_and(|id| message.keywords & (1 << id) != 0)
|
||||
}
|
||||
|
||||
fn received_at_order(&self) -> Option<AHashMap<u32, u32>> {
|
||||
// Ranks rather than the dates themselves: a sorted set holds u32
|
||||
// positions, and dates past 2106 would otherwise tie
|
||||
self.emails.has_received_at.then(|| {
|
||||
let mut order = self
|
||||
.emails
|
||||
.items
|
||||
.iter()
|
||||
.map(|m| (m.received_at, m.document_id))
|
||||
.collect::<Vec<_>>();
|
||||
order.sort_unstable();
|
||||
let mut rank = 0;
|
||||
let mut last = None;
|
||||
order
|
||||
.into_iter()
|
||||
.map(|(received_at, document_id)| {
|
||||
if last != Some(received_at) {
|
||||
last = Some(received_at);
|
||||
rank += 1;
|
||||
}
|
||||
(document_id, rank)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
fn received_between(&self, from: u64, to: u64) -> Option<RoaringBitmap> {
|
||||
self.emails.has_received_at.then(|| {
|
||||
self.emails
|
||||
.items
|
||||
.iter()
|
||||
.filter(|m| (from..to).contains(&m.received_at))
|
||||
.map(|m| m.document_id)
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) {
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use super::{FromModSeq, ToModSeq};
|
||||
@@ -331,8 +333,13 @@ impl<T: SessionStream> SessionData<T> {
|
||||
.map(|m| m.document_id),
|
||||
)));
|
||||
}
|
||||
// inbuxa: date filters run on the cache when it carries the
|
||||
// received dates (see MessagesCache)
|
||||
Filter::Before(date) => {
|
||||
filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date));
|
||||
filters.push(match cache.received_between(0, date.max(0) as u64) {
|
||||
Some(set) => SearchFilter::is_in_set(set),
|
||||
None => SearchFilter::lt(EmailSearchField::ReceivedAt, date),
|
||||
});
|
||||
}
|
||||
Filter::Deleted => {
|
||||
filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter(
|
||||
@@ -360,10 +367,16 @@ impl<T: SessionStream> SessionData<T> {
|
||||
filters.push(SearchFilter::gt(EmailSearchField::Size, size));
|
||||
}
|
||||
Filter::On(date) => {
|
||||
filters.push(SearchFilter::And);
|
||||
filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date));
|
||||
filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400));
|
||||
filters.push(SearchFilter::End);
|
||||
if let Some(set) =
|
||||
cache.received_between(date.max(0) as u64, date.max(0) as u64 + 86400)
|
||||
{
|
||||
filters.push(SearchFilter::is_in_set(set));
|
||||
} else {
|
||||
filters.push(SearchFilter::And);
|
||||
filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date));
|
||||
filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400));
|
||||
filters.push(SearchFilter::End);
|
||||
}
|
||||
}
|
||||
Filter::Seen => {
|
||||
filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter(
|
||||
@@ -383,7 +396,10 @@ impl<T: SessionStream> SessionData<T> {
|
||||
filters.push(SearchFilter::ge(EmailSearchField::SentAt, date));
|
||||
}
|
||||
Filter::Since(date) => {
|
||||
filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date));
|
||||
filters.push(match cache.received_between(date.max(0) as u64, u64::MAX) {
|
||||
Some(set) => SearchFilter::is_in_set(set),
|
||||
None => SearchFilter::ge(EmailSearchField::ReceivedAt, date),
|
||||
});
|
||||
}
|
||||
Filter::Smaller(size) => {
|
||||
filters.push(SearchFilter::lt(EmailSearchField::Size, size));
|
||||
@@ -448,16 +464,18 @@ impl<T: SessionStream> SessionData<T> {
|
||||
filters.push(SearchFilter::End);*/
|
||||
}
|
||||
Filter::Older(secs) => {
|
||||
filters.push(SearchFilter::le(
|
||||
EmailSearchField::ReceivedAt,
|
||||
now().saturating_sub(secs as u64),
|
||||
));
|
||||
let date = now().saturating_sub(secs as u64);
|
||||
filters.push(match cache.received_between(0, date + 1) {
|
||||
Some(set) => SearchFilter::is_in_set(set),
|
||||
None => SearchFilter::le(EmailSearchField::ReceivedAt, date),
|
||||
});
|
||||
}
|
||||
Filter::Younger(secs) => {
|
||||
filters.push(SearchFilter::ge(
|
||||
EmailSearchField::ReceivedAt,
|
||||
now().saturating_sub(secs as u64),
|
||||
));
|
||||
let date = now().saturating_sub(secs as u64);
|
||||
filters.push(match cache.received_between(date, u64::MAX) {
|
||||
Some(set) => SearchFilter::is_in_set(set),
|
||||
None => SearchFilter::ge(EmailSearchField::ReceivedAt, date),
|
||||
});
|
||||
}
|
||||
Filter::ModSeq((modseq, _)) => {
|
||||
let mut set = RoaringBitmap::new();
|
||||
@@ -659,9 +677,16 @@ impl<T: SessionStream> SessionData<T> {
|
||||
let mut comparators = Vec::with_capacity(imap_comparator.len());
|
||||
for comparator in imap_comparator {
|
||||
comparators.push(match comparator.sort {
|
||||
search::Sort::Arrival => {
|
||||
SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending)
|
||||
}
|
||||
// inbuxa: sorted on the cache when it carries the dates
|
||||
search::Sort::Arrival => match cache.received_at_order() {
|
||||
Some(set) => SearchComparator::SortedSet {
|
||||
set,
|
||||
ascending: comparator.ascending,
|
||||
},
|
||||
None => {
|
||||
SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending)
|
||||
}
|
||||
},
|
||||
search::Sort::Cc => {
|
||||
return Err(trc::ImapEvent::Error
|
||||
.into_err()
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState};
|
||||
@@ -181,14 +183,22 @@ impl EmailQuery for Server {
|
||||
}),
|
||||
)));
|
||||
}
|
||||
EmailFilter::Before(date) => filters.push(SearchFilter::lt(
|
||||
EmailSearchField::ReceivedAt,
|
||||
date.timestamp(),
|
||||
)),
|
||||
EmailFilter::After(date) => filters.push(SearchFilter::gt(
|
||||
EmailSearchField::ReceivedAt,
|
||||
date.timestamp(),
|
||||
)),
|
||||
// inbuxa: received dates come from the cache when it has
|
||||
// them, so these filters cost no query (see MessagesCache)
|
||||
EmailFilter::Before(date) => {
|
||||
let date = date.timestamp().max(0) as u64;
|
||||
filters.push(match cached_messages.received_between(0, date) {
|
||||
Some(set) => SearchFilter::is_in_set(set),
|
||||
None => SearchFilter::lt(EmailSearchField::ReceivedAt, date),
|
||||
})
|
||||
}
|
||||
EmailFilter::After(date) => {
|
||||
let date = date.timestamp().max(0) as u64;
|
||||
filters.push(match cached_messages.received_between(date + 1, u64::MAX) {
|
||||
Some(set) => SearchFilter::is_in_set(set),
|
||||
None => SearchFilter::gt(EmailSearchField::ReceivedAt, date),
|
||||
})
|
||||
}
|
||||
EmailFilter::MinSize(size) => {
|
||||
filters.push(SearchFilter::ge(EmailSearchField::Size, size))
|
||||
}
|
||||
@@ -282,9 +292,19 @@ impl EmailQuery for Server {
|
||||
.unwrap_or_default()
|
||||
{
|
||||
comparators.push(match comparator.property {
|
||||
EmailComparator::ReceivedAt => {
|
||||
SearchComparator::field(EmailSearchField::ReceivedAt, comparator.is_ascending)
|
||||
}
|
||||
// inbuxa: a local sort when the cache carries received dates;
|
||||
// upstream pulled every message id of the account, ordered,
|
||||
// from the search store on each query
|
||||
EmailComparator::ReceivedAt => match cached_messages.received_at_order() {
|
||||
Some(set) => SearchComparator::SortedSet {
|
||||
set,
|
||||
ascending: comparator.is_ascending,
|
||||
},
|
||||
None => SearchComparator::field(
|
||||
EmailSearchField::ReceivedAt,
|
||||
comparator.is_ascending,
|
||||
),
|
||||
},
|
||||
EmailComparator::Size => {
|
||||
SearchComparator::field(EmailSearchField::Size, comparator.is_ascending)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*
|
||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||
*/
|
||||
|
||||
use crate::changes::state::StateManager;
|
||||
@@ -95,9 +97,16 @@ impl ThreadGet for Server {
|
||||
SearchQuery::new(SearchIndex::Email)
|
||||
.with_account_id(account_id)
|
||||
.with_mask(all_ids)
|
||||
.with_comparator(SearchComparator::Field {
|
||||
field: SearchField::Email(EmailSearchField::ReceivedAt),
|
||||
ascending: true,
|
||||
// inbuxa: sorted on the cache when it has the dates
|
||||
.with_comparator(match cache.received_at_order() {
|
||||
Some(set) => SearchComparator::SortedSet {
|
||||
set,
|
||||
ascending: true,
|
||||
},
|
||||
None => SearchComparator::Field {
|
||||
field: SearchField::Email(EmailSearchField::ReceivedAt),
|
||||
ascending: true,
|
||||
},
|
||||
}),
|
||||
)
|
||||
.await?,
|
||||
|
||||
@@ -15,8 +15,8 @@ use crate::{
|
||||
},
|
||||
},
|
||||
search::{
|
||||
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
|
||||
SearchQuery, SearchValue,
|
||||
IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter,
|
||||
SearchOperator, SearchQuery, SearchValue,
|
||||
},
|
||||
write::SearchIndex,
|
||||
};
|
||||
@@ -115,6 +115,38 @@ impl MysqlStore {
|
||||
bounded(conn, result, limit)
|
||||
}
|
||||
|
||||
/// inbuxa: every document's value of one unsigned field in an account,
|
||||
/// in one round trip (see the PostgreSQL backend).
|
||||
pub async fn unsigned_values(
|
||||
&self,
|
||||
index: SearchIndex,
|
||||
field: SearchField,
|
||||
account_id: u32,
|
||||
) -> trc::Result<Vec<(u32, u64)>> {
|
||||
let query = format!(
|
||||
"SELECT {}, {} FROM {} WHERE {} = ?",
|
||||
SearchField::DocumentId.column(),
|
||||
field.column(),
|
||||
index.mysql_table(),
|
||||
SearchField::AccountId.column()
|
||||
);
|
||||
let mut conn = self.conn().await?;
|
||||
let limit = self.timeouts.query;
|
||||
let result = tokio::time::timeout(limit, async {
|
||||
let s = conn.prep(query).await.map_err(into_error)?;
|
||||
conn.exec::<(i32, i64), _, _>(s, (account_id as i32,))
|
||||
.await
|
||||
.map(|rows| {
|
||||
rows.into_iter()
|
||||
.map(|(id, value)| (id as u32, value as u64))
|
||||
.collect()
|
||||
})
|
||||
.map_err(into_error)
|
||||
})
|
||||
.await;
|
||||
bounded(conn, result, limit)
|
||||
}
|
||||
|
||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||
let table = filter.index.mysql_table();
|
||||
let mut query = format!("DELETE FROM {table} ");
|
||||
|
||||
@@ -14,7 +14,7 @@ use crate::{
|
||||
tls::MakeRustlsConnect,
|
||||
},
|
||||
search::{
|
||||
CalendarSearchField, ContactSearchField, EmailSearchField, SearchableField,
|
||||
CalendarSearchField, ContactSearchField, EmailSearchField, SearchField, SearchableField,
|
||||
TracingSearchField,
|
||||
},
|
||||
*,
|
||||
@@ -304,6 +304,16 @@ async fn create_search_tables<T: SearchableField + PsqlSearchField + 'static>(
|
||||
conn.execute(&create_index_query, &[])
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
// inbuxa: every sorted query is for one account, so an index led
|
||||
// by the account column answers "this account, in this order"
|
||||
// with an ordered scan instead of sorting the account's rows
|
||||
let account_column = SearchField::AccountId.column();
|
||||
let create_index_query = format!(
|
||||
"CREATE INDEX IF NOT EXISTS idx_{table_name}_{account_column}_{column_name} ON {table_name}({account_column}, {column_name})",
|
||||
);
|
||||
conn.execute(&create_index_query, &[])
|
||||
.await
|
||||
.map_err(into_error)?;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ use crate::{
|
||||
},
|
||||
},
|
||||
search::{
|
||||
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
|
||||
SearchQuery, SearchValue,
|
||||
IndexDocument, SearchComparator, SearchDocumentId, SearchField, SearchFilter,
|
||||
SearchOperator, SearchQuery, SearchValue,
|
||||
},
|
||||
write::SearchIndex,
|
||||
};
|
||||
@@ -197,6 +197,44 @@ impl PostgresStore {
|
||||
bounded(conn, result, limit)
|
||||
}
|
||||
|
||||
/// inbuxa: every document's value of one unsigned field in an account,
|
||||
/// in one round trip. Feeds the message cache's received dates so
|
||||
/// sorting by receivedAt needs no query at all.
|
||||
pub async fn unsigned_values(
|
||||
&self,
|
||||
index: SearchIndex,
|
||||
field: SearchField,
|
||||
account_id: u32,
|
||||
) -> trc::Result<Vec<(u32, u64)>> {
|
||||
let query = format!(
|
||||
"SELECT {}, {} FROM {} WHERE {} = $1",
|
||||
SearchField::DocumentId.column(),
|
||||
field.column(),
|
||||
index.psql_table(),
|
||||
SearchField::AccountId.column()
|
||||
);
|
||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||
let limit = self.timeouts.query;
|
||||
let result = tokio::time::timeout(limit, async {
|
||||
let s = conn.prepare_cached(&query).await.map_err(into_error)?;
|
||||
conn.query(&s, &[&(account_id as i32)])
|
||||
.await
|
||||
.and_then(|rows| {
|
||||
rows.into_iter()
|
||||
.map(|row| {
|
||||
Ok((
|
||||
row.try_get::<_, i32>(0)? as u32,
|
||||
row.try_get::<_, i64>(1)? as u64,
|
||||
))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
})
|
||||
.map_err(into_error)
|
||||
})
|
||||
.await;
|
||||
bounded(conn, result, limit)
|
||||
}
|
||||
|
||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||
debug_assert!(!filter.filters.is_empty());
|
||||
let table = filter.index.psql_table();
|
||||
|
||||
@@ -9,8 +9,8 @@
|
||||
use crate::{
|
||||
SearchStore, Store,
|
||||
search::{
|
||||
IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery,
|
||||
SearchValue,
|
||||
IndexDocument, QueryResults, SearchComparator, SearchField, SearchFilter, SearchOperator,
|
||||
SearchQuery, SearchValue,
|
||||
split::{SplitFilter, split_filters},
|
||||
},
|
||||
write::SearchIndex,
|
||||
@@ -113,10 +113,17 @@ impl SearchStore {
|
||||
};
|
||||
|
||||
// Merge results locally
|
||||
let results = SearchQuery::new(query.index)
|
||||
.with_filters(filters)
|
||||
.with_mask(query.mask)
|
||||
.filter();
|
||||
let results = if has_local_filters || has_external_filters {
|
||||
SearchQuery::new(query.index)
|
||||
.with_filters(filters)
|
||||
.with_mask(query.mask)
|
||||
.filter()
|
||||
} else {
|
||||
// inbuxa: only the account filter, which the local filter skips,
|
||||
// and it would answer with nothing: the result is the mask,
|
||||
// sorted by the local comparators below
|
||||
QueryResults::new(query.mask, Vec::new())
|
||||
};
|
||||
|
||||
let total_results = results.results().len();
|
||||
match total_results.cmp(&1) {
|
||||
@@ -308,6 +315,57 @@ impl SearchStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// inbuxa: every document's value of one unsigned field in an account
|
||||
/// (one round trip), or `None` when the search store is not SQL and
|
||||
/// callers must keep sorting through the store. ST-6, ST-12: a replica
|
||||
/// answers in a read scope.
|
||||
pub async fn unsigned_values(
|
||||
&self,
|
||||
index: SearchIndex,
|
||||
field: SearchField,
|
||||
account_id: u32,
|
||||
) -> trc::Result<Option<Vec<(u32, u64)>>> {
|
||||
if let SearchStore::Store(Store::Replicated(store)) = self {
|
||||
if !store.primary.is_pg_or_mysql() {
|
||||
return Ok(None);
|
||||
}
|
||||
return match store.read_target(crate::SUBSPACE_SEARCH_INDEX).await {
|
||||
Some(replica) => match crate::sql_backend!(
|
||||
&store.replicas[replica].store,
|
||||
db => db.unsigned_values(index, field.clone(), account_id).await
|
||||
) {
|
||||
Ok(values) => {
|
||||
store.served(replica);
|
||||
Ok(Some(values))
|
||||
}
|
||||
Err(err) => {
|
||||
store.failed(replica, err);
|
||||
crate::sql_backend!(&store.primary, db => db.unsigned_values(index, field, account_id).await)
|
||||
.map(Some)
|
||||
}
|
||||
},
|
||||
None => {
|
||||
crate::sql_backend!(&store.primary, db => db.unsigned_values(index, field, account_id).await)
|
||||
.map(Some)
|
||||
}
|
||||
};
|
||||
}
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
SearchStore::Store(Store::PostgreSQL(store)) => store
|
||||
.unsigned_values(index, field, account_id)
|
||||
.await
|
||||
.map(Some),
|
||||
#[cfg(feature = "mysql")]
|
||||
SearchStore::Store(Store::MySQL(store)) => store
|
||||
.unsigned_values(index, field, account_id)
|
||||
.await
|
||||
.map(Some),
|
||||
_ => Ok(None),
|
||||
}
|
||||
.caused_by(trc::location!())
|
||||
}
|
||||
|
||||
pub fn internal_fts(&self) -> Option<&Store> {
|
||||
match self {
|
||||
SearchStore::Store(store) => match store {
|
||||
|
||||
Reference in new issue
Block a user