Import upstream v0.16.22, stripped
Upstream commit: 474dd0229cb20cf513036619781ed97bd8073c3f Enterprise-only files removed or emptied: 63 Enterprise-only snippets removed: 117 in 50 files Dangling module declarations removed: 5 Cargo edits turning enterprise off: 14 Verification: clean Enterprise feature gates left for rebuilt features: 19 in 18 files Produced by tools/fork/strip.py. The full report is in docs/fork/strip-reports/ on main.
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
SearchStore,
|
||||
backend::meili::{MeiliSearchStore, Task, TaskStatus, TaskUid},
|
||||
search::{
|
||||
CalendarSearchField, ContactSearchField, EmailSearchField, SearchField, SearchableField,
|
||||
TracingSearchField,
|
||||
},
|
||||
write::now,
|
||||
};
|
||||
use registry::schema::structs;
|
||||
use reqwest::{Error, Response, Url};
|
||||
use serde_json::{Value, json};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
const UNCONFIRMED_TASK_RECHECK_DELAY: u64 = 600;
|
||||
pub(crate) const MAX_TOTAL_HITS: u64 = 100_000;
|
||||
|
||||
impl MeiliSearchStore {
|
||||
pub async fn open(config: structs::MeilisearchStore) -> Result<SearchStore, String> {
|
||||
let client = config
|
||||
.http_auth
|
||||
.build_http_client(
|
||||
config.http_headers,
|
||||
"application/json".into(),
|
||||
config.timeout,
|
||||
config.allow_invalid_certs,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Url::parse(&config.url).map_err(|e| format!("Invalid URL: {e}",))?;
|
||||
|
||||
let ms = Self {
|
||||
client,
|
||||
url: config.url,
|
||||
task_poll_interval: Duration::from_millis(500),
|
||||
task_poll_retries: 120,
|
||||
task_fail_on_timeout: true,
|
||||
};
|
||||
|
||||
if let Err(err) = ms.create_indexes().await {
|
||||
return Err(format!("Failed to create indexes: {err}"));
|
||||
}
|
||||
|
||||
Ok(SearchStore::MeiliSearch(Arc::new(MeiliSearchStore {
|
||||
client: ms.client,
|
||||
url: ms.url,
|
||||
task_poll_interval: config.poll_interval.into_inner(),
|
||||
task_poll_retries: config.max_retries as usize,
|
||||
task_fail_on_timeout: config.fail_on_timeout,
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn create_indexes(&self) -> trc::Result<()> {
|
||||
self.create_index::<EmailSearchField>().await?;
|
||||
self.create_index::<CalendarSearchField>().await?;
|
||||
self.create_index::<ContactSearchField>().await?;
|
||||
self.create_index::<TracingSearchField>().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn index_exists(&self, index_uid: &str) -> trc::Result<bool> {
|
||||
let response = self
|
||||
.client
|
||||
.get(format!("{}/indexes/{}", self.url, index_uid))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?;
|
||||
|
||||
match response.status().as_u16() {
|
||||
200..=299 => Ok(true),
|
||||
404 => Ok(false),
|
||||
status => {
|
||||
let text = response.text().await.unwrap_or_default();
|
||||
Err(trc::StoreEvent::MeilisearchError
|
||||
.reason(text)
|
||||
.ctx(trc::Key::Code, status))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_index<T: SearchableField>(&self) -> trc::Result<()> {
|
||||
let index_name = T::index().index_name();
|
||||
|
||||
if self.index_exists(index_name).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let response = assert_success(
|
||||
self.client
|
||||
.post(format!("{}/indexes", self.url))
|
||||
.body(
|
||||
json!({
|
||||
"uid": index_name,
|
||||
"primaryKey": "id",
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if !self.wait_for_task(response).await? {
|
||||
// Index already exists
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut searchable = Vec::new();
|
||||
let mut filterable = Vec::new();
|
||||
let mut sortable = Vec::new();
|
||||
|
||||
for field in T::all_fields() {
|
||||
if field.is_indexed() {
|
||||
sortable.push(Value::String(field.field_name().to_string()));
|
||||
}
|
||||
if field.is_text() {
|
||||
searchable.push(Value::String(field.field_name().to_string()));
|
||||
} else {
|
||||
filterable.push(Value::String(field.field_name().to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
for key in T::primary_keys() {
|
||||
filterable.push(Value::String(key.field_name().to_string()));
|
||||
if matches!(key, SearchField::Id) {
|
||||
sortable.push(Value::String(key.field_name().to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
filterable.push(Value::String("bcc".into()));
|
||||
|
||||
if !searchable.is_empty() {
|
||||
self.update_index_settings(
|
||||
index_name,
|
||||
"searchable-attributes",
|
||||
Value::Array(searchable),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !filterable.is_empty() {
|
||||
self.update_index_settings(
|
||||
index_name,
|
||||
"filterable-attributes",
|
||||
Value::Array(filterable),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !sortable.is_empty() {
|
||||
self.update_index_settings(index_name, "sortable-attributes", Value::Array(sortable))
|
||||
.await?;
|
||||
}
|
||||
|
||||
self.update_index_pagination(index_name).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn update_index_pagination(&self, index_uid: &str) -> trc::Result<bool> {
|
||||
let response = assert_success(
|
||||
self.client
|
||||
.patch(format!(
|
||||
"{}/indexes/{}/settings/pagination",
|
||||
self.url, index_uid
|
||||
))
|
||||
.body(json!({ "maxTotalHits": MAX_TOTAL_HITS }).to_string())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.wait_for_task(response).await
|
||||
}
|
||||
|
||||
async fn update_index_settings(
|
||||
&self,
|
||||
index_uid: &str,
|
||||
setting: &str,
|
||||
value: Value,
|
||||
) -> trc::Result<bool> {
|
||||
let response = assert_success(
|
||||
self.client
|
||||
.put(format!(
|
||||
"{}/indexes/{}/settings/{}",
|
||||
self.url, index_uid, setting
|
||||
))
|
||||
.body(value.to_string())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
self.wait_for_task(response).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
pub async fn drop_indexes(&self) -> trc::Result<()> {
|
||||
use crate::write::SearchIndex;
|
||||
|
||||
for index in &[
|
||||
SearchIndex::Email,
|
||||
SearchIndex::Calendar,
|
||||
SearchIndex::Contacts,
|
||||
SearchIndex::Tracing,
|
||||
] {
|
||||
let response = self
|
||||
.client
|
||||
.delete(format!("{}/indexes/{}", self.url, index.index_name()))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?;
|
||||
|
||||
match response.status().as_u16() {
|
||||
200..=299 => {
|
||||
self.wait_for_task(response).await?;
|
||||
}
|
||||
400..=499 => {
|
||||
// Index does not exist
|
||||
return Ok(());
|
||||
}
|
||||
_ => {
|
||||
let status = response.status();
|
||||
let msg = response.text().await.unwrap_or_default();
|
||||
return Err(trc::StoreEvent::MeilisearchError
|
||||
.reason(msg)
|
||||
.ctx(trc::Key::Code, status.as_u16()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_task(&self, response: Response) -> trc::Result<bool> {
|
||||
let response_body = response.text().await.map_err(|err| {
|
||||
trc::StoreEvent::MeilisearchError
|
||||
.reason(err)
|
||||
.details("Request failed")
|
||||
})?;
|
||||
let task_uid = serde_json::from_str::<TaskUid>(&response_body)
|
||||
.map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?
|
||||
.task_uid;
|
||||
|
||||
let mut loop_count = 0;
|
||||
let url = format!("{}/tasks/{}", self.url, task_uid);
|
||||
|
||||
while loop_count < self.task_poll_retries {
|
||||
let resp = assert_success(self.client.get(&url).send().await).await?;
|
||||
|
||||
let text = resp
|
||||
.text()
|
||||
.await
|
||||
.map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?;
|
||||
|
||||
let task = serde_json::from_str::<Task>(&text).map_err(|err| {
|
||||
trc::StoreEvent::MeilisearchError
|
||||
.reason(err)
|
||||
.details(text.clone())
|
||||
})?;
|
||||
|
||||
match task.status {
|
||||
TaskStatus::Succeeded => return Ok(true),
|
||||
TaskStatus::Failed => {
|
||||
let (code, message) = task
|
||||
.error
|
||||
.map(|e| (e.code, Some(e.message)))
|
||||
.unwrap_or((None, None));
|
||||
return if matches!(code.as_deref(), Some("index_already_exists")) {
|
||||
Ok(false)
|
||||
} else {
|
||||
Err(trc::StoreEvent::MeilisearchError
|
||||
.reason("Meilisearch task failed.")
|
||||
.id(task_uid)
|
||||
.code(code)
|
||||
.details(message))
|
||||
};
|
||||
}
|
||||
TaskStatus::Canceled => {
|
||||
return Err(trc::StoreEvent::MeilisearchError
|
||||
.reason("Meilisearch task was canceled")
|
||||
.id(task_uid));
|
||||
}
|
||||
TaskStatus::Enqueued | TaskStatus::Processing => {
|
||||
loop_count += 1;
|
||||
tokio::time::sleep(self.task_poll_interval).await;
|
||||
}
|
||||
TaskStatus::Unknown => {
|
||||
return Err(trc::StoreEvent::MeilisearchError
|
||||
.reason("Meilisearch task returned an unknown status")
|
||||
.id(task_uid)
|
||||
.details(text));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let err = trc::StoreEvent::MeilisearchError
|
||||
.reason("Timed out waiting for Meilisearch task")
|
||||
.id(task_uid);
|
||||
|
||||
Err(if self.task_fail_on_timeout {
|
||||
err
|
||||
} else {
|
||||
err.ctx(
|
||||
trc::Key::NextRetry,
|
||||
now().saturating_add(UNCONFIRMED_TASK_RECHECK_DELAY),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn assert_success(response: Result<Response, Error>) -> trc::Result<Response> {
|
||||
match response {
|
||||
Ok(response) => {
|
||||
let status = response.status();
|
||||
if status.is_success() {
|
||||
Ok(response)
|
||||
} else {
|
||||
Err(trc::StoreEvent::MeilisearchError
|
||||
.reason(response.text().await.unwrap_or_default())
|
||||
.ctx(trc::Key::Code, status.as_u16()))
|
||||
}
|
||||
}
|
||||
Err(err) => Err(trc::StoreEvent::MeilisearchError.reason(err)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use reqwest::Client;
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
|
||||
pub mod main;
|
||||
pub mod search;
|
||||
|
||||
pub struct MeiliSearchStore {
|
||||
client: Client,
|
||||
url: String,
|
||||
task_poll_interval: Duration,
|
||||
task_poll_retries: usize,
|
||||
task_fail_on_timeout: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct TaskUid {
|
||||
#[serde(rename = "taskUid")]
|
||||
pub task_uid: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TaskError {
|
||||
message: String,
|
||||
#[serde(default)]
|
||||
code: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Task {
|
||||
//#[serde(rename = "uid")]
|
||||
//uid: u64,
|
||||
status: TaskStatus,
|
||||
#[serde(default)]
|
||||
error: Option<TaskError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
enum TaskStatus {
|
||||
Enqueued,
|
||||
Processing,
|
||||
Succeeded,
|
||||
Failed,
|
||||
Canceled,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct MeiliSearchResponse {
|
||||
hits: Vec<MeiliHit>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct MeiliDocumentsResponse {
|
||||
results: Vec<MeiliHit>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct MeiliHit {
|
||||
id: u64,
|
||||
}
|
||||
@@ -0,0 +1,561 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use crate::{
|
||||
backend::meili::{
|
||||
MeiliDocumentsResponse, MeiliSearchResponse, MeiliSearchStore,
|
||||
main::{MAX_TOTAL_HITS, assert_success},
|
||||
},
|
||||
search::*,
|
||||
write::SearchIndex,
|
||||
};
|
||||
use ahash::AHashSet;
|
||||
use serde_json::{Map, Value, json};
|
||||
use std::fmt::{Display, Write};
|
||||
|
||||
const MAX_SEARCH_RESULTS: usize = 10_000;
|
||||
|
||||
impl MeiliSearchStore {
|
||||
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
||||
let mut index_documents: [String; 5] = [
|
||||
String::new(),
|
||||
String::new(),
|
||||
String::new(),
|
||||
String::new(),
|
||||
String::new(),
|
||||
];
|
||||
|
||||
for document in documents {
|
||||
let request = &mut index_documents[document.index.array_pos()];
|
||||
if !request.is_empty() {
|
||||
request.push(',');
|
||||
} else {
|
||||
request.reserve(1024);
|
||||
request.push('[');
|
||||
}
|
||||
|
||||
json_serialize(request, &document);
|
||||
}
|
||||
|
||||
for (mut payload, index) in index_documents.into_iter().zip([
|
||||
SearchIndex::Email,
|
||||
SearchIndex::Calendar,
|
||||
SearchIndex::Contacts,
|
||||
SearchIndex::Tracing,
|
||||
SearchIndex::File,
|
||||
]) {
|
||||
if payload.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
payload.push(']');
|
||||
|
||||
let response = assert_success(
|
||||
self.client
|
||||
.put(format!(
|
||||
"{}/indexes/{}/documents",
|
||||
self.url,
|
||||
index.index_name()
|
||||
))
|
||||
.body(payload)
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
self.wait_for_task(response).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn query<R: SearchDocumentId>(
|
||||
&self,
|
||||
index: SearchIndex,
|
||||
filters: &[SearchFilter],
|
||||
sort: &[SearchComparator],
|
||||
) -> trc::Result<Vec<R>> {
|
||||
let filter_group = build_query(filters);
|
||||
|
||||
if filter_group.q.is_empty() && sort.is_empty() {
|
||||
return self.fetch_documents(index, &filter_group.filter).await;
|
||||
}
|
||||
|
||||
let mut body = Map::new();
|
||||
body.insert("limit".to_string(), Value::from(MAX_SEARCH_RESULTS));
|
||||
body.insert(
|
||||
"attributesToRetrieve".to_string(),
|
||||
Value::Array(vec![Value::String("id".to_string())]),
|
||||
);
|
||||
|
||||
if !filter_group.filter.is_empty() {
|
||||
body.insert("filter".to_string(), Value::String(filter_group.filter));
|
||||
}
|
||||
|
||||
if !filter_group.q.is_empty() {
|
||||
body.insert("q".to_string(), Value::String(filter_group.q));
|
||||
body.insert(
|
||||
"matchingStrategy".to_string(),
|
||||
Value::String("all".to_string()),
|
||||
);
|
||||
|
||||
if !filter_group.search_on.is_empty() {
|
||||
body.insert(
|
||||
"attributesToSearchOn".to_string(),
|
||||
Value::Array(
|
||||
filter_group
|
||||
.search_on
|
||||
.into_iter()
|
||||
.map(|field| Value::String(field.to_string()))
|
||||
.collect(),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if !sort.is_empty() {
|
||||
let sort_arr: Vec<Value> = sort
|
||||
.iter()
|
||||
.filter_map(|comp| match comp {
|
||||
SearchComparator::Field { field, ascending } => Some(Value::String(format!(
|
||||
"{}:{}",
|
||||
field.field_name(),
|
||||
if *ascending { "asc" } else { "desc" }
|
||||
))),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
if !sort_arr.is_empty() {
|
||||
body.insert("sort".to_string(), Value::Array(sort_arr));
|
||||
}
|
||||
}
|
||||
|
||||
let url = format!("{}/indexes/{}/search", self.url, index.index_name());
|
||||
let mut results = Vec::new();
|
||||
let mut offset = 0;
|
||||
|
||||
loop {
|
||||
body.insert("offset".to_string(), Value::from(offset));
|
||||
|
||||
let resp = assert_success(
|
||||
self.client
|
||||
.post(&url)
|
||||
.body(Value::Object(body.clone()).to_string())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let text = resp
|
||||
.text()
|
||||
.await
|
||||
.map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?;
|
||||
|
||||
let hits = serde_json::from_str::<MeiliSearchResponse>(&text)
|
||||
.map_err(|err| {
|
||||
trc::StoreEvent::MeilisearchError
|
||||
.reason(err)
|
||||
.details(text.clone())
|
||||
})?
|
||||
.hits;
|
||||
|
||||
let total = hits.len();
|
||||
results.extend(hits.into_iter().map(|hit| R::from_u64(hit.id)));
|
||||
|
||||
if total < MAX_SEARCH_RESULTS {
|
||||
break;
|
||||
}
|
||||
|
||||
offset += total;
|
||||
|
||||
if offset >= MAX_TOTAL_HITS as usize {
|
||||
trc::event!(
|
||||
Store(trc::StoreEvent::MeilisearchError),
|
||||
Reason = "Search results were truncated",
|
||||
Collection = index.index_name(),
|
||||
Total = offset,
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
async fn fetch_documents<R: SearchDocumentId>(
|
||||
&self,
|
||||
index: SearchIndex,
|
||||
filter: &str,
|
||||
) -> trc::Result<Vec<R>> {
|
||||
let url = format!(
|
||||
"{}/indexes/{}/documents/fetch",
|
||||
self.url,
|
||||
index.index_name()
|
||||
);
|
||||
let mut results = Vec::new();
|
||||
let mut offset = 0;
|
||||
|
||||
loop {
|
||||
let mut body = Map::new();
|
||||
body.insert("limit".to_string(), Value::from(MAX_SEARCH_RESULTS));
|
||||
body.insert("offset".to_string(), Value::from(offset));
|
||||
body.insert(
|
||||
"fields".to_string(),
|
||||
Value::Array(vec![Value::String("id".to_string())]),
|
||||
);
|
||||
|
||||
if !filter.is_empty() {
|
||||
body.insert("filter".to_string(), Value::String(filter.to_string()));
|
||||
}
|
||||
|
||||
let resp = assert_success(
|
||||
self.client
|
||||
.post(&url)
|
||||
.body(Value::Object(body).to_string())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let text = resp
|
||||
.text()
|
||||
.await
|
||||
.map_err(|err| trc::StoreEvent::MeilisearchError.reason(err))?;
|
||||
|
||||
let documents = serde_json::from_str::<MeiliDocumentsResponse>(&text)
|
||||
.map_err(|err| {
|
||||
trc::StoreEvent::MeilisearchError
|
||||
.reason(err)
|
||||
.details(text.clone())
|
||||
})?
|
||||
.results;
|
||||
|
||||
let total = documents.len();
|
||||
results.extend(documents.into_iter().map(|hit| R::from_u64(hit.id)));
|
||||
|
||||
if total < MAX_SEARCH_RESULTS {
|
||||
break;
|
||||
}
|
||||
|
||||
offset += total;
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||
let filter_group = build_query(&filter.filters);
|
||||
|
||||
if filter_group.filter.is_empty() {
|
||||
return Err(trc::StoreEvent::MeilisearchError.reason(
|
||||
"Meilisearch delete-by-filter requires structured (non-text) filters only",
|
||||
));
|
||||
}
|
||||
|
||||
let url = format!(
|
||||
"{}/indexes/{}/documents/delete",
|
||||
self.url,
|
||||
filter.index.index_name()
|
||||
);
|
||||
|
||||
let response = assert_success(
|
||||
self.client
|
||||
.post(url)
|
||||
.body(json!({ "filter": filter_group.filter }).to_string())
|
||||
.send()
|
||||
.await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.wait_for_task(response).await?;
|
||||
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Debug)]
|
||||
struct FilterGroup {
|
||||
q: String,
|
||||
filter: String,
|
||||
search_on: AHashSet<&'static str>,
|
||||
}
|
||||
|
||||
fn build_query(filters: &[SearchFilter]) -> FilterGroup {
|
||||
if filters.is_empty() {
|
||||
return FilterGroup::default();
|
||||
}
|
||||
let mut operator_stack = Vec::new();
|
||||
let mut operator = &SearchFilter::And;
|
||||
let mut is_first = true;
|
||||
let mut filter = String::new();
|
||||
let mut queries = AHashSet::new();
|
||||
let mut search_on = AHashSet::new();
|
||||
|
||||
for f in filters {
|
||||
match f {
|
||||
SearchFilter::Operator { field, op, value } => {
|
||||
if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains)
|
||||
{
|
||||
let value = match value {
|
||||
SearchValue::Text { value, .. } => value,
|
||||
_ => {
|
||||
debug_assert!(
|
||||
false,
|
||||
"Text field search with non-text value is not supported"
|
||||
);
|
||||
""
|
||||
}
|
||||
};
|
||||
|
||||
search_on.insert(field.field_name());
|
||||
|
||||
if matches!(op, SearchOperator::Equal) {
|
||||
queries.insert(format!("{value:?}"));
|
||||
} else {
|
||||
for token in value.split_whitespace() {
|
||||
queries.insert(token.to_string());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if !filter.is_empty() && !filter.ends_with('(') {
|
||||
match operator {
|
||||
SearchFilter::And => filter.push_str(" AND "),
|
||||
SearchFilter::Or => filter.push_str(" OR "),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
match value {
|
||||
SearchValue::Text { value, .. } => {
|
||||
filter.push_str(field.field_name());
|
||||
filter.push(' ');
|
||||
op.write_meli_op(&mut filter, format!("{value:?}"));
|
||||
}
|
||||
SearchValue::KeyValues(kv) => {
|
||||
let (key, value) = kv.iter().next().unwrap();
|
||||
filter.push_str(field.field_name());
|
||||
filter.push('.');
|
||||
filter.push_str(key);
|
||||
filter.push(' ');
|
||||
op.write_meli_op(&mut filter, format!("{value:?}"));
|
||||
}
|
||||
SearchValue::Int(v) => {
|
||||
filter.push_str(field.field_name());
|
||||
filter.push(' ');
|
||||
op.write_meli_op(&mut filter, v);
|
||||
}
|
||||
SearchValue::Uint(v) => {
|
||||
filter.push_str(field.field_name());
|
||||
filter.push(' ');
|
||||
op.write_meli_op(&mut filter, v);
|
||||
}
|
||||
SearchValue::Boolean(v) => {
|
||||
filter.push_str(field.field_name());
|
||||
filter.push(' ');
|
||||
op.write_meli_op(&mut filter, v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
SearchFilter::And | SearchFilter::Or => {
|
||||
if !filter.is_empty() && !filter.ends_with('(') {
|
||||
match operator {
|
||||
SearchFilter::And => filter.push_str(" AND "),
|
||||
SearchFilter::Or => filter.push_str(" OR "),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
operator_stack.push((operator, is_first));
|
||||
operator = f;
|
||||
is_first = true;
|
||||
filter.push('(');
|
||||
}
|
||||
SearchFilter::Not => {
|
||||
if !filter.is_empty() && !filter.ends_with('(') {
|
||||
match operator {
|
||||
SearchFilter::And => filter.push_str(" AND "),
|
||||
SearchFilter::Or => filter.push_str(" OR "),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
operator_stack.push((operator, is_first));
|
||||
operator = &SearchFilter::And;
|
||||
is_first = true;
|
||||
filter.push_str("NOT (");
|
||||
}
|
||||
SearchFilter::End => {
|
||||
let p = operator_stack.pop().unwrap_or((&SearchFilter::And, true));
|
||||
operator = p.0;
|
||||
is_first = p.1;
|
||||
|
||||
if !filter.ends_with('(') {
|
||||
filter.push(')');
|
||||
} else {
|
||||
filter.pop();
|
||||
if filter.ends_with("NOT ") {
|
||||
let len = filter.len();
|
||||
filter.truncate(len - 4);
|
||||
}
|
||||
if filter.ends_with(" AND ") {
|
||||
let len = filter.len();
|
||||
filter.truncate(len - 5);
|
||||
is_first = true;
|
||||
} else if filter.ends_with(" OR ") {
|
||||
let len = filter.len();
|
||||
filter.truncate(len - 4);
|
||||
is_first = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
SearchFilter::DocumentSet(_) => {
|
||||
debug_assert!(false, "DocumentSet filters are not supported")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut q = String::new();
|
||||
if !queries.is_empty() {
|
||||
for (idx, term) in queries.into_iter().enumerate() {
|
||||
if idx > 0 {
|
||||
q.push(' ');
|
||||
}
|
||||
q.push_str(&term);
|
||||
}
|
||||
}
|
||||
|
||||
FilterGroup {
|
||||
q,
|
||||
filter,
|
||||
search_on,
|
||||
}
|
||||
}
|
||||
|
||||
impl SearchOperator {
|
||||
fn write_meli_op(&self, query: &mut String, value: impl Display) {
|
||||
match self {
|
||||
SearchOperator::LowerThan => {
|
||||
let _ = write!(query, "< {value}");
|
||||
}
|
||||
SearchOperator::LowerEqualThan => {
|
||||
let _ = write!(query, "<= {value}");
|
||||
}
|
||||
SearchOperator::GreaterThan => {
|
||||
let _ = write!(query, "> {value}");
|
||||
}
|
||||
SearchOperator::GreaterEqualThan => {
|
||||
let _ = write!(query, ">= {value}");
|
||||
}
|
||||
SearchOperator::Equal | SearchOperator::Contains => {
|
||||
let _ = write!(query, "= {value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn json_serialize(request: &mut String, document: &IndexDocument) {
|
||||
let mut id = 0u64;
|
||||
let mut is_first = true;
|
||||
request.push('{');
|
||||
for (k, v) in document.fields.iter() {
|
||||
match k {
|
||||
SearchField::AccountId => {
|
||||
if let SearchValue::Uint(account_id) = v {
|
||||
id |= account_id << 32;
|
||||
}
|
||||
}
|
||||
SearchField::DocumentId => {
|
||||
if let SearchValue::Uint(doc_id) = v {
|
||||
id |= doc_id;
|
||||
}
|
||||
}
|
||||
SearchField::Id => {
|
||||
if let SearchValue::Uint(doc_id) = v {
|
||||
id = *doc_id;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if !is_first {
|
||||
request.push(',');
|
||||
} else {
|
||||
is_first = false;
|
||||
}
|
||||
|
||||
let _ = write!(request, "{:?}:", k.field_name());
|
||||
match v {
|
||||
SearchValue::Text { value, .. } => {
|
||||
json_serialize_str(request, value);
|
||||
}
|
||||
SearchValue::KeyValues(map) => {
|
||||
request.push('{');
|
||||
for (i, (key, value)) in map.iter().enumerate() {
|
||||
if i > 0 {
|
||||
request.push(',');
|
||||
}
|
||||
json_serialize_str(request, key);
|
||||
request.push(':');
|
||||
json_serialize_str(request, value);
|
||||
}
|
||||
request.push('}');
|
||||
}
|
||||
SearchValue::Int(v) => {
|
||||
let _ = write!(request, "{}", v);
|
||||
}
|
||||
SearchValue::Uint(v) => {
|
||||
let _ = write!(request, "{}", v);
|
||||
}
|
||||
SearchValue::Boolean(v) => {
|
||||
let _ = write!(request, "{}", v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*if id == 0 {
|
||||
debug_assert!(false, "Document is missing required ID fields");
|
||||
}*/
|
||||
|
||||
let _ = write!(request, ",\"id\":{id}}}");
|
||||
}
|
||||
|
||||
fn json_serialize_str(request: &mut String, value: &str) {
|
||||
request.push('"');
|
||||
for c in value.chars() {
|
||||
match c {
|
||||
'"' => request.push_str("\\\""),
|
||||
'\\' => request.push_str("\\\\"),
|
||||
'\n' => request.push_str("\\n"),
|
||||
'\r' => request.push_str("\\r"),
|
||||
'\t' => request.push_str("\\t"),
|
||||
'\u{0008}' => request.push_str("\\b"), // backspace
|
||||
'\u{000C}' => request.push_str("\\f"), // form feed
|
||||
_ => {
|
||||
if !c.is_control() {
|
||||
request.push(c);
|
||||
} else {
|
||||
let _ = write!(request, "\\u{:04x}", c as u32);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
request.push('"');
|
||||
}
|
||||
|
||||
impl SearchIndex {
|
||||
#[inline(always)]
|
||||
fn array_pos(&self) -> usize {
|
||||
match self {
|
||||
SearchIndex::Email => 0,
|
||||
SearchIndex::Calendar => 1,
|
||||
SearchIndex::Contacts => 2,
|
||||
SearchIndex::Tracing => 3,
|
||||
SearchIndex::File => 4,
|
||||
SearchIndex::InMemory => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user