Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a80aaea08 |
@@ -31,11 +31,8 @@
|
|||||||
# crates/types/src/branding.rs, not Cargo.toml, and the image is tagged
|
# crates/types/src/branding.rs, not Cargo.toml, and the image is tagged
|
||||||
# with it, so a tag beside an unbumped macro would publish an image that
|
# with it, so a tag beside an unbumped macro would publish an image that
|
||||||
# reports a different version from its tag.
|
# reports a different version from its tag.
|
||||||
# * the tag must be on main or on a release/* branch, so an image never
|
# * the tag must be on main, so an image never describes code that was never
|
||||||
# describes code that was never reviewed onto one of them. A release/*
|
# reviewed onto the default branch.
|
||||||
# branch carries a hotfix: it starts at an earlier release tag, takes
|
|
||||||
# fixes through pull requests into it, and is tagged there, so production
|
|
||||||
# can get a fix without everything that has landed on main since.
|
|
||||||
#
|
#
|
||||||
# :latest moves with every published tag: tags are cut by the weekly release
|
# :latest moves with every published tag: tags are cut by the weekly release
|
||||||
# (or by hand for a real release); there are no prerelease tags here.
|
# (or by hand for a real release); there are no prerelease tags here.
|
||||||
@@ -77,13 +74,8 @@ jobs:
|
|||||||
echo "Refusing to publish an image that would report the wrong version." >&2
|
echo "Refusing to publish an image that would report the wrong version." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
commit="$(git rev-parse "${TAG}^{commit}")"
|
git merge-base --is-ancestor "$(git rev-parse "${TAG}^{commit}")" origin/main \
|
||||||
on=""
|
|| { echo "$TAG is not on main" >&2; exit 1; }
|
||||||
for ref in origin/main $(git for-each-ref --format='%(refname:short)' 'refs/remotes/origin/release/*'); do
|
|
||||||
if git merge-base --is-ancestor "$commit" "$ref"; then on="$ref"; break; fi
|
|
||||||
done
|
|
||||||
[ -n "$on" ] || { echo "$TAG is not on main or a release/* branch" >&2; exit 1; }
|
|
||||||
echo "$TAG is on $on"
|
|
||||||
echo "version=$V" >> "$GITHUB_OUTPUT"
|
echo "version=$V" >> "$GITHUB_OUTPUT"
|
||||||
echo "version $V"
|
echo "version $V"
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::auth::AccessToken;
|
use crate::auth::AccessToken;
|
||||||
@@ -20,16 +18,6 @@ impl Server {
|
|||||||
access_token: &AccessToken,
|
access_token: &AccessToken,
|
||||||
addr: IpAddr,
|
addr: IpAddr,
|
||||||
) -> trc::Result<Option<InFlight>> {
|
) -> trc::Result<Option<InFlight>> {
|
||||||
// inbuxa: an account with unlimited requests passes both limits
|
|
||||||
// below anyway, so don't count its requests. The count is a write to
|
|
||||||
// one counter per account in the in-memory store, and concurrent
|
|
||||||
// requests from one account queue on that key (a row lock on SQL,
|
|
||||||
// conflict retries on RocksDB): in a cluster rehearsal ten parallel
|
|
||||||
// admin writes were accepted one after another, about 33 ms apart.
|
|
||||||
if access_token.has_permission(Permission::UnlimitedRequests) {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
let rate_reset = if let Some(rate) = &self.core.network.http.rate_authenticated {
|
let rate_reset = if let Some(rate) = &self.core.network.http.rate_authenticated {
|
||||||
if self.is_ip_allowed(addr) {
|
if self.is_ip_allowed(addr) {
|
||||||
None
|
None
|
||||||
|
|||||||
Vendored
+17
-133
@@ -7,7 +7,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
BuildServer, Core, Server,
|
Core, Server,
|
||||||
config::{
|
config::{
|
||||||
server::{Listeners, tls::parse_certificates},
|
server::{Listeners, tls::parse_certificates},
|
||||||
storage::Storage,
|
storage::Storage,
|
||||||
@@ -245,73 +245,21 @@ fn error_object(error: &Error) -> Option<ObjectId> {
|
|||||||
// until someone reloaded. Writes to objects the settings are built from now
|
// until someone reloaded. Writes to objects the settings are built from now
|
||||||
// reload them, here and across the cluster, as ReloadSettings does.
|
// reload them, here and across the cluster, as ReloadSettings does.
|
||||||
|
|
||||||
/// Coalesces the full reloads that registry writes trigger. A write waits
|
/// Coalesces the full reloads that registry writes trigger: a write waits for
|
||||||
/// for more writes before a reload starts (see [`WRITE_QUIET`]), then
|
/// a reload that started after it was stored, and joins one if it can, so a
|
||||||
/// takes the result of the first reload that started after it was stored,
|
/// burst of writes costs a reload or two rather than one each.
|
||||||
/// so a burst of writes, or a request with many objects, costs one reload
|
#[derive(Default)]
|
||||||
/// or two rather than one each.
|
|
||||||
pub struct SettingsReloadGate {
|
pub struct SettingsReloadGate {
|
||||||
requested: std::sync::atomic::AtomicU64,
|
requested: std::sync::atomic::AtomicU64,
|
||||||
reloads: std::sync::atomic::AtomicU64,
|
state: tokio::sync::Mutex<SettingsReloadState>,
|
||||||
state: parking_lot::Mutex<SettingsReloadState>,
|
|
||||||
completed: tokio::sync::watch::Sender<u64>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct SettingsReloadState {
|
struct SettingsReloadState {
|
||||||
/// A reload is waiting for writes to settle, or running.
|
completed: u64,
|
||||||
scheduled: bool,
|
refused: Option<String>,
|
||||||
/// When the oldest write not yet covered by a reload was stored, and
|
|
||||||
/// the newest.
|
|
||||||
first_write: Option<std::time::Instant>,
|
|
||||||
last_write: Option<std::time::Instant>,
|
|
||||||
/// Recent reloads, oldest first: the last write each covered, and why
|
|
||||||
/// it was refused, if it was.
|
|
||||||
results: std::collections::VecDeque<(u64, Option<String>)>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for SettingsReloadGate {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
requested: Default::default(),
|
|
||||||
reloads: Default::default(),
|
|
||||||
state: Default::default(),
|
|
||||||
completed: tokio::sync::watch::Sender::new(0),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SettingsReloadGate {
|
|
||||||
/// How many full reloads registry writes have run.
|
|
||||||
pub fn reloads(&self) -> u64 {
|
|
||||||
self.reloads.load(std::sync::atomic::Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SettingsReloadState {
|
|
||||||
/// The result of the reload that covered write `ticket`, once it ran.
|
|
||||||
fn result_for(&self, ticket: u64) -> Option<Result<(), String>> {
|
|
||||||
self.results
|
|
||||||
.iter()
|
|
||||||
.find(|(covers, _)| *covers >= ticket)
|
|
||||||
.map(|(_, refused)| refused.clone().map_or(Ok(()), Err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// How long a full reload waits after the last registry write for another.
|
|
||||||
/// Parallel requests reach the server tens of milliseconds apart (in a
|
|
||||||
/// cluster rehearsal, ten x:<Object>/set requests sent at once arrived about
|
|
||||||
/// 33 ms apart and each got a reload of its own), so the window is a little
|
|
||||||
/// over twice that. A single write pays it once, on top of the reload.
|
|
||||||
pub const WRITE_QUIET: std::time::Duration = std::time::Duration::from_millis(75);
|
|
||||||
|
|
||||||
/// The longest a full reload waits after the first write it covers, so a
|
|
||||||
/// steady stream of writes still reloads at least this often.
|
|
||||||
pub const WRITE_MAX_WAIT: std::time::Duration = std::time::Duration::from_millis(250);
|
|
||||||
|
|
||||||
/// How many past reload results a waiting write can look up.
|
|
||||||
const RELOAD_RESULTS: usize = 64;
|
|
||||||
|
|
||||||
/// The reload a write to `object` calls for: the object to reload, or None
|
/// The reload a write to `object` calls for: the object to reload, or None
|
||||||
/// when the running settings don't hold that object (accounts, domains and
|
/// when the running settings don't hold that object (accounts, domains and
|
||||||
/// other data read as needed, stores, which take a restart, and objects with
|
/// other data read as needed, stores, which take a restart, and objects with
|
||||||
@@ -422,85 +370,21 @@ impl Server {
|
|||||||
return Some(result);
|
return Some(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: #39 joined only writes that queued behind a running
|
|
||||||
// reload; requests that arrive tens of milliseconds apart never
|
|
||||||
// overlapped one, so each got a reload of its own. The reload now
|
|
||||||
// waits until writes settle (WRITE_QUIET after the last one, at
|
|
||||||
// most WRITE_MAX_WAIT after the first) and covers them all. It runs
|
|
||||||
// in a task of its own, so a request that goes away doesn't take
|
|
||||||
// it with it; each write then takes the result of the reload that
|
|
||||||
// started after it was stored.
|
|
||||||
let gate = &self.inner.data.settings_reload;
|
let gate = &self.inner.data.settings_reload;
|
||||||
let ticket = gate
|
let ticket = gate
|
||||||
.requested
|
.requested
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
|
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
|
||||||
+ 1;
|
+ 1;
|
||||||
let now = std::time::Instant::now();
|
let mut state = gate.state.lock().await;
|
||||||
{
|
if state.completed >= ticket {
|
||||||
let mut state = gate.state.lock();
|
// A reload that started after this write was stored has run
|
||||||
state.first_write.get_or_insert(now);
|
return Some(state.refused.clone().map_or(Ok(()), Err));
|
||||||
state.last_write = Some(now);
|
|
||||||
}
|
}
|
||||||
|
let covers = gate.requested.load(std::sync::atomic::Ordering::SeqCst);
|
||||||
loop {
|
let result = self.reload_and_broadcast(change).await;
|
||||||
let mut completed = {
|
state.completed = covers;
|
||||||
let mut state = gate.state.lock();
|
state.refused = result.clone().err();
|
||||||
if let Some(result) = state.result_for(ticket) {
|
Some(result)
|
||||||
return Some(result);
|
|
||||||
}
|
|
||||||
if !state.scheduled {
|
|
||||||
state.scheduled = true;
|
|
||||||
let server = self.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
server.run_write_reload(change).await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
gate.completed.subscribe()
|
|
||||||
};
|
|
||||||
if completed.changed().await.is_err() {
|
|
||||||
return Some(Err("The settings reload was interrupted".to_string()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for registry writes to settle, then reloads the settings once
|
|
||||||
/// for all the writes stored so far.
|
|
||||||
async fn run_write_reload(&self, change: RegistryChange) {
|
|
||||||
let gate = &self.inner.data.settings_reload;
|
|
||||||
loop {
|
|
||||||
let deadline = {
|
|
||||||
let state = gate.state.lock();
|
|
||||||
let now = std::time::Instant::now();
|
|
||||||
let first = state.first_write.unwrap_or(now);
|
|
||||||
let last = state.last_write.unwrap_or(now);
|
|
||||||
(last + WRITE_QUIET).min(first + WRITE_MAX_WAIT)
|
|
||||||
};
|
|
||||||
if deadline <= std::time::Instant::now() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
tokio::time::sleep_until(deadline.into()).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Writes stored from here on wait for the next reload
|
|
||||||
let covers = {
|
|
||||||
let mut state = gate.state.lock();
|
|
||||||
state.first_write = None;
|
|
||||||
state.last_write = None;
|
|
||||||
gate.requested.load(std::sync::atomic::Ordering::SeqCst)
|
|
||||||
};
|
|
||||||
gate.reloads
|
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
||||||
let result = self.inner.build_server().reload_and_broadcast(change).await;
|
|
||||||
|
|
||||||
{
|
|
||||||
let mut state = gate.state.lock();
|
|
||||||
if state.results.len() == RELOAD_RESULTS {
|
|
||||||
state.results.pop_front();
|
|
||||||
}
|
|
||||||
state.results.push_back((covers, result.err()));
|
|
||||||
state.scheduled = false;
|
|
||||||
}
|
|
||||||
gate.completed.send_replace(covers);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn reload_and_broadcast(&self, change: RegistryChange) -> Result<(), String> {
|
async fn reload_and_broadcast(&self, change: RegistryChange) -> Result<(), String> {
|
||||||
|
|||||||
@@ -94,7 +94,6 @@ impl Data {
|
|||||||
span_id_gen: id_generator,
|
span_id_gen: id_generator,
|
||||||
queue_status: true.into(),
|
queue_status: true.into(),
|
||||||
settings_reload: Default::default(),
|
settings_reload: Default::default(),
|
||||||
store_health: Default::default(),
|
|
||||||
applications,
|
applications,
|
||||||
logos: Default::default(),
|
logos: Default::default(),
|
||||||
smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"),
|
smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"),
|
||||||
@@ -238,7 +237,6 @@ impl Default for Data {
|
|||||||
registry_id_gen: Default::default(),
|
registry_id_gen: Default::default(),
|
||||||
queue_status: true.into(),
|
queue_status: true.into(),
|
||||||
settings_reload: Default::default(),
|
settings_reload: Default::default(),
|
||||||
store_health: Default::default(),
|
|
||||||
applications: WebApplications::new(),
|
applications: WebApplications::new(),
|
||||||
logos: Default::default(),
|
logos: Default::default(),
|
||||||
smtp_connectors: TlsConnectors::try_new().unwrap(),
|
smtp_connectors: TlsConnectors::try_new().unwrap(),
|
||||||
|
|||||||
@@ -31,10 +31,6 @@ pub struct TelemetrySubscriber {
|
|||||||
pub interests: Interests,
|
pub interests: Interests,
|
||||||
pub typ: TelemetrySubscriberType,
|
pub typ: TelemetrySubscriberType,
|
||||||
pub lossy: bool,
|
pub lossy: bool,
|
||||||
/// inbuxa: a hash of the settings the running tracer is built from
|
|
||||||
/// (everything but its events, level and lossiness, which change in
|
|
||||||
/// place), so a reload can tell which tracers to start over.
|
|
||||||
pub settings: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::large_enum_variant)]
|
#[allow(clippy::large_enum_variant)]
|
||||||
@@ -171,7 +167,6 @@ impl Tracers {
|
|||||||
for tracer in bp.list_infallible::<Tracer>().await {
|
for tracer in bp.list_infallible::<Tracer>().await {
|
||||||
let id = tracer.id;
|
let id = tracer.id;
|
||||||
let tracer = tracer.object;
|
let tracer = tracer.object;
|
||||||
let settings = tracer_settings(&tracer);
|
|
||||||
let level;
|
let level;
|
||||||
let lossy;
|
let lossy;
|
||||||
let events;
|
let events;
|
||||||
@@ -384,7 +379,6 @@ impl Tracers {
|
|||||||
interests: Default::default(),
|
interests: Default::default(),
|
||||||
lossy,
|
lossy,
|
||||||
typ,
|
typ,
|
||||||
settings,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Parse disabled events
|
// Parse disabled events
|
||||||
@@ -432,7 +426,6 @@ impl Tracers {
|
|||||||
for hook in bp.list_infallible::<WebHook>().await {
|
for hook in bp.list_infallible::<WebHook>().await {
|
||||||
let id = hook.id;
|
let id = hook.id;
|
||||||
let hook = hook.object;
|
let hook = hook.object;
|
||||||
let settings = webhook_settings(&hook);
|
|
||||||
|
|
||||||
if !hook.enable {
|
if !hook.enable {
|
||||||
continue;
|
continue;
|
||||||
@@ -455,7 +448,6 @@ impl Tracers {
|
|||||||
id: format!("w_{}", id.id()),
|
id: format!("w_{}", id.id()),
|
||||||
interests: Default::default(),
|
interests: Default::default(),
|
||||||
lossy: hook.lossy,
|
lossy: hook.lossy,
|
||||||
settings,
|
|
||||||
typ: TelemetrySubscriberType::Webhook(WebhookTracer {
|
typ: TelemetrySubscriberType::Webhook(WebhookTracer {
|
||||||
url: hook.url,
|
url: hook.url,
|
||||||
timeout: hook.timeout.into_inner(),
|
timeout: hook.timeout.into_inner(),
|
||||||
@@ -524,8 +516,6 @@ impl Tracers {
|
|||||||
data: storage.data.clone(),
|
data: storage.data.clone(),
|
||||||
}),
|
}),
|
||||||
lossy: true,
|
lossy: true,
|
||||||
// Stores take a restart
|
|
||||||
settings: 0,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -551,7 +541,6 @@ impl Tracers {
|
|||||||
buffered: true,
|
buffered: true,
|
||||||
}),
|
}),
|
||||||
lossy: false,
|
lossy: false,
|
||||||
settings: 0,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -579,7 +568,6 @@ impl Tracers {
|
|||||||
buffered: true,
|
buffered: true,
|
||||||
}),
|
}),
|
||||||
lossy: false,
|
lossy: false,
|
||||||
settings: 0,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -713,42 +701,6 @@ impl Metrics {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: what a tracer is built from, less what changes in place
|
|
||||||
macro_rules! in_place_reset {
|
|
||||||
($tracer:expr) => {{
|
|
||||||
$tracer.enable = true;
|
|
||||||
$tracer.level = Default::default();
|
|
||||||
$tracer.lossy = false;
|
|
||||||
$tracer.events = Default::default();
|
|
||||||
$tracer.events_policy = Default::default();
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
|
|
||||||
fn settings_hash(settings: &impl std::fmt::Debug) -> u64 {
|
|
||||||
use std::hash::{Hash, Hasher};
|
|
||||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
|
||||||
format!("{settings:?}").hash(&mut hasher);
|
|
||||||
hasher.finish()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tracer_settings(tracer: &Tracer) -> u64 {
|
|
||||||
let mut tracer = tracer.clone();
|
|
||||||
match &mut tracer {
|
|
||||||
Tracer::Log(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::Stdout(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::Journal(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::OtelHttp(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::OtelGrpc(tracer) => in_place_reset!(tracer),
|
|
||||||
}
|
|
||||||
settings_hash(&tracer)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn webhook_settings(hook: &WebHook) -> u64 {
|
|
||||||
let mut hook = hook.clone();
|
|
||||||
in_place_reset!(hook);
|
|
||||||
settings_hash(&hook)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn apply_events(
|
fn apply_events(
|
||||||
event_types: impl IntoIterator<Item = EventType>,
|
event_types: impl IntoIterator<Item = EventType>,
|
||||||
policy: EventPolicy,
|
policy: EventPolicy,
|
||||||
|
|||||||
@@ -163,8 +163,6 @@ pub struct Data {
|
|||||||
pub queue_status: AtomicBool,
|
pub queue_status: AtomicBool,
|
||||||
// inbuxa: coalesces the settings reloads registry writes trigger
|
// inbuxa: coalesces the settings reloads registry writes trigger
|
||||||
pub settings_reload: cache::reload::SettingsReloadGate,
|
pub settings_reload: cache::reload::SettingsReloadGate,
|
||||||
// inbuxa: the readiness probe's cached answer
|
|
||||||
pub store_health: storage::ready::StoreHealth,
|
|
||||||
|
|
||||||
pub applications: WebApplications,
|
pub applications: WebApplications,
|
||||||
pub logos: Mutex<AHashMap<Box<str>, LogoCache>>,
|
pub logos: Mutex<AHashMap<Box<str>, LogoCache>>,
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ pub mod document;
|
|||||||
pub mod encryption;
|
pub mod encryption;
|
||||||
pub mod index;
|
pub mod index;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
pub mod ready; // inbuxa: readiness follows the data store
|
|
||||||
pub mod state;
|
pub mod state;
|
||||||
pub mod transaction;
|
pub mod transaction;
|
||||||
|
|
||||||
|
|||||||
@@ -1,83 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Readiness that reflects the data store.
|
|
||||||
//!
|
|
||||||
//! /healthz/ready used to answer 200 whenever a data store was configured,
|
|
||||||
//! so a load balancer kept sending traffic to a node through a database
|
|
||||||
//! outage. It now reads one key from the data store, with a short time
|
|
||||||
//! limit, and caches the answer for a couple of seconds so probes can't load
|
|
||||||
//! the database. Liveness stays 200: restarting a node doesn't bring its
|
|
||||||
//! database back, and an orchestrator that restarts on failed liveness would
|
|
||||||
//! otherwise restart every node at once.
|
|
||||||
|
|
||||||
use crate::Server;
|
|
||||||
use parking_lot::Mutex;
|
|
||||||
use std::{
|
|
||||||
sync::atomic::{AtomicBool, Ordering},
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{ValueKey, write::ValueClass};
|
|
||||||
|
|
||||||
/// How long a probe's answer is reused.
|
|
||||||
pub const READY_CACHE: Duration = Duration::from_secs(2);
|
|
||||||
/// How long a probe waits for the data store.
|
|
||||||
pub const READY_PROBE_TIMEOUT: Duration = Duration::from_secs(2);
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct StoreHealth {
|
|
||||||
last: Mutex<Option<(Instant, bool)>>,
|
|
||||||
probing: AtomicBool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Clears the probing flag even when the request is dropped mid-probe.
|
|
||||||
struct ProbeGuard<'x>(&'x AtomicBool);
|
|
||||||
|
|
||||||
impl Drop for ProbeGuard<'_> {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.0.store(false, Ordering::Release);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Server {
|
|
||||||
/// Whether the data store answers: a cached result younger than
|
|
||||||
/// READY_CACHE, or a fresh read bounded by READY_PROBE_TIMEOUT. While
|
|
||||||
/// one probe is running, other callers get the last answer.
|
|
||||||
pub async fn is_data_store_ready(&self) -> bool {
|
|
||||||
let store = &self.core.storage.data;
|
|
||||||
if store.is_none() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let health = &self.inner.data.store_health;
|
|
||||||
let last = *health.last.lock();
|
|
||||||
if let Some((at, ready)) = last
|
|
||||||
&& at.elapsed() < READY_CACHE
|
|
||||||
{
|
|
||||||
return ready;
|
|
||||||
}
|
|
||||||
if health.probing.swap(true, Ordering::AcqRel) {
|
|
||||||
return last.is_none_or(|(_, ready)| ready);
|
|
||||||
}
|
|
||||||
let _guard = ProbeGuard(&health.probing);
|
|
||||||
|
|
||||||
let ready = tokio::time::timeout(
|
|
||||||
READY_PROBE_TIMEOUT,
|
|
||||||
store.get_value::<u64>(ValueKey::from(ValueClass::Property(0))),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.is_ok_and(|result| result.is_ok());
|
|
||||||
// Say so once per outage, not on every probe
|
|
||||||
if !ready && last.is_none_or(|(_, ready)| ready) {
|
|
||||||
trc::event!(
|
|
||||||
Store(trc::StoreEvent::UnexpectedError),
|
|
||||||
Details = "Readiness probe: the data store didn't answer",
|
|
||||||
Limit = READY_PROBE_TIMEOUT,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
*health.last.lock() = Some((Instant::now(), ready));
|
|
||||||
ready
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,26 +14,15 @@ pub mod webhooks;
|
|||||||
use tracers::log::spawn_log_tracer;
|
use tracers::log::spawn_log_tracer;
|
||||||
use tracers::otel::spawn_otel_tracer;
|
use tracers::otel::spawn_otel_tracer;
|
||||||
use tracers::stdout::spawn_console_tracer;
|
use tracers::stdout::spawn_console_tracer;
|
||||||
use ahash::AHashMap;
|
|
||||||
use parking_lot::Mutex;
|
|
||||||
use trc::{Collector, ipc::subscriber::SubscriberBuilder};
|
use trc::{Collector, ipc::subscriber::SubscriberBuilder};
|
||||||
use webhooks::spawn_webhook_tracer;
|
use webhooks::spawn_webhook_tracer;
|
||||||
|
|
||||||
use crate::config::telemetry::{Telemetry, TelemetrySubscriberType};
|
use crate::config::telemetry::{Telemetry, TelemetrySubscriberType};
|
||||||
|
|
||||||
/// inbuxa: the tracers this server started, by subscriber id, with the
|
|
||||||
/// settings each was built from. Live-tracing streams and other subscribers
|
|
||||||
/// registered elsewhere aren't listed, so a reload leaves them running.
|
|
||||||
static RUNNING_TRACERS: Mutex<Option<AHashMap<String, u64>>> = Mutex::new(None);
|
|
||||||
|
|
||||||
impl Telemetry {
|
impl Telemetry {
|
||||||
pub fn enable(self) {
|
pub fn enable(self) {
|
||||||
let mut running = RUNNING_TRACERS.lock();
|
|
||||||
let running = running.get_or_insert_with(AHashMap::new);
|
|
||||||
|
|
||||||
// Spawn tracers
|
// Spawn tracers
|
||||||
for tracer in self.tracers.subscribers {
|
for tracer in self.tracers.subscribers {
|
||||||
running.insert(tracer.id.clone(), tracer.settings);
|
|
||||||
tracer.typ.spawn(
|
tracer.typ.spawn(
|
||||||
SubscriberBuilder::new(tracer.id)
|
SubscriberBuilder::new(tracer.id)
|
||||||
.with_interests(tracer.interests)
|
.with_interests(tracer.interests)
|
||||||
@@ -48,39 +37,25 @@ impl Telemetry {
|
|||||||
Collector::reload();
|
Collector::reload();
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: upstream only refreshed the events, level and lossiness of a
|
|
||||||
// tracer that was already running, so a Log tracer moved to another
|
|
||||||
// path (or any tracer whose own settings changed) kept going as it was
|
|
||||||
// built until a restart, while the reload reported the change applied.
|
|
||||||
// A tracer whose settings changed is now started over: the new one is
|
|
||||||
// registered under the same id and the collector swaps it in at an
|
|
||||||
// event boundary, so no event is lost or written twice (see
|
|
||||||
// Update::RegisterSubscriber); the old one writes what it has queued
|
|
||||||
// and stops.
|
|
||||||
pub fn update(self) {
|
pub fn update(self) {
|
||||||
let mut running = RUNNING_TRACERS.lock();
|
|
||||||
let running = running.get_or_insert_with(AHashMap::new);
|
|
||||||
|
|
||||||
// Remove tracers that are no longer active
|
// Remove tracers that are no longer active
|
||||||
running.retain(|id, _| {
|
let active_subscribers = Collector::get_subscribers();
|
||||||
let keep = self
|
for subscribed_id in &active_subscribers {
|
||||||
|
if !self
|
||||||
.tracers
|
.tracers
|
||||||
.subscribers
|
.subscribers
|
||||||
.iter()
|
.iter()
|
||||||
.any(|tracer| tracer.id == *id);
|
.any(|tracer| tracer.id == *subscribed_id)
|
||||||
if !keep {
|
{
|
||||||
Collector::remove_subscriber(id.clone());
|
Collector::remove_subscriber(subscribed_id.clone());
|
||||||
}
|
}
|
||||||
keep
|
}
|
||||||
});
|
|
||||||
|
|
||||||
// Start new tracers, start over those whose settings changed and
|
// Activate new tracers or update existing ones
|
||||||
// update the rest in place
|
|
||||||
for tracer in self.tracers.subscribers {
|
for tracer in self.tracers.subscribers {
|
||||||
if running.get(&tracer.id) == Some(&tracer.settings) {
|
if active_subscribers.contains(&tracer.id) {
|
||||||
Collector::update_subscriber(tracer.id, tracer.interests, tracer.lossy);
|
Collector::update_subscriber(tracer.id, tracer.interests, tracer.lossy);
|
||||||
} else {
|
} else {
|
||||||
running.insert(tracer.id.clone(), tracer.settings);
|
|
||||||
tracer.typ.spawn(
|
tracer.typ.spawn(
|
||||||
SubscriberBuilder::new(tracer.id)
|
SubscriberBuilder::new(tracer.id)
|
||||||
.with_interests(tracer.interests)
|
.with_interests(tracer.interests)
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::{path::PathBuf, time::SystemTime};
|
use std::{path::PathBuf, time::SystemTime};
|
||||||
@@ -17,27 +15,9 @@ use tokio::{
|
|||||||
};
|
};
|
||||||
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
||||||
|
|
||||||
// inbuxa: when a Log tracer is started over on the same files (its rotation
|
|
||||||
// or format changed), the new one waits for the old one to write what it
|
|
||||||
// has queued, so their lines don't interleave. Keyed by path and prefix;
|
|
||||||
// each entry is the last tracer's "done" signal, sent when it ends.
|
|
||||||
type LogFileOwners = ahash::AHashMap<(String, String), tokio::sync::oneshot::Receiver<()>>;
|
|
||||||
static LOG_FILE_OWNERS: parking_lot::Mutex<Option<LogFileOwners>> = parking_lot::Mutex::new(None);
|
|
||||||
|
|
||||||
pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) {
|
pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) {
|
||||||
let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>();
|
|
||||||
let previous = LOG_FILE_OWNERS
|
|
||||||
.lock()
|
|
||||||
.get_or_insert_with(Default::default)
|
|
||||||
.insert((settings.path.clone(), settings.prefix.clone()), done_rx);
|
|
||||||
let (_, mut rx) = builder.register();
|
let (_, mut rx) = builder.register();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
// Dropped when this tracer ends, however it ends
|
|
||||||
let _done = done_tx;
|
|
||||||
if let Some(previous) = previous {
|
|
||||||
let _ = previous.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(writer) = settings.build_writer().await {
|
if let Some(writer) = settings.build_writer().await {
|
||||||
let mut buf = FmtWriter::new(writer)
|
let mut buf = FmtWriter::new(writer)
|
||||||
.with_ansi(settings.ansi)
|
.with_ansi(settings.ansi)
|
||||||
|
|||||||
@@ -47,10 +47,6 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer
|
|||||||
let mut pending_spans = Vec::new();
|
let mut pending_spans = Vec::new();
|
||||||
|
|
||||||
let mut active_spans = AHashMap::new();
|
let mut active_spans = AHashMap::new();
|
||||||
let mut closing = false;
|
|
||||||
let started = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
|
||||||
.map_or(0, |d| d.as_secs());
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
// Wait for the next event or timeout
|
// Wait for the next event or timeout
|
||||||
@@ -79,26 +75,12 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer
|
|||||||
events.iter().chain(std::iter::once(&event)),
|
events.iter().chain(std::iter::once(&event)),
|
||||||
&instrumentation,
|
&instrumentation,
|
||||||
));
|
));
|
||||||
} else if span.inner.timestamp < started {
|
|
||||||
// inbuxa: a span that was open when this
|
|
||||||
// tracer replaced another one (its settings
|
|
||||||
// changed) is exported with its end event
|
|
||||||
// rather than dropped
|
|
||||||
pending_spans.push(build_span_data(
|
|
||||||
span,
|
|
||||||
&event,
|
|
||||||
std::iter::once(&event),
|
|
||||||
&instrumentation,
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
// inbuxa: the tracer was removed or replaced; export
|
break;
|
||||||
// what is pending now rather than drop it
|
|
||||||
closing = true;
|
|
||||||
next_delivery = Instant::now();
|
|
||||||
}
|
}
|
||||||
Err(_) => (),
|
Err(_) => (),
|
||||||
}
|
}
|
||||||
@@ -149,9 +131,6 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if closing {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER);
|
wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer};
|
use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer};
|
||||||
@@ -27,11 +25,6 @@ use trc::{
|
|||||||
|
|
||||||
pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) {
|
pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) {
|
||||||
let (tx, mut rx) = builder.register();
|
let (tx, mut rx) = builder.register();
|
||||||
// inbuxa: failed deliveries come back through a weak sender, so the
|
|
||||||
// channel closes when the collector drops this webhook (removed, or
|
|
||||||
// replaced after a settings change) and the task ends; upstream held a
|
|
||||||
// sender here and the task outlived its subscription
|
|
||||||
let tx = tx.downgrade();
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let settings = Arc::new(settings);
|
let settings = Arc::new(settings);
|
||||||
let mut wakeup_time = LONG_1Y_SLUMBER;
|
let mut wakeup_time = LONG_1Y_SLUMBER;
|
||||||
@@ -65,15 +58,6 @@ pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: Webhook
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
// inbuxa: deliver what is pending rather than drop it
|
|
||||||
if !pending_events.is_empty() {
|
|
||||||
spawn_webhook_handler(
|
|
||||||
settings.clone(),
|
|
||||||
in_flight.clone(),
|
|
||||||
std::mem::take(&mut pending_events),
|
|
||||||
tx.clone(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(_) => (),
|
Err(_) => (),
|
||||||
@@ -118,7 +102,7 @@ fn spawn_webhook_handler(
|
|||||||
settings: Arc<WebhookTracer>,
|
settings: Arc<WebhookTracer>,
|
||||||
in_flight: Arc<AtomicBool>,
|
in_flight: Arc<AtomicBool>,
|
||||||
events: EventBatch,
|
events: EventBatch,
|
||||||
webhook_tx: mpsc::WeakSender<EventBatch>,
|
webhook_tx: mpsc::Sender<EventBatch>,
|
||||||
) {
|
) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
in_flight.store(true, Ordering::Relaxed);
|
in_flight.store(true, Ordering::Relaxed);
|
||||||
@@ -129,11 +113,7 @@ fn spawn_webhook_handler(
|
|||||||
if let Err(err) = post_webhook_events(&settings, &wrapper).await {
|
if let Err(err) = post_webhook_events(&settings, &wrapper).await {
|
||||||
trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err);
|
trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err);
|
||||||
|
|
||||||
let sent = match webhook_tx.upgrade() {
|
if webhook_tx.send(wrapper.events.into_inner()).await.is_err() {
|
||||||
Some(webhook_tx) => webhook_tx.send(wrapper.events.into_inner()).await.is_ok(),
|
|
||||||
None => false,
|
|
||||||
};
|
|
||||||
if !sent {
|
|
||||||
trc::event!(
|
trc::event!(
|
||||||
Server(ServerEvent::ThreadError),
|
Server(ServerEvent::ThreadError),
|
||||||
Details = "Failed to send failed webhook events back to main thread",
|
Details = "Failed to send failed webhook events back to main thread",
|
||||||
|
|||||||
@@ -553,10 +553,8 @@ impl ParseHttp for Server {
|
|||||||
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
|
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
|
||||||
}
|
}
|
||||||
"ready" => {
|
"ready" => {
|
||||||
// inbuxa: ready only while the data store answers
|
|
||||||
// (a cached, time-limited read); liveness stays 200
|
|
||||||
return Ok(JsonProblemResponse({
|
return Ok(JsonProblemResponse({
|
||||||
if self.is_data_store_ready().await {
|
if !self.core.storage.data.is_none() {
|
||||||
StatusCode::OK
|
StatusCode::OK
|
||||||
} else {
|
} else {
|
||||||
StatusCode::SERVICE_UNAVAILABLE
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
@@ -17,42 +15,22 @@ use jmap_proto::{error::set::SetError, types::state::State};
|
|||||||
use jmap_tools::{Key, Value};
|
use jmap_tools::{Key, Value};
|
||||||
use registry::{
|
use registry::{
|
||||||
jmap::IntoValue,
|
jmap::IntoValue,
|
||||||
schema::{
|
schema::prelude::{Object, ObjectInner, ObjectType, Property},
|
||||||
prelude::{Object, ObjectInner, ObjectType, Property},
|
|
||||||
structs::Task,
|
|
||||||
},
|
|
||||||
types::{EnumImpl, datetime::UTCDateTime},
|
types::{EnumImpl, datetime::UTCDateTime},
|
||||||
};
|
};
|
||||||
use services::task_manager::lock::TaskLockManager;
|
|
||||||
use smtp::reporting::index::{ExternalReportIndex, InternalReportIndex};
|
use smtp::reporting::index::{ExternalReportIndex, InternalReportIndex};
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use store::{
|
use store::{
|
||||||
U64_LEN, ValueKey,
|
U64_LEN, ValueKey,
|
||||||
registry::{RegistryFilter, RegistryFilterValue, RegistryQuery},
|
registry::{RegistryFilter, RegistryFilterValue, RegistryQuery},
|
||||||
write::{BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, key::KeySerializer},
|
write::{BatchBuilder, RegistryClass, ValueClass, key::KeySerializer},
|
||||||
};
|
};
|
||||||
use trc::AddContext;
|
use trc::AddContext;
|
||||||
use types::id::Id;
|
use types::id::Id;
|
||||||
|
|
||||||
pub(crate) async fn report_set(
|
pub(crate) async fn report_set(
|
||||||
set: RegistrySetResponse<'_>,
|
mut set: RegistrySetResponse<'_>,
|
||||||
) -> trc::Result<RegistrySetResponse<'_>> {
|
) -> trc::Result<RegistrySetResponse<'_>> {
|
||||||
// inbuxa: task locks taken to reschedule reports are released however
|
|
||||||
// the request ends; a held lock is renewed, so a leaked one would keep
|
|
||||||
// the report's task from ever running
|
|
||||||
let server = set.server;
|
|
||||||
let mut locked_tasks = Vec::new();
|
|
||||||
let result = report_set_locked(set, &mut locked_tasks).await;
|
|
||||||
for task_id in locked_tasks {
|
|
||||||
server.remove_index_lock(task_id).await;
|
|
||||||
}
|
|
||||||
result
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn report_set_locked<'x>(
|
|
||||||
mut set: RegistrySetResponse<'x>,
|
|
||||||
locked_tasks: &mut Vec<u64>,
|
|
||||||
) -> trc::Result<RegistrySetResponse<'x>> {
|
|
||||||
let object_id = set.object_type.to_id();
|
let object_id = set.object_type.to_id();
|
||||||
|
|
||||||
// Reports cannot be created
|
// Reports cannot be created
|
||||||
@@ -111,45 +89,12 @@ async fn report_set_locked<'x>(
|
|||||||
.get_value::<Object>(ValueKey::from(key.clone()))
|
.get_value::<Object>(ValueKey::from(key.clone()))
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
// inbuxa: the report's task shares its id. Hold the task
|
|
||||||
// while its queue rows move, as x:Task/set does, and move the
|
|
||||||
// row the task is actually queued under
|
|
||||||
if !set.server.try_lock_task(item_id).await {
|
|
||||||
set.response.not_updated.append(
|
|
||||||
id,
|
|
||||||
SetError::forbidden().with_description(
|
|
||||||
"The report is being sent and cannot be rescheduled".to_string(),
|
|
||||||
),
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
locked_tasks.push(item_id);
|
|
||||||
let queued = set
|
|
||||||
.server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id: item_id },
|
|
||||||
)))
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
match &mut report_obj.inner {
|
match &mut report_obj.inner {
|
||||||
ObjectInner::DmarcInternalReport(report) => {
|
ObjectInner::DmarcInternalReport(report) => {
|
||||||
report.reschedule_ops(
|
report.reschedule_ops(&mut batch, item_id, report_obj.revision, deliver_at);
|
||||||
&mut batch,
|
|
||||||
item_id,
|
|
||||||
report_obj.revision,
|
|
||||||
deliver_at,
|
|
||||||
queued.as_ref(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
ObjectInner::TlsInternalReport(report) => {
|
ObjectInner::TlsInternalReport(report) => {
|
||||||
report.reschedule_ops(
|
report.reschedule_ops(&mut batch, item_id, report_obj.revision, deliver_at);
|
||||||
&mut batch,
|
|
||||||
item_id,
|
|
||||||
report_obj.revision,
|
|
||||||
deliver_at,
|
|
||||||
queued.as_ref(),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
@@ -211,9 +156,6 @@ async fn report_set_locked<'x>(
|
|||||||
.write(batch.build_all())
|
.write(batch.build_all())
|
||||||
.await
|
.await
|
||||||
.caused_by(trc::location!())?;
|
.caused_by(trc::location!())?;
|
||||||
// inbuxa: a rescheduled report may now be due sooner than the task
|
|
||||||
// manager's next scan
|
|
||||||
set.server.notify_task_queue();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(set)
|
Ok(set)
|
||||||
|
|||||||
@@ -463,10 +463,15 @@ pub(crate) async fn task_query(
|
|||||||
.set_values(typ.is_some()),
|
.set_values(typ.is_some()),
|
||||||
|key, value| {
|
|key, value| {
|
||||||
if let Some(typ) = typ {
|
if let Some(typ) = typ {
|
||||||
// inbuxa: a row whose type can't be read matches no type
|
let task_type =
|
||||||
// filter; the task manager logs and repairs it
|
TaskType::from_id(value.deserialize_be_u16(0)?).ok_or_else(|| {
|
||||||
let task_type = value.deserialize_be_u16(0).ok().and_then(TaskType::from_id);
|
trc::StoreEvent::DataCorruption
|
||||||
if task_type != Some(typ) {
|
.into_err()
|
||||||
|
.ctx(trc::Key::Key, key.to_vec())
|
||||||
|
.ctx(trc::Key::Value, value.to_vec())
|
||||||
|
.caused_by(trc::location!())
|
||||||
|
})?;
|
||||||
|
if task_type != typ {
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ use common::network::limiter::ConcurrencyLimiter;
|
|||||||
use common::network::{ServerInstance, TcpAcceptor};
|
use common::network::{ServerInstance, TcpAcceptor};
|
||||||
use common::{Inner, Server};
|
use common::{Inner, Server};
|
||||||
use registry::schema::enums::TaskType;
|
use registry::schema::enums::TaskType;
|
||||||
use registry::schema::prelude::ObjectType;
|
|
||||||
use registry::schema::structs::{
|
use registry::schema::structs::{
|
||||||
Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry,
|
Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry,
|
||||||
};
|
};
|
||||||
@@ -299,7 +298,6 @@ impl TaskQueueManager for Server {
|
|||||||
|
|
||||||
// Retrieve tasks pending to be processed
|
// Retrieve tasks pending to be processed
|
||||||
let mut tasks = Vec::new();
|
let mut tasks = Vec::new();
|
||||||
let mut unreadable = Vec::new();
|
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut next_event = None;
|
let mut next_event = None;
|
||||||
ipc.revision += 1;
|
ipc.revision += 1;
|
||||||
@@ -313,21 +311,12 @@ impl TaskQueueManager for Server {
|
|||||||
let task_id = key.deserialize_be_u64(U64_LEN)?;
|
let task_id = key.deserialize_be_u64(U64_LEN)?;
|
||||||
|
|
||||||
if task_due <= now_timestamp {
|
if task_due <= now_timestamp {
|
||||||
// inbuxa: a row whose task type can't be read is
|
let task_type_idx = value.deserialize_be_u16(0)?;
|
||||||
// set aside, not allowed to end the scan: every
|
let task_type = TaskType::from_id(task_type_idx).ok_or_else(|| {
|
||||||
// task due after it would wait behind it
|
trc::StoreEvent::DataCorruption
|
||||||
let Some((task_type_idx, task_type)) = value
|
.caused_by(trc::location!())
|
||||||
.deserialize_be_u16(0)
|
.ctx(trc::Key::Value, value)
|
||||||
.ok()
|
})?;
|
||||||
.and_then(|idx| TaskType::from_id(idx).map(|typ| (idx, typ)))
|
|
||||||
else {
|
|
||||||
unreadable.push(UnreadableDueRow {
|
|
||||||
due: task_due,
|
|
||||||
id: task_id,
|
|
||||||
value: value.to_vec(),
|
|
||||||
});
|
|
||||||
return Ok(true);
|
|
||||||
};
|
|
||||||
// inbuxa: running here under a lease this node
|
// inbuxa: running here under a lease this node
|
||||||
// renews; don't hand it to a worker again
|
// renews; don't hand it to a worker again
|
||||||
if task_locks.is_held(task_id) {
|
if task_locks.is_held(task_id) {
|
||||||
@@ -400,11 +389,6 @@ impl TaskQueueManager for Server {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
if !unreadable.is_empty() && repair_due_rows(self, unreadable).await {
|
|
||||||
// Look again at once for the rows that were rewritten
|
|
||||||
self.notify_task_queue();
|
|
||||||
}
|
|
||||||
|
|
||||||
if !tasks.is_empty() {
|
if !tasks.is_empty() {
|
||||||
trc::event!(
|
trc::event!(
|
||||||
TaskManager(TaskManagerEvent::TaskAcquired),
|
TaskManager(TaskManagerEvent::TaskAcquired),
|
||||||
@@ -835,114 +819,3 @@ impl TaskResult {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: a task queue row whose task type could not be read.
|
|
||||||
struct UnreadableDueRow {
|
|
||||||
due: u64,
|
|
||||||
id: u64,
|
|
||||||
value: Vec<u8>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: logs each unreadable queue row and repairs it from the task it
|
|
||||||
/// schedules. The task row says what the task is, so the queue row is
|
|
||||||
/// rewritten with that task's type; a row with no task behind it is removed.
|
|
||||||
///
|
|
||||||
/// Rescheduling an internal DMARC or TLS report wrote the report's object
|
|
||||||
/// type into the queue row instead of the task type. Such a row is the time
|
|
||||||
/// an administrator chose, so the task is moved to it as the reschedule
|
|
||||||
/// meant to do: the task row takes that due, and a queue row left at the
|
|
||||||
/// task's previous due is removed. Returns whether any row was repaired.
|
|
||||||
async fn repair_due_rows(server: &Server, rows: Vec<UnreadableDueRow>) -> bool {
|
|
||||||
let mut repaired = false;
|
|
||||||
for row in rows {
|
|
||||||
let UnreadableDueRow { due, id, value } = row;
|
|
||||||
trc::error!(
|
|
||||||
trc::StoreEvent::DataCorruption
|
|
||||||
.into_err()
|
|
||||||
.id(id)
|
|
||||||
.ctx(trc::Key::Due, trc::Value::Timestamp(due))
|
|
||||||
.ctx(
|
|
||||||
trc::Key::Key,
|
|
||||||
[due.to_be_bytes(), id.to_be_bytes()].concat()
|
|
||||||
)
|
|
||||||
.ctx(trc::Key::Value, value.clone())
|
|
||||||
.details("Unreadable task queue row skipped")
|
|
||||||
.caused_by(trc::location!())
|
|
||||||
);
|
|
||||||
|
|
||||||
let task_key = ValueClass::TaskQueue(TaskQueueClass::Task { id });
|
|
||||||
let due_key = ValueClass::TaskQueue(TaskQueueClass::Due { id, due });
|
|
||||||
let task = match server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(task_key.clone()))
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(task) => task,
|
|
||||||
Err(err) => {
|
|
||||||
trc::error!(
|
|
||||||
err.id(id)
|
|
||||||
.details("Failed to read the task of an unreadable queue row.")
|
|
||||||
.caused_by(trc::location!())
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
let action = if let Some(mut task) = task {
|
|
||||||
let task_type = task.object_type();
|
|
||||||
batch.assert_value(task_key.clone(), AssertValue::Some);
|
|
||||||
if rescheduled_report_type(&value) == Some(task_type) {
|
|
||||||
let old_due = task.due_timestamp();
|
|
||||||
if old_due != due {
|
|
||||||
batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id,
|
|
||||||
due: old_due,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
task.set_status(TaskStatus::at(due as i64));
|
|
||||||
}
|
|
||||||
batch
|
|
||||||
.set(due_key, task_type.to_id().serialize())
|
|
||||||
.set(task_key, task.to_pickled_vec());
|
|
||||||
"Rewrote the queue row from its task."
|
|
||||||
} else {
|
|
||||||
batch.clear(due_key);
|
|
||||||
"Removed a queue row with no task."
|
|
||||||
};
|
|
||||||
|
|
||||||
match server.store().write(batch.build_all()).await {
|
|
||||||
Ok(_) => {
|
|
||||||
repaired = true;
|
|
||||||
trc::event!(
|
|
||||||
TaskManager(TaskManagerEvent::TaskIgnored),
|
|
||||||
Id = id,
|
|
||||||
Due = trc::Value::Timestamp(due),
|
|
||||||
Reason = action,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(err) if err.matches(trc::EventType::Store(trc::StoreEvent::AssertValueFailed)) => {
|
|
||||||
// The task went away meanwhile; the next scan looks again
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
trc::error!(
|
|
||||||
err.id(id)
|
|
||||||
.details("Failed to repair an unreadable queue row.")
|
|
||||||
.caused_by(trc::location!())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
repaired
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: the task type a report reschedule meant, when a queue row holds
|
|
||||||
/// an internal report's object type (the value that reschedule wrote).
|
|
||||||
fn rescheduled_report_type(value: &[u8]) -> Option<TaskType> {
|
|
||||||
let id = u16::from_be_bytes(value.get(..2)?.try_into().ok()?);
|
|
||||||
match ObjectType::from_id(id)? {
|
|
||||||
ObjectType::DmarcInternalReport => Some(TaskType::DmarcReport),
|
|
||||||
ObjectType::TlsInternalReport => Some(TaskType::TlsReport),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -47,10 +47,8 @@ impl SpawnQueueManager for IpcReceivers {
|
|||||||
// inbuxa: upstream started these only when the node's role included
|
// inbuxa: upstream started these only when the node's role included
|
||||||
// outboundMta at boot, so turning the role on later did nothing and
|
// outboundMta at boot, so turning the role on later did nothing and
|
||||||
// turning it off left them delivering until a restart. They now run
|
// turning it off left them delivering until a restart. They now run
|
||||||
// on every node: the queue follows the role live (see Queue::start),
|
// on every node and follow the role live (see Queue::start and the
|
||||||
// and the report scheduler records DMARC and TLS results on every
|
// report scheduler). This also drains the queue channel on nodes
|
||||||
// node, whatever its role (see reporting/scheduler.rs). This also
|
|
||||||
// drains the queue channel on nodes
|
|
||||||
// without the role, where every queued message's refresh used to sit
|
// without the role, where every queued message's refresh used to sit
|
||||||
// in a channel nobody read until it filled and queueing blocked.
|
// in a channel nobody read until it filled and queueing blocked.
|
||||||
if !core.storage.registry.is_recovery_mode() {
|
if !core.storage.registry.is_recovery_mode() {
|
||||||
|
|||||||
@@ -2,12 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::AggregateTimestamp;
|
use super::AggregateTimestamp;
|
||||||
use super::shared::{MAX_WRITE_RETRIES, Revisioned, write_retry_pause};
|
|
||||||
use crate::{
|
use crate::{
|
||||||
core::Session,
|
core::Session,
|
||||||
queue::RecipientDomain,
|
queue::RecipientDomain,
|
||||||
@@ -352,44 +349,30 @@ impl DmarcReporting for Server {
|
|||||||
let object_id = ObjectType::DmarcInternalReport.to_id();
|
let object_id = ObjectType::DmarcInternalReport.to_id();
|
||||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
||||||
|
|
||||||
// Delete report. inbuxa: only the version read here, so a record
|
let Some(report) = self
|
||||||
// another node appends meanwhile is sent with it rather than lost
|
.store()
|
||||||
let mut attempt = 0;
|
.get_value::<DmarcInternalReport>(ValueKey::from(key.clone()))
|
||||||
let report = loop {
|
.await
|
||||||
let Some(Revisioned {
|
.caused_by(trc::location!())?
|
||||||
revision,
|
else {
|
||||||
value: report,
|
return Ok(());
|
||||||
}) = self
|
|
||||||
.store()
|
|
||||||
.get_value::<Revisioned<DmarcInternalReport>>(ValueKey::from(key.clone()))
|
|
||||||
.await
|
|
||||||
.caused_by(trc::location!())?
|
|
||||||
else {
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.assert_value(key.clone(), AssertValue::Hash(revision))
|
|
||||||
.clear(key.clone())
|
|
||||||
.clear(RegistryClass::PrimaryKey {
|
|
||||||
object_id: object_id.into(),
|
|
||||||
index_id: Property::Domain.to_id(),
|
|
||||||
key: KeySerializer::new(report.domain.len() + U64_LEN)
|
|
||||||
.write(&report.domain)
|
|
||||||
.write(report.policy_identifier)
|
|
||||||
.finalize(),
|
|
||||||
});
|
|
||||||
match self.store().write(batch.build_all()).await {
|
|
||||||
Ok(_) => break report,
|
|
||||||
Err(err) if err.is_assertion_failure() && attempt < MAX_WRITE_RETRIES => {
|
|
||||||
attempt += 1;
|
|
||||||
write_retry_pause(attempt).await;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Delete report
|
||||||
|
let mut batch = BatchBuilder::new();
|
||||||
|
batch.clear(key).clear(RegistryClass::PrimaryKey {
|
||||||
|
object_id: object_id.into(),
|
||||||
|
index_id: Property::Domain.to_id(),
|
||||||
|
key: KeySerializer::new(report.domain.len() + U64_LEN)
|
||||||
|
.write(&report.domain)
|
||||||
|
.write(report.policy_identifier)
|
||||||
|
.finalize(),
|
||||||
|
});
|
||||||
|
self.store()
|
||||||
|
.write(batch.build_all())
|
||||||
|
.await
|
||||||
|
.caused_by(trc::location!())?;
|
||||||
|
|
||||||
let span_id = self.inner.data.span_id_gen.generate();
|
let span_id = self.inner.data.span_id_gen.generate();
|
||||||
let event_from = report.report.date_range_begin.timestamp() as u64;
|
let event_from = report.report.date_range_begin.timestamp() as u64;
|
||||||
let event_to = report.report.date_range_end.timestamp() as u64;
|
let event_to = report.report.date_range_end.timestamp() as u64;
|
||||||
@@ -693,11 +676,8 @@ impl DmarcReporting for Server {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
// inbuxa: another node appended first; try again
|
if err.is_assertion_failure() && rety_count < 3 {
|
||||||
// after a short pause
|
|
||||||
if err.is_assertion_failure() && rety_count < MAX_WRITE_RETRIES {
|
|
||||||
rety_count += 1;
|
rety_count += 1;
|
||||||
write_retry_pause(rety_count).await;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
trc::error!(
|
trc::error!(
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use registry::{
|
use registry::{
|
||||||
@@ -42,49 +40,35 @@ pub trait InternalReportIndex: ObjectImpl {
|
|||||||
|
|
||||||
fn primary_key(&self) -> ValueClass;
|
fn primary_key(&self) -> ValueClass;
|
||||||
|
|
||||||
/// Moves the report's delivery, and its queued task, to `at`.
|
|
||||||
///
|
|
||||||
/// inbuxa: the new queue row carries the task's type, as
|
|
||||||
/// `schedule_task_with_id` writes it, and the task row gets the new due
|
|
||||||
/// too. `queued` is the task as stored: its due, not the report's
|
|
||||||
/// `deliverAt`, is the queue row that exists (they differ once the task
|
|
||||||
/// has been retried).
|
|
||||||
fn reschedule_ops(
|
fn reschedule_ops(
|
||||||
&mut self,
|
&mut self,
|
||||||
batch: &mut BatchBuilder,
|
batch: &mut BatchBuilder,
|
||||||
item_id: u64,
|
item_id: u64,
|
||||||
revision: u64,
|
revision: u64,
|
||||||
at: UTCDateTime,
|
at: UTCDateTime,
|
||||||
queued: Option<&Task>,
|
|
||||||
) {
|
) {
|
||||||
let current_deliver_at = self.deliver_at();
|
let current_deliver_at = self.deliver_at();
|
||||||
let current_due = current_deliver_at.timestamp() as u64;
|
|
||||||
let queued_due = queued.map_or(current_due, |task| task.due_timestamp());
|
|
||||||
let new_due = at.timestamp() as u64;
|
|
||||||
|
|
||||||
if current_deliver_at != at || queued_due != new_due {
|
if current_deliver_at != at {
|
||||||
let object = Self::OBJECT;
|
let object = Self::OBJECT;
|
||||||
let object_id = object.to_id();
|
let object_id = object.to_id();
|
||||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
||||||
|
|
||||||
self.set_deliver_at(at);
|
self.set_deliver_at(at);
|
||||||
|
|
||||||
batch.assert_value(key.clone(), AssertValue::Hash(revision));
|
|
||||||
if queued_due != new_due {
|
|
||||||
batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id: item_id,
|
|
||||||
due: queued_due,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
// A row an earlier reschedule left at the report's deliverAt
|
|
||||||
if current_due != new_due && current_due != queued_due {
|
|
||||||
batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id: item_id,
|
|
||||||
due: current_due,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
batch
|
batch
|
||||||
.schedule_task_with_id(item_id, self.task(item_id))
|
.assert_value(key.clone(), AssertValue::Hash(revision))
|
||||||
|
.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
||||||
|
id: item_id,
|
||||||
|
due: current_deliver_at.timestamp() as u64,
|
||||||
|
}))
|
||||||
|
.set(
|
||||||
|
ValueClass::TaskQueue(TaskQueueClass::Due {
|
||||||
|
id: item_id,
|
||||||
|
due: at.timestamp() as u64,
|
||||||
|
}),
|
||||||
|
object_id.serialize(),
|
||||||
|
)
|
||||||
.set(key, self.to_pickled_vec());
|
.set(key, self.to_pickled_vec());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use common::config::smtp::report::AggregateFrequency;
|
use common::config::smtp::report::AggregateFrequency;
|
||||||
@@ -17,7 +15,6 @@ pub mod inbound;
|
|||||||
pub mod index;
|
pub mod index;
|
||||||
pub mod scheduler;
|
pub mod scheduler;
|
||||||
pub mod send;
|
pub mod send;
|
||||||
pub mod shared; // inbuxa: reports written by every node
|
|
||||||
pub mod spf;
|
pub mod spf;
|
||||||
pub mod tls;
|
pub mod tls;
|
||||||
|
|
||||||
|
|||||||
@@ -20,17 +20,14 @@ impl SpawnReport for mpsc::Receiver<ReportingEvent> {
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
while let Some(event) = self.recv().await {
|
while let Some(event) = self.recv().await {
|
||||||
let server = inner.build_server();
|
let server = inner.build_server();
|
||||||
// inbuxa: every node records what it received, whatever its
|
// inbuxa: reports are the outbound MTA's business, as at
|
||||||
// role. An aggregate report covers all of a domain's mail,
|
// boot, but the role is read per event so a change applies
|
||||||
// whichever node took it, and recording is a store write
|
// without a restart. Events that arrive while the role is
|
||||||
// that nodes already share: the report's primary key is
|
// off are dropped, as they were on a node started without it
|
||||||
// versioned, so concurrent appends from several nodes retry
|
if !matches!(event, ReportingEvent::Stop) && !server.core.network.roles.outbound_mta
|
||||||
// rather than overwrite. Only building and sending the
|
{
|
||||||
// report (the DmarcReport and TlsReport tasks) belongs to
|
continue;
|
||||||
// the outbound MTA; the task manager keeps those to nodes
|
}
|
||||||
// with that role. Upstream ran this only on outbound MTA
|
|
||||||
// nodes, so mail received anywhere else never reached a
|
|
||||||
// report.
|
|
||||||
match event {
|
match event {
|
||||||
ReportingEvent::Dmarc(event) => server.schedule_dmarc(event).await,
|
ReportingEvent::Dmarc(event) => server.schedule_dmarc(event).await,
|
||||||
ReportingEvent::Tls(event) => server.schedule_tls(event).await,
|
ReportingEvent::Tls(event) => server.schedule_tls(event).await,
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! inbuxa: internal DMARC and TLS reports are shared by every node. Any node
|
|
||||||
//! that receives mail appends to them, so several nodes can write one report
|
|
||||||
//! at once, and the node that sends it may do so while another is appending.
|
|
||||||
//! Appends already guard the report's versioned primary key and retry when
|
|
||||||
//! another writer got there first; these helpers give those retries room and
|
|
||||||
//! let the sender delete exactly the report it read.
|
|
||||||
|
|
||||||
use rand::RngExt;
|
|
||||||
use std::time::Duration;
|
|
||||||
use store::{Deserialize, xxhash_rust::xxh3::xxh3_64};
|
|
||||||
|
|
||||||
/// How many times a report write that lost to another writer is retried.
|
|
||||||
/// Upstream retried three times, when only outbound MTA nodes wrote.
|
|
||||||
pub(crate) const MAX_WRITE_RETRIES: u32 = 10;
|
|
||||||
|
|
||||||
/// A short random pause, longer on each attempt, before retrying a report
|
|
||||||
/// write that lost to another node, so the writers spread out instead of
|
|
||||||
/// colliding again.
|
|
||||||
pub(crate) async fn write_retry_pause(attempt: u32) {
|
|
||||||
let ms = rand::rng().random_range(5..=25u64) * u64::from(attempt.max(1));
|
|
||||||
tokio::time::sleep(Duration::from_millis(ms)).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A stored value with the hash of the bytes it was read from, for
|
|
||||||
/// `AssertValue::Hash`: a write asserting it fails if anyone changed the
|
|
||||||
/// value since.
|
|
||||||
pub(crate) struct Revisioned<T> {
|
|
||||||
pub revision: u64,
|
|
||||||
pub value: T,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: Deserialize> Deserialize for Revisioned<T> {
|
|
||||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
|
||||||
Ok(Revisioned {
|
|
||||||
revision: xxh3_64(bytes),
|
|
||||||
value: T::deserialize(bytes)?,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,12 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::AggregateTimestamp;
|
use super::AggregateTimestamp;
|
||||||
use super::shared::{MAX_WRITE_RETRIES, Revisioned, write_retry_pause};
|
|
||||||
use crate::{
|
use crate::{
|
||||||
queue::RecipientDomain,
|
queue::RecipientDomain,
|
||||||
reporting::{index::InternalReportIndex, send::MtaReportSend},
|
reporting::{index::InternalReportIndex, send::MtaReportSend},
|
||||||
@@ -73,41 +70,29 @@ impl TlsReporting for Server {
|
|||||||
let object_id = ObjectType::TlsInternalReport.to_id();
|
let object_id = ObjectType::TlsInternalReport.to_id();
|
||||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
||||||
|
|
||||||
// Delete report. inbuxa: only the version read here, so a result
|
let Some(report) = self
|
||||||
// another node appends meanwhile is sent with it rather than lost
|
.store()
|
||||||
let mut attempt = 0;
|
.get_value::<TlsInternalReport>(ValueKey::from(key.clone()))
|
||||||
let report = loop {
|
.await
|
||||||
let Some(Revisioned {
|
.caused_by(trc::location!())?
|
||||||
revision,
|
else {
|
||||||
value: report,
|
return Ok(());
|
||||||
}) = self
|
|
||||||
.store()
|
|
||||||
.get_value::<Revisioned<TlsInternalReport>>(ValueKey::from(key.clone()))
|
|
||||||
.await
|
|
||||||
.caused_by(trc::location!())?
|
|
||||||
else {
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.assert_value(key.clone(), AssertValue::Hash(revision))
|
|
||||||
.clear(key.clone())
|
|
||||||
.clear(RegistryClass::PrimaryKey {
|
|
||||||
object_id: object_id.into(),
|
|
||||||
index_id: Property::Domain.to_id(),
|
|
||||||
key: report.domain.as_bytes().to_vec(),
|
|
||||||
});
|
|
||||||
match self.core.storage.data.write(batch.build_all()).await {
|
|
||||||
Ok(_) => break report,
|
|
||||||
Err(err) if err.is_assertion_failure() && attempt < MAX_WRITE_RETRIES => {
|
|
||||||
attempt += 1;
|
|
||||||
write_retry_pause(attempt).await;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Delete report
|
||||||
|
let mut batch = BatchBuilder::new();
|
||||||
|
batch.clear(key).clear(RegistryClass::PrimaryKey {
|
||||||
|
object_id: object_id.into(),
|
||||||
|
index_id: Property::Domain.to_id(),
|
||||||
|
key: report.domain.as_bytes().to_vec(),
|
||||||
|
});
|
||||||
|
self.core
|
||||||
|
.storage
|
||||||
|
.data
|
||||||
|
.write(batch.build_all())
|
||||||
|
.await
|
||||||
|
.caused_by(trc::location!())?;
|
||||||
|
|
||||||
let domain_name = report.domain.as_str();
|
let domain_name = report.domain.as_str();
|
||||||
let event_from = report.report.date_range_start.timestamp() as u64;
|
let event_from = report.report.date_range_start.timestamp() as u64;
|
||||||
let event_to = report.report.date_range_end.timestamp() as u64;
|
let event_to = report.report.date_range_end.timestamp() as u64;
|
||||||
@@ -492,11 +477,8 @@ impl TlsReporting for Server {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
// inbuxa: another node appended first; try again
|
if err.is_assertion_failure() && rety_count < 3 {
|
||||||
// after a short pause
|
|
||||||
if err.is_assertion_failure() && rety_count < MAX_WRITE_RETRIES {
|
|
||||||
rety_count += 1;
|
rety_count += 1;
|
||||||
write_retry_pause(rety_count).await;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
trc::error!(
|
trc::error!(
|
||||||
|
|||||||
@@ -30,9 +30,6 @@ pub mod s3;
|
|||||||
pub mod sqlite;
|
pub mod sqlite;
|
||||||
// inbuxa: scale-out storage (sharded stores)
|
// inbuxa: scale-out storage (sharded stores)
|
||||||
pub mod scaleout;
|
pub mod scaleout;
|
||||||
// inbuxa: client-side SQL query limits
|
|
||||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
|
||||||
pub mod query_timeout;
|
|
||||||
|
|
||||||
|
|
||||||
pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize;
|
pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize;
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use std::ops::Range;
|
|||||||
|
|
||||||
use mysql_async::prelude::Queryable;
|
use mysql_async::prelude::Queryable;
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, into_error};
|
use super::{MysqlStore, into_error};
|
||||||
|
|
||||||
impl MysqlStore {
|
impl MysqlStore {
|
||||||
pub(crate) async fn get_blob(
|
pub(crate) async fn get_blob(
|
||||||
@@ -19,63 +19,48 @@ impl MysqlStore {
|
|||||||
range: Range<usize>,
|
range: Range<usize>,
|
||||||
) -> trc::Result<Option<Vec<u8>>> {
|
) -> trc::Result<Option<Vec<u8>>> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prep("SELECT v FROM t WHERE k = ?")
|
||||||
let s = conn
|
.await
|
||||||
.prep("SELECT v FROM t WHERE k = ?")
|
.map_err(into_error)?;
|
||||||
.await
|
conn.exec_first::<Vec<u8>, _, _>(&s, (key,))
|
||||||
.map_err(into_error)?;
|
.await
|
||||||
conn.exec_first::<Vec<u8>, _, _>(&s, (key,))
|
.map(|bytes| {
|
||||||
.await
|
if range.start == 0 && range.end == usize::MAX {
|
||||||
.map(|bytes| {
|
bytes
|
||||||
if range.start == 0 && range.end == usize::MAX {
|
} else {
|
||||||
|
bytes.map(|bytes| {
|
||||||
bytes
|
bytes
|
||||||
} else {
|
.get(range.start..std::cmp::min(bytes.len(), range.end))
|
||||||
bytes.map(|bytes| {
|
.unwrap_or_default()
|
||||||
bytes
|
.to_vec()
|
||||||
.get(range.start..std::cmp::min(bytes.len(), range.end))
|
})
|
||||||
.unwrap_or_default()
|
}
|
||||||
.to_vec()
|
})
|
||||||
})
|
.map_err(into_error)
|
||||||
}
|
|
||||||
})
|
|
||||||
.map_err(into_error)
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)")
|
||||||
let s = conn
|
.await
|
||||||
.prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)")
|
.map_err(into_error)?;
|
||||||
.await
|
conn.exec_drop(&s, (key, data))
|
||||||
.map_err(into_error)?;
|
.await
|
||||||
conn.exec_drop(&s, (key, data))
|
.map_err(into_error)
|
||||||
.await
|
.map(|_| ())
|
||||||
.map_err(into_error)
|
|
||||||
.map(|_| ())
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prep("DELETE FROM t WHERE k = ?")
|
||||||
let s = conn
|
.await
|
||||||
.prep("DELETE FROM t WHERE k = ?")
|
.map_err(into_error)?;
|
||||||
.await
|
conn.exec_iter(&s, (key,))
|
||||||
.map_err(into_error)?;
|
.await
|
||||||
conn.exec_iter(&s, (key,))
|
.map_err(into_error)
|
||||||
.await
|
.map(|hits| hits.affected_rows() > 0)
|
||||||
.map_err(into_error)
|
|
||||||
.map(|hits| hits.affected_rows() > 0)
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use mysql_async::{Params, Row, prelude::Queryable};
|
|||||||
|
|
||||||
use crate::{IntoRows, QueryResult, QueryType, Value};
|
use crate::{IntoRows, QueryResult, QueryType, Value};
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, into_error};
|
use super::{MysqlStore, into_error};
|
||||||
|
|
||||||
impl MysqlStore {
|
impl MysqlStore {
|
||||||
pub(crate) async fn sql_query<T: QueryResult>(
|
pub(crate) async fn sql_query<T: QueryResult>(
|
||||||
@@ -19,32 +19,27 @@ impl MysqlStore {
|
|||||||
params: &[Value<'_>],
|
params: &[Value<'_>],
|
||||||
) -> trc::Result<T> {
|
) -> trc::Result<T> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn.prep(query).await.map_err(into_error)?;
|
||||||
let result = tokio::time::timeout(limit, async {
|
let params = Params::Positional(params.iter().map(Into::into).collect());
|
||||||
let s = conn.prep(query).await.map_err(into_error)?;
|
|
||||||
let params = Params::Positional(params.iter().map(Into::into).collect());
|
|
||||||
|
|
||||||
match T::query_type() {
|
match T::query_type() {
|
||||||
QueryType::Execute => conn.exec_drop(s, params).await.map_or_else(
|
QueryType::Execute => conn.exec_drop(s, params).await.map_or_else(
|
||||||
|e| Err(into_error(e)),
|
|e| Err(into_error(e)),
|
||||||
|_| Ok(T::from_exec(conn.affected_rows() as usize)),
|
|_| Ok(T::from_exec(conn.affected_rows() as usize)),
|
||||||
),
|
),
|
||||||
QueryType::Exists => conn
|
QueryType::Exists => conn
|
||||||
.exec_first::<Row, _, _>(s, params)
|
.exec_first::<Row, _, _>(s, params)
|
||||||
.await
|
.await
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some()))),
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some()))),
|
||||||
QueryType::QueryOne => conn
|
QueryType::QueryOne => conn
|
||||||
.exec_first::<Row, _, _>(s, params)
|
.exec_first::<Row, _, _>(s, params)
|
||||||
.await
|
.await
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))),
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))),
|
||||||
QueryType::QueryAll => conn
|
QueryType::QueryAll => conn
|
||||||
.exec::<Row, _, _>(s, params)
|
.exec::<Row, _, _>(s, params)
|
||||||
.await
|
.await
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, into_error};
|
use super::{MysqlStore, into_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
backend::mysql::MysqlSearchField,
|
backend::mysql::MysqlSearchField,
|
||||||
search::{
|
search::{
|
||||||
@@ -72,7 +72,6 @@ impl MysqlStore {
|
|||||||
.db_name(Some(replica.database.clone()))
|
.db_name(Some(replica.database.clone()))
|
||||||
.tcp_port(replica.port as u16),
|
.tcp_port(replica.port as u16),
|
||||||
),
|
),
|
||||||
timeouts: Default::default(),
|
|
||||||
})),
|
})),
|
||||||
replica.host,
|
replica.host,
|
||||||
replica.port as u16,
|
replica.port as u16,
|
||||||
@@ -82,7 +81,6 @@ impl MysqlStore {
|
|||||||
|
|
||||||
let primary = Store::MySQL(Arc::new(MysqlStore {
|
let primary = Store::MySQL(Arc::new(MysqlStore {
|
||||||
conn_pool: Pool::new(opts),
|
conn_pool: Pool::new(opts),
|
||||||
timeouts: Default::default(),
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// ST-1: no replicas, no change
|
// ST-1: no replicas, no change
|
||||||
@@ -101,96 +99,88 @@ impl MysqlStore {
|
|||||||
|
|
||||||
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for table in [
|
|
||||||
SUBSPACE_ACL,
|
|
||||||
SUBSPACE_TASK_QUEUE,
|
|
||||||
SUBSPACE_DELETED_ITEMS,
|
|
||||||
SUBSPACE_SPAM_SAMPLES,
|
|
||||||
crate::SUBSPACE_INBUXA, // inbuxa: masked email
|
|
||||||
SUBSPACE_BLOB_LINK,
|
|
||||||
SUBSPACE_IN_MEMORY_VALUE,
|
|
||||||
SUBSPACE_PROPERTY,
|
|
||||||
SUBSPACE_REGISTRY,
|
|
||||||
SUBSPACE_REGISTRY_PK,
|
|
||||||
SUBSPACE_DIRECTORY,
|
|
||||||
SUBSPACE_QUEUE_MESSAGE,
|
|
||||||
SUBSPACE_QUEUE_EVENT,
|
|
||||||
SUBSPACE_REPORT_OUT,
|
|
||||||
SUBSPACE_REPORT_IN,
|
|
||||||
SUBSPACE_LOGS,
|
|
||||||
SUBSPACE_TELEMETRY_SPAN,
|
|
||||||
SUBSPACE_TELEMETRY_METRIC,
|
|
||||||
] {
|
|
||||||
let table = char::from(table);
|
|
||||||
conn.query_drop(format!(
|
|
||||||
"CREATE TABLE IF NOT EXISTS {table} (
|
|
||||||
k VARBINARY(255) NOT NULL,
|
|
||||||
v MEDIUMBLOB NOT NULL,
|
|
||||||
PRIMARY KEY (k)
|
|
||||||
) ENGINE=InnoDB"
|
|
||||||
))
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
for table in [
|
||||||
|
SUBSPACE_ACL,
|
||||||
|
SUBSPACE_TASK_QUEUE,
|
||||||
|
SUBSPACE_DELETED_ITEMS,
|
||||||
|
SUBSPACE_SPAM_SAMPLES,
|
||||||
|
crate::SUBSPACE_INBUXA, // inbuxa: masked email
|
||||||
|
SUBSPACE_BLOB_LINK,
|
||||||
|
SUBSPACE_IN_MEMORY_VALUE,
|
||||||
|
SUBSPACE_PROPERTY,
|
||||||
|
SUBSPACE_REGISTRY,
|
||||||
|
SUBSPACE_REGISTRY_PK,
|
||||||
|
SUBSPACE_DIRECTORY,
|
||||||
|
SUBSPACE_QUEUE_MESSAGE,
|
||||||
|
SUBSPACE_QUEUE_EVENT,
|
||||||
|
SUBSPACE_REPORT_OUT,
|
||||||
|
SUBSPACE_REPORT_IN,
|
||||||
|
SUBSPACE_LOGS,
|
||||||
|
SUBSPACE_TELEMETRY_SPAN,
|
||||||
|
SUBSPACE_TELEMETRY_METRIC,
|
||||||
|
] {
|
||||||
|
let table = char::from(table);
|
||||||
conn.query_drop(format!(
|
conn.query_drop(format!(
|
||||||
"CREATE TABLE IF NOT EXISTS {} (
|
"CREATE TABLE IF NOT EXISTS {table} (
|
||||||
k VARBINARY(255) NOT NULL,
|
k VARBINARY(255) NOT NULL,
|
||||||
v LONGBLOB NOT NULL,
|
v MEDIUMBLOB NOT NULL,
|
||||||
PRIMARY KEY (k)
|
PRIMARY KEY (k)
|
||||||
) ENGINE=InnoDB",
|
) ENGINE=InnoDB"
|
||||||
char::from(SUBSPACE_BLOBS),
|
|
||||||
))
|
))
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
|
}
|
||||||
|
|
||||||
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
|
conn.query_drop(format!(
|
||||||
let table = char::from(table);
|
"CREATE TABLE IF NOT EXISTS {} (
|
||||||
conn.query_drop(format!(
|
k VARBINARY(255) NOT NULL,
|
||||||
"CREATE TABLE IF NOT EXISTS {table} (
|
v LONGBLOB NOT NULL,
|
||||||
k BLOB,
|
PRIMARY KEY (k)
|
||||||
PRIMARY KEY (k(400))
|
) ENGINE=InnoDB",
|
||||||
) ENGINE=InnoDB"
|
char::from(SUBSPACE_BLOBS),
|
||||||
))
|
))
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
}
|
|
||||||
|
|
||||||
for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] {
|
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
|
||||||
conn.query_drop(format!(
|
let table = char::from(table);
|
||||||
"CREATE TABLE IF NOT EXISTS {} (
|
conn.query_drop(format!(
|
||||||
k VARBINARY(255) NOT NULL,
|
"CREATE TABLE IF NOT EXISTS {table} (
|
||||||
v BIGINT NOT NULL DEFAULT 0,
|
k BLOB,
|
||||||
PRIMARY KEY (k)
|
PRIMARY KEY (k(400))
|
||||||
) ENGINE=InnoDB",
|
) ENGINE=InnoDB"
|
||||||
char::from(table)
|
))
|
||||||
))
|
.await
|
||||||
.await
|
.map_err(into_error)?;
|
||||||
.map_err(into_error)?;
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||||
})
|
conn.query_drop(format!(
|
||||||
.await;
|
"CREATE TABLE IF NOT EXISTS {} (
|
||||||
bounded(conn, result, limit)
|
k VARBINARY(255) NOT NULL,
|
||||||
|
v BIGINT NOT NULL DEFAULT 0,
|
||||||
|
PRIMARY KEY (k)
|
||||||
|
) ENGINE=InnoDB",
|
||||||
|
char::from(table)
|
||||||
|
))
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
create_search_tables::<EmailSearchField>(&mut conn).await?;
|
|
||||||
create_search_tables::<CalendarSearchField>(&mut conn).await?;
|
|
||||||
create_search_tables::<ContactSearchField>(&mut conn).await?;
|
|
||||||
//create_search_tables::<FileSearchField>(&mut conn).await?;
|
|
||||||
create_search_tables::<TracingSearchField>(&mut conn).await?;
|
|
||||||
|
|
||||||
Ok(())
|
create_search_tables::<EmailSearchField>(&mut conn).await?;
|
||||||
})
|
create_search_tables::<CalendarSearchField>(&mut conn).await?;
|
||||||
.await;
|
create_search_tables::<ContactSearchField>(&mut conn).await?;
|
||||||
bounded(conn, result, limit)
|
//create_search_tables::<FileSearchField>(&mut conn).await?;
|
||||||
|
create_search_tables::<TracingSearchField>(&mut conn).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::backend::query_timeout::QueryTimeouts;
|
|
||||||
use crate::{
|
use crate::{
|
||||||
search::{
|
search::{
|
||||||
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
||||||
@@ -15,7 +14,7 @@ use crate::{
|
|||||||
write::SearchIndex,
|
write::SearchIndex,
|
||||||
};
|
};
|
||||||
use mysql_async::Pool;
|
use mysql_async::Pool;
|
||||||
use std::{fmt::Display, time::Duration};
|
use std::fmt::Display;
|
||||||
|
|
||||||
pub mod blob;
|
pub mod blob;
|
||||||
pub mod lookup;
|
pub mod lookup;
|
||||||
@@ -26,8 +25,6 @@ pub mod write;
|
|||||||
|
|
||||||
pub struct MysqlStore {
|
pub struct MysqlStore {
|
||||||
pub(crate) conn_pool: Pool,
|
pub(crate) conn_pool: Pool,
|
||||||
/// inbuxa: client-side query limits (see backend::query_timeout)
|
|
||||||
pub(crate) timeouts: QueryTimeouts,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: how long a request waits for a pooled connection (including
|
/// inbuxa: how long a request waits for a pooled connection (including
|
||||||
@@ -57,43 +54,6 @@ pub(crate) async fn pool_conn(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: the error for an operation that ran past its time limit.
|
|
||||||
pub(crate) fn query_timeout_error(limit: Duration) -> trc::Error {
|
|
||||||
trc::StoreEvent::MysqlError
|
|
||||||
.reason("Query timed out")
|
|
||||||
.details(format!(
|
|
||||||
"No answer from the database within {} s",
|
|
||||||
limit.as_secs()
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: ends an operation run on `conn` under `limit`. When it ran out,
|
|
||||||
/// the connection is closed rather than returned to the pool: a query may
|
|
||||||
/// still be in flight on it, or a transaction open. Conn::disconnect marks
|
|
||||||
/// the connection closed before it sends anything, so even when the server
|
|
||||||
/// doesn't answer and the attempt is dropped, the pool discards it instead
|
|
||||||
/// of waiting to clean it up.
|
|
||||||
pub(crate) fn bounded<T>(
|
|
||||||
conn: mysql_async::Conn,
|
|
||||||
result: Result<trc::Result<T>, tokio::time::error::Elapsed>,
|
|
||||||
limit: Duration,
|
|
||||||
) -> trc::Result<T> {
|
|
||||||
match result {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(_) => {
|
|
||||||
discard(conn);
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: closes a connection whose state is unknown (see bounded).
|
|
||||||
pub(crate) fn discard(conn: mysql_async::Conn) {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let _ = tokio::time::timeout(Duration::from_secs(1), conn.disconnect()).await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub(crate) fn into_error(err: impl Display) -> trc::Error {
|
pub(crate) fn into_error(err: impl Display) -> trc::Error {
|
||||||
trc::StoreEvent::MysqlError.reason(err)
|
trc::StoreEvent::MysqlError.reason(err)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, discard, into_error, is_timeout_error, query_timeout_error};
|
use super::{MysqlStore, into_error, is_timeout_error};
|
||||||
use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass};
|
use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass};
|
||||||
use futures::TryStreamExt;
|
use futures::TryStreamExt;
|
||||||
use mysql_async::{Row, prelude::Queryable};
|
use mysql_async::{Row, prelude::Queryable};
|
||||||
@@ -17,50 +17,40 @@ impl MysqlStore {
|
|||||||
U: Deserialize + 'static,
|
U: Deserialize + 'static,
|
||||||
{
|
{
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prep(format!(
|
||||||
let s = conn
|
"SELECT v FROM {} WHERE k = ?",
|
||||||
.prep(format!(
|
char::from(key.subspace())
|
||||||
"SELECT v FROM {} WHERE k = ?",
|
))
|
||||||
char::from(key.subspace())
|
.await
|
||||||
))
|
.map_err(into_error)?;
|
||||||
.await
|
let key = key.serialize(0);
|
||||||
.map_err(into_error)?;
|
conn.exec_first::<Vec<u8>, _, _>(&s, (&key,))
|
||||||
let key = key.serialize(0);
|
.await
|
||||||
conn.exec_first::<Vec<u8>, _, _>(&s, (&key,))
|
.map_err(into_error)
|
||||||
.await
|
.and_then(|r| {
|
||||||
.map_err(into_error)
|
if let Some(r) = r {
|
||||||
.and_then(|r| {
|
Ok(Some(U::deserialize_owned_with_key(&key, r)?))
|
||||||
if let Some(r) = r {
|
} else {
|
||||||
Ok(Some(U::deserialize_owned_with_key(&key, r)?))
|
Ok(None)
|
||||||
} else {
|
}
|
||||||
Ok(None)
|
})
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prep(format!(
|
||||||
let s = conn
|
"SELECT 1 FROM {} WHERE k = ?",
|
||||||
.prep(format!(
|
char::from(key.subspace())
|
||||||
"SELECT 1 FROM {} WHERE k = ?",
|
))
|
||||||
char::from(key.subspace())
|
.await
|
||||||
))
|
.map_err(into_error)?;
|
||||||
.await
|
let key = key.serialize(0);
|
||||||
.map_err(into_error)?;
|
conn.exec_first::<u8, _, _>(&s, (&key,))
|
||||||
let key = key.serialize(0);
|
.await
|
||||||
conn.exec_first::<u8, _, _>(&s, (&key,))
|
.map_err(into_error)
|
||||||
.await
|
.map(|r| r.is_some())
|
||||||
.map_err(into_error)
|
|
||||||
.map(|r| r.is_some())
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn iterate<T: Key>(
|
pub(crate) async fn iterate<T: Key>(
|
||||||
@@ -74,36 +64,28 @@ impl MysqlStore {
|
|||||||
let end = params.end.serialize(0);
|
let end = params.end.serialize(0);
|
||||||
let keys = if params.values { "k, v" } else { "k" };
|
let keys = if params.values { "k, v" } else { "k" };
|
||||||
|
|
||||||
// inbuxa: a scan may run for hours, so the query limit bounds each
|
let s = conn
|
||||||
// wait for the database (preparing, the query starting, the next
|
.prep(&match (params.first, params.ascending) {
|
||||||
// row) rather than the scan. A wait that runs out closes the
|
(true, true) => {
|
||||||
// connection.
|
format!(
|
||||||
let limit = self.timeouts.query;
|
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1"
|
||||||
let query = match (params.first, params.ascending) {
|
)
|
||||||
(true, true) => {
|
}
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1")
|
(true, false) => {
|
||||||
}
|
format!(
|
||||||
(true, false) => {
|
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1"
|
||||||
format!(
|
)
|
||||||
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC LIMIT 1"
|
}
|
||||||
)
|
(false, true) => {
|
||||||
}
|
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC")
|
||||||
(false, true) => {
|
}
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC")
|
(false, false) => {
|
||||||
}
|
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC")
|
||||||
(false, false) => {
|
}
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC")
|
})
|
||||||
}
|
.await
|
||||||
};
|
.map_err(into_error)?;
|
||||||
let s = match tokio::time::timeout(limit, conn.prep(&query)).await {
|
|
||||||
Ok(s) => s.map_err(into_error)?,
|
|
||||||
Err(_) => {
|
|
||||||
discard(conn);
|
|
||||||
return Err(query_timeout_error(limit));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let mut from = begin;
|
let mut from = begin;
|
||||||
let mut stalled = false;
|
|
||||||
let mut to = end;
|
let mut to = end;
|
||||||
let mut resume_key = None;
|
let mut resume_key = None;
|
||||||
|
|
||||||
@@ -112,26 +94,13 @@ impl MysqlStore {
|
|||||||
let mut timed_out = false;
|
let mut timed_out = false;
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut rows = match tokio::time::timeout(
|
let mut rows = conn
|
||||||
limit,
|
.exec_stream::<Row, _, _>(&s, (from.clone(), to.clone()))
|
||||||
conn.exec_stream::<Row, _, _>(&s, (from.clone(), to.clone())),
|
.await
|
||||||
)
|
.map_err(into_error)?;
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(rows) => rows.map_err(into_error)?,
|
|
||||||
// Leaves the scan loop for the timeout below
|
|
||||||
Err(_) => break,
|
|
||||||
};
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let next = match tokio::time::timeout(limit, rows.try_next()).await {
|
match rows.try_next().await {
|
||||||
Ok(next) => next,
|
|
||||||
Err(_) => {
|
|
||||||
stalled = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
match next {
|
|
||||||
Ok(Some(mut row)) => {
|
Ok(Some(mut row)) => {
|
||||||
let value = if params.values {
|
let value = if params.values {
|
||||||
row.take_opt::<Vec<u8>, _>(1)
|
row.take_opt::<Vec<u8>, _>(1)
|
||||||
@@ -167,10 +136,6 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if stalled {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
match last_key {
|
match last_key {
|
||||||
Some(last_key) if timed_out => {
|
Some(last_key) if timed_out => {
|
||||||
if params.ascending {
|
if params.ascending {
|
||||||
@@ -183,9 +148,6 @@ impl MysqlStore {
|
|||||||
_ => return Ok(()),
|
_ => return Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
discard(conn);
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn get_counter(
|
pub(crate) async fn get_counter(
|
||||||
@@ -196,19 +158,14 @@ impl MysqlStore {
|
|||||||
let table = char::from(key.subspace());
|
let table = char::from(key.subspace());
|
||||||
let key = key.serialize(0);
|
let key = key.serialize(0);
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prep(format!("SELECT v FROM {table} WHERE k = ?"))
|
||||||
let s = conn
|
.await
|
||||||
.prep(format!("SELECT v FROM {table} WHERE k = ?"))
|
.map_err(into_error)?;
|
||||||
.await
|
match conn.exec_first::<i64, _, _>(&s, (key,)).await {
|
||||||
.map_err(into_error)?;
|
Ok(Some(num)) => Ok(num),
|
||||||
match conn.exec_first::<i64, _, _>(&s, (key,)).await {
|
Ok(None) => Ok(0),
|
||||||
Ok(Some(num)) => Ok(num),
|
Err(e) => Err(into_error(e)),
|
||||||
Ok(None) => Ok(0),
|
}
|
||||||
Err(e) => Err(into_error(e)),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ use crate::{
|
|||||||
backend::{
|
backend::{
|
||||||
MAX_TOKEN_LENGTH,
|
MAX_TOKEN_LENGTH,
|
||||||
mysql::{
|
mysql::{
|
||||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, bounded,
|
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, into_error,
|
||||||
into_error, is_timeout_error,
|
is_timeout_error,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
search::{
|
search::{
|
||||||
@@ -27,62 +27,57 @@ use std::fmt::Write;
|
|||||||
impl MysqlStore {
|
impl MysqlStore {
|
||||||
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let mut tx_opts = TxOpts::default();
|
||||||
let result = tokio::time::timeout(limit, async {
|
tx_opts
|
||||||
let mut tx_opts = TxOpts::default();
|
.with_consistent_snapshot(false)
|
||||||
tx_opts
|
.with_isolation_level(IsolationLevel::ReadCommitted);
|
||||||
.with_consistent_snapshot(false)
|
let mut trx = conn.start_transaction(tx_opts).await.map_err(into_error)?;
|
||||||
.with_isolation_level(IsolationLevel::ReadCommitted);
|
|
||||||
let mut trx = conn.start_transaction(tx_opts).await.map_err(into_error)?;
|
|
||||||
|
|
||||||
for document in documents {
|
for document in documents {
|
||||||
let index = document.index;
|
let index = document.index;
|
||||||
let primary_keys = index.primary_keys();
|
let primary_keys = index.primary_keys();
|
||||||
let all_fields = index.all_fields();
|
let all_fields = index.all_fields();
|
||||||
let mut fields = document.fields;
|
let mut fields = document.fields;
|
||||||
let mut values = Vec::with_capacity(fields.len() + 2);
|
let mut values = Vec::with_capacity(fields.len() + 2);
|
||||||
let mut query = format!("INSERT INTO {} (", index.mysql_table());
|
let mut query = format!("INSERT INTO {} (", index.mysql_table());
|
||||||
|
|
||||||
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
query.push(',');
|
query.push(',');
|
||||||
}
|
|
||||||
query.push_str(field.column());
|
|
||||||
}
|
}
|
||||||
|
query.push_str(field.column());
|
||||||
query.push_str(") VALUES (");
|
|
||||||
|
|
||||||
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
|
||||||
if i > 0 {
|
|
||||||
query.push(',');
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(value) = fields.remove(field) {
|
|
||||||
query.push('?');
|
|
||||||
values.push(value);
|
|
||||||
} else {
|
|
||||||
query.push_str("NULL");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
query.push_str(") ON DUPLICATE KEY UPDATE ");
|
|
||||||
for (i, field) in all_fields.iter().enumerate() {
|
|
||||||
if i > 0 {
|
|
||||||
query.push(',');
|
|
||||||
}
|
|
||||||
let column = field.column();
|
|
||||||
let _ = write!(&mut query, "{column} = VALUES({column})");
|
|
||||||
}
|
|
||||||
|
|
||||||
let s = trx.prep(&query).await.map_err(into_error)?;
|
|
||||||
|
|
||||||
trx.exec_drop(&s, values).await.map_err(into_error)?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
trx.commit().await.map_err(into_error)
|
query.push_str(") VALUES (");
|
||||||
})
|
|
||||||
.await;
|
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
||||||
bounded(conn, result, limit)
|
if i > 0 {
|
||||||
|
query.push(',');
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(value) = fields.remove(field) {
|
||||||
|
query.push('?');
|
||||||
|
values.push(value);
|
||||||
|
} else {
|
||||||
|
query.push_str("NULL");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
query.push_str(") ON DUPLICATE KEY UPDATE ");
|
||||||
|
for (i, field) in all_fields.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
query.push(',');
|
||||||
|
}
|
||||||
|
let column = field.column();
|
||||||
|
let _ = write!(&mut query, "{column} = VALUES({column})");
|
||||||
|
}
|
||||||
|
|
||||||
|
let s = trx.prep(&query).await.map_err(into_error)?;
|
||||||
|
|
||||||
|
trx.exec_drop(&s, values).await.map_err(into_error)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
trx.commit().await.map_err(into_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn query<R: SearchDocumentId>(
|
pub async fn query<R: SearchDocumentId>(
|
||||||
@@ -102,17 +97,12 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn.prep(query).await.map_err(into_error)?;
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prep(query).await.map_err(into_error)?;
|
|
||||||
|
|
||||||
conn.exec::<i64, _, _>(s, params)
|
conn.exec::<i64, _, _>(s, params)
|
||||||
.await
|
.await
|
||||||
.map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect())
|
.map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect())
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||||
@@ -121,47 +111,40 @@ impl MysqlStore {
|
|||||||
let params = build_filter(&mut query, &filter.filters);
|
let params = build_filter(&mut query, &filter.filters);
|
||||||
|
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.maintenance;
|
let s = conn.prep(&query).await.map_err(into_error)?;
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prep(&query).await.map_err(into_error)?;
|
|
||||||
|
|
||||||
match conn.exec_drop(s, params.clone()).await {
|
match conn.exec_drop(s, params.clone()).await {
|
||||||
Ok(_) => return Ok(conn.affected_rows()),
|
Ok(_) => return Ok(conn.affected_rows()),
|
||||||
Err(err) if is_timeout_error(&err) => (),
|
Err(err) if is_timeout_error(&err) => (),
|
||||||
Err(err) => return Err(into_error(err)),
|
Err(err) => return Err(into_error(err)),
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||||
let mut deleted = 0;
|
let mut deleted = 0;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let s = conn
|
||||||
|
.prep(format!("{query} LIMIT {chunk_size}"))
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let s = conn
|
match conn.exec_drop(&s, params.clone()).await {
|
||||||
.prep(format!("{query} LIMIT {chunk_size}"))
|
Ok(_) => {
|
||||||
.await
|
let affected = conn.affected_rows();
|
||||||
.map_err(into_error)?;
|
if affected == 0 {
|
||||||
|
return Ok(deleted);
|
||||||
loop {
|
|
||||||
match conn.exec_drop(&s, params.clone()).await {
|
|
||||||
Ok(_) => {
|
|
||||||
let affected = conn.affected_rows();
|
|
||||||
if affected == 0 {
|
|
||||||
return Ok(deleted);
|
|
||||||
}
|
|
||||||
deleted += affected;
|
|
||||||
}
|
}
|
||||||
Err(err)
|
deleted += affected;
|
||||||
if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
|
|
||||||
{
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
}
|
||||||
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,9 +6,7 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{
|
use super::{DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, into_error, is_timeout_error};
|
||||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, bounded, into_error, is_timeout_error,
|
|
||||||
};
|
|
||||||
use crate::{
|
use crate::{
|
||||||
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
||||||
SUBSPACE_REGISTRY_IDX,
|
SUBSPACE_REGISTRY_IDX,
|
||||||
@@ -34,45 +32,41 @@ impl MysqlStore {
|
|||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let mut retry_count = 0;
|
let mut retry_count = 0;
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
loop {
|
|
||||||
let err = match self.write_trx(&mut conn, &mut batch).await {
|
|
||||||
Ok(result) => {
|
|
||||||
return Ok(result);
|
|
||||||
}
|
|
||||||
Err(err) => err,
|
|
||||||
};
|
|
||||||
|
|
||||||
let _ = conn.query_drop("ROLLBACK;").await;
|
loop {
|
||||||
|
let err = match self.write_trx(&mut conn, &mut batch).await {
|
||||||
match err {
|
Ok(result) => {
|
||||||
CommitError::Mysql(Error::Server(err))
|
return Ok(result);
|
||||||
if [1062, 1213].contains(&err.code)
|
|
||||||
&& retry_count < MAX_COMMIT_ATTEMPTS
|
|
||||||
&& start.elapsed() < MAX_COMMIT_TIME => {}
|
|
||||||
/*CommitError::Retry => {
|
|
||||||
if retry_count > MAX_COMMIT_ATTEMPTS || start.elapsed() > MAX_COMMIT_TIME {
|
|
||||||
return Err(trc::StoreEvent::AssertValueFailed
|
|
||||||
.into_err()
|
|
||||||
.caused_by(trc::location!()));
|
|
||||||
}
|
|
||||||
}*/
|
|
||||||
CommitError::Mysql(err) => {
|
|
||||||
return Err(into_error(err));
|
|
||||||
}
|
|
||||||
CommitError::Internal(err) => {
|
|
||||||
return Err(err);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
Err(err) => err,
|
||||||
|
};
|
||||||
|
|
||||||
let backoff = rand::rng().random_range(50..=300);
|
let _ = conn.query_drop("ROLLBACK;").await;
|
||||||
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
|
||||||
retry_count += 1;
|
match err {
|
||||||
|
CommitError::Mysql(Error::Server(err))
|
||||||
|
if [1062, 1213].contains(&err.code)
|
||||||
|
&& retry_count < MAX_COMMIT_ATTEMPTS
|
||||||
|
&& start.elapsed() < MAX_COMMIT_TIME => {}
|
||||||
|
/*CommitError::Retry => {
|
||||||
|
if retry_count > MAX_COMMIT_ATTEMPTS || start.elapsed() > MAX_COMMIT_TIME {
|
||||||
|
return Err(trc::StoreEvent::AssertValueFailed
|
||||||
|
.into_err()
|
||||||
|
.caused_by(trc::location!()));
|
||||||
|
}
|
||||||
|
}*/
|
||||||
|
CommitError::Mysql(err) => {
|
||||||
|
return Err(into_error(err));
|
||||||
|
}
|
||||||
|
CommitError::Internal(err) => {
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
let backoff = rand::rng().random_range(50..=300);
|
||||||
bounded(conn, result, limit)
|
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
||||||
|
retry_count += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn write_trx(
|
async fn write_trx(
|
||||||
@@ -391,81 +385,71 @@ impl MysqlStore {
|
|||||||
|
|
||||||
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.maintenance;
|
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||||
let result = tokio::time::timeout(limit, async {
|
purge_table(&mut conn, char::from(subspace)).await?;
|
||||||
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
}
|
||||||
purge_table(&mut conn, char::from(subspace)).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn().await?;
|
||||||
let limit = self.timeouts.maintenance;
|
let table = char::from(from.subspace());
|
||||||
let result = tokio::time::timeout(limit, async {
|
let mut from = from.serialize(0);
|
||||||
let table = char::from(from.subspace());
|
let to = to.serialize(0);
|
||||||
let mut from = from.serialize(0);
|
|
||||||
let to = to.serialize(0);
|
|
||||||
|
|
||||||
let delete = conn
|
let delete = conn
|
||||||
.prep(format!("DELETE FROM {table} WHERE k >= ? AND k < ?"))
|
.prep(format!("DELETE FROM {table} WHERE k >= ? AND k < ?"))
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
|
||||||
|
match conn.exec_drop(&delete, (&from, &to)).await {
|
||||||
|
Ok(_) => return Ok(()),
|
||||||
|
Err(err) if is_timeout_error(&err) => (),
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let boundary = conn
|
||||||
|
.prep(format!(
|
||||||
|
"SELECT k FROM {table} WHERE k >= ? AND k < ? ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}"
|
||||||
|
))
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
|
|
||||||
match conn.exec_drop(&delete, (&from, &to)).await {
|
|
||||||
Ok(_) => return Ok(()),
|
|
||||||
Err(err) if is_timeout_error(&err) => (),
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let boundary = conn
|
let next = match conn
|
||||||
.prep(format!(
|
.exec_first::<Vec<u8>, _, _>(&boundary, (&from, &to))
|
||||||
"SELECT k FROM {table} WHERE k >= ? AND k < ? ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}"
|
|
||||||
))
|
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
{
|
||||||
|
Ok(next) => next,
|
||||||
loop {
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
let next = match conn
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
.exec_first::<Vec<u8>, _, _>(&boundary, (&from, &to))
|
break;
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(next) => next,
|
|
||||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
};
|
|
||||||
|
|
||||||
match conn
|
|
||||||
.exec_drop(&delete, (&from, next.as_ref().unwrap_or(&to)))
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(_) => (),
|
|
||||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
}
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
};
|
||||||
|
|
||||||
match next {
|
match conn
|
||||||
Some(next) => from = next,
|
.exec_drop(&delete, (&from, next.as_ref().unwrap_or(&to)))
|
||||||
None => return Ok(()),
|
.await
|
||||||
|
{
|
||||||
|
Ok(_) => (),
|
||||||
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
}
|
||||||
|
|
||||||
|
match next {
|
||||||
|
Some(next) => from = next,
|
||||||
|
None => return Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,15 +2,13 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::ops::Range;
|
use std::ops::Range;
|
||||||
|
|
||||||
use crate::backend::postgres::into_pool_error;
|
use crate::backend::postgres::into_pool_error;
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error};
|
use super::{PostgresStore, into_error};
|
||||||
|
|
||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub(crate) async fn get_blob(
|
pub(crate) async fn get_blob(
|
||||||
@@ -19,68 +17,53 @@ impl PostgresStore {
|
|||||||
range: Range<usize>,
|
range: Range<usize>,
|
||||||
) -> trc::Result<Option<Vec<u8>>> {
|
) -> trc::Result<Option<Vec<u8>>> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached("SELECT v FROM t WHERE k = $1")
|
||||||
let s = conn
|
.await
|
||||||
.prepare_cached("SELECT v FROM t WHERE k = $1")
|
.map_err(into_error)?;
|
||||||
.await
|
conn.query_opt(&s, &[&key])
|
||||||
.map_err(into_error)?;
|
.await
|
||||||
conn.query_opt(&s, &[&key])
|
.and_then(|row| {
|
||||||
.await
|
if let Some(row) = row {
|
||||||
.and_then(|row| {
|
Ok(Some(if range.start == 0 && range.end == usize::MAX {
|
||||||
if let Some(row) = row {
|
row.try_get::<_, Vec<u8>>(0)?
|
||||||
Ok(Some(if range.start == 0 && range.end == usize::MAX {
|
|
||||||
row.try_get::<_, Vec<u8>>(0)?
|
|
||||||
} else {
|
|
||||||
let bytes = row.try_get::<_, &[u8]>(0)?;
|
|
||||||
bytes
|
|
||||||
.get(range.start..std::cmp::min(bytes.len(), range.end))
|
|
||||||
.unwrap_or_default()
|
|
||||||
.to_vec()
|
|
||||||
}))
|
|
||||||
} else {
|
} else {
|
||||||
Ok(None)
|
let bytes = row.try_get::<_, &[u8]>(0)?;
|
||||||
}
|
bytes
|
||||||
})
|
.get(range.start..std::cmp::min(bytes.len(), range.end))
|
||||||
.map_err(into_error)
|
.unwrap_or_default()
|
||||||
})
|
.to_vec()
|
||||||
.await;
|
}))
|
||||||
bounded(conn, result, limit)
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.map_err(into_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached(
|
||||||
let s = conn
|
"INSERT INTO t (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v",
|
||||||
.prepare_cached(
|
)
|
||||||
"INSERT INTO t (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v",
|
.await
|
||||||
)
|
.map_err(into_error)?;
|
||||||
.await
|
conn.execute(&s, &[&key, &data])
|
||||||
.map_err(into_error)?;
|
.await
|
||||||
conn.execute(&s, &[&key, &data])
|
.map_err(into_error)
|
||||||
.await
|
.map(|_| ())
|
||||||
.map_err(into_error)
|
|
||||||
.map(|_| ())
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached("DELETE FROM t WHERE k = $1")
|
||||||
let s = conn
|
.await
|
||||||
.prepare_cached("DELETE FROM t WHERE k = $1")
|
.map_err(into_error)?;
|
||||||
.await
|
conn.execute(&s, &[&key])
|
||||||
.map_err(into_error)?;
|
.await
|
||||||
conn.execute(&s, &[&key])
|
.map_err(into_error)
|
||||||
.await
|
.map(|hits| hits > 0)
|
||||||
.map_err(into_error)
|
|
||||||
.map(|hits| hits > 0)
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{QueryResult, QueryType, backend::postgres::into_pool_error};
|
use crate::{QueryResult, QueryType, backend::postgres::into_pool_error};
|
||||||
@@ -14,7 +12,7 @@ use tokio_postgres::types::{FromSql, ToSql, Type};
|
|||||||
|
|
||||||
use crate::IntoRows;
|
use crate::IntoRows;
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error};
|
use super::{PostgresStore, into_error};
|
||||||
|
|
||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub(crate) async fn sql_query<T: QueryResult>(
|
pub(crate) async fn sql_query<T: QueryResult>(
|
||||||
@@ -23,38 +21,33 @@ impl PostgresStore {
|
|||||||
params_: &[crate::Value<'_>],
|
params_: &[crate::Value<'_>],
|
||||||
) -> trc::Result<T> {
|
) -> trc::Result<T> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn.prepare_cached(query).await.map_err(into_error)?;
|
||||||
let result = tokio::time::timeout(limit, async {
|
let params = params_
|
||||||
let s = conn.prepare_cached(query).await.map_err(into_error)?;
|
.iter()
|
||||||
let params = params_
|
.map(|v| v as &(dyn tokio_postgres::types::ToSql + Sync))
|
||||||
.iter()
|
.collect::<Vec<_>>();
|
||||||
.map(|v| v as &(dyn tokio_postgres::types::ToSql + Sync))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
match T::query_type() {
|
match T::query_type() {
|
||||||
QueryType::Execute => conn
|
QueryType::Execute => conn
|
||||||
.execute(&s, params.as_slice())
|
.execute(&s, params.as_slice())
|
||||||
|
.await
|
||||||
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exec(r as usize))),
|
||||||
|
QueryType::Exists => {
|
||||||
|
let rows = conn.query_raw(&s, params).await.map_err(into_error)?;
|
||||||
|
pin_mut!(rows);
|
||||||
|
rows.try_next()
|
||||||
.await
|
.await
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exec(r as usize))),
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some())))
|
||||||
QueryType::Exists => {
|
|
||||||
let rows = conn.query_raw(&s, params).await.map_err(into_error)?;
|
|
||||||
pin_mut!(rows);
|
|
||||||
rows.try_next()
|
|
||||||
.await
|
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_exists(r.is_some())))
|
|
||||||
}
|
|
||||||
QueryType::QueryOne => conn
|
|
||||||
.query_opt(&s, params.as_slice())
|
|
||||||
.await
|
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))),
|
|
||||||
QueryType::QueryAll => conn
|
|
||||||
.query(&s, params.as_slice())
|
|
||||||
.await
|
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
|
||||||
}
|
}
|
||||||
})
|
QueryType::QueryOne => conn
|
||||||
.await;
|
.query_opt(&s, params.as_slice())
|
||||||
bounded(conn, result, limit)
|
.await
|
||||||
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_one(r))),
|
||||||
|
QueryType::QueryAll => conn
|
||||||
|
.query(&s, params.as_slice())
|
||||||
|
.await
|
||||||
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error};
|
use super::{PostgresStore, into_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
backend::postgres::{
|
backend::postgres::{
|
||||||
PsqlSearchField, into_pool_error,
|
PsqlSearchField, into_pool_error,
|
||||||
@@ -119,7 +119,6 @@ impl PostgresStore {
|
|||||||
Store::PostgreSQL(Arc::new(PostgresStore {
|
Store::PostgreSQL(Arc::new(PostgresStore {
|
||||||
conn_pool: pool,
|
conn_pool: pool,
|
||||||
ts_configs: ts_configs.clone(),
|
ts_configs: ts_configs.clone(),
|
||||||
timeouts: Default::default(),
|
|
||||||
})),
|
})),
|
||||||
replica.host,
|
replica.host,
|
||||||
replica.port as u16,
|
replica.port as u16,
|
||||||
@@ -130,7 +129,6 @@ impl PostgresStore {
|
|||||||
let primary = Store::PostgreSQL(Arc::new(PostgresStore {
|
let primary = Store::PostgreSQL(Arc::new(PostgresStore {
|
||||||
conn_pool: primary_pool,
|
conn_pool: primary_pool,
|
||||||
ts_configs,
|
ts_configs,
|
||||||
timeouts: Default::default(),
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// ST-1: no replicas, no change
|
// ST-1: no replicas, no change
|
||||||
@@ -149,92 +147,84 @@ impl PostgresStore {
|
|||||||
|
|
||||||
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for table in [
|
|
||||||
SUBSPACE_ACL,
|
|
||||||
SUBSPACE_TASK_QUEUE,
|
|
||||||
SUBSPACE_DELETED_ITEMS,
|
|
||||||
SUBSPACE_SPAM_SAMPLES,
|
|
||||||
crate::SUBSPACE_INBUXA, // inbuxa: masked email
|
|
||||||
SUBSPACE_BLOB_LINK,
|
|
||||||
SUBSPACE_IN_MEMORY_VALUE,
|
|
||||||
SUBSPACE_PROPERTY,
|
|
||||||
SUBSPACE_REGISTRY,
|
|
||||||
SUBSPACE_REGISTRY_PK,
|
|
||||||
SUBSPACE_QUEUE_MESSAGE,
|
|
||||||
SUBSPACE_QUEUE_EVENT,
|
|
||||||
SUBSPACE_REPORT_OUT,
|
|
||||||
SUBSPACE_REPORT_IN,
|
|
||||||
SUBSPACE_LOGS,
|
|
||||||
SUBSPACE_BLOBS,
|
|
||||||
SUBSPACE_DIRECTORY,
|
|
||||||
SUBSPACE_TELEMETRY_SPAN,
|
|
||||||
SUBSPACE_TELEMETRY_METRIC,
|
|
||||||
] {
|
|
||||||
let table = char::from(table);
|
|
||||||
conn.execute(
|
|
||||||
&format!(
|
|
||||||
"CREATE TABLE IF NOT EXISTS {table} (
|
|
||||||
k BYTEA PRIMARY KEY,
|
|
||||||
v BYTEA NOT NULL
|
|
||||||
)"
|
|
||||||
),
|
|
||||||
&[],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
|
for table in [
|
||||||
let table = char::from(table);
|
SUBSPACE_ACL,
|
||||||
conn.execute(
|
SUBSPACE_TASK_QUEUE,
|
||||||
&format!(
|
SUBSPACE_DELETED_ITEMS,
|
||||||
"CREATE TABLE IF NOT EXISTS {table} (
|
SUBSPACE_SPAM_SAMPLES,
|
||||||
k BYTEA PRIMARY KEY
|
crate::SUBSPACE_INBUXA, // inbuxa: masked email
|
||||||
)"
|
SUBSPACE_BLOB_LINK,
|
||||||
),
|
SUBSPACE_IN_MEMORY_VALUE,
|
||||||
&[],
|
SUBSPACE_PROPERTY,
|
||||||
)
|
SUBSPACE_REGISTRY,
|
||||||
.await
|
SUBSPACE_REGISTRY_PK,
|
||||||
.map_err(into_error)?;
|
SUBSPACE_QUEUE_MESSAGE,
|
||||||
}
|
SUBSPACE_QUEUE_EVENT,
|
||||||
|
SUBSPACE_REPORT_OUT,
|
||||||
for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] {
|
SUBSPACE_REPORT_IN,
|
||||||
conn.execute(
|
SUBSPACE_LOGS,
|
||||||
&format!(
|
SUBSPACE_BLOBS,
|
||||||
"CREATE TABLE IF NOT EXISTS {} (
|
SUBSPACE_DIRECTORY,
|
||||||
|
SUBSPACE_TELEMETRY_SPAN,
|
||||||
|
SUBSPACE_TELEMETRY_METRIC,
|
||||||
|
] {
|
||||||
|
let table = char::from(table);
|
||||||
|
conn.execute(
|
||||||
|
&format!(
|
||||||
|
"CREATE TABLE IF NOT EXISTS {table} (
|
||||||
k BYTEA PRIMARY KEY,
|
k BYTEA PRIMARY KEY,
|
||||||
v BIGINT NOT NULL DEFAULT 0
|
v BYTEA NOT NULL
|
||||||
)",
|
)"
|
||||||
char::from(table)
|
),
|
||||||
),
|
&[],
|
||||||
&[],
|
)
|
||||||
)
|
.await
|
||||||
.await
|
.map_err(into_error)?;
|
||||||
.map_err(into_error)?;
|
}
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
for table in [SUBSPACE_INDEXES, SUBSPACE_REGISTRY_IDX] {
|
||||||
})
|
let table = char::from(table);
|
||||||
.await;
|
conn.execute(
|
||||||
bounded(conn, result, limit)
|
&format!(
|
||||||
|
"CREATE TABLE IF NOT EXISTS {table} (
|
||||||
|
k BYTEA PRIMARY KEY
|
||||||
|
)"
|
||||||
|
),
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
for table in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||||
|
conn.execute(
|
||||||
|
&format!(
|
||||||
|
"CREATE TABLE IF NOT EXISTS {} (
|
||||||
|
k BYTEA PRIMARY KEY,
|
||||||
|
v BIGINT NOT NULL DEFAULT 0
|
||||||
|
)",
|
||||||
|
char::from(table)
|
||||||
|
),
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
create_search_tables::<EmailSearchField>(&conn).await?;
|
|
||||||
create_search_tables::<CalendarSearchField>(&conn).await?;
|
|
||||||
create_search_tables::<ContactSearchField>(&conn).await?;
|
|
||||||
//create_search_tables::<FileSearchField>(&conn).await?;
|
|
||||||
create_search_tables::<TracingSearchField>(&conn).await?;
|
|
||||||
|
|
||||||
Ok(())
|
create_search_tables::<EmailSearchField>(&conn).await?;
|
||||||
})
|
create_search_tables::<CalendarSearchField>(&conn).await?;
|
||||||
.await;
|
create_search_tables::<ContactSearchField>(&conn).await?;
|
||||||
bounded(conn, result, limit)
|
//create_search_tables::<FileSearchField>(&conn).await?;
|
||||||
|
create_search_tables::<TracingSearchField>(&conn).await?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,21 +265,12 @@ async fn create_search_tables<T: SearchableField + PsqlSearchField + 'static>(
|
|||||||
for field in T::all_fields() {
|
for field in T::all_fields() {
|
||||||
if field.is_text() || field.is_json() {
|
if field.is_text() || field.is_json() {
|
||||||
let column_name = field.column();
|
let column_name = field.column();
|
||||||
// inbuxa: with GIN's default fastupdate=on, new entries wait in
|
|
||||||
// an unindexed pending list that every search scans in full
|
|
||||||
// until a VACUUM (or 4 MB of backlog) merges it. On a mailbox
|
|
||||||
// taking steady mail that list never drains and searches slow
|
|
||||||
// from milliseconds to hundreds of them. Pay the index update
|
|
||||||
// at insert time instead.
|
|
||||||
let index_name = format!("gin_{table_name}_{column_name}");
|
|
||||||
let create_index_query = format!(
|
let create_index_query = format!(
|
||||||
"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} USING GIN({column_name}) WITH (fastupdate = off)",
|
"CREATE INDEX IF NOT EXISTS gin_{table_name}_{column_name} ON {table_name} USING GIN({column_name})",
|
||||||
);
|
);
|
||||||
conn.execute(&create_index_query, &[])
|
conn.execute(&create_index_query, &[])
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
// Indexes made before this change keep fastupdate=on
|
|
||||||
disable_gin_fastupdate(conn, &index_name).await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if field.is_indexed() {
|
if field.is_indexed() {
|
||||||
@@ -306,69 +287,6 @@ async fn create_search_tables<T: SearchableField + PsqlSearchField + 'static>(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: turns fastupdate off on a GIN index made with the default and
|
|
||||||
/// merges the pending list it has built up. Idempotent: an index that already
|
|
||||||
/// has the option is left alone, so this costs one catalog read per index at
|
|
||||||
/// startup. A failure is logged and startup goes on, since search still works,
|
|
||||||
/// only slower.
|
|
||||||
async fn disable_gin_fastupdate(conn: &Object, index_name: &str) {
|
|
||||||
if let Err(err) = try_disable_gin_fastupdate(conn, index_name).await {
|
|
||||||
trc::event!(
|
|
||||||
Store(trc::StoreEvent::PostgresqlError),
|
|
||||||
Details = format!("Failed to turn off fastupdate on search index {index_name}"),
|
|
||||||
Reason = err.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn try_disable_gin_fastupdate(conn: &Object, index_name: &str) -> trc::Result<()> {
|
|
||||||
let options = conn
|
|
||||||
.query_opt(
|
|
||||||
"SELECT COALESCE(reloptions, '{}')::text[] FROM pg_class WHERE oid = to_regclass($1)",
|
|
||||||
&[&index_name],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?
|
|
||||||
.map(|row| row.try_get::<_, Vec<String>>(0))
|
|
||||||
.transpose()
|
|
||||||
.map_err(into_error)?;
|
|
||||||
let Some(options) = options else {
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
if gin_fastupdate_is_off(&options) {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
// SET (fastupdate) takes a SHARE UPDATE EXCLUSIVE lock, which doesn't
|
|
||||||
// block reads or writes. Turning it off stops new entries going to the
|
|
||||||
// pending list but doesn't flush the entries already there.
|
|
||||||
conn.execute(
|
|
||||||
&format!("ALTER INDEX {index_name} SET (fastupdate = off)"),
|
|
||||||
&[],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
conn.query_one(
|
|
||||||
"SELECT gin_clean_pending_list($1::text::regclass)",
|
|
||||||
&[&index_name],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether a relation's reloptions turn GIN's fastupdate off.
|
|
||||||
fn gin_fastupdate_is_off(options: &[String]) -> bool {
|
|
||||||
options.iter().any(|option| {
|
|
||||||
option.split_once('=').is_some_and(|(name, value)| {
|
|
||||||
name.trim().eq_ignore_ascii_case("fastupdate")
|
|
||||||
&& matches!(
|
|
||||||
value.trim().to_ascii_lowercase().as_str(),
|
|
||||||
"off" | "false" | "no" | "0" | "f" | "n"
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> {
|
async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> {
|
||||||
let mut ts_configs = AHashSet::from_iter([PG_FALLBACK_LANG, PG_UNSTEMMED_LANG]);
|
let mut ts_configs = AHashSet::from_iter([PG_FALLBACK_LANG, PG_UNSTEMMED_LANG]);
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::backend::query_timeout::QueryTimeouts;
|
|
||||||
use crate::{
|
use crate::{
|
||||||
search::{
|
search::{
|
||||||
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
||||||
@@ -15,8 +14,7 @@ use crate::{
|
|||||||
write::SearchIndex,
|
write::SearchIndex,
|
||||||
};
|
};
|
||||||
use ahash::AHashSet;
|
use ahash::AHashSet;
|
||||||
use deadpool_postgres::{Object, Pool};
|
use deadpool_postgres::Pool;
|
||||||
use std::time::Duration;
|
|
||||||
use tokio_postgres::error::SqlState;
|
use tokio_postgres::error::SqlState;
|
||||||
|
|
||||||
pub mod blob;
|
pub mod blob;
|
||||||
@@ -30,8 +28,6 @@ pub mod write;
|
|||||||
pub struct PostgresStore {
|
pub struct PostgresStore {
|
||||||
pub(crate) conn_pool: Pool,
|
pub(crate) conn_pool: Pool,
|
||||||
pub(crate) ts_configs: AHashSet<&'static str>,
|
pub(crate) ts_configs: AHashSet<&'static str>,
|
||||||
/// inbuxa: client-side query limits (see backend::query_timeout)
|
|
||||||
pub(crate) timeouts: QueryTimeouts,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
@@ -76,34 +72,6 @@ pub(crate) fn is_timeout_error(err: &tokio_postgres::Error) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: the error for an operation that ran past its time limit.
|
|
||||||
pub(crate) fn query_timeout_error(limit: Duration) -> trc::Error {
|
|
||||||
trc::StoreEvent::PostgresqlError
|
|
||||||
.reason("Query timed out")
|
|
||||||
.details(format!(
|
|
||||||
"No answer from the database within {} s",
|
|
||||||
limit.as_secs()
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: ends an operation run on `conn` under `limit`. When it ran out,
|
|
||||||
/// the connection is taken out of the pool and closed: a query may still be
|
|
||||||
/// in flight on it, or a transaction open, so it can't be handed to the
|
|
||||||
/// next caller.
|
|
||||||
pub(crate) fn bounded<T>(
|
|
||||||
conn: Object,
|
|
||||||
result: Result<trc::Result<T>, tokio::time::error::Elapsed>,
|
|
||||||
limit: Duration,
|
|
||||||
) -> trc::Result<T> {
|
|
||||||
match result {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(_) => {
|
|
||||||
drop(Object::take(conn));
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub(crate) fn into_pool_error(err: deadpool_postgres::PoolError) -> trc::Error {
|
pub(crate) fn into_pool_error(err: deadpool_postgres::PoolError) -> trc::Error {
|
||||||
match err {
|
match err {
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error, is_timeout_error, query_timeout_error};
|
use super::{PostgresStore, into_error, is_timeout_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
Deserialize, IterateParams, Key, ValueKey, backend::postgres::into_pool_error,
|
Deserialize, IterateParams, Key, ValueKey, backend::postgres::into_pool_error,
|
||||||
write::ValueClass,
|
write::ValueClass,
|
||||||
@@ -19,50 +17,40 @@ impl PostgresStore {
|
|||||||
U: Deserialize + 'static,
|
U: Deserialize + 'static,
|
||||||
{
|
{
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached(&format!(
|
||||||
let s = conn
|
"SELECT v FROM {} WHERE k = $1",
|
||||||
.prepare_cached(&format!(
|
char::from(key.subspace())
|
||||||
"SELECT v FROM {} WHERE k = $1",
|
))
|
||||||
char::from(key.subspace())
|
.await
|
||||||
))
|
.map_err(into_error)?;
|
||||||
.await
|
let key = key.serialize(0);
|
||||||
.map_err(into_error)?;
|
conn.query_opt(&s, &[&key])
|
||||||
let key = key.serialize(0);
|
.await
|
||||||
conn.query_opt(&s, &[&key])
|
.map_err(into_error)
|
||||||
.await
|
.and_then(|r| {
|
||||||
.map_err(into_error)
|
if let Some(r) = r {
|
||||||
.and_then(|r| {
|
Ok(Some(U::deserialize_with_key(&key, r.get(0))?))
|
||||||
if let Some(r) = r {
|
} else {
|
||||||
Ok(Some(U::deserialize_with_key(&key, r.get(0))?))
|
Ok(None)
|
||||||
} else {
|
}
|
||||||
Ok(None)
|
})
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached(&format!(
|
||||||
let s = conn
|
"SELECT 1 FROM {} WHERE k = $1",
|
||||||
.prepare_cached(&format!(
|
char::from(key.subspace())
|
||||||
"SELECT 1 FROM {} WHERE k = $1",
|
))
|
||||||
char::from(key.subspace())
|
.await
|
||||||
))
|
.map_err(into_error)?;
|
||||||
.await
|
let key = key.serialize(0);
|
||||||
.map_err(into_error)?;
|
conn.query_opt(&s, &[&key])
|
||||||
let key = key.serialize(0);
|
.await
|
||||||
conn.query_opt(&s, &[&key])
|
.map_err(into_error)
|
||||||
.await
|
.map(|r| r.is_some())
|
||||||
.map_err(into_error)
|
|
||||||
.map(|r| r.is_some())
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn iterate<T: Key>(
|
pub(crate) async fn iterate<T: Key>(
|
||||||
@@ -76,65 +64,44 @@ impl PostgresStore {
|
|||||||
let end = params.end.serialize(0);
|
let end = params.end.serialize(0);
|
||||||
let keys = if params.values { "k, v" } else { "k" };
|
let keys = if params.values { "k, v" } else { "k" };
|
||||||
|
|
||||||
// inbuxa: a scan may run for hours, so the query limit bounds each
|
let s = conn
|
||||||
// wait for the database (preparing, the query starting, the next
|
.prepare_cached(&match (params.first, params.ascending) {
|
||||||
// row) rather than the scan. A wait that runs out closes the
|
(true, true) => {
|
||||||
// connection.
|
format!(
|
||||||
let limit = self.timeouts.query;
|
"SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC LIMIT 1"
|
||||||
let query = match (params.first, params.ascending) {
|
)
|
||||||
(true, true) => {
|
}
|
||||||
format!(
|
(true, false) => {
|
||||||
"SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC LIMIT 1"
|
format!(
|
||||||
)
|
|
||||||
}
|
|
||||||
(true, false) => {
|
|
||||||
format!(
|
|
||||||
"SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC LIMIT 1"
|
"SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC LIMIT 1"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
(false, true) => {
|
(false, true) => {
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC")
|
format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC")
|
||||||
}
|
}
|
||||||
(false, false) => {
|
(false, false) => {
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC")
|
format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC")
|
||||||
}
|
}
|
||||||
};
|
})
|
||||||
let s = match tokio::time::timeout(limit, conn.prepare_cached(&query)).await {
|
.await.map_err(into_error)?;
|
||||||
Ok(s) => s.map_err(into_error)?,
|
|
||||||
Err(_) => {
|
|
||||||
drop(deadpool_postgres::Object::take(conn));
|
|
||||||
return Err(query_timeout_error(limit));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let mut from = begin;
|
let mut from = begin;
|
||||||
let mut to = end;
|
let mut to = end;
|
||||||
let mut resume_key: Option<Vec<u8>> = None;
|
let mut resume_key: Option<Vec<u8>> = None;
|
||||||
|
|
||||||
let mut stalled = false;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let mut last_key = None;
|
let mut last_key = None;
|
||||||
let mut timed_out = false;
|
let mut timed_out = false;
|
||||||
|
|
||||||
{
|
{
|
||||||
let rows =
|
let rows = conn
|
||||||
match tokio::time::timeout(limit, conn.query_raw(&s, &[&from, &to])).await {
|
.query_raw(&s, &[&from, &to])
|
||||||
Ok(rows) => rows.map_err(into_error)?,
|
.await
|
||||||
// Leaves the scan loop for the timeout below
|
.map_err(into_error)?;
|
||||||
Err(_) => break,
|
|
||||||
};
|
|
||||||
|
|
||||||
pin_mut!(rows);
|
pin_mut!(rows);
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let next = match tokio::time::timeout(limit, rows.try_next()).await {
|
match rows.try_next().await {
|
||||||
Ok(next) => next,
|
|
||||||
Err(_) => {
|
|
||||||
stalled = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
match next {
|
|
||||||
Ok(Some(row)) => {
|
Ok(Some(row)) => {
|
||||||
let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?;
|
let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?;
|
||||||
let value = if params.values {
|
let value = if params.values {
|
||||||
@@ -165,10 +132,6 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if stalled {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
match last_key {
|
match last_key {
|
||||||
Some(last_key) if timed_out => {
|
Some(last_key) if timed_out => {
|
||||||
if params.ascending {
|
if params.ascending {
|
||||||
@@ -181,9 +144,6 @@ impl PostgresStore {
|
|||||||
_ => return Ok(()),
|
_ => return Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
drop(deadpool_postgres::Object::take(conn));
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn get_counter(
|
pub(crate) async fn get_counter(
|
||||||
@@ -195,19 +155,14 @@ impl PostgresStore {
|
|||||||
let key = key.serialize(0);
|
let key = key.serialize(0);
|
||||||
|
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached(&format!("SELECT v FROM {table} WHERE k = $1"))
|
||||||
let s = conn
|
.await
|
||||||
.prepare_cached(&format!("SELECT v FROM {table} WHERE k = $1"))
|
.map_err(into_error)?;
|
||||||
.await
|
match conn.query_opt(&s, &[&key]).await {
|
||||||
.map_err(into_error)?;
|
Ok(Some(row)) => row.try_get(0).map_err(into_error),
|
||||||
match conn.query_opt(&s, &[&key]).await {
|
Ok(None) => Ok(0),
|
||||||
Ok(Some(row)) => row.try_get(0).map_err(into_error),
|
Err(e) => Err(into_error(e)),
|
||||||
Ok(None) => Ok(0),
|
}
|
||||||
Err(e) => Err(into_error(e)),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ use crate::{
|
|||||||
backend::{
|
backend::{
|
||||||
MAX_TOKEN_LENGTH,
|
MAX_TOKEN_LENGTH,
|
||||||
postgres::{
|
postgres::{
|
||||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, PostgresStore, PsqlSearchField, bounded,
|
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, PostgresStore, PsqlSearchField, into_error,
|
||||||
into_error, into_pool_error, is_timeout_error,
|
into_pool_error, is_timeout_error,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
search::{
|
search::{
|
||||||
@@ -36,135 +36,125 @@ impl PostgresStore {
|
|||||||
|
|
||||||
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
||||||
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let trx = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.build_transaction()
|
||||||
let trx = conn
|
.isolation_level(IsolationLevel::ReadCommitted)
|
||||||
.build_transaction()
|
.start()
|
||||||
.isolation_level(IsolationLevel::ReadCommitted)
|
.await
|
||||||
.start()
|
.map_err(into_error)?;
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
|
|
||||||
for document in documents {
|
for document in documents {
|
||||||
let index = document.index;
|
let index = document.index;
|
||||||
let primary_keys = index.primary_keys();
|
let primary_keys = index.primary_keys();
|
||||||
let all_fields = index.all_fields();
|
let all_fields = index.all_fields();
|
||||||
let fields = document.fields;
|
let fields = document.fields;
|
||||||
// inbuxa: keyword text (addresses, contact fields, ...) is split into
|
// inbuxa: keyword text (addresses, contact fields, ...) is split into
|
||||||
// words before it reaches the text parser, see keyword_terms();
|
// words before it reaches the text parser, see keyword_terms().
|
||||||
// language text gets the words inside its URLs, host names and
|
let keywords = primary_keys
|
||||||
// file names added, see url_terms().
|
.iter()
|
||||||
let keywords = primary_keys
|
.chain(all_fields)
|
||||||
.iter()
|
.map(|field| match fields.get(field) {
|
||||||
.chain(all_fields)
|
Some(SearchValue::Text {
|
||||||
.map(|field| match fields.get(field) {
|
value,
|
||||||
Some(SearchValue::Text {
|
language: Language::None,
|
||||||
value,
|
}) if field.is_text() => Some(keyword_terms(value)),
|
||||||
language: Language::None,
|
_ => None,
|
||||||
}) if field.is_text() => Some(keyword_terms(value)),
|
})
|
||||||
Some(SearchValue::Text { value, .. }) if field.is_text() => {
|
.collect::<Vec<_>>();
|
||||||
url_terms(value)
|
let mut values = Vec::with_capacity(fields.len() + 2);
|
||||||
}
|
let mut query = format!("INSERT INTO {} (", index.psql_table());
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let mut values = Vec::with_capacity(fields.len() + 2);
|
|
||||||
let mut query = format!("INSERT INTO {} (", index.psql_table());
|
|
||||||
|
|
||||||
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
query.push(',');
|
query.push(',');
|
||||||
}
|
|
||||||
query.push_str(field.column());
|
|
||||||
|
|
||||||
if let Some(sort_column) = field.sort_column() {
|
|
||||||
query.push(',');
|
|
||||||
query.push_str(sort_column);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
query.push_str(field.column());
|
||||||
|
|
||||||
query.push_str(") VALUES (");
|
if let Some(sort_column) = field.sort_column() {
|
||||||
|
query.push(',');
|
||||||
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
query.push_str(sort_column);
|
||||||
if i > 0 {
|
|
||||||
query.push(',');
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(value) = fields.get(field) {
|
|
||||||
let value_ref = format!("${}", values.len() + 1);
|
|
||||||
let (text_len, language) =
|
|
||||||
if let SearchValue::Text { value, language } = value {
|
|
||||||
(value.len(), self.ts_config(language))
|
|
||||||
} else {
|
|
||||||
(0, PG_UNSTEMMED_LANG)
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Some(keywords) = &keywords[i] {
|
|
||||||
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
|
||||||
values.push(keywords as &(dyn ToSql + Sync));
|
|
||||||
if field.sort_column().is_some() {
|
|
||||||
let value_ref = format!("${}", values.len() + 1);
|
|
||||||
if text_len > 255 {
|
|
||||||
let _ = write!(&mut query, ",left({value_ref},255)");
|
|
||||||
} else {
|
|
||||||
let _ = write!(&mut query, ",{value_ref}");
|
|
||||||
}
|
|
||||||
values.push(value as &(dyn ToSql + Sync));
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
} else if field.is_text() {
|
|
||||||
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
|
||||||
} else if text_len > 512 {
|
|
||||||
query.push_str("left(");
|
|
||||||
query.push_str(&value_ref);
|
|
||||||
query.push_str(",512)");
|
|
||||||
} else {
|
|
||||||
query.push_str(&value_ref);
|
|
||||||
}
|
|
||||||
|
|
||||||
if field.sort_column().is_some() {
|
|
||||||
if text_len > 255 {
|
|
||||||
query.push_str(",left(");
|
|
||||||
query.push_str(&value_ref);
|
|
||||||
query.push_str(",255)");
|
|
||||||
} else {
|
|
||||||
query.push(',');
|
|
||||||
query.push_str(&value_ref);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
values.push(value as &(dyn ToSql + Sync));
|
|
||||||
} else {
|
|
||||||
query.push_str("NULL");
|
|
||||||
if field.sort_column().is_some() {
|
|
||||||
query.push_str(",NULL");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
query.push_str(") ON CONFLICT (");
|
|
||||||
for (i, pkey) in primary_keys.iter().enumerate() {
|
|
||||||
if i > 0 {
|
|
||||||
query.push(',');
|
|
||||||
}
|
|
||||||
query.push_str(pkey.column());
|
|
||||||
}
|
|
||||||
query.push_str(") DO UPDATE SET ");
|
|
||||||
for (i, field) in all_fields.iter().enumerate() {
|
|
||||||
if i > 0 {
|
|
||||||
query.push(',');
|
|
||||||
}
|
|
||||||
let column = field.column();
|
|
||||||
let _ = write!(&mut query, "{column} = EXCLUDED.{column}");
|
|
||||||
}
|
|
||||||
|
|
||||||
trx.execute(&query, &values).await.map_err(into_error)?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
trx.commit().await.map_err(into_error)
|
query.push_str(") VALUES (");
|
||||||
})
|
|
||||||
.await;
|
for (i, field) in primary_keys.iter().chain(all_fields).enumerate() {
|
||||||
bounded(conn, result, limit)
|
if i > 0 {
|
||||||
|
query.push(',');
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(value) = fields.get(field) {
|
||||||
|
let value_ref = format!("${}", values.len() + 1);
|
||||||
|
let (text_len, language) = if let SearchValue::Text { value, language } = value
|
||||||
|
{
|
||||||
|
(value.len(), self.ts_config(language))
|
||||||
|
} else {
|
||||||
|
(0, PG_UNSTEMMED_LANG)
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Some(keywords) = &keywords[i] {
|
||||||
|
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
||||||
|
values.push(keywords as &(dyn ToSql + Sync));
|
||||||
|
if field.sort_column().is_some() {
|
||||||
|
let value_ref = format!("${}", values.len() + 1);
|
||||||
|
if text_len > 255 {
|
||||||
|
let _ = write!(&mut query, ",left({value_ref},255)");
|
||||||
|
} else {
|
||||||
|
let _ = write!(&mut query, ",{value_ref}");
|
||||||
|
}
|
||||||
|
values.push(value as &(dyn ToSql + Sync));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
} else if field.is_text() {
|
||||||
|
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
||||||
|
} else if text_len > 512 {
|
||||||
|
query.push_str("left(");
|
||||||
|
query.push_str(&value_ref);
|
||||||
|
query.push_str(",512)");
|
||||||
|
} else {
|
||||||
|
query.push_str(&value_ref);
|
||||||
|
}
|
||||||
|
|
||||||
|
if field.sort_column().is_some() {
|
||||||
|
if text_len > 255 {
|
||||||
|
query.push_str(",left(");
|
||||||
|
query.push_str(&value_ref);
|
||||||
|
query.push_str(",255)");
|
||||||
|
} else {
|
||||||
|
query.push(',');
|
||||||
|
query.push_str(&value_ref);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
values.push(value as &(dyn ToSql + Sync));
|
||||||
|
} else {
|
||||||
|
query.push_str("NULL");
|
||||||
|
if field.sort_column().is_some() {
|
||||||
|
query.push_str(",NULL");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
query.push_str(") ON CONFLICT (");
|
||||||
|
for (i, pkey) in primary_keys.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
query.push(',');
|
||||||
|
}
|
||||||
|
query.push_str(pkey.column());
|
||||||
|
}
|
||||||
|
query.push_str(") DO UPDATE SET ");
|
||||||
|
for (i, field) in all_fields.iter().enumerate() {
|
||||||
|
if i > 0 {
|
||||||
|
query.push(',');
|
||||||
|
}
|
||||||
|
let column = field.column();
|
||||||
|
let _ = write!(&mut query, "{column} = EXCLUDED.{column}");
|
||||||
|
}
|
||||||
|
|
||||||
|
trx.execute(&query, &values).await.map_err(into_error)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
trx.commit().await.map_err(into_error)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn query<R: SearchDocumentId>(
|
pub async fn query<R: SearchDocumentId>(
|
||||||
@@ -180,21 +170,16 @@ impl PostgresStore {
|
|||||||
build_sort(&mut query, sort);
|
build_sort(&mut query, sort);
|
||||||
}
|
}
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let s = conn.prepare_cached(&query).await.map_err(into_error)?;
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prepare_cached(&query).await.map_err(into_error)?;
|
|
||||||
|
|
||||||
conn.query(&s, params.as_slice())
|
conn.query(&s, params.as_slice())
|
||||||
.await
|
.await
|
||||||
.and_then(|rows| {
|
.and_then(|rows| {
|
||||||
rows.into_iter()
|
rows.into_iter()
|
||||||
.map(|row| row.try_get::<_, DocId>(0).map(|v| R::from_u64(v.0)))
|
.map(|row| row.try_get::<_, DocId>(0).map(|v| R::from_u64(v.0)))
|
||||||
.collect::<Result<Vec<R>, _>>()
|
.collect::<Result<Vec<R>, _>>()
|
||||||
})
|
})
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||||
@@ -204,45 +189,40 @@ impl PostgresStore {
|
|||||||
let params = self.build_filter(&mut where_clause, &filter.filters);
|
let params = self.build_filter(&mut where_clause, &filter.filters);
|
||||||
let params = params.iter().map(SqlParam::as_sql).collect::<Vec<_>>();
|
let params = params.iter().map(SqlParam::as_sql).collect::<Vec<_>>();
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
let s = conn
|
||||||
let result = tokio::time::timeout(limit, async {
|
.prepare_cached(&format!("DELETE FROM {table}{where_clause}"))
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
|
||||||
|
match conn.execute(&s, params.as_slice()).await {
|
||||||
|
Ok(deleted) => return Ok(deleted),
|
||||||
|
Err(err) if is_timeout_error(&err) => (),
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||||
|
let mut deleted = 0;
|
||||||
|
|
||||||
|
loop {
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached(&format!("DELETE FROM {table}{where_clause}"))
|
.prepare_cached(&format!(
|
||||||
|
"DELETE FROM {table} WHERE ctid IN (SELECT ctid FROM {table}{where_clause} LIMIT {chunk_size})"
|
||||||
|
))
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
|
|
||||||
match conn.execute(&s, params.as_slice()).await {
|
|
||||||
Ok(deleted) => return Ok(deleted),
|
|
||||||
Err(err) if is_timeout_error(&err) => (),
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
|
||||||
let mut deleted = 0;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let s = conn
|
match conn.execute(&s, params.as_slice()).await {
|
||||||
.prepare_cached(&format!(
|
Ok(0) => return Ok(deleted),
|
||||||
"DELETE FROM {table} WHERE ctid IN (SELECT ctid FROM {table}{where_clause} LIMIT {chunk_size})"
|
Ok(affected) => deleted += affected,
|
||||||
))
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
.await
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
.map_err(into_error)?;
|
break;
|
||||||
|
|
||||||
loop {
|
|
||||||
match conn.execute(&s, params.as_slice()).await {
|
|
||||||
Ok(0) => return Ok(deleted),
|
|
||||||
Ok(affected) => deleted += affected,
|
|
||||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
}
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_filter<'x>(
|
fn build_filter<'x>(
|
||||||
@@ -295,36 +275,14 @@ impl PostgresStore {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// inbuxa: a query word written as a URL, host,
|
|
||||||
// file or hyphenated word also matches as its word
|
|
||||||
// parts, which url_terms() indexes
|
|
||||||
let parts = match value {
|
|
||||||
SearchValue::Text { value, .. } => query_url_terms(value),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
let parts_pos = value_pos + 1;
|
|
||||||
let _ = write!(query, "@@ ({method}('{config}', ${value_pos})");
|
let _ = write!(query, "@@ ({method}('{config}', ${value_pos})");
|
||||||
if parts.is_some() {
|
|
||||||
let _ = write!(query, " || {method}('{config}', ${parts_pos})");
|
|
||||||
}
|
|
||||||
for fallback in [PG_FALLBACK_LANG, PG_UNSTEMMED_LANG] {
|
for fallback in [PG_FALLBACK_LANG, PG_UNSTEMMED_LANG] {
|
||||||
if fallback != config && self.ts_configs.contains(fallback) {
|
if fallback != config && self.ts_configs.contains(fallback) {
|
||||||
let _ =
|
let _ =
|
||||||
write!(query, " || {method}('{fallback}', ${value_pos})");
|
write!(query, " || {method}('{fallback}', ${value_pos})");
|
||||||
if parts.is_some() {
|
|
||||||
let _ = write!(
|
|
||||||
query,
|
|
||||||
" || {method}('{fallback}', ${parts_pos})"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
query.push(')');
|
query.push(')');
|
||||||
values.push(SqlParam::Ref(value));
|
|
||||||
if let Some(parts) = parts {
|
|
||||||
values.push(SqlParam::Owned(parts));
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
values.push(SqlParam::Ref(value));
|
values.push(SqlParam::Ref(value));
|
||||||
} else if let SearchValue::KeyValues(kv) = value {
|
} else if let SearchValue::KeyValues(kv) = value {
|
||||||
@@ -418,75 +376,6 @@ pub(crate) fn keyword_terms(value: &str) -> String {
|
|||||||
terms
|
terms
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: in language text (subject, body, attachments) PostgreSQL's parser
|
|
||||||
// keeps a URL, a host name, a path or a file name as tokens of its own:
|
|
||||||
// "https://x.example/shipping-support/" gives a url, a host and a url_path,
|
|
||||||
// "invoice-2024.pdf" a file, so a body search for "shipping" or "invoice"
|
|
||||||
// missed messages where the word appears only there, while the built-in index
|
|
||||||
// splits them into words. The text is indexed as it was, followed by the word
|
|
||||||
// parts of each such token (SpaceTokenizer, as keyword_terms() splits), so
|
|
||||||
// they go through the same configuration and stemming as the words around
|
|
||||||
// them. On sample mail the text vector grows by about 15% for a newsletter
|
|
||||||
// full of tracking links and 30% for a short order notice with three links.
|
|
||||||
// Plain words, and words that only carry punctuation ("end.", "(see"),
|
|
||||||
// add nothing; hyphenated words are already split by the parser. Returns None
|
|
||||||
// when there is nothing to add, so most text is indexed exactly as before.
|
|
||||||
/// Characters that join the parts of a URL, host, path, address or file name.
|
|
||||||
const URL_SEPARATORS: [char; 13] = [
|
|
||||||
'/', '.', '@', ':', '?', '=', '&', '#', '_', '%', '+', '~', '\\',
|
|
||||||
];
|
|
||||||
|
|
||||||
pub(crate) fn url_terms(value: &str) -> Option<String> {
|
|
||||||
let mut terms = String::new();
|
|
||||||
// Each word is added once: a phrase search still finds the first URL it
|
|
||||||
// is in, and a newsletter's hundred tracking links don't add a hundred
|
|
||||||
// positions for "utm" and "campaign"
|
|
||||||
let mut seen = std::collections::HashSet::new();
|
|
||||||
for token in value.split(|c: char| {
|
|
||||||
c.is_whitespace() || matches!(c, '<' | '>' | '"' | '(' | ')' | '[' | ']' | '{' | '}')
|
|
||||||
}) {
|
|
||||||
let token = token.trim_matches(|c: char| !c.is_alphanumeric());
|
|
||||||
if token.contains(URL_SEPARATORS) {
|
|
||||||
for word in SpaceTokenizer::new(token, MAX_TOKEN_LENGTH) {
|
|
||||||
if !seen.insert(word.clone()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if terms.is_empty() {
|
|
||||||
terms.reserve(value.len() + 64);
|
|
||||||
terms.push_str(value);
|
|
||||||
terms.push('\n');
|
|
||||||
} else {
|
|
||||||
terms.push(' ');
|
|
||||||
}
|
|
||||||
terms.push_str(&word);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
(!terms.is_empty()).then_some(terms)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The query side of url_terms(): each query word that is a URL, host, file
|
|
||||||
/// name or hyphenated word replaced by its word parts, or None when there is
|
|
||||||
/// none. It is searched in addition to the query as written, so documents
|
|
||||||
/// indexed before url_terms() still match as they did.
|
|
||||||
pub(crate) fn query_url_terms(value: &str) -> Option<String> {
|
|
||||||
let mut terms = String::with_capacity(value.len());
|
|
||||||
let mut changed = false;
|
|
||||||
for token in value.split_whitespace() {
|
|
||||||
let word = token.trim_matches(|c: char| !c.is_alphanumeric());
|
|
||||||
if !terms.is_empty() {
|
|
||||||
terms.push(' ');
|
|
||||||
}
|
|
||||||
if word.contains(URL_SEPARATORS) || word.contains('-') {
|
|
||||||
changed = true;
|
|
||||||
terms.push_str(&keyword_terms(word));
|
|
||||||
} else {
|
|
||||||
terms.push_str(token);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
changed.then_some(terms)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) enum SqlParam<'x> {
|
pub(super) enum SqlParam<'x> {
|
||||||
Ref(&'x (dyn ToSql + Sync)),
|
Ref(&'x (dyn ToSql + Sync)),
|
||||||
Owned(String),
|
Owned(String),
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error, is_timeout_error};
|
use super::{PostgresStore, into_error, is_timeout_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
||||||
SUBSPACE_REGISTRY_IDX,
|
SUBSPACE_REGISTRY_IDX,
|
||||||
@@ -32,53 +30,48 @@ enum CommitError {
|
|||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
||||||
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
let start = Instant::now();
|
||||||
let result = tokio::time::timeout(limit, async {
|
let mut retry_count = 0;
|
||||||
let start = Instant::now();
|
|
||||||
let mut retry_count = 0;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
match self.write_trx(&mut conn, &mut batch).await {
|
match self.write_trx(&mut conn, &mut batch).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
return Ok(result);
|
return Ok(result);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
match err {
|
||||||
|
CommitError::Postgres(err) => match err.code() {
|
||||||
|
Some(
|
||||||
|
&SqlState::T_R_SERIALIZATION_FAILURE
|
||||||
|
| &SqlState::T_R_DEADLOCK_DETECTED,
|
||||||
|
) if retry_count < MAX_COMMIT_ATTEMPTS
|
||||||
|
&& start.elapsed() < MAX_COMMIT_TIME => {}
|
||||||
|
Some(&SqlState::UNIQUE_VIOLATION) => {
|
||||||
|
return Err(trc::StoreEvent::AssertValueFailed
|
||||||
|
.into_err()
|
||||||
|
.reason("Unique violation")
|
||||||
|
.caused_by(trc::location!()));
|
||||||
|
}
|
||||||
|
_ => return Err(into_error(err)),
|
||||||
|
},
|
||||||
|
CommitError::Internal(err) => return Err(err),
|
||||||
|
/*CommitError::Retry => {
|
||||||
|
if retry_count > MAX_COMMIT_ATTEMPTS
|
||||||
|
|| start.elapsed() > MAX_COMMIT_TIME
|
||||||
|
{
|
||||||
|
return Err(trc::StoreEvent::AssertValueFailed
|
||||||
|
.into_err()
|
||||||
|
.caused_by(trc::location!()));
|
||||||
|
}
|
||||||
|
}*/
|
||||||
}
|
}
|
||||||
Err(err) => {
|
|
||||||
match err {
|
|
||||||
CommitError::Postgres(err) => match err.code() {
|
|
||||||
Some(
|
|
||||||
&SqlState::T_R_SERIALIZATION_FAILURE
|
|
||||||
| &SqlState::T_R_DEADLOCK_DETECTED,
|
|
||||||
) if retry_count < MAX_COMMIT_ATTEMPTS
|
|
||||||
&& start.elapsed() < MAX_COMMIT_TIME => {}
|
|
||||||
Some(&SqlState::UNIQUE_VIOLATION) => {
|
|
||||||
return Err(trc::StoreEvent::AssertValueFailed
|
|
||||||
.into_err()
|
|
||||||
.reason("Unique violation")
|
|
||||||
.caused_by(trc::location!()));
|
|
||||||
}
|
|
||||||
_ => return Err(into_error(err)),
|
|
||||||
},
|
|
||||||
CommitError::Internal(err) => return Err(err),
|
|
||||||
/*CommitError::Retry => {
|
|
||||||
if retry_count > MAX_COMMIT_ATTEMPTS
|
|
||||||
|| start.elapsed() > MAX_COMMIT_TIME
|
|
||||||
{
|
|
||||||
return Err(trc::StoreEvent::AssertValueFailed
|
|
||||||
.into_err()
|
|
||||||
.caused_by(trc::location!()));
|
|
||||||
}
|
|
||||||
}*/
|
|
||||||
}
|
|
||||||
|
|
||||||
let backoff = rand::rng().random_range(50..=300);
|
let backoff = rand::rng().random_range(50..=300);
|
||||||
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
||||||
retry_count += 1;
|
retry_count += 1;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn write_trx(
|
async fn write_trx(
|
||||||
@@ -400,81 +393,72 @@ impl PostgresStore {
|
|||||||
|
|
||||||
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
|
||||||
purge_table(&conn, char::from(subspace)).await?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||||
})
|
purge_table(&conn, char::from(subspace)).await?;
|
||||||
.await;
|
}
|
||||||
bounded(conn, result, limit)
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
let table = char::from(from.subspace());
|
||||||
let result = tokio::time::timeout(limit, async {
|
let mut from = from.serialize(0);
|
||||||
let table = char::from(from.subspace());
|
let to = to.serialize(0);
|
||||||
let mut from = from.serialize(0);
|
|
||||||
let to = to.serialize(0);
|
|
||||||
|
|
||||||
let delete = conn
|
let delete = conn
|
||||||
.prepare_cached(&format!("DELETE FROM {table} WHERE k >= $1 AND k < $2"))
|
.prepare_cached(&format!("DELETE FROM {table} WHERE k >= $1 AND k < $2"))
|
||||||
|
.await
|
||||||
|
.map_err(into_error)?;
|
||||||
|
|
||||||
|
match conn.execute(&delete, &[&from, &to]).await {
|
||||||
|
Ok(_) => return Ok(()),
|
||||||
|
Err(err) if is_timeout_error(&err) => (),
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut chunk_size = DELETE_CHUNK_SIZE;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let boundary = conn
|
||||||
|
.prepare_cached(&format!(
|
||||||
|
"SELECT k FROM {table} WHERE k >= $1 AND k < $2 ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}"
|
||||||
|
))
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
|
|
||||||
match conn.execute(&delete, &[&from, &to]).await {
|
|
||||||
Ok(_) => return Ok(()),
|
|
||||||
Err(err) if is_timeout_error(&err) => (),
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut chunk_size = DELETE_CHUNK_SIZE;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let boundary = conn
|
let next = match conn.query_opt(&boundary, &[&from, &to]).await {
|
||||||
.prepare_cached(&format!(
|
Ok(next) => match next {
|
||||||
"SELECT k FROM {table} WHERE k >= $1 AND k < $2 ORDER BY k ASC LIMIT 1 OFFSET {chunk_size}"
|
Some(row) => Some(row.try_get::<_, Vec<u8>>(0).map_err(into_error)?),
|
||||||
))
|
None => None,
|
||||||
|
},
|
||||||
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
};
|
||||||
|
|
||||||
|
match conn
|
||||||
|
.execute(&delete, &[&from, next.as_ref().unwrap_or(&to)])
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
{
|
||||||
|
Ok(_) => (),
|
||||||
loop {
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
let next = match conn.query_opt(&boundary, &[&from, &to]).await {
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
Ok(next) => match next {
|
break;
|
||||||
Some(row) => Some(row.try_get::<_, Vec<u8>>(0).map_err(into_error)?),
|
|
||||||
None => None,
|
|
||||||
},
|
|
||||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
};
|
|
||||||
|
|
||||||
match conn
|
|
||||||
.execute(&delete, &[&from, next.as_ref().unwrap_or(&to)])
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(_) => (),
|
|
||||||
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(into_error(err)),
|
|
||||||
}
|
}
|
||||||
|
Err(err) => return Err(into_error(err)),
|
||||||
|
}
|
||||||
|
|
||||||
match next {
|
match next {
|
||||||
Some(next) => from = next,
|
Some(next) => from = next,
|
||||||
None => return Ok(()),
|
None => return Ok(()),
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Client-side limits on SQL queries.
|
|
||||||
//!
|
|
||||||
//! The pool timeouts bound getting a connection, not using one. A database
|
|
||||||
//! that stops answering while the TCP connection stays up (a paused
|
|
||||||
//! container, a hung server whose kernel still acknowledges keepalives)
|
|
||||||
//! left a query on a checked-out connection waiting for as long as it took.
|
|
||||||
//! A server-side statement_timeout can't help there: the server that would
|
|
||||||
//! enforce it is the one not answering. So each operation on a PostgreSQL
|
|
||||||
//! or MySQL connection runs under a time limit here, and a connection whose
|
|
||||||
//! operation ran out is closed rather than put back in the pool, since its
|
|
||||||
//! protocol state is unknown.
|
|
||||||
//!
|
|
||||||
//! Two limits:
|
|
||||||
//! - `query`, two minutes, for request-path work: reads, writes, blob
|
|
||||||
//! transfers, search queries and document indexing. Those take
|
|
||||||
//! milliseconds; two minutes leaves room for a large blob over a slow
|
|
||||||
//! link and still ends a hang.
|
|
||||||
//! - `maintenance`, thirty minutes, for work that legitimately runs long in
|
|
||||||
//! one statement: range deletes (account removal, purges), unindexing,
|
|
||||||
//! and creating tables and indexes at startup.
|
|
||||||
//!
|
|
||||||
//! Iterating over a range (exports, reindexing, maintenance scans) can run
|
|
||||||
//! for hours, so there the `query` limit applies to each wait for the next
|
|
||||||
//! row instead of the whole scan.
|
|
||||||
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub struct QueryTimeouts {
|
|
||||||
pub query: Duration,
|
|
||||||
pub maintenance: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl QueryTimeouts {
|
|
||||||
pub const QUERY: Duration = Duration::from_secs(120);
|
|
||||||
pub const MAINTENANCE: Duration = Duration::from_secs(30 * 60);
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for QueryTimeouts {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
query: Self::QUERY,
|
|
||||||
maintenance: Self::MAINTENANCE,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "test_mode")]
|
|
||||||
impl crate::Store {
|
|
||||||
/// Sets the query limits of a SQL store that was just built (tests only:
|
|
||||||
/// the limits aren't configurable).
|
|
||||||
pub fn with_query_timeouts(self, timeouts: QueryTimeouts) -> Self {
|
|
||||||
match self {
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
crate::Store::PostgreSQL(mut store) => {
|
|
||||||
std::sync::Arc::get_mut(&mut store)
|
|
||||||
.expect("store already shared")
|
|
||||||
.timeouts = timeouts;
|
|
||||||
crate::Store::PostgreSQL(store)
|
|
||||||
}
|
|
||||||
#[cfg(feature = "mysql")]
|
|
||||||
crate::Store::MySQL(mut store) => {
|
|
||||||
std::sync::Arc::get_mut(&mut store)
|
|
||||||
.expect("store already shared")
|
|
||||||
.timeouts = timeouts;
|
|
||||||
crate::Store::MySQL(store)
|
|
||||||
}
|
|
||||||
store => store,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -245,27 +245,9 @@ impl Collector {
|
|||||||
Update::RegisterReceiver { receiver } => {
|
Update::RegisterReceiver { receiver } => {
|
||||||
self.receivers.push(receiver);
|
self.receivers.push(receiver);
|
||||||
}
|
}
|
||||||
Update::RegisterSubscriber { mut subscriber } => {
|
Update::RegisterSubscriber { subscriber } => {
|
||||||
// inbuxa: a subscriber registered under the id of a
|
ACTIVE_SUBSCRIBERS.lock().push(subscriber.id.clone());
|
||||||
// running one replaces it (a tracer whose settings
|
self.subscribers.push(subscriber);
|
||||||
// changed). Every event collected so far went to the old
|
|
||||||
// one, every later event goes to the new one: the old
|
|
||||||
// one's batch is sent first (anything its full channel
|
|
||||||
// can't take moves over, rather than being dropped), and
|
|
||||||
// dropping it closes its channel, so its task writes
|
|
||||||
// what is queued and ends.
|
|
||||||
if let Some(old) = self.subscribers.iter_mut().find(|s| s.id == subscriber.id) {
|
|
||||||
let _ = old.send_batch();
|
|
||||||
if !old.batch.is_empty() {
|
|
||||||
let mut batch = std::mem::take(&mut old.batch);
|
|
||||||
batch.append(&mut subscriber.batch);
|
|
||||||
subscriber.batch = batch;
|
|
||||||
}
|
|
||||||
*old = subscriber;
|
|
||||||
} else {
|
|
||||||
ACTIVE_SUBSCRIBERS.lock().push(subscriber.id.clone());
|
|
||||||
self.subscribers.push(subscriber);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Update::UnregisterSubscriber { id } => {
|
Update::UnregisterSubscriber { id } => {
|
||||||
ACTIVE_SUBSCRIBERS.lock().retain(|s| s != &id);
|
ACTIVE_SUBSCRIBERS.lock().retain(|s| s != &id);
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -107,9 +105,6 @@ impl SubscriberBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Registers the subscriber with the collector. inbuxa: one registered
|
|
||||||
/// under the id of a running subscriber replaces it, handing over at an
|
|
||||||
/// event boundary; the old one's channel then closes.
|
|
||||||
pub fn register(self) -> (mpsc::Sender<EventBatch>, mpsc::Receiver<EventBatch>) {
|
pub fn register(self) -> (mpsc::Sender<EventBatch>, mpsc::Receiver<EventBatch>) {
|
||||||
let (tx, rx) = mpsc::channel(8192);
|
let (tx, rx) = mpsc::channel(8192);
|
||||||
|
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
|
|||||||
#[macro_export]
|
#[macro_export]
|
||||||
macro_rules! brand_version {
|
macro_rules! brand_version {
|
||||||
() => {
|
() => {
|
||||||
"2026.9.25"
|
"2026.9.24.3"
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,317 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! DMARC results recorded on a node without outboundMta reach the aggregate
|
|
||||||
//! report, which a node with outboundMta builds and sends. Before, a front
|
|
||||||
//! node's results were dropped (or, before live roles, left in a channel
|
|
||||||
//! nobody read), so the report covered only the mail the outbound nodes
|
|
||||||
//! received. Also checks that nodes appending to one report at once lose
|
|
||||||
//! nothing. Needs a store the nodes can share (STORE=PostgreSql or MySql).
|
|
||||||
|
|
||||||
use crate::{smtp::inbound::TestMessage, utils::server::TestServerBuilder};
|
|
||||||
use common::{Server, config::smtp::report::AggregateFrequency, ipc::DmarcEvent};
|
|
||||||
use mail_auth::{
|
|
||||||
common::parse::TxtRecordParser,
|
|
||||||
dmarc::Dmarc,
|
|
||||||
report::{ActionDisposition, DmarcResult, Record, Report},
|
|
||||||
};
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::ClusterTaskType,
|
|
||||||
prelude::{ObjectType, Property},
|
|
||||||
structs::{
|
|
||||||
ClusterListenerGroup, ClusterRole, ClusterTaskGroup, ClusterTaskGroupProperties,
|
|
||||||
DmarcInternalReport, DmarcReportSettings, Expression, Task, TaskDmarcReport,
|
|
||||||
TaskStatus,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
types::{EnumImpl, map::Map},
|
|
||||||
};
|
|
||||||
use smtp::reporting::{dmarc::DmarcReporting, send::MtaReportSend};
|
|
||||||
use std::{
|
|
||||||
collections::BTreeSet,
|
|
||||||
net::IpAddr,
|
|
||||||
sync::Arc,
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{
|
|
||||||
ValueKey,
|
|
||||||
registry::{RegistryFilter, RegistryFilterValue, RegistryQuery},
|
|
||||||
write::{BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, now},
|
|
||||||
};
|
|
||||||
use types::id::Id;
|
|
||||||
|
|
||||||
const FRONT_ROLE: &str = "front_reports_front";
|
|
||||||
const MTA_ROLE: &str = "front_reports_mta";
|
|
||||||
const DOMAIN: &str = "front-reports.example";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn front_node_report_tests() {
|
|
||||||
if matches!(
|
|
||||||
std::env::var("STORE").as_deref(),
|
|
||||||
Ok("RocksDb" | "Sqlite") | Err(_)
|
|
||||||
) {
|
|
||||||
println!("Skipping front node report tests: they need a store the nodes can share.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!(
|
|
||||||
"Running front node report tests on {}...",
|
|
||||||
std::env::var("STORE").unwrap_or_default()
|
|
||||||
);
|
|
||||||
|
|
||||||
// A front role without outboundMta, an MTA role with it
|
|
||||||
let seed = TestServerBuilder::new("front_reports_seed").await;
|
|
||||||
seed.insert_object(role(FRONT_ROLE, &[ClusterTaskType::PushNotifications]))
|
|
||||||
.await;
|
|
||||||
seed.insert_object(role(MTA_ROLE, &[ClusterTaskType::OutboundMta]))
|
|
||||||
.await;
|
|
||||||
seed.insert_object(DmarcReportSettings {
|
|
||||||
aggregate_max_report_size: Expression {
|
|
||||||
else_: "1048576".into(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let seed = seed.disable_services().build().await;
|
|
||||||
|
|
||||||
// The front node receives mail from two sources: the events its SMTP
|
|
||||||
// sessions hand the report scheduler
|
|
||||||
let front = TestServerBuilder::new_with_role(
|
|
||||||
"front_reports_front",
|
|
||||||
"front.front-reports.example".into(),
|
|
||||||
Some(FRONT_ROLE.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await;
|
|
||||||
let front_server = front.server.clone();
|
|
||||||
assert!(!front_server.core.network.roles.outbound_mta);
|
|
||||||
for ip in ["192.0.2.1", "192.0.2.2"] {
|
|
||||||
front_server.schedule_report(event(ip)).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both are recorded in the shared report. Upstream, and main after live
|
|
||||||
// roles, left the front node's results out
|
|
||||||
let report_id = wait_for_report(&front_server, 2).await;
|
|
||||||
|
|
||||||
// Make the report due now. The front node leaves it alone: building and
|
|
||||||
// sending it is the outbound MTA's
|
|
||||||
move_task(&front_server, report_id, TaskStatus::now()).await;
|
|
||||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
|
||||||
front_server.notify_task_queue();
|
|
||||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
||||||
assert!(
|
|
||||||
task_exists(&front_server, report_id).await,
|
|
||||||
"the front node ran the report task"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Several writers append to the report at once, from both nodes: none
|
|
||||||
// of their records is lost. The report waits in the future meanwhile, or
|
|
||||||
// the MTA node would send it as soon as it starts
|
|
||||||
move_task(
|
|
||||||
&front_server,
|
|
||||||
report_id,
|
|
||||||
TaskStatus::at(now() as i64 + 3600),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let mut mta = TestServerBuilder::new_with_role(
|
|
||||||
"front_reports_mta",
|
|
||||||
"mta.front-reports.example".into(),
|
|
||||||
Some(MTA_ROLE.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.capture_queue()
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await;
|
|
||||||
let mta_server = mta.server.clone();
|
|
||||||
assert!(mta_server.core.network.roles.outbound_mta);
|
|
||||||
|
|
||||||
let concurrent: Vec<String> = (10..18).map(|n| format!("192.0.2.{n}")).collect();
|
|
||||||
let mut handles = Vec::new();
|
|
||||||
for (n, ip) in concurrent.iter().enumerate() {
|
|
||||||
let server = if n % 2 == 0 {
|
|
||||||
front_server.clone()
|
|
||||||
} else {
|
|
||||||
mta_server.clone()
|
|
||||||
};
|
|
||||||
let ip = ip.clone();
|
|
||||||
handles.push(tokio::spawn(async move {
|
|
||||||
server.schedule_dmarc(Box::new(event(&ip))).await;
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
for handle in handles {
|
|
||||||
handle.await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Due again, the MTA node sends the report with every record in it
|
|
||||||
move_task(&mta_server, report_id, TaskStatus::now()).await;
|
|
||||||
let message = mta.expect_message().await;
|
|
||||||
let report =
|
|
||||||
Report::parse_rfc5322(message.read_message(&mta).await.as_bytes(), usize::MAX).unwrap();
|
|
||||||
assert_eq!(report.domain(), DOMAIN);
|
|
||||||
let sent: BTreeSet<IpAddr> = report
|
|
||||||
.records()
|
|
||||||
.iter()
|
|
||||||
.map(|r| r.source_ip().unwrap())
|
|
||||||
.collect();
|
|
||||||
let expected: BTreeSet<IpAddr> = ["192.0.2.1", "192.0.2.2"]
|
|
||||||
.into_iter()
|
|
||||||
.map(String::from)
|
|
||||||
.chain(concurrent)
|
|
||||||
.map(|ip| ip.parse().unwrap())
|
|
||||||
.collect();
|
|
||||||
assert_eq!(sent, expected);
|
|
||||||
wait_for(Duration::from_secs(20), "report task to finish", || async {
|
|
||||||
!task_exists(&mta_server, report_id).await
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
assert!(reports(&mta_server).await.is_empty());
|
|
||||||
|
|
||||||
if seed.is_reset() {
|
|
||||||
seed.temp_dir.delete();
|
|
||||||
front.temp_dir.delete();
|
|
||||||
mta.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn role(name: &str, tasks: &[ClusterTaskType]) -> ClusterRole {
|
|
||||||
ClusterRole {
|
|
||||||
name: name.into(),
|
|
||||||
description: None,
|
|
||||||
listeners: ClusterListenerGroup::EnableAll,
|
|
||||||
tasks: ClusterTaskGroup::EnableSome(ClusterTaskGroupProperties {
|
|
||||||
task_types: Map::new(tasks.to_vec()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn event(ip: &str) -> DmarcEvent {
|
|
||||||
DmarcEvent {
|
|
||||||
domain: DOMAIN.to_string(),
|
|
||||||
report_record: Record::new()
|
|
||||||
.with_source_ip(ip.parse().unwrap())
|
|
||||||
.with_action_disposition(ActionDisposition::Pass)
|
|
||||||
.with_dmarc_dkim_result(DmarcResult::Pass)
|
|
||||||
.with_dmarc_spf_result(DmarcResult::Pass)
|
|
||||||
.with_envelope_from("sender.example")
|
|
||||||
.with_header_from("sender.example"),
|
|
||||||
dmarc_record: Arc::new(
|
|
||||||
Dmarc::parse(format!("v=DMARC1; p=reject; rua=mailto:reports@{DOMAIN}").as_bytes())
|
|
||||||
.unwrap(),
|
|
||||||
),
|
|
||||||
interval: AggregateFrequency::Daily,
|
|
||||||
span_id: 0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reports(server: &Server) -> Vec<(u64, DmarcInternalReport)> {
|
|
||||||
let ids = server
|
|
||||||
.registry()
|
|
||||||
.query::<Vec<Id>>(RegistryQuery::new(ObjectType::DmarcInternalReport).filter(
|
|
||||||
RegistryFilter::greater_than(
|
|
||||||
Property::Domain,
|
|
||||||
RegistryFilterValue::Bytes(vec![]),
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let mut reports = Vec::new();
|
|
||||||
for id in ids {
|
|
||||||
if let Some(report) = server
|
|
||||||
.store()
|
|
||||||
.get_value::<DmarcInternalReport>(ValueKey::from(ValueClass::Registry(
|
|
||||||
RegistryClass::Item {
|
|
||||||
object_id: ObjectType::DmarcInternalReport.to_id(),
|
|
||||||
item_id: id.id(),
|
|
||||||
},
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
{
|
|
||||||
reports.push((id.id(), report));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reports
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for the report for `DOMAIN` to hold `records` records; returns its id.
|
|
||||||
async fn wait_for_report(server: &Server, records: usize) -> u64 {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let found = reports(server)
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.find(|(_, report)| report.domain == DOMAIN);
|
|
||||||
if let Some((id, report)) = &found
|
|
||||||
&& report.report.records.len() == records
|
|
||||||
{
|
|
||||||
return *id;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(10),
|
|
||||||
"no report with {records} records for {DOMAIN}: {found:?}"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reschedules the report's task.
|
|
||||||
async fn move_task(server: &Server, id: u64, status: TaskStatus) {
|
|
||||||
let task = server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.expect("report task missing");
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id,
|
|
||||||
due: task.due_timestamp(),
|
|
||||||
}))
|
|
||||||
.schedule_task_with_id(
|
|
||||||
id,
|
|
||||||
Task::DmarcReport(TaskDmarcReport {
|
|
||||||
report_id: id.into(),
|
|
||||||
status,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server.notify_task_queue();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn task_exists(server: &Server, id: u64) -> bool {
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for<F, Fut>(within: Duration, what: &str, mut check: F)
|
|
||||||
where
|
|
||||||
F: FnMut() -> Fut,
|
|
||||||
Fut: Future<Output = bool>,
|
|
||||||
{
|
|
||||||
let started = Instant::now();
|
|
||||||
while !check().await {
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < within,
|
|
||||||
"still waiting for the {what} after {:?}",
|
|
||||||
started.elapsed()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,7 +7,6 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
pub mod broadcast;
|
pub mod broadcast;
|
||||||
pub mod front_reports; // inbuxa: every node records DMARC and TLS results
|
|
||||||
pub mod live_roles; // inbuxa: role edits apply without a restart
|
pub mod live_roles; // inbuxa: role edits apply without a restart
|
||||||
#[cfg(feature = "nats")]
|
#[cfg(feature = "nats")]
|
||||||
pub mod coordinator; // inbuxa: coordinator reconnects
|
pub mod coordinator; // inbuxa: coordinator reconnects
|
||||||
|
|||||||
@@ -2,12 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
pub mod analyze;
|
pub mod analyze;
|
||||||
pub mod dmarc;
|
pub mod dmarc;
|
||||||
pub mod reschedule; // inbuxa: report reschedules and unreadable queue rows
|
|
||||||
pub mod scheduler;
|
pub mod scheduler;
|
||||||
pub mod tls;
|
pub mod tls;
|
||||||
|
|||||||
@@ -1,370 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Rescheduling an internal DMARC or TLS report over JMAP moves its task: the
|
|
||||||
//! task runs at the new time, x:Task/get shows the new due, and tasks due
|
|
||||||
//! after it still run. A task queue row whose type can't be read is logged
|
|
||||||
//! and repaired rather than stopping every task due after it, including the
|
|
||||||
//! rows an earlier reschedule wrote with the report's object type.
|
|
||||||
|
|
||||||
use crate::utils::server::{TestServer, TestServerBuilder};
|
|
||||||
use common::{
|
|
||||||
Server,
|
|
||||||
config::smtp::report::AggregateFrequency,
|
|
||||||
ipc::{DmarcEvent, PolicyType, TlsEvent},
|
|
||||||
};
|
|
||||||
use mail_auth::{
|
|
||||||
common::parse::TxtRecordParser,
|
|
||||||
dmarc::Dmarc,
|
|
||||||
mta_sts::TlsRpt,
|
|
||||||
report::{ActionDisposition, DmarcResult, Record},
|
|
||||||
};
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::{TaskStoreMaintenanceType, TaskType},
|
|
||||||
prelude::{ObjectType, Property},
|
|
||||||
structs::{
|
|
||||||
DmarcInternalReport, DmarcReportSettings, Expression, Task, TaskStatus,
|
|
||||||
TaskStoreMaintenance, TlsInternalReport, TlsReportSettings,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
types::{EnumImpl, ObjectImpl, datetime::UTCDateTime},
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
use smtp::reporting::{index::InternalReportIndex, send::MtaReportSend};
|
|
||||||
use std::{
|
|
||||||
sync::Arc,
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{
|
|
||||||
SerializeInfallible, ValueKey,
|
|
||||||
write::{BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, now},
|
|
||||||
};
|
|
||||||
use types::id::Id;
|
|
||||||
use utils::snowflake::SnowflakeIdGenerator;
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
#[serial_test::serial]
|
|
||||||
async fn report_reschedule() {
|
|
||||||
let mut test = TestServerBuilder::new("smtp_report_reschedule")
|
|
||||||
.await
|
|
||||||
.with_http_listener(19057)
|
|
||||||
.await
|
|
||||||
.capture_queue()
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let admin = test.account("admin");
|
|
||||||
admin
|
|
||||||
.registry_create_object(TlsReportSettings {
|
|
||||||
max_report_size: Expression {
|
|
||||||
else_: "1024".into(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
admin
|
|
||||||
.registry_create_object(DmarcReportSettings {
|
|
||||||
aggregate_max_report_size: Expression {
|
|
||||||
else_: "1024".into(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
admin.reload_settings().await;
|
|
||||||
test.reload_core();
|
|
||||||
test.expect_reload_settings().await;
|
|
||||||
let admin = test.account("admin");
|
|
||||||
|
|
||||||
// A daily DMARC and TLS report, due a day from now
|
|
||||||
schedule_dmarc(&test, "foobar.org").await;
|
|
||||||
schedule_tls(&test, "foobar.org").await;
|
|
||||||
let dmarc_id = wait_for_report::<DmarcInternalReport>(&test, "foobar.org").await;
|
|
||||||
let tls_id = wait_for_report::<TlsInternalReport>(&test, "foobar.org").await;
|
|
||||||
|
|
||||||
// Reschedule both to a few seconds from now, with a task due after them
|
|
||||||
let at = now() + 3;
|
|
||||||
let later = marker_task(&test.server, at + 3).await;
|
|
||||||
for (object, id, task_type) in [
|
|
||||||
(
|
|
||||||
ObjectType::DmarcInternalReport,
|
|
||||||
dmarc_id,
|
|
||||||
TaskType::DmarcReport,
|
|
||||||
),
|
|
||||||
(ObjectType::TlsInternalReport, tls_id, TaskType::TlsReport),
|
|
||||||
] {
|
|
||||||
admin
|
|
||||||
.registry_update_object(
|
|
||||||
object,
|
|
||||||
id,
|
|
||||||
json!({
|
|
||||||
Property::DeliverAt: UTCDateTime::from_timestamp(at as i64),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
// x:Task/get shows the new due, and the queue row carries the task's
|
|
||||||
// type. Upstream wrote the report's object type there and left the
|
|
||||||
// task at its old due
|
|
||||||
let task = admin.registry_get::<Task>(id).await;
|
|
||||||
assert_eq!(task.object_type(), task_type);
|
|
||||||
assert_eq!(
|
|
||||||
task.due_timestamp(),
|
|
||||||
at,
|
|
||||||
"{object:?} task due not moved: {task:?}"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
queue_row(&test.server, id.id(), at).await,
|
|
||||||
Some(task_type.to_id().serialize()),
|
|
||||||
"{object:?} queue row"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both reports go out at the new time, and the later task still runs
|
|
||||||
wait_until_run(&test.server, &[dmarc_id.id(), tls_id.id(), later]).await;
|
|
||||||
assert!(now() >= at, "the reports went out before their new time");
|
|
||||||
assert!(
|
|
||||||
admin
|
|
||||||
.registry_get_all::<DmarcInternalReport>()
|
|
||||||
.await
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
admin
|
|
||||||
.registry_get_all::<TlsInternalReport>()
|
|
||||||
.await
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
|
|
||||||
// Rows an earlier reschedule may have left in a store: one with the
|
|
||||||
// report's object type and the task left at its old due, and one that
|
|
||||||
// is unreadable and has no task behind it. Neither may hold back a task
|
|
||||||
// due after them.
|
|
||||||
schedule_dmarc(&test, "foobar.net").await;
|
|
||||||
let dmarc_id = wait_for_report::<DmarcInternalReport>(&test, "foobar.net").await;
|
|
||||||
let at = now() + 2;
|
|
||||||
let old_due = old_style_reschedule(&test.server, dmarc_id.id(), at).await;
|
|
||||||
let orphan = SnowflakeIdGenerator::global_id().unwrap();
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch.set(
|
|
||||||
ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id: orphan,
|
|
||||||
due: at,
|
|
||||||
}),
|
|
||||||
vec![0xff, 0xff],
|
|
||||||
);
|
|
||||||
test.server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
let later = marker_task(&test.server, at + 2).await;
|
|
||||||
|
|
||||||
wait_until_run(&test.server, &[dmarc_id.id(), later]).await;
|
|
||||||
assert!(
|
|
||||||
admin
|
|
||||||
.registry_get_all::<DmarcInternalReport>()
|
|
||||||
.await
|
|
||||||
.is_empty()
|
|
||||||
);
|
|
||||||
assert_eq!(queue_row(&test.server, orphan, at).await, None);
|
|
||||||
assert_eq!(queue_row(&test.server, dmarc_id.id(), at).await, None);
|
|
||||||
assert_eq!(queue_row(&test.server, dmarc_id.id(), old_due).await, None);
|
|
||||||
|
|
||||||
// x:Task/query by type skips an unreadable row rather than failing
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
let due = now() + 3600;
|
|
||||||
batch.set(
|
|
||||||
ValueClass::TaskQueue(TaskQueueClass::Due { id: orphan, due }),
|
|
||||||
vec![0xff, 0xff],
|
|
||||||
);
|
|
||||||
test.server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
admin
|
|
||||||
.registry_query_ids(
|
|
||||||
ObjectType::Task,
|
|
||||||
vec![(Property::Type, TaskType::DmarcReport.as_str())],
|
|
||||||
Vec::<&str>::new(),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id: orphan,
|
|
||||||
due,
|
|
||||||
}));
|
|
||||||
test.server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn schedule_dmarc(test: &TestServer, domain: &str) {
|
|
||||||
test.server
|
|
||||||
.schedule_report(DmarcEvent {
|
|
||||||
domain: domain.to_string(),
|
|
||||||
report_record: Record::new()
|
|
||||||
.with_source_ip("192.168.1.2".parse().unwrap())
|
|
||||||
.with_action_disposition(ActionDisposition::Pass)
|
|
||||||
.with_dmarc_dkim_result(DmarcResult::Pass)
|
|
||||||
.with_dmarc_spf_result(DmarcResult::Fail)
|
|
||||||
.with_envelope_from("[email protected]")
|
|
||||||
.with_envelope_to("[email protected]")
|
|
||||||
.with_header_from("[email protected]"),
|
|
||||||
dmarc_record: Arc::new(
|
|
||||||
Dmarc::parse(format!("v=DMARC1; p=reject; rua=mailto:reports@{domain}").as_bytes())
|
|
||||||
.unwrap(),
|
|
||||||
),
|
|
||||||
interval: AggregateFrequency::Daily,
|
|
||||||
span_id: 0,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn schedule_tls(test: &TestServer, domain: &str) {
|
|
||||||
test.server
|
|
||||||
.schedule_report(TlsEvent {
|
|
||||||
domain: domain.to_string(),
|
|
||||||
policy: PolicyType::None,
|
|
||||||
failure: None,
|
|
||||||
tls_record: Arc::new(
|
|
||||||
TlsRpt::parse(format!("v=TLSRPTv1;rua=mailto:reports@{domain}").as_bytes())
|
|
||||||
.unwrap(),
|
|
||||||
),
|
|
||||||
interval: AggregateFrequency::Daily,
|
|
||||||
span_id: 0,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
trait ReportDomain: ObjectImpl {
|
|
||||||
fn report_domain(&self) -> &str;
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReportDomain for DmarcInternalReport {
|
|
||||||
fn report_domain(&self) -> &str {
|
|
||||||
&self.domain
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ReportDomain for TlsInternalReport {
|
|
||||||
fn report_domain(&self) -> &str {
|
|
||||||
&self.domain
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for_report<T: ReportDomain>(test: &TestServer, domain: &str) -> Id {
|
|
||||||
let admin = test.account("admin");
|
|
||||||
for _ in 0..100 {
|
|
||||||
if let Some((id, _)) = admin
|
|
||||||
.registry_get_all::<T>()
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.find(|(_, report)| report.report_domain() == domain)
|
|
||||||
{
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
||||||
}
|
|
||||||
panic!("No {} for {domain}", T::OBJECT.as_str());
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A task that succeeds when it runs, due at `due`.
|
|
||||||
async fn marker_task(server: &Server, due: u64) -> u64 {
|
|
||||||
let id = SnowflakeIdGenerator::global_id().unwrap();
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch.schedule_task_with_id(
|
|
||||||
id,
|
|
||||||
Task::StoreMaintenance(TaskStoreMaintenance {
|
|
||||||
maintenance_type: TaskStoreMaintenanceType::RemoveLockDav,
|
|
||||||
shard_index: Some(0),
|
|
||||||
status: TaskStatus::at(due as i64),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server.notify_task_queue();
|
|
||||||
id
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What the reschedule before this fix wrote: the report's object type in
|
|
||||||
/// the new queue row, and the task row left at its old due. Returns that
|
|
||||||
/// old due.
|
|
||||||
async fn old_style_reschedule(server: &Server, item_id: u64, at: u64) -> u64 {
|
|
||||||
let object_id = ObjectType::DmarcInternalReport.to_id();
|
|
||||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
|
||||||
let mut report = server
|
|
||||||
.store()
|
|
||||||
.get_value::<DmarcInternalReport>(ValueKey::from(key.clone()))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
let old_due = report.deliver_at().timestamp() as u64;
|
|
||||||
report.set_deliver_at(UTCDateTime::from_timestamp(at as i64));
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id: item_id,
|
|
||||||
due: old_due,
|
|
||||||
}))
|
|
||||||
.set(
|
|
||||||
ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id: item_id,
|
|
||||||
due: at,
|
|
||||||
}),
|
|
||||||
object_id.serialize(),
|
|
||||||
)
|
|
||||||
.set(key, report.to_pickled_vec());
|
|
||||||
server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server.notify_task_queue();
|
|
||||||
old_due
|
|
||||||
}
|
|
||||||
|
|
||||||
struct RawValue(Vec<u8>);
|
|
||||||
|
|
||||||
impl store::Deserialize for RawValue {
|
|
||||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
|
||||||
Ok(RawValue(bytes.to_vec()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn queue_row(server: &Server, id: u64, due: u64) -> Option<Vec<u8>> {
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.get_value::<RawValue>(ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id,
|
|
||||||
due,
|
|
||||||
})))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.map(|raw| raw.0)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn task_exists(server: &Server, id: u64) -> bool {
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_until_run(server: &Server, ids: &[u64]) {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let mut pending = Vec::new();
|
|
||||||
for id in ids {
|
|
||||||
if task_exists(server, *id).await {
|
|
||||||
pending.push(*id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if pending.is_empty() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if started.elapsed() > Duration::from_secs(30) {
|
|
||||||
panic!("tasks {pending:?} never ran");
|
|
||||||
}
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -21,8 +21,6 @@ pub mod replica_mysql; // inbuxa: read replicas on MySQL
|
|||||||
#[cfg(all(feature = "postgres", feature = "redis"))]
|
#[cfg(all(feature = "postgres", feature = "redis"))]
|
||||||
pub mod replica_cluster; // inbuxa: read replicas across nodes
|
pub mod replica_cluster; // inbuxa: read replicas across nodes
|
||||||
pub mod scaleout; // inbuxa: scale-out storage
|
pub mod scaleout; // inbuxa: scale-out storage
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
pub mod search_gin; // inbuxa: GIN indexes without a pending list
|
|
||||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
||||||
pub mod sql_timeout;
|
pub mod sql_timeout;
|
||||||
pub mod task_locks; // inbuxa: task locks across nodes
|
pub mod task_locks; // inbuxa: task locks across nodes
|
||||||
|
|||||||
@@ -9,31 +9,11 @@
|
|||||||
//! the pool's timeouts. Upstream's pools had none, so the worker waited for
|
//! the pool's timeouts. Upstream's pools had none, so the worker waited for
|
||||||
//! good. No database is needed: a local listener that never answers plays
|
//! good. No database is needed: a local listener that never answers plays
|
||||||
//! the server.
|
//! the server.
|
||||||
//!
|
|
||||||
//! inbuxa: the same for a database that stops answering while connections
|
|
||||||
//! are already open (a paused container): a query on a checked-out
|
|
||||||
//! connection ends within the query limit, the store works again once the
|
|
||||||
//! database is back, and /healthz/ready says 503 in between while
|
|
||||||
//! /healthz/live stays 200. These need the local test databases; a proxy
|
|
||||||
//! that can stop forwarding plays the pause.
|
|
||||||
|
|
||||||
use registry::schema::structs::DataStore;
|
use registry::schema::structs::DataStore;
|
||||||
use std::{
|
use std::time::{Duration, Instant};
|
||||||
sync::{
|
use store::{Store, ValueKey, write::ValueClass};
|
||||||
Arc,
|
use tokio::net::TcpListener;
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
},
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{
|
|
||||||
IterateParams, Store, ValueKey,
|
|
||||||
backend::query_timeout::QueryTimeouts,
|
|
||||||
write::{BatchBuilder, ValueClass},
|
|
||||||
};
|
|
||||||
use tokio::{
|
|
||||||
io::{AsyncReadExt, AsyncWriteExt},
|
|
||||||
net::{TcpListener, TcpStream},
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Accepts connections on a local port and never sends a byte.
|
/// Accepts connections on a local port and never sends a byte.
|
||||||
async fn silent_server() -> u16 {
|
async fn silent_server() -> u16 {
|
||||||
@@ -114,263 +94,3 @@ pub async fn mysql_pool_timeout() {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A TCP proxy to a local port that can stop forwarding, in both
|
|
||||||
/// directions, while keeping every connection open: a paused server whose
|
|
||||||
/// kernel still keeps the connections up.
|
|
||||||
struct PausableProxy {
|
|
||||||
port: u16,
|
|
||||||
paused: Arc<AtomicBool>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PausableProxy {
|
|
||||||
async fn start(upstream: u16) -> Self {
|
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let port = listener.local_addr().unwrap().port();
|
|
||||||
let paused = Arc::new(AtomicBool::new(false));
|
|
||||||
let paused_ = paused.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
while let Ok((client, _)) = listener.accept().await {
|
|
||||||
let Ok(server) = TcpStream::connect(("127.0.0.1", upstream)).await else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let (client_rx, client_tx) = client.into_split();
|
|
||||||
let (server_rx, server_tx) = server.into_split();
|
|
||||||
tokio::spawn(forward(client_rx, server_tx, paused_.clone()));
|
|
||||||
tokio::spawn(forward(server_rx, client_tx, paused_.clone()));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
PausableProxy { port, paused }
|
|
||||||
}
|
|
||||||
|
|
||||||
fn pause(&self, paused: bool) {
|
|
||||||
self.paused.store(paused, Ordering::SeqCst);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn forward(
|
|
||||||
mut from: tokio::net::tcp::OwnedReadHalf,
|
|
||||||
mut to: tokio::net::tcp::OwnedWriteHalf,
|
|
||||||
paused: Arc<AtomicBool>,
|
|
||||||
) {
|
|
||||||
let mut buf = vec![0u8; 16384];
|
|
||||||
loop {
|
|
||||||
while paused.load(Ordering::SeqCst) {
|
|
||||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
||||||
}
|
|
||||||
let n = match from.read(&mut buf).await {
|
|
||||||
Ok(0) | Err(_) => return,
|
|
||||||
Ok(n) => n,
|
|
||||||
};
|
|
||||||
// Hold what arrived while paused until the pause ends
|
|
||||||
while paused.load(Ordering::SeqCst) {
|
|
||||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
||||||
}
|
|
||||||
if to.write_all(&buf[..n]).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const TEST_LIMITS: QueryTimeouts = QueryTimeouts {
|
|
||||||
query: Duration::from_secs(2),
|
|
||||||
maintenance: Duration::from_secs(3),
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Opens `connections` pooled connections at once, so the operations that
|
|
||||||
/// follow find one idle and check it out.
|
|
||||||
async fn warm(store: &Store, connections: usize) {
|
|
||||||
let reads = (0..connections).map(|_| async {
|
|
||||||
store
|
|
||||||
.get_value::<u64>(ValueKey::from(ValueClass::Property(0)))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
});
|
|
||||||
futures::future::join_all(reads).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// With the database paused, reads, scans and writes on connections the
|
|
||||||
/// pool already holds end in an error within the query limit; once it is
|
|
||||||
/// back, the store works again.
|
|
||||||
async fn assert_queries_time_out(store: Store, proxy: &PausableProxy) {
|
|
||||||
store.create_tables().await.unwrap();
|
|
||||||
warm(&store, 4).await;
|
|
||||||
// mysql_async resets a connection on its way back to the pool; let
|
|
||||||
// those finish, or the connections are stuck in the reset when the
|
|
||||||
// pause starts and the pool's own wait timeout answers instead
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
proxy.pause(true);
|
|
||||||
|
|
||||||
let key = || ValueKey::from(ValueClass::Property(0));
|
|
||||||
let limit = TEST_LIMITS.query;
|
|
||||||
for (what, op) in [("read", 0), ("scan", 1), ("write", 2)] {
|
|
||||||
let started = Instant::now();
|
|
||||||
let result = tokio::time::timeout(Duration::from_secs(20), async {
|
|
||||||
match op {
|
|
||||||
0 => store.get_value::<u64>(key()).await.map(|_| ()),
|
|
||||||
1 => {
|
|
||||||
store
|
|
||||||
.iterate(
|
|
||||||
IterateParams::new(
|
|
||||||
ValueKey::from(ValueClass::Property(0)),
|
|
||||||
ValueKey::from(ValueClass::Property(u8::MAX)),
|
|
||||||
),
|
|
||||||
|_, _| Ok(true),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.with_account_id(u32::MAX - 7)
|
|
||||||
.with_collection(types::collection::Collection::Email)
|
|
||||||
.with_document(0)
|
|
||||||
.set(ValueClass::Property(0), 1u64.to_be_bytes().to_vec());
|
|
||||||
store.write(batch.build_all()).await.map(|_| ())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let elapsed = started.elapsed();
|
|
||||||
match result {
|
|
||||||
Ok(Err(err)) => {
|
|
||||||
let err = format!("{err:?}");
|
|
||||||
println!("Paused database, {what}: {err} after {elapsed:?}");
|
|
||||||
assert!(err.contains("Query timed out"), "{what}: {err}");
|
|
||||||
assert!(
|
|
||||||
elapsed >= limit && elapsed < limit * 3,
|
|
||||||
"{what} ended after {elapsed:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(Ok(())) => panic!("{what} succeeded against a paused database"),
|
|
||||||
Err(_) => panic!("{what} still waiting after {elapsed:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
proxy.pause(false);
|
|
||||||
tokio::time::timeout(Duration::from_secs(20), store.get_value::<u64>(key()))
|
|
||||||
.await
|
|
||||||
.expect("still waiting after the database came back")
|
|
||||||
.expect("the store didn't recover");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_query_timeout() {
|
|
||||||
println!("Running PostgreSQL query timeout test...");
|
|
||||||
let DataStore::PostgreSql(mut config) =
|
|
||||||
crate::utils::storage::build_data_store("PostgreSql", "").await
|
|
||||||
else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
let proxy = PausableProxy::start(config.port as u16).await;
|
|
||||||
config.host = "127.0.0.1".into();
|
|
||||||
config.port = proxy.port as u64;
|
|
||||||
// New connections through the paused proxy give up as quickly
|
|
||||||
config.timeout = Some(TEST_LIMITS.query.into());
|
|
||||||
let store = Store::build(DataStore::PostgreSql(config))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.with_query_timeouts(TEST_LIMITS);
|
|
||||||
assert_queries_time_out(store, &proxy).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "mysql")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn mysql_query_timeout() {
|
|
||||||
println!("Running MySQL query timeout test...");
|
|
||||||
let DataStore::MySql(mut config) = crate::utils::storage::build_data_store("MySql", "").await
|
|
||||||
else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
let proxy = PausableProxy::start(config.port as u16).await;
|
|
||||||
config.host = "127.0.0.1".into();
|
|
||||||
config.port = proxy.port as u64;
|
|
||||||
let store = Store::build(DataStore::MySql(config))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.with_query_timeouts(TEST_LIMITS);
|
|
||||||
assert_queries_time_out(store, &proxy).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// /healthz/ready follows the data store; /healthz/live doesn't.
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_readiness() {
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
|
||||||
use registry::schema::enums::NetworkListenerProtocol;
|
|
||||||
|
|
||||||
const HTTP_PORT: u16 = 11_320;
|
|
||||||
if std::env::var("STORE").as_deref() != Ok("PostgreSql") {
|
|
||||||
println!("Skipping the readiness test: it runs with STORE=PostgreSql.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!("Running readiness test...");
|
|
||||||
|
|
||||||
let test = TestServerBuilder::new("postgres_readiness")
|
|
||||||
.await
|
|
||||||
.with_listener(NetworkListenerProtocol::Http, "http", HTTP_PORT, true)
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
// Point the running node's data store at the database through the proxy
|
|
||||||
let DataStore::PostgreSql(mut config) =
|
|
||||||
crate::utils::storage::build_data_store("PostgreSql", "").await
|
|
||||||
else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
let proxy = PausableProxy::start(config.port as u16).await;
|
|
||||||
config.host = "127.0.0.1".into();
|
|
||||||
config.port = proxy.port as u64;
|
|
||||||
config.timeout = Some(TEST_LIMITS.query.into());
|
|
||||||
let store = Store::build(DataStore::PostgreSql(config))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.with_query_timeouts(TEST_LIMITS);
|
|
||||||
let inner = &test.server.inner;
|
|
||||||
let mut core = inner.shared_core.load_full().as_ref().clone();
|
|
||||||
core.storage.data = store;
|
|
||||||
inner.shared_core.store(Arc::new(core));
|
|
||||||
|
|
||||||
let health = |path: &'static str| async move {
|
|
||||||
reqwest::Client::builder()
|
|
||||||
.danger_accept_invalid_certs(true)
|
|
||||||
.timeout(Duration::from_secs(10))
|
|
||||||
.build()
|
|
||||||
.unwrap()
|
|
||||||
.get(format!("https://127.0.0.1:{HTTP_PORT}/healthz/{path}"))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.status()
|
|
||||||
.as_u16()
|
|
||||||
};
|
|
||||||
let wait_for = |path: &'static str, status: u16| async move {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let got = health(path).await;
|
|
||||||
if got == status {
|
|
||||||
println!("/healthz/{path}: {got} after {:?}", started.elapsed());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(20),
|
|
||||||
"/healthz/{path} still {got}, expected {status}"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
wait_for("ready", 200).await;
|
|
||||||
proxy.pause(true);
|
|
||||||
wait_for("ready", 503).await;
|
|
||||||
assert_eq!(health("live").await, 200);
|
|
||||||
proxy.pause(false);
|
|
||||||
wait_for("ready", 200).await;
|
|
||||||
assert_eq!(health("live").await, 200);
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -133,11 +133,6 @@ pub async fn test(test: &TestServer) {
|
|||||||
println!("Running address search tests...");
|
println!("Running address search tests...");
|
||||||
test_address_search(store.clone()).await;
|
test_address_search(store.clone()).await;
|
||||||
|
|
||||||
// inbuxa: words inside URLs, host names and file names in body text
|
|
||||||
// are found on every backend
|
|
||||||
println!("Running URL word search tests...");
|
|
||||||
test_url_word_search(store.clone()).await;
|
|
||||||
|
|
||||||
// Large document insert test
|
// Large document insert test
|
||||||
println!("Running large document insert tests...");
|
println!("Running large document insert tests...");
|
||||||
let mut large_text = String::with_capacity(20 * 1024 * 1024);
|
let mut large_text = String::with_capacity(20 * 1024 * 1024);
|
||||||
@@ -1134,79 +1129,3 @@ async fn test_address_search(store: SearchStore) {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn test_url_word_search(store: SearchStore) {
|
|
||||||
const ACCOUNT_ID: u32 = 8;
|
|
||||||
let bodies = [
|
|
||||||
"Track your parcel here: https://x.example/shipping-support/ and reply.",
|
|
||||||
"Reset it at https://mail.example.com/login/?password=reset&user=jane now.",
|
|
||||||
"Attached is invoice-2024.pdf for your records.",
|
|
||||||
"Shipping was fast, thanks again.",
|
|
||||||
"Nothing to see at www.example.org/about-us, really.",
|
|
||||||
];
|
|
||||||
|
|
||||||
let mut documents = Vec::new();
|
|
||||||
let mut mask = RoaringBitmap::new();
|
|
||||||
for (document_id, body) in bodies.iter().enumerate() {
|
|
||||||
let mut document = IndexDocument::new(SearchIndex::Email)
|
|
||||||
.with_account_id(ACCOUNT_ID)
|
|
||||||
.with_document_id(document_id as u32);
|
|
||||||
document.index_text(EmailSearchField::Body, body, Language::English);
|
|
||||||
document.index_unsigned(EmailSearchField::ReceivedAt, document_id as u64);
|
|
||||||
documents.push(document);
|
|
||||||
mask.insert(document_id as u32);
|
|
||||||
}
|
|
||||||
store.index(documents).await.unwrap();
|
|
||||||
if let SearchStore::ElasticSearch(store) = &store {
|
|
||||||
store.refresh_index(SearchIndex::Email).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (text, expected) in [
|
|
||||||
// only inside a URL path, a query string or a file name
|
|
||||||
("shipping", vec![0u32, 3]),
|
|
||||||
("support", vec![0]),
|
|
||||||
("password", vec![1]),
|
|
||||||
("login", vec![1]),
|
|
||||||
("jane", vec![1]),
|
|
||||||
("invoice", vec![2]),
|
|
||||||
("pdf", vec![2]),
|
|
||||||
("2024", vec![2]),
|
|
||||||
// host names
|
|
||||||
("example", vec![0, 1, 4]),
|
|
||||||
("mail", vec![1]),
|
|
||||||
// written as they appear
|
|
||||||
("https://x.example/shipping-support/", vec![0]),
|
|
||||||
("shipping-support", vec![0]),
|
|
||||||
("invoice-2024.pdf", vec![2]),
|
|
||||||
("mail.example.com", vec![1]),
|
|
||||||
// plain words are unaffected
|
|
||||||
("parcel", vec![0]),
|
|
||||||
("records", vec![2]),
|
|
||||||
("thanks", vec![3]),
|
|
||||||
// no match
|
|
||||||
("billing", vec![]),
|
|
||||||
("example.net", vec![]),
|
|
||||||
] {
|
|
||||||
let ids = store
|
|
||||||
.query_account(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filters(vec![
|
|
||||||
SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID),
|
|
||||||
SearchFilter::has_english_text(EmailSearchField::Body, text),
|
|
||||||
])
|
|
||||||
.with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt))
|
|
||||||
.with_mask(mask.clone()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(ids, expected, "Body {text:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
store
|
|
||||||
.unindex(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filter(SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,159 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! PostgreSQL full-text GIN indexes are built with fastupdate off, and an
|
|
||||||
//! index made earlier with the default is switched over at startup. With
|
|
||||||
//! fastupdate on, new entries wait in a pending list that every search scans
|
|
||||||
//! in full until VACUUM merges it.
|
|
||||||
|
|
||||||
use crate::utils::storage::build_data_store;
|
|
||||||
use registry::schema::structs::DataStore;
|
|
||||||
use store::{Rows, SearchStore, Store};
|
|
||||||
|
|
||||||
const SCHEMA: &str = "gin_fastupdate_test";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_gin_fastupdate() {
|
|
||||||
println!("Running PostgreSQL GIN fastupdate test...");
|
|
||||||
|
|
||||||
// Work in a schema of our own so the shared search tables are untouched
|
|
||||||
let admin = Store::build(build_data_store("PostgreSql", "").await)
|
|
||||||
.await
|
|
||||||
.expect("Failed to connect to PostgreSQL");
|
|
||||||
for query in [
|
|
||||||
format!("DROP SCHEMA IF EXISTS {SCHEMA} CASCADE"),
|
|
||||||
format!("CREATE SCHEMA {SCHEMA}"),
|
|
||||||
] {
|
|
||||||
admin.sql_query::<usize>(&query, vec![]).await.unwrap();
|
|
||||||
}
|
|
||||||
let DataStore::PostgreSql(mut config) = build_data_store("PostgreSql", "").await else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
config.options = Some(format!("-c search_path={SCHEMA}"));
|
|
||||||
let store = Store::build(DataStore::PostgreSql(config))
|
|
||||||
.await
|
|
||||||
.expect("Failed to connect to PostgreSQL");
|
|
||||||
let search = SearchStore::Store(store.clone());
|
|
||||||
|
|
||||||
// A fresh schema
|
|
||||||
search.create_indexes().await.unwrap();
|
|
||||||
let indexes = gin_indexes(&admin).await;
|
|
||||||
assert!(
|
|
||||||
indexes.len() >= 4,
|
|
||||||
"expected the search GIN indexes, found {indexes:?}"
|
|
||||||
);
|
|
||||||
for (name, options) in &indexes {
|
|
||||||
assert!(
|
|
||||||
options.contains("fastupdate=off"),
|
|
||||||
"fresh index {name} has options {options:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// A schema from before the change: the same indexes, made with the
|
|
||||||
// default fastupdate=on, and a pending list with something in it
|
|
||||||
for (name, _) in &indexes {
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>(
|
|
||||||
&format!("ALTER INDEX {SCHEMA}.{name} RESET (fastupdate)"),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
for (name, options) in gin_indexes(&admin).await {
|
|
||||||
assert!(
|
|
||||||
!options.contains("fastupdate"),
|
|
||||||
"index {name} still has options {options:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>(
|
|
||||||
&format!(
|
|
||||||
"INSERT INTO {SCHEMA}.s_email (accid, docid, subj, body) \
|
|
||||||
SELECT 1, n, to_tsvector('simple', 'pending subject ' || n), \
|
|
||||||
to_tsvector('simple', 'pending body text ' || n) \
|
|
||||||
FROM generate_series(1, 500) n"
|
|
||||||
),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(
|
|
||||||
pending_tuples(&admin, "gin_s_email_body").await > 0,
|
|
||||||
"no pending list to merge"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Startup on the existing schema switches every index over and merges
|
|
||||||
// what was pending
|
|
||||||
search.create_indexes().await.unwrap();
|
|
||||||
for (name, options) in gin_indexes(&admin).await {
|
|
||||||
assert!(
|
|
||||||
options.contains("fastupdate=off"),
|
|
||||||
"existing index {name} has options {options:?} after startup"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert_eq!(pending_tuples(&admin, "gin_s_email_body").await, 0);
|
|
||||||
|
|
||||||
// And a second startup changes nothing
|
|
||||||
search.create_indexes().await.unwrap();
|
|
||||||
for (name, options) in gin_indexes(&admin).await {
|
|
||||||
assert!(options.contains("fastupdate=off"), "{name}: {options:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>(&format!("DROP SCHEMA {SCHEMA} CASCADE"), vec![])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The GIN indexes in the test schema with their reloptions.
|
|
||||||
async fn gin_indexes(admin: &Store) -> Vec<(String, String)> {
|
|
||||||
admin
|
|
||||||
.sql_query::<Rows>(
|
|
||||||
&format!(
|
|
||||||
"SELECT c.relname::text, COALESCE(array_to_string(c.reloptions, ','), '') \
|
|
||||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \
|
|
||||||
JOIN pg_am a ON a.oid = c.relam \
|
|
||||||
WHERE n.nspname = '{SCHEMA}' AND c.relkind = 'i' AND a.amname = 'gin' \
|
|
||||||
ORDER BY 1"
|
|
||||||
),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.rows
|
|
||||||
.into_iter()
|
|
||||||
.map(|row| {
|
|
||||||
let mut values = row.values.into_iter();
|
|
||||||
(
|
|
||||||
values.next().unwrap().to_str().into_owned(),
|
|
||||||
values.next().unwrap().to_str().into_owned(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Tuples waiting in a GIN index's pending list (pgstattuple is a contrib
|
|
||||||
/// extension the test database has).
|
|
||||||
async fn pending_tuples(admin: &Store, index: &str) -> i64 {
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>("CREATE EXTENSION IF NOT EXISTS pgstattuple", vec![])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
admin
|
|
||||||
.sql_query::<Rows>(
|
|
||||||
&format!("SELECT pending_tuples FROM pgstatginindex('{SCHEMA}.{index}'::regclass)"),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.rows
|
|
||||||
.into_iter()
|
|
||||||
.next()
|
|
||||||
.and_then(|row| row.values.into_iter().next())
|
|
||||||
.map(|value| value.to_str().parse::<i64>().unwrap())
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
@@ -116,64 +116,12 @@ async fn test_write_applies(test: &TestServer) {
|
|||||||
for name in &names {
|
for name in &names {
|
||||||
assert!(has_schedule(test, name), "{name} missing");
|
assert!(has_schedule(test, name), "{name} missing");
|
||||||
}
|
}
|
||||||
// A burst of separate requests shares a reload or two: each arrives
|
|
||||||
// tens of milliseconds after the last, so none overlaps a running
|
|
||||||
// reload, and the reload waits for writes to settle instead
|
|
||||||
let reloads = test.server.inner.data.settings_reload.reloads();
|
|
||||||
let started = std::time::Instant::now();
|
|
||||||
let burst = (0..10)
|
|
||||||
.map(|i| format!("autoreload-burst-{i}"))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let mut writes = Vec::new();
|
|
||||||
for name in &burst {
|
|
||||||
writes.push(admin.registry_create([MtaDeliverySchedule {
|
|
||||||
name: name.clone(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
}]));
|
|
||||||
}
|
|
||||||
for response in futures::future::join_all(writes).await {
|
|
||||||
assert_applied(&response);
|
|
||||||
schedule_ids.push(response.created_id(0));
|
|
||||||
}
|
|
||||||
let burst_reloads = test.server.inner.data.settings_reload.reloads() - reloads;
|
|
||||||
println!(
|
|
||||||
"10 concurrent writes: {burst_reloads} reload(s), {} ms",
|
|
||||||
started.elapsed().as_millis()
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
(1..=2).contains(&burst_reloads),
|
|
||||||
"{burst_reloads} reloads for 10 concurrent writes"
|
|
||||||
);
|
|
||||||
for name in &burst {
|
|
||||||
assert!(has_schedule(test, name), "{name} missing");
|
|
||||||
}
|
|
||||||
|
|
||||||
// A single write still reloads promptly
|
|
||||||
let reloads = test.server.inner.data.settings_reload.reloads();
|
|
||||||
let started = std::time::Instant::now();
|
|
||||||
let response = admin
|
|
||||||
.registry_create([MtaDeliverySchedule {
|
|
||||||
name: "autoreload-single".into(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
}])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
schedule_ids.push(response.created_id(0));
|
|
||||||
println!("1 write: {} ms", started.elapsed().as_millis());
|
|
||||||
assert_eq!(
|
|
||||||
test.server.inner.data.settings_reload.reloads() - reloads,
|
|
||||||
1
|
|
||||||
);
|
|
||||||
assert!(has_schedule(test, "autoreload-single"));
|
|
||||||
|
|
||||||
// Several objects in one request: one reload
|
// Several objects in one request: one reload
|
||||||
let response = admin
|
let response = admin
|
||||||
.registry_destroy(ObjectType::MtaDeliverySchedule, schedule_ids.iter())
|
.registry_destroy(ObjectType::MtaDeliverySchedule, schedule_ids.iter())
|
||||||
.await;
|
.await;
|
||||||
assert_applied(&response);
|
assert_applied(&response);
|
||||||
for name in names.iter().chain(&burst) {
|
for name in &names {
|
||||||
assert!(!has_schedule(test, name), "{name} still present");
|
assert!(!has_schedule(test, name), "{name} still present");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ pub mod quota;
|
|||||||
pub mod reload; // inbuxa: reloads and build errors
|
pub mod reload; // inbuxa: reloads and build errors
|
||||||
pub mod security;
|
pub mod security;
|
||||||
pub mod task;
|
pub mod task;
|
||||||
pub mod tracer_reload; // inbuxa: tracers start over when their settings change
|
|
||||||
pub mod tenant;
|
pub mod tenant;
|
||||||
pub mod undelete;
|
pub mod undelete;
|
||||||
|
|
||||||
|
|||||||
@@ -1,204 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
// inbuxa: a tracer whose own settings change is started over by the reload
|
|
||||||
// that follows the write: a Log tracer moved to another directory writes
|
|
||||||
// there from then on, and no event is lost or written twice on the way.
|
|
||||||
|
|
||||||
use crate::utils::{
|
|
||||||
jmap::JmapResponse,
|
|
||||||
server::{TestServer, TestServerBuilder},
|
|
||||||
};
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::{EventPolicy, LogRotateFrequency, TracingLevel},
|
|
||||||
prelude::ObjectType,
|
|
||||||
structs::{Expression, MtaStageAuth, Tracer, TracerLog},
|
|
||||||
},
|
|
||||||
types::map::Map,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
use std::{
|
|
||||||
path::{Path, PathBuf},
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use trc::{EventType, ServerEvent};
|
|
||||||
|
|
||||||
const PREFIX: &str = "tracer-reload";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn tracer_reload_tests() {
|
|
||||||
let mut test = TestServerBuilder::new("tracer_reload_tests")
|
|
||||||
.await
|
|
||||||
.with_default_listeners()
|
|
||||||
.await
|
|
||||||
.with_object(MtaStageAuth {
|
|
||||||
require: Expression {
|
|
||||||
else_: "false".to_string(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let admin = test
|
|
||||||
.create_user_account(
|
|
||||||
"admin",
|
|
||||||
"[email protected]",
|
|
||||||
"these_pretzels_are_making_me_thirsty",
|
|
||||||
&[],
|
|
||||||
"Admin",
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
test.account("admin")
|
|
||||||
.assign_roles_to_account(admin.id(), &["user", "system"])
|
|
||||||
.await;
|
|
||||||
test.insert_account(admin);
|
|
||||||
|
|
||||||
test_log_tracer_moves(&test).await;
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn test_log_tracer_moves(test: &TestServer) {
|
|
||||||
println!("Running Log tracer path change...");
|
|
||||||
let admin = test.account("[email protected]");
|
|
||||||
let old_dir = test.temp_dir.path.join("tracer-old");
|
|
||||||
let new_dir = test.temp_dir.path.join("tracer-new");
|
|
||||||
for dir in [&old_dir, &new_dir] {
|
|
||||||
let _ = std::fs::remove_dir_all(dir);
|
|
||||||
std::fs::create_dir_all(dir).unwrap();
|
|
||||||
}
|
|
||||||
let old_file = old_dir.join(PREFIX);
|
|
||||||
let new_file = new_dir.join(PREFIX);
|
|
||||||
|
|
||||||
// A Log tracer for one event type, written to the old directory
|
|
||||||
let response = admin
|
|
||||||
.registry_create([Tracer::Log(TracerLog {
|
|
||||||
path: old_dir.to_string_lossy().into_owned(),
|
|
||||||
prefix: PREFIX.into(),
|
|
||||||
rotate: LogRotateFrequency::Never,
|
|
||||||
ansi: false,
|
|
||||||
multiline: false,
|
|
||||||
enable: true,
|
|
||||||
level: TracingLevel::Trace,
|
|
||||||
lossy: false,
|
|
||||||
events: Map::new(vec![EventType::Server(ServerEvent::Licensing)]),
|
|
||||||
events_policy: EventPolicy::Include,
|
|
||||||
})])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
let tracer_id = response.created_id(0);
|
|
||||||
emit("marker-before");
|
|
||||||
wait_for(&old_file, "marker-before").await;
|
|
||||||
|
|
||||||
// Events keep coming while the path changes
|
|
||||||
let stream = tokio::spawn(async {
|
|
||||||
for i in 0..2000u32 {
|
|
||||||
emit(&format!("seq-{i:05}-end"));
|
|
||||||
if i % 50 == 0 {
|
|
||||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
||||||
let response = admin
|
|
||||||
.registry_update(
|
|
||||||
ObjectType::Tracer,
|
|
||||||
[(tracer_id, json!({"path": new_dir.to_string_lossy()}))],
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
stream.await.unwrap();
|
|
||||||
|
|
||||||
// Once the reload has run, events go to the new file only
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
emit("marker-after");
|
|
||||||
wait_for(&new_file, "marker-after").await;
|
|
||||||
wait_for(&new_file, "seq-01999-end").await;
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
let old = read(&old_file);
|
|
||||||
let new = read(&new_file);
|
|
||||||
assert!(!old.contains("marker-after"), "old file still written to");
|
|
||||||
assert!(!new.contains("marker-before"));
|
|
||||||
|
|
||||||
// Every event written once, in one file or the other
|
|
||||||
let old_seq = count_seq(&old);
|
|
||||||
let new_seq = count_seq(&new);
|
|
||||||
println!(
|
|
||||||
"{} events in the old file, {} in the new one",
|
|
||||||
old_seq.iter().filter(|c| **c > 0).count(),
|
|
||||||
new_seq.iter().filter(|c| **c > 0).count()
|
|
||||||
);
|
|
||||||
for i in 0..2000 {
|
|
||||||
assert_eq!(
|
|
||||||
old_seq[i] + new_seq[i],
|
|
||||||
1,
|
|
||||||
"seq-{i:05} written {} + {} times",
|
|
||||||
old_seq[i],
|
|
||||||
new_seq[i]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
new_seq.iter().any(|c| *c > 0),
|
|
||||||
"no event of the stream reached the new file"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Removing the tracer stops it
|
|
||||||
let response = admin
|
|
||||||
.registry_destroy(ObjectType::Tracer, [tracer_id])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
emit("marker-removed");
|
|
||||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
||||||
assert!(!read(&new_file).contains("marker-removed"));
|
|
||||||
assert!(!read(&old_file).contains("marker-removed"));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emit(marker: &str) {
|
|
||||||
trc::event!(Server(ServerEvent::Licensing), Details = marker.to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read(path: &Path) -> String {
|
|
||||||
std::fs::read_to_string(path).unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn count_seq(text: &str) -> Vec<u32> {
|
|
||||||
let mut counts = vec![0u32; 2000];
|
|
||||||
for part in text.split("seq-").skip(1) {
|
|
||||||
if let Some(n) = part.get(..5).and_then(|n| n.parse::<usize>().ok())
|
|
||||||
&& part[5..].starts_with("-end")
|
|
||||||
{
|
|
||||||
counts[n] += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
counts
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for(path: &PathBuf, marker: &str) {
|
|
||||||
let started = Instant::now();
|
|
||||||
while !read(path).contains(marker) {
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(10),
|
|
||||||
"{marker} not in {}",
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn assert_applied(response: &JmapResponse) {
|
|
||||||
assert_eq!(
|
|
||||||
response.pointer("/methodResponses/0/1/x:settingsReload"),
|
|
||||||
Some(&json!({"applied": true})),
|
|
||||||
"{response:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user