Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f398d95062 | ||
|
|
81ec917d74 | ||
|
|
e2ab26ad19 | ||
|
|
0b4aa9c084 | ||
|
|
4e6c8b916e |
@@ -3,28 +3,11 @@
|
|||||||
# whether a person pushed it or weekly-release.yml created it through the
|
# whether a person pushed it or weekly-release.yml created it through the
|
||||||
# releases API.
|
# releases API.
|
||||||
#
|
#
|
||||||
# The image is multi-arch (linux/amd64, linux/arm64), built by two jobs on
|
# The image is multi-arch (linux/amd64, linux/arm64) as before, but built in
|
||||||
# the image-build runner rather than one buildx run for both. The Dockerfile's
|
# one buildx run on host1 instead of one native runner per architecture: the
|
||||||
# builder stage runs on the build platform and cross-compiles with an aarch64
|
# Dockerfile's builder stage runs on the build platform and cross-compiles
|
||||||
# linker, so only the small final stage (apt, setcap) goes through QEMU for
|
# with an aarch64 linker, so only the small final stage (apt, setcap) goes
|
||||||
# arm64 -- but two release builds (LTO, one codegen unit) side by side on one
|
# through QEMU for arm64. No digest-joining job is needed.
|
||||||
# machine each take twice as long. Production runs amd64, so amd64 goes first
|
|
||||||
# and on its own:
|
|
||||||
# * publish-amd64 pushes :<version>-amd64 and :<version>, a plain amd64
|
|
||||||
# image, as soon as its build is done. A deploy can start from it.
|
|
||||||
# * publish-arm64 then builds arm64, pushes :<version>-arm64, and replaces
|
|
||||||
# :<version> with the two-platform index. :latest moves only here, so it
|
|
||||||
# never names an image without arm64.
|
|
||||||
#
|
|
||||||
# Both jobs use one BuildKit builder, `gitea-builder`, whose container
|
|
||||||
# (buildx_buildkit_gitea-builder0) and state volume stay on the runner's host
|
|
||||||
# between jobs: a job container's `buildx create` finds the existing container
|
|
||||||
# and reuses it and its cache. The dependency build (`cargo chef cook`) is
|
|
||||||
# keyed on the recipe, which only a dependency change alters, so a release
|
|
||||||
# normally compiles just the workspace. Removing that container or its volume
|
|
||||||
# costs the next release a cold build, nothing more. The planner and dependency
|
|
||||||
# layers for the build platform are shared, so arm64 also reuses what amd64
|
|
||||||
# just did where it can.
|
|
||||||
#
|
#
|
||||||
# Two guards before anything is pushed:
|
# Two guards before anything is pushed:
|
||||||
# * the tag must be v<brand_version!>. The version is a string in
|
# * the tag must be v<brand_version!>. The version is a string in
|
||||||
@@ -87,7 +70,7 @@ jobs:
|
|||||||
echo "version=$V" >> "$GITHUB_OUTPUT"
|
echo "version=$V" >> "$GITHUB_OUTPUT"
|
||||||
echo "version $V"
|
echo "version $V"
|
||||||
|
|
||||||
publish-amd64:
|
publish:
|
||||||
needs: [version]
|
needs: [version]
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
container:
|
container:
|
||||||
@@ -106,15 +89,16 @@ jobs:
|
|||||||
test -n "$REGISTRY" && test -n "$VERSION"
|
test -n "$REGISTRY" && test -n "$VERSION"
|
||||||
test -n "$PACKAGE_TOKEN" || { echo "PACKAGE_TOKEN secret is not set on this repository" >&2; exit 1; }
|
test -n "$PACKAGE_TOKEN" || { echo "PACKAGE_TOKEN secret is not set on this repository" >&2; exit 1; }
|
||||||
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
|
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
|
||||||
|
docker run --privileged --rm tonistiigi/binfmt --install arm64
|
||||||
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
|
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
|
||||||
# Attestations off, as before: they add manifests of their own, and the
|
# Attestations off, as before: they add manifests of their own to the
|
||||||
# index should hold the two images and nothing else.
|
# index, and the index should hold the two images and nothing else.
|
||||||
- run: |
|
- run: |
|
||||||
docker buildx build \
|
docker buildx build \
|
||||||
--platform linux/amd64 \
|
--platform linux/amd64,linux/arm64 \
|
||||||
--provenance=false --sbom=false \
|
--provenance=false --sbom=false \
|
||||||
--tag "$IMAGE:$VERSION-amd64" \
|
|
||||||
--tag "$IMAGE:$VERSION" \
|
--tag "$IMAGE:$VERSION" \
|
||||||
|
--tag "$IMAGE:latest" \
|
||||||
--push .
|
--push .
|
||||||
docker buildx imagetools inspect "$IMAGE:$VERSION"
|
docker buildx imagetools inspect "$IMAGE:$VERSION"
|
||||||
# Gitea keeps a container package on its owner; linking it shows it on
|
# Gitea keeps a container package on its owner; linking it shows it on
|
||||||
@@ -127,47 +111,11 @@ jobs:
|
|||||||
- if: always()
|
- if: always()
|
||||||
run: docker logout "$REGISTRY" || true
|
run: docker logout "$REGISTRY" || true
|
||||||
|
|
||||||
publish-arm64:
|
|
||||||
needs: [version, publish-amd64]
|
|
||||||
runs-on: docker
|
|
||||||
container:
|
|
||||||
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
|
|
||||||
volumes:
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
|
||||||
env:
|
|
||||||
DOCKER_BUILDKIT: "1"
|
|
||||||
REGISTRY: ${{ vars.REGISTRY }}
|
|
||||||
IMAGE: ${{ vars.REGISTRY }}/${{ github.repository }}
|
|
||||||
VERSION: ${{ needs.version.outputs.version }}
|
|
||||||
PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }}
|
|
||||||
steps:
|
|
||||||
- uses: coffey-labs/actions/checkout@fab0c4d45e0162963965f1555df27b7bed5e20ec
|
|
||||||
- run: |
|
|
||||||
echo "$PACKAGE_TOKEN" | docker login -u jcoffey-dev --password-stdin "$REGISTRY"
|
|
||||||
docker run --privileged --rm tonistiigi/binfmt --install arm64
|
|
||||||
docker buildx create --use --name gitea-builder --driver docker-container || docker buildx use gitea-builder
|
|
||||||
# The index is built from the two per-architecture tags rather than from
|
|
||||||
# :<version>, which by now is the amd64 image and would be read as such.
|
|
||||||
- run: |
|
|
||||||
docker buildx build \
|
|
||||||
--platform linux/arm64 \
|
|
||||||
--provenance=false --sbom=false \
|
|
||||||
--tag "$IMAGE:$VERSION-arm64" \
|
|
||||||
--push .
|
|
||||||
docker buildx imagetools create \
|
|
||||||
--tag "$IMAGE:$VERSION" \
|
|
||||||
--tag "$IMAGE:latest" \
|
|
||||||
"$IMAGE:$VERSION-amd64" "$IMAGE:$VERSION-arm64"
|
|
||||||
docker buildx imagetools inspect "$IMAGE:$VERSION"
|
|
||||||
- if: always()
|
|
||||||
run: docker logout "$REGISTRY" || true
|
|
||||||
|
|
||||||
# The weekly release creates its Release (and so the tag) first; a tag
|
# The weekly release creates its Release (and so the tag) first; a tag
|
||||||
# pushed by hand has none. Either way the tag ends up with exactly one
|
# pushed by hand has none. Either way the tag ends up with exactly one
|
||||||
# Release, created once the amd64 image exists so its pull instructions
|
# Release, created after the image exists so its pull instructions work.
|
||||||
# work; arm64 and the binaries follow.
|
|
||||||
release:
|
release:
|
||||||
needs: [version, publish-amd64]
|
needs: [version, publish]
|
||||||
runs-on: light
|
runs-on: light
|
||||||
container:
|
container:
|
||||||
image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim
|
image: python:3.13-slim@sha256:8d9d0b8bcf6506481eae4907c18f5e3e7902e629f5f6d684f9e7c32e85e3ddf0 # 3.13-slim
|
||||||
@@ -191,9 +139,7 @@ jobs:
|
|||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code != 404: raise
|
if e.code != 404: raise
|
||||||
image = f"{os.environ['REGISTRY']}/{os.environ['REPO']}:{version}"
|
image = f"{os.environ['REGISTRY']}/{os.environ['REPO']}:{version}"
|
||||||
body = (f"Container image: `{image}` (linux/amd64, linux/arm64); also `:latest`. "
|
body = (f"Container image: `{image}` (linux/amd64, linux/arm64); also `:latest`.\n\n"
|
||||||
"amd64 is published first; arm64 is added to the same tag when its build "
|
|
||||||
"finishes, and `:latest` moves then.\n\n"
|
|
||||||
"Binaries for a host install are attached: `inbuxa-linux-amd64.tar.gz` and "
|
"Binaries for a host install are attached: `inbuxa-linux-amd64.tar.gz` and "
|
||||||
"`inbuxa-linux-arm64.tar.gz`, with `SHA256SUMS`. Each is the binary out of this "
|
"`inbuxa-linux-arm64.tar.gz`, with `SHA256SUMS`. Each is the binary out of this "
|
||||||
"release's image for that architecture, so it is the same build. The image "
|
"release's image for that architecture, so it is the same build. The image "
|
||||||
@@ -216,7 +162,7 @@ jobs:
|
|||||||
# `docker create` does not start anything, so pulling an arm64 image on an
|
# `docker create` does not start anything, so pulling an arm64 image on an
|
||||||
# amd64 runner and copying a file out of it needs no emulation.
|
# amd64 runner and copying a file out of it needs no emulation.
|
||||||
binaries:
|
binaries:
|
||||||
needs: [version, publish-arm64, release]
|
needs: [version, publish, release]
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
container:
|
container:
|
||||||
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
|
image: docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d # 28-cli
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::auth::AccessToken;
|
use crate::auth::AccessToken;
|
||||||
@@ -20,16 +18,6 @@ impl Server {
|
|||||||
access_token: &AccessToken,
|
access_token: &AccessToken,
|
||||||
addr: IpAddr,
|
addr: IpAddr,
|
||||||
) -> trc::Result<Option<InFlight>> {
|
) -> trc::Result<Option<InFlight>> {
|
||||||
// inbuxa: an account with unlimited requests passes both limits
|
|
||||||
// below anyway, so don't count its requests. The count is a write to
|
|
||||||
// one counter per account in the in-memory store, and concurrent
|
|
||||||
// requests from one account queue on that key (a row lock on SQL,
|
|
||||||
// conflict retries on RocksDB): in a cluster rehearsal ten parallel
|
|
||||||
// admin writes were accepted one after another, about 33 ms apart.
|
|
||||||
if access_token.has_permission(Permission::UnlimitedRequests) {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
|
|
||||||
let rate_reset = if let Some(rate) = &self.core.network.http.rate_authenticated {
|
let rate_reset = if let Some(rate) = &self.core.network.http.rate_authenticated {
|
||||||
if self.is_ip_allowed(addr) {
|
if self.is_ip_allowed(addr) {
|
||||||
None
|
None
|
||||||
|
|||||||
Vendored
+14
-390
@@ -7,33 +7,26 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
BuildServer, Core, Server,
|
Core, Server,
|
||||||
config::{
|
config::{
|
||||||
server::{Listeners, tls::parse_certificates},
|
server::{Listeners, tls::parse_certificates},
|
||||||
storage::Storage,
|
storage::Storage,
|
||||||
telemetry::Telemetry,
|
telemetry::Telemetry,
|
||||||
},
|
},
|
||||||
ipc::{BroadcastEvent, QueueEvent, RegistryChange},
|
ipc::{QueueEvent, RegistryChange},
|
||||||
network::security::{BlockedIps, IpWithTtl},
|
network::security::{BlockedIps, IpWithTtl},
|
||||||
};
|
};
|
||||||
use ahash::AHashMap;
|
use ahash::AHashMap;
|
||||||
use directory::Directories;
|
use directory::Directories;
|
||||||
use registry::{
|
use registry::{
|
||||||
schema::{prelude::ObjectType, structs::BlockedIp},
|
schema::{prelude::ObjectType, structs::BlockedIp},
|
||||||
types::{
|
types::error::{Error, Warning},
|
||||||
error::{Error, Warning},
|
|
||||||
id::ObjectId,
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use store::{LookupStores, registry::bootstrap::Bootstrap, write::now};
|
use store::{LookupStores, registry::bootstrap::Bootstrap, write::now};
|
||||||
|
|
||||||
pub struct ReloadResult {
|
pub struct ReloadResult {
|
||||||
/// Errors that kept the reload from being applied.
|
|
||||||
pub errors: Vec<Error>,
|
pub errors: Vec<Error>,
|
||||||
/// inbuxa: errors in objects that already failed when the running
|
|
||||||
/// settings were built; logged, but they don't refuse a reload.
|
|
||||||
pub known_errors: Vec<Error>,
|
|
||||||
pub warnings: Vec<Warning>,
|
pub warnings: Vec<Warning>,
|
||||||
pub replaced_core: bool,
|
pub replaced_core: bool,
|
||||||
}
|
}
|
||||||
@@ -121,30 +114,24 @@ impl Server {
|
|||||||
directories: directory.directories,
|
directories: directory.directories,
|
||||||
};
|
};
|
||||||
|
|
||||||
// inbuxa: upstream swapped the core only when the whole build
|
// Parse tracers
|
||||||
// was free of errors, while boot runs with whatever built. So one
|
|
||||||
// object that failed (a DNS lookup that timed out, say) refused
|
|
||||||
// every later reload, cluster-wide when the reload came from
|
|
||||||
// ReloadSettings, and the running settings went stale. Now a
|
|
||||||
// reload is refused only for errors in objects that built when
|
|
||||||
// the running settings were built: those would be lost by
|
|
||||||
// applying it. Objects that already failed then are missing
|
|
||||||
// from the running settings anyway, as at boot, so their
|
|
||||||
// errors are reported but don't hold the reload back.
|
|
||||||
let tracers = Telemetry::parse(&mut bootstrap, &storage).await;
|
let tracers = Telemetry::parse(&mut bootstrap, &storage).await;
|
||||||
let core = Box::pin(Core::parse(&mut bootstrap, storage)).await;
|
|
||||||
let mut servers = Listeners::parse(&mut bootstrap).await;
|
|
||||||
|
|
||||||
if !self.has_new_build_errors(&bootstrap.errors) {
|
if bootstrap.errors.is_empty() {
|
||||||
|
let core = Box::pin(Core::parse(&mut bootstrap, storage)).await;
|
||||||
|
|
||||||
|
if bootstrap.errors.is_empty() {
|
||||||
|
let mut servers = Listeners::parse(&mut bootstrap).await;
|
||||||
servers
|
servers
|
||||||
.parse_tcp_acceptors(&mut bootstrap, self.inner.clone())
|
.parse_tcp_acceptors(&mut bootstrap, self.inner.clone())
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
if !self.has_new_build_errors(&bootstrap.errors) {
|
if bootstrap.errors.is_empty() {
|
||||||
// Update core
|
// Update core
|
||||||
self.inner.shared_core.store(core.into());
|
self.inner.shared_core.store(core.into());
|
||||||
|
|
||||||
// Update tracers
|
// Update tracers
|
||||||
|
|
||||||
tracers.update();
|
tracers.update();
|
||||||
|
|
||||||
// Reload queue settings
|
// Reload queue settings
|
||||||
@@ -155,32 +142,14 @@ impl Server {
|
|||||||
.await
|
.await
|
||||||
.ok();
|
.ok();
|
||||||
|
|
||||||
// inbuxa: the task manager reads the node's role on
|
|
||||||
// every scan; scan now, so a role that gained task
|
|
||||||
// types starts claiming them without waiting out the
|
|
||||||
// refresh interval
|
|
||||||
self.inner.ipc.task_tx.notify_one();
|
|
||||||
|
|
||||||
self.record_build_errors(&bootstrap.errors);
|
|
||||||
|
|
||||||
return Ok(ReloadResult {
|
return Ok(ReloadResult {
|
||||||
errors: Vec::new(),
|
errors: bootstrap.errors,
|
||||||
known_errors: bootstrap.errors,
|
|
||||||
warnings: bootstrap.warnings,
|
warnings: bootstrap.warnings,
|
||||||
replaced_core: true,
|
replaced_core: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
let (known_errors, errors) = std::mem::take(&mut bootstrap.errors)
|
|
||||||
.into_iter()
|
|
||||||
.partition(|error| self.is_known_build_error(error));
|
|
||||||
return Ok(ReloadResult {
|
|
||||||
errors,
|
|
||||||
known_errors,
|
|
||||||
warnings: bootstrap.warnings,
|
|
||||||
replaced_core: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -194,7 +163,7 @@ impl ReloadResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn log(&self) {
|
pub fn log(&self) {
|
||||||
for error in self.errors.iter().chain(&self.known_errors) {
|
for error in &self.errors {
|
||||||
error.log();
|
error.log();
|
||||||
}
|
}
|
||||||
for warning in &self.warnings {
|
for warning in &self.warnings {
|
||||||
@@ -207,353 +176,8 @@ impl From<Bootstrap> for ReloadResult {
|
|||||||
fn from(bootstrap: Bootstrap) -> Self {
|
fn from(bootstrap: Bootstrap) -> Self {
|
||||||
Self {
|
Self {
|
||||||
errors: bootstrap.errors,
|
errors: bootstrap.errors,
|
||||||
known_errors: Vec::new(),
|
|
||||||
warnings: bootstrap.warnings,
|
warnings: bootstrap.warnings,
|
||||||
replaced_core: false,
|
replaced_core: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: which objects failed to build for the running settings
|
|
||||||
impl Server {
|
|
||||||
/// Records the objects that failed to build for the settings now running.
|
|
||||||
pub fn record_build_errors(&self, errors: &[Error]) {
|
|
||||||
*self.inner.data.build_errors.lock() = errors.iter().filter_map(error_object).collect();
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_known_build_error(&self, error: &Error) -> bool {
|
|
||||||
error_object(error).is_some_and(|id| self.inner.data.build_errors.lock().contains(&id))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn has_new_build_errors(&self, errors: &[Error]) -> bool {
|
|
||||||
errors.iter().any(|error| !self.is_known_build_error(error))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn error_object(error: &Error) -> Option<ObjectId> {
|
|
||||||
match error {
|
|
||||||
Error::Validation { object_id, .. }
|
|
||||||
| Error::Build { object_id, .. }
|
|
||||||
| Error::NotFound { object_id } => Some(*object_id),
|
|
||||||
Error::Internal { object_id, .. } => *object_id,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// inbuxa: upstream applied a registry write to the running settings only on
|
|
||||||
// an explicit x:Action ReloadSettings (Directory and Authentication aside), so
|
|
||||||
// a new MtaDeliverySchedule, say, stayed unknown ("Queue strategy not found")
|
|
||||||
// until someone reloaded. Writes to objects the settings are built from now
|
|
||||||
// reload them, here and across the cluster, as ReloadSettings does.
|
|
||||||
|
|
||||||
/// Coalesces the full reloads that registry writes trigger. A write waits
|
|
||||||
/// for more writes before a reload starts (see [`WRITE_QUIET`]), then
|
|
||||||
/// takes the result of the first reload that started after it was stored,
|
|
||||||
/// so a burst of writes, or a request with many objects, costs one reload
|
|
||||||
/// or two rather than one each.
|
|
||||||
pub struct SettingsReloadGate {
|
|
||||||
requested: std::sync::atomic::AtomicU64,
|
|
||||||
reloads: std::sync::atomic::AtomicU64,
|
|
||||||
state: parking_lot::Mutex<SettingsReloadState>,
|
|
||||||
completed: tokio::sync::watch::Sender<u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
struct SettingsReloadState {
|
|
||||||
/// A reload is waiting for writes to settle, or running.
|
|
||||||
scheduled: bool,
|
|
||||||
/// When the oldest write not yet covered by a reload was stored, and
|
|
||||||
/// the newest.
|
|
||||||
first_write: Option<std::time::Instant>,
|
|
||||||
last_write: Option<std::time::Instant>,
|
|
||||||
/// Recent reloads, oldest first: the last write each covered, and why
|
|
||||||
/// it was refused, if it was.
|
|
||||||
results: std::collections::VecDeque<(u64, Option<String>)>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for SettingsReloadGate {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
requested: Default::default(),
|
|
||||||
reloads: Default::default(),
|
|
||||||
state: Default::default(),
|
|
||||||
completed: tokio::sync::watch::Sender::new(0),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SettingsReloadGate {
|
|
||||||
/// How many full reloads registry writes have run.
|
|
||||||
pub fn reloads(&self) -> u64 {
|
|
||||||
self.reloads.load(std::sync::atomic::Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SettingsReloadState {
|
|
||||||
/// The result of the reload that covered write `ticket`, once it ran.
|
|
||||||
fn result_for(&self, ticket: u64) -> Option<Result<(), String>> {
|
|
||||||
self.results
|
|
||||||
.iter()
|
|
||||||
.find(|(covers, _)| *covers >= ticket)
|
|
||||||
.map(|(_, refused)| refused.clone().map_or(Ok(()), Err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// How long a full reload waits after the last registry write for another.
|
|
||||||
/// Parallel requests reach the server tens of milliseconds apart (in a
|
|
||||||
/// cluster rehearsal, ten x:<Object>/set requests sent at once arrived about
|
|
||||||
/// 33 ms apart and each got a reload of its own), so the window is a little
|
|
||||||
/// over twice that. A single write pays it once, on top of the reload.
|
|
||||||
pub const WRITE_QUIET: std::time::Duration = std::time::Duration::from_millis(75);
|
|
||||||
|
|
||||||
/// The longest a full reload waits after the first write it covers, so a
|
|
||||||
/// steady stream of writes still reloads at least this often.
|
|
||||||
pub const WRITE_MAX_WAIT: std::time::Duration = std::time::Duration::from_millis(250);
|
|
||||||
|
|
||||||
/// How many past reload results a waiting write can look up.
|
|
||||||
const RELOAD_RESULTS: usize = 64;
|
|
||||||
|
|
||||||
/// The reload a write to `object` calls for: the object to reload, or None
|
|
||||||
/// when the running settings don't hold that object (accounts, domains and
|
|
||||||
/// other data read as needed, stores, which take a restart, and objects with
|
|
||||||
/// reload actions of their own, such as applications). Blocked IPs have a
|
|
||||||
/// reload of their own; allowed IPs take the full one.
|
|
||||||
pub fn write_reload_target(object: ObjectType) -> Option<ObjectType> {
|
|
||||||
match object {
|
|
||||||
ObjectType::Certificate => Some(ObjectType::Certificate),
|
|
||||||
ObjectType::MemoryLookupKey
|
|
||||||
| ObjectType::MemoryLookupKeyValue
|
|
||||||
| ObjectType::HttpLookup
|
|
||||||
| ObjectType::StoreLookup => Some(ObjectType::StoreLookup),
|
|
||||||
ObjectType::BlockedIp => Some(ObjectType::BlockedIp),
|
|
||||||
// Allowed IPs are part of the core's security settings
|
|
||||||
// (Security::parse), which only a full reload rebuilds; the blocked-IP
|
|
||||||
// reload doesn't touch them
|
|
||||||
ObjectType::AllowedIp
|
|
||||||
| ObjectType::AcmeProvider
|
|
||||||
| ObjectType::AddressBook
|
|
||||||
| ObjectType::AiModel
|
|
||||||
| ObjectType::Asn
|
|
||||||
| ObjectType::Authentication
|
|
||||||
| ObjectType::Cache
|
|
||||||
| ObjectType::Calendar
|
|
||||||
| ObjectType::CalendarAlarm
|
|
||||||
| ObjectType::CalendarScheduling
|
|
||||||
| ObjectType::ClusterRole
|
|
||||||
| ObjectType::DataRetention
|
|
||||||
| ObjectType::Directory
|
|
||||||
| ObjectType::DkimReportSettings
|
|
||||||
| ObjectType::DmarcReportSettings
|
|
||||||
| ObjectType::DnsResolver
|
|
||||||
| ObjectType::DsnReportSettings
|
|
||||||
| ObjectType::Email
|
|
||||||
| ObjectType::EventTracingLevel
|
|
||||||
| ObjectType::FileStorage
|
|
||||||
| ObjectType::Http
|
|
||||||
| ObjectType::HttpForm
|
|
||||||
| ObjectType::Imap
|
|
||||||
| ObjectType::Jmap
|
|
||||||
| ObjectType::Metrics
|
|
||||||
| ObjectType::MtaConnectionStrategy
|
|
||||||
| ObjectType::MtaDeliverySchedule
|
|
||||||
| ObjectType::MtaExtensions
|
|
||||||
| ObjectType::MtaHook
|
|
||||||
| ObjectType::MtaInboundSession
|
|
||||||
| ObjectType::MtaInboundThrottle
|
|
||||||
| ObjectType::MtaMilter
|
|
||||||
| ObjectType::MtaOutboundStrategy
|
|
||||||
| ObjectType::MtaOutboundThrottle
|
|
||||||
| ObjectType::MtaQueueQuota
|
|
||||||
| ObjectType::MtaRoute
|
|
||||||
| ObjectType::MtaStageAuth
|
|
||||||
| ObjectType::MtaStageConnect
|
|
||||||
| ObjectType::MtaStageData
|
|
||||||
| ObjectType::MtaStageEhlo
|
|
||||||
| ObjectType::MtaStageMail
|
|
||||||
| ObjectType::MtaStageRcpt
|
|
||||||
| ObjectType::MtaSts
|
|
||||||
| ObjectType::MtaTlsStrategy
|
|
||||||
| ObjectType::MtaVirtualQueue
|
|
||||||
| ObjectType::NetworkListener
|
|
||||||
| ObjectType::OidcProvider
|
|
||||||
| ObjectType::ReportSettings
|
|
||||||
| ObjectType::Search
|
|
||||||
| ObjectType::Security
|
|
||||||
| ObjectType::SenderAuth
|
|
||||||
| ObjectType::Sharing
|
|
||||||
| ObjectType::SieveSystemInterpreter
|
|
||||||
| ObjectType::SieveSystemScript
|
|
||||||
| ObjectType::SieveUserInterpreter
|
|
||||||
| ObjectType::SieveUserScript
|
|
||||||
| ObjectType::SpamClassifier
|
|
||||||
| ObjectType::SpamDnsblServer
|
|
||||||
| ObjectType::SpamDnsblSettings
|
|
||||||
| ObjectType::SpamFileExtension
|
|
||||||
| ObjectType::SpamPyzor
|
|
||||||
| ObjectType::SpamRule
|
|
||||||
| ObjectType::SpamSettings
|
|
||||||
| ObjectType::SpamTag
|
|
||||||
| ObjectType::SpfReportSettings
|
|
||||||
| ObjectType::SystemSettings
|
|
||||||
| ObjectType::TaskManager
|
|
||||||
| ObjectType::TlsReportSettings
|
|
||||||
| ObjectType::Tracer
|
|
||||||
| ObjectType::WebDav
|
|
||||||
| ObjectType::WebHook => Some(object),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Server {
|
|
||||||
/// Applies a stored registry write to `object` to the running settings,
|
|
||||||
/// and on success tells the other nodes to do the same. Returns None when
|
|
||||||
/// the write needs no reload, Some(Ok(())) when it was applied, and
|
|
||||||
/// Some(Err(reason)) when the reload was refused (the write stays stored;
|
|
||||||
/// ReloadSettings reports the same errors).
|
|
||||||
pub async fn reload_after_write(&self, object: ObjectType) -> Option<Result<(), String>> {
|
|
||||||
let target = write_reload_target(object)?;
|
|
||||||
let change = RegistryChange::Reload(target);
|
|
||||||
|
|
||||||
if matches!(
|
|
||||||
target,
|
|
||||||
ObjectType::Certificate | ObjectType::StoreLookup | ObjectType::BlockedIp
|
|
||||||
) {
|
|
||||||
// Cheap, and limited to their own objects
|
|
||||||
let result = self.reload_and_broadcast(change).await;
|
|
||||||
return Some(result);
|
|
||||||
}
|
|
||||||
|
|
||||||
// inbuxa: #39 joined only writes that queued behind a running
|
|
||||||
// reload; requests that arrive tens of milliseconds apart never
|
|
||||||
// overlapped one, so each got a reload of its own. The reload now
|
|
||||||
// waits until writes settle (WRITE_QUIET after the last one, at
|
|
||||||
// most WRITE_MAX_WAIT after the first) and covers them all. It runs
|
|
||||||
// in a task of its own, so a request that goes away doesn't take
|
|
||||||
// it with it; each write then takes the result of the reload that
|
|
||||||
// started after it was stored.
|
|
||||||
let gate = &self.inner.data.settings_reload;
|
|
||||||
let ticket = gate
|
|
||||||
.requested
|
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst)
|
|
||||||
+ 1;
|
|
||||||
let now = std::time::Instant::now();
|
|
||||||
{
|
|
||||||
let mut state = gate.state.lock();
|
|
||||||
state.first_write.get_or_insert(now);
|
|
||||||
state.last_write = Some(now);
|
|
||||||
}
|
|
||||||
|
|
||||||
loop {
|
|
||||||
let mut completed = {
|
|
||||||
let mut state = gate.state.lock();
|
|
||||||
if let Some(result) = state.result_for(ticket) {
|
|
||||||
return Some(result);
|
|
||||||
}
|
|
||||||
if !state.scheduled {
|
|
||||||
state.scheduled = true;
|
|
||||||
let server = self.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
server.run_write_reload(change).await;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
gate.completed.subscribe()
|
|
||||||
};
|
|
||||||
if completed.changed().await.is_err() {
|
|
||||||
return Some(Err("The settings reload was interrupted".to_string()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for registry writes to settle, then reloads the settings once
|
|
||||||
/// for all the writes stored so far.
|
|
||||||
async fn run_write_reload(&self, change: RegistryChange) {
|
|
||||||
let gate = &self.inner.data.settings_reload;
|
|
||||||
loop {
|
|
||||||
let deadline = {
|
|
||||||
let state = gate.state.lock();
|
|
||||||
let now = std::time::Instant::now();
|
|
||||||
let first = state.first_write.unwrap_or(now);
|
|
||||||
let last = state.last_write.unwrap_or(now);
|
|
||||||
(last + WRITE_QUIET).min(first + WRITE_MAX_WAIT)
|
|
||||||
};
|
|
||||||
if deadline <= std::time::Instant::now() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
tokio::time::sleep_until(deadline.into()).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Writes stored from here on wait for the next reload
|
|
||||||
let covers = {
|
|
||||||
let mut state = gate.state.lock();
|
|
||||||
state.first_write = None;
|
|
||||||
state.last_write = None;
|
|
||||||
gate.requested.load(std::sync::atomic::Ordering::SeqCst)
|
|
||||||
};
|
|
||||||
gate.reloads
|
|
||||||
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
|
||||||
let result = self.inner.build_server().reload_and_broadcast(change).await;
|
|
||||||
|
|
||||||
{
|
|
||||||
let mut state = gate.state.lock();
|
|
||||||
if state.results.len() == RELOAD_RESULTS {
|
|
||||||
state.results.pop_front();
|
|
||||||
}
|
|
||||||
state.results.push_back((covers, result.err()));
|
|
||||||
state.scheduled = false;
|
|
||||||
}
|
|
||||||
gate.completed.send_replace(covers);
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reload_and_broadcast(&self, change: RegistryChange) -> Result<(), String> {
|
|
||||||
match Box::pin(self.reload_registry(change)).await {
|
|
||||||
Ok(reload) if !reload.has_errors() => {
|
|
||||||
reload.log();
|
|
||||||
self.cluster_broadcast(BroadcastEvent::RegistryChange(change))
|
|
||||||
.await;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
Ok(reload) => {
|
|
||||||
reload.log();
|
|
||||||
let reason = describe_reload_errors(&reload.errors);
|
|
||||||
trc::event!(
|
|
||||||
Registry(trc::RegistryEvent::BuildWarning),
|
|
||||||
Details = "Settings didn't reload after a registry write",
|
|
||||||
Reason = reason.clone(),
|
|
||||||
);
|
|
||||||
Err(reason)
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
let reason = err.to_string();
|
|
||||||
trc::error!(err.details("Failed to reload settings after a registry write"));
|
|
||||||
Err(reason)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: a refused reload's errors in a sentence: the first one, naming its
|
|
||||||
/// object, and how many more there are.
|
|
||||||
pub fn describe_reload_errors(errors: &[Error]) -> String {
|
|
||||||
let mut description = match errors.first() {
|
|
||||||
Some(Error::Build { object_id, message }) => format!("{object_id}: {message}"),
|
|
||||||
Some(Error::Validation { object_id, errors }) => format!(
|
|
||||||
"{object_id}: {}",
|
|
||||||
errors
|
|
||||||
.iter()
|
|
||||||
.map(|err| err.to_string())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("; ")
|
|
||||||
),
|
|
||||||
Some(Error::Internal {
|
|
||||||
object_id: Some(object_id),
|
|
||||||
error,
|
|
||||||
}) => format!("{object_id}: {error}"),
|
|
||||||
Some(Error::Internal { error, .. }) => error.to_string(),
|
|
||||||
Some(Error::NotFound { object_id }) => format!("{object_id} was not found"),
|
|
||||||
None => String::new(),
|
|
||||||
};
|
|
||||||
let more = errors.len().saturating_sub(1);
|
|
||||||
if more > 0 {
|
|
||||||
description.push_str(&format!(" ({more} more in the server log.)"));
|
|
||||||
}
|
|
||||||
description
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -93,12 +93,9 @@ impl Data {
|
|||||||
registry_id_gen: id_generator.clone(),
|
registry_id_gen: id_generator.clone(),
|
||||||
span_id_gen: id_generator,
|
span_id_gen: id_generator,
|
||||||
queue_status: true.into(),
|
queue_status: true.into(),
|
||||||
settings_reload: Default::default(),
|
|
||||||
store_health: Default::default(),
|
|
||||||
applications,
|
applications,
|
||||||
logos: Default::default(),
|
logos: Default::default(),
|
||||||
smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"),
|
smtp_connectors: TlsConnectors::try_new().failed("Failed to build TLS connectors"),
|
||||||
build_errors: Default::default(),
|
|
||||||
asn_geo_data: Default::default(),
|
asn_geo_data: Default::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -237,12 +234,9 @@ impl Default for Data {
|
|||||||
span_id_gen: Default::default(),
|
span_id_gen: Default::default(),
|
||||||
registry_id_gen: Default::default(),
|
registry_id_gen: Default::default(),
|
||||||
queue_status: true.into(),
|
queue_status: true.into(),
|
||||||
settings_reload: Default::default(),
|
|
||||||
store_health: Default::default(),
|
|
||||||
applications: WebApplications::new(),
|
applications: WebApplications::new(),
|
||||||
logos: Default::default(),
|
logos: Default::default(),
|
||||||
smtp_connectors: TlsConnectors::try_new().unwrap(),
|
smtp_connectors: TlsConnectors::try_new().unwrap(),
|
||||||
build_errors: Default::default(),
|
|
||||||
asn_geo_data: Default::default(),
|
asn_geo_data: Default::default(),
|
||||||
lookup_stores: Default::default(),
|
lookup_stores: Default::default(),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use mail_auth::common::resolver::ToReverseName;
|
|||||||
use nlp::classifier::model::{CcfhClassifier, FhClassifier};
|
use nlp::classifier::model::{CcfhClassifier, FhClassifier};
|
||||||
use registry::schema::{
|
use registry::schema::{
|
||||||
enums::{ExpressionVariable, ModelSize},
|
enums::{ExpressionVariable, ModelSize},
|
||||||
|
prelude::ObjectType,
|
||||||
structs::{
|
structs::{
|
||||||
self, SpamDnsblServer, SpamDnsblSettings, SpamFileExtension, SpamPyzor, SpamRule,
|
self, SpamDnsblServer, SpamDnsblSettings, SpamFileExtension, SpamPyzor, SpamRule,
|
||||||
SpamSettings, SpamTag,
|
SpamSettings, SpamTag,
|
||||||
@@ -24,10 +25,10 @@ use registry::schema::{
|
|||||||
use sieve::SpamStatus;
|
use sieve::SpamStatus;
|
||||||
use std::{
|
use std::{
|
||||||
net::{IpAddr, SocketAddr},
|
net::{IpAddr, SocketAddr},
|
||||||
sync::Arc,
|
time::Duration,
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
};
|
||||||
use store::registry::{RegistryObject, bootstrap::Bootstrap};
|
use store::registry::{RegistryObject, bootstrap::Bootstrap};
|
||||||
|
use tokio::net::lookup_host;
|
||||||
use utils::{cache::CacheItemWeight, glob::GlobMap};
|
use utils::{cache::CacheItemWeight, glob::GlobMap};
|
||||||
|
|
||||||
#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
|
#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)]
|
||||||
@@ -156,11 +157,7 @@ pub struct FtrlParameters {
|
|||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct PyzorConfig {
|
pub struct PyzorConfig {
|
||||||
// inbuxa: the server is resolved when a message is checked, not while the
|
pub address: SocketAddr,
|
||||||
// settings are built (see PyzorConfig::address)
|
|
||||||
pub host: String,
|
|
||||||
pub port: u16,
|
|
||||||
pub resolved: Arc<parking_lot::Mutex<Option<(SocketAddr, Instant)>>>,
|
|
||||||
pub timeout: Duration,
|
pub timeout: Duration,
|
||||||
pub min_count: u64,
|
pub min_count: u64,
|
||||||
pub min_wl_count: u64,
|
pub min_wl_count: u64,
|
||||||
@@ -477,15 +474,31 @@ impl PyzorConfig {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: upstream resolved the host here and reported a failed lookup
|
let port = pyzor.port;
|
||||||
// as a build error, so a DNS hiccup on one node refused every settings
|
let host = pyzor.host;
|
||||||
// reload on it (and, from the node that ran ReloadSettings, across the
|
let address = match lookup_host(format!("{host}:{port}"))
|
||||||
// cluster). The lookup now happens when a message is checked; a
|
.await
|
||||||
// failure there is logged as a Pyzor error for that message.
|
.map(|mut a| a.next())
|
||||||
|
{
|
||||||
|
Ok(Some(address)) => address,
|
||||||
|
Ok(None) => {
|
||||||
|
bp.build_error(
|
||||||
|
ObjectType::SpamPyzor.singleton(),
|
||||||
|
"Invalid address: No addresses found.",
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
bp.build_error(
|
||||||
|
ObjectType::SpamPyzor.singleton(),
|
||||||
|
format!("Invalid address: {}", err),
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
PyzorConfig {
|
PyzorConfig {
|
||||||
host: pyzor.host,
|
address,
|
||||||
port: pyzor.port as u16,
|
|
||||||
resolved: Default::default(),
|
|
||||||
timeout: pyzor.timeout.into_inner(),
|
timeout: pyzor.timeout.into_inner(),
|
||||||
min_count: pyzor.block_count,
|
min_count: pyzor.block_count,
|
||||||
min_wl_count: pyzor.allow_count,
|
min_wl_count: pyzor.allow_count,
|
||||||
@@ -495,35 +508,6 @@ impl PyzorConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: how long a resolved Pyzor address is reused
|
|
||||||
const PYZOR_RESOLVE_TTL: Duration = Duration::from_secs(300);
|
|
||||||
|
|
||||||
impl PyzorConfig {
|
|
||||||
/// The server's address: the host itself when it is an IP address,
|
|
||||||
/// otherwise the first address it resolves to, reused for five minutes.
|
|
||||||
pub async fn address(&self) -> std::io::Result<SocketAddr> {
|
|
||||||
if let Ok(ip) = self.host.parse::<IpAddr>() {
|
|
||||||
return Ok(SocketAddr::new(ip, self.port));
|
|
||||||
}
|
|
||||||
if let Some((address, resolved_at)) = *self.resolved.lock()
|
|
||||||
&& resolved_at.elapsed() < PYZOR_RESOLVE_TTL
|
|
||||||
{
|
|
||||||
return Ok(address);
|
|
||||||
}
|
|
||||||
let address = tokio::net::lookup_host((self.host.as_str(), self.port))
|
|
||||||
.await?
|
|
||||||
.next()
|
|
||||||
.ok_or_else(|| {
|
|
||||||
std::io::Error::new(
|
|
||||||
std::io::ErrorKind::NotFound,
|
|
||||||
format!("{} has no addresses", self.host),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
*self.resolved.lock() = Some((address, Instant::now()));
|
|
||||||
Ok(address)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ClassifierConfig {
|
impl ClassifierConfig {
|
||||||
pub async fn parse(bp: &mut Bootstrap) -> Option<Self> {
|
pub async fn parse(bp: &mut Bootstrap) -> Option<Self> {
|
||||||
let classifier = bp.setting_infallible::<structs::SpamClassifier>().await;
|
let classifier = bp.setting_infallible::<structs::SpamClassifier>().await;
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use self::resolver::Policy;
|
use self::resolver::Policy;
|
||||||
@@ -24,7 +22,7 @@ use registry::schema::{
|
|||||||
};
|
};
|
||||||
use smtp_proto::*;
|
use smtp_proto::*;
|
||||||
use std::{
|
use std::{
|
||||||
net::{IpAddr, SocketAddr},
|
net::{SocketAddr, ToSocketAddrs},
|
||||||
str::FromStr,
|
str::FromStr,
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
@@ -386,16 +384,19 @@ impl SessionConfig {
|
|||||||
Some(Milter {
|
Some(Milter {
|
||||||
enable: bp.compile_expr(id, &milter.ctx_enable()),
|
enable: bp.compile_expr(id, &milter.ctx_enable()),
|
||||||
id,
|
id,
|
||||||
// inbuxa: upstream resolved the hostname here (a
|
addrs: format!("{}:{}", milter.hostname, milter.port)
|
||||||
// blocking lookup) and made a failure a build error,
|
.to_socket_addrs()
|
||||||
// which refused the whole settings reload. An IP
|
.map_err(|err| {
|
||||||
// address is kept as is; a name is resolved on each
|
bp.build_error(
|
||||||
// connection (MilterClient::connect).
|
id,
|
||||||
addrs: milter
|
format!(
|
||||||
.hostname
|
"Unable to resolve milter hostname {}: {}",
|
||||||
.parse::<IpAddr>()
|
milter.hostname, err
|
||||||
.map(|ip| vec![SocketAddr::new(ip, milter.port as u16)])
|
),
|
||||||
.unwrap_or_default(),
|
)
|
||||||
|
})
|
||||||
|
.ok()?
|
||||||
|
.collect(),
|
||||||
hostname: milter.hostname,
|
hostname: milter.hostname,
|
||||||
port: milter.port as u16,
|
port: milter.port as u16,
|
||||||
timeout_connect: milter.timeout_connect.into_inner(),
|
timeout_connect: milter.timeout_connect.into_inner(),
|
||||||
|
|||||||
@@ -31,10 +31,6 @@ pub struct TelemetrySubscriber {
|
|||||||
pub interests: Interests,
|
pub interests: Interests,
|
||||||
pub typ: TelemetrySubscriberType,
|
pub typ: TelemetrySubscriberType,
|
||||||
pub lossy: bool,
|
pub lossy: bool,
|
||||||
/// inbuxa: a hash of the settings the running tracer is built from
|
|
||||||
/// (everything but its events, level and lossiness, which change in
|
|
||||||
/// place), so a reload can tell which tracers to start over.
|
|
||||||
pub settings: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::large_enum_variant)]
|
#[allow(clippy::large_enum_variant)]
|
||||||
@@ -171,7 +167,6 @@ impl Tracers {
|
|||||||
for tracer in bp.list_infallible::<Tracer>().await {
|
for tracer in bp.list_infallible::<Tracer>().await {
|
||||||
let id = tracer.id;
|
let id = tracer.id;
|
||||||
let tracer = tracer.object;
|
let tracer = tracer.object;
|
||||||
let settings = tracer_settings(&tracer);
|
|
||||||
let level;
|
let level;
|
||||||
let lossy;
|
let lossy;
|
||||||
let events;
|
let events;
|
||||||
@@ -384,7 +379,6 @@ impl Tracers {
|
|||||||
interests: Default::default(),
|
interests: Default::default(),
|
||||||
lossy,
|
lossy,
|
||||||
typ,
|
typ,
|
||||||
settings,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Parse disabled events
|
// Parse disabled events
|
||||||
@@ -432,7 +426,6 @@ impl Tracers {
|
|||||||
for hook in bp.list_infallible::<WebHook>().await {
|
for hook in bp.list_infallible::<WebHook>().await {
|
||||||
let id = hook.id;
|
let id = hook.id;
|
||||||
let hook = hook.object;
|
let hook = hook.object;
|
||||||
let settings = webhook_settings(&hook);
|
|
||||||
|
|
||||||
if !hook.enable {
|
if !hook.enable {
|
||||||
continue;
|
continue;
|
||||||
@@ -455,7 +448,6 @@ impl Tracers {
|
|||||||
id: format!("w_{}", id.id()),
|
id: format!("w_{}", id.id()),
|
||||||
interests: Default::default(),
|
interests: Default::default(),
|
||||||
lossy: hook.lossy,
|
lossy: hook.lossy,
|
||||||
settings,
|
|
||||||
typ: TelemetrySubscriberType::Webhook(WebhookTracer {
|
typ: TelemetrySubscriberType::Webhook(WebhookTracer {
|
||||||
url: hook.url,
|
url: hook.url,
|
||||||
timeout: hook.timeout.into_inner(),
|
timeout: hook.timeout.into_inner(),
|
||||||
@@ -524,8 +516,6 @@ impl Tracers {
|
|||||||
data: storage.data.clone(),
|
data: storage.data.clone(),
|
||||||
}),
|
}),
|
||||||
lossy: true,
|
lossy: true,
|
||||||
// Stores take a restart
|
|
||||||
settings: 0,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -551,7 +541,6 @@ impl Tracers {
|
|||||||
buffered: true,
|
buffered: true,
|
||||||
}),
|
}),
|
||||||
lossy: false,
|
lossy: false,
|
||||||
settings: 0,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -579,7 +568,6 @@ impl Tracers {
|
|||||||
buffered: true,
|
buffered: true,
|
||||||
}),
|
}),
|
||||||
lossy: false,
|
lossy: false,
|
||||||
settings: 0,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -713,42 +701,6 @@ impl Metrics {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: what a tracer is built from, less what changes in place
|
|
||||||
macro_rules! in_place_reset {
|
|
||||||
($tracer:expr) => {{
|
|
||||||
$tracer.enable = true;
|
|
||||||
$tracer.level = Default::default();
|
|
||||||
$tracer.lossy = false;
|
|
||||||
$tracer.events = Default::default();
|
|
||||||
$tracer.events_policy = Default::default();
|
|
||||||
}};
|
|
||||||
}
|
|
||||||
|
|
||||||
fn settings_hash(settings: &impl std::fmt::Debug) -> u64 {
|
|
||||||
use std::hash::{Hash, Hasher};
|
|
||||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
|
||||||
format!("{settings:?}").hash(&mut hasher);
|
|
||||||
hasher.finish()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tracer_settings(tracer: &Tracer) -> u64 {
|
|
||||||
let mut tracer = tracer.clone();
|
|
||||||
match &mut tracer {
|
|
||||||
Tracer::Log(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::Stdout(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::Journal(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::OtelHttp(tracer) => in_place_reset!(tracer),
|
|
||||||
Tracer::OtelGrpc(tracer) => in_place_reset!(tracer),
|
|
||||||
}
|
|
||||||
settings_hash(&tracer)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn webhook_settings(hook: &WebHook) -> u64 {
|
|
||||||
let mut hook = hook.clone();
|
|
||||||
in_place_reset!(hook);
|
|
||||||
settings_hash(&hook)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn apply_events(
|
fn apply_events(
|
||||||
event_types: impl IntoIterator<Item = EventType>,
|
event_types: impl IntoIterator<Item = EventType>,
|
||||||
policy: EventPolicy,
|
policy: EventPolicy,
|
||||||
|
|||||||
@@ -335,72 +335,3 @@ impl EmailPush {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: the task locks this node holds, so a graceful stop can hand them
|
|
||||||
/// back instead of leaving the tasks blocked until the locks expire.
|
|
||||||
pub struct TaskLocks {
|
|
||||||
held: parking_lot::Mutex<ahash::AHashSet<u64>>,
|
|
||||||
stopping: AtomicBool,
|
|
||||||
expiry: std::sync::atomic::AtomicU64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TaskLocks {
|
|
||||||
/// How long a task lock lasts, in seconds, unless it is released first
|
|
||||||
/// or renewed. inbuxa: upstream held a lock for an hour, so a killed
|
|
||||||
/// node's tasks waited that long; the lock is now a five-minute lease
|
|
||||||
/// that the task manager renews every third of it while the task runs
|
|
||||||
/// (renew_task_locks), so a dead node's tasks run elsewhere within
|
|
||||||
/// minutes.
|
|
||||||
pub const DEFAULT_EXPIRY: u64 = 5 * 60;
|
|
||||||
|
|
||||||
pub fn is_stopping(&self) -> bool {
|
|
||||||
self.stopping.load(Ordering::Acquire)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Stops new claims and returns the ids of every lock still held.
|
|
||||||
pub fn stop(&self) -> Vec<u64> {
|
|
||||||
self.stopping.store(true, Ordering::Release);
|
|
||||||
self.held.lock().drain().collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn insert(&self, id: u64) {
|
|
||||||
self.held.lock().insert(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn remove(&self, id: u64) {
|
|
||||||
self.held.lock().remove(&id);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn held(&self) -> usize {
|
|
||||||
self.held.lock().len()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: the tasks this node holds, to renew their locks.
|
|
||||||
pub fn held_ids(&self) -> Vec<u64> {
|
|
||||||
self.held.lock().iter().copied().collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: whether this node holds (and is running) the task.
|
|
||||||
pub fn is_held(&self, id: u64) -> bool {
|
|
||||||
self.held.lock().contains(&id)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn expiry(&self) -> u64 {
|
|
||||||
self.expiry.load(Ordering::Relaxed)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Changes the lock lifetime; the tests shorten it.
|
|
||||||
pub fn set_expiry(&self, seconds: u64) {
|
|
||||||
self.expiry.store(seconds.max(1), Ordering::Relaxed);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for TaskLocks {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
held: Default::default(),
|
|
||||||
stopping: AtomicBool::new(false),
|
|
||||||
expiry: std::sync::atomic::AtomicU64::new(Self::DEFAULT_EXPIRY),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -161,19 +161,11 @@ pub struct Data {
|
|||||||
pub span_id_gen: SnowflakeIdGenerator,
|
pub span_id_gen: SnowflakeIdGenerator,
|
||||||
pub registry_id_gen: SnowflakeIdGenerator,
|
pub registry_id_gen: SnowflakeIdGenerator,
|
||||||
pub queue_status: AtomicBool,
|
pub queue_status: AtomicBool,
|
||||||
// inbuxa: coalesces the settings reloads registry writes trigger
|
|
||||||
pub settings_reload: cache::reload::SettingsReloadGate,
|
|
||||||
// inbuxa: the readiness probe's cached answer
|
|
||||||
pub store_health: storage::ready::StoreHealth,
|
|
||||||
|
|
||||||
pub applications: WebApplications,
|
pub applications: WebApplications,
|
||||||
pub logos: Mutex<AHashMap<Box<str>, LogoCache>>,
|
pub logos: Mutex<AHashMap<Box<str>, LogoCache>>,
|
||||||
|
|
||||||
pub smtp_connectors: TlsConnectors,
|
pub smtp_connectors: TlsConnectors,
|
||||||
|
|
||||||
// inbuxa: the objects that failed to build when the running settings
|
|
||||||
// were built, at boot or by the last applied reload (see reload_registry)
|
|
||||||
pub build_errors: Mutex<AHashSet<registry::types::id::ObjectId>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -287,8 +279,6 @@ pub struct HttpAuthCache {
|
|||||||
pub struct Ipc {
|
pub struct Ipc {
|
||||||
pub push_tx: mpsc::Sender<PushEvent>,
|
pub push_tx: mpsc::Sender<PushEvent>,
|
||||||
pub task_tx: Arc<Notify>,
|
pub task_tx: Arc<Notify>,
|
||||||
// inbuxa: task locks held by this node, released on a graceful stop
|
|
||||||
pub task_locks: Arc<crate::ipc::TaskLocks>,
|
|
||||||
pub queue_tx: mpsc::Sender<QueueEvent>,
|
pub queue_tx: mpsc::Sender<QueueEvent>,
|
||||||
pub report_tx: mpsc::Sender<ReportingEvent>,
|
pub report_tx: mpsc::Sender<ReportingEvent>,
|
||||||
pub broadcast_tx: Option<mpsc::Sender<BroadcastEvent>>,
|
pub broadcast_tx: Option<mpsc::Sender<BroadcastEvent>>,
|
||||||
|
|||||||
@@ -23,13 +23,6 @@ use utils::{UnwrapFailure, codec::leb128::Leb128_};
|
|||||||
|
|
||||||
pub(super) const MAGIC_MARKER: u8 = 123;
|
pub(super) const MAGIC_MARKER: u8 = 123;
|
||||||
|
|
||||||
// inbuxa: blobs kept under a fixed name instead of a content hash. Nothing
|
|
||||||
// links to them, so the export names them outright.
|
|
||||||
const NAMED_BLOBS: &[&[u8]] = &[
|
|
||||||
crate::manager::SPAM_CLASSIFIER_KEY,
|
|
||||||
crate::manager::SPAM_TRAINER_KEY,
|
|
||||||
];
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
|
||||||
pub(super) enum Family {
|
pub(super) enum Family {
|
||||||
Data = 0,
|
Data = 0,
|
||||||
@@ -150,21 +143,15 @@ impl Core {
|
|||||||
.await
|
.await
|
||||||
.failed("Failed to iterate over data store");
|
.failed("Failed to iterate over data store");
|
||||||
|
|
||||||
// inbuxa: the trained spam classifier and its trainer state are
|
for hash in blobs {
|
||||||
// blobs stored under fixed names with no blob link, so the walk
|
|
||||||
// over links above never reaches them.
|
|
||||||
let named = NAMED_BLOBS.iter().map(|key| key.to_vec());
|
|
||||||
for key in blobs
|
|
||||||
.into_iter()
|
|
||||||
.map(|hash| hash.as_slice().to_vec())
|
|
||||||
.chain(named)
|
|
||||||
{
|
|
||||||
if let Some(blob) = blob_store
|
if let Some(blob) = blob_store
|
||||||
.get_blob(&key, 0..usize::MAX)
|
.get_blob(hash.as_slice(), 0..usize::MAX)
|
||||||
.await
|
.await
|
||||||
.failed("Failed to get blob")
|
.failed("Failed to get blob")
|
||||||
{
|
{
|
||||||
writer.send((key, blob)).failed("Failed to send key");
|
writer
|
||||||
|
.send((hash.as_slice().to_vec(), blob))
|
||||||
|
.failed("Failed to send key");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
@@ -336,13 +323,7 @@ impl Family {
|
|||||||
SUBSPACE_REGISTRY_IDX,
|
SUBSPACE_REGISTRY_IDX,
|
||||||
SUBSPACE_REGISTRY_PK,
|
SUBSPACE_REGISTRY_PK,
|
||||||
SUBSPACE_DIRECTORY,
|
SUBSPACE_DIRECTORY,
|
||||||
// inbuxa: registry objects the upstream list left out, so an
|
store::SUBSPACE_INBUXA, // inbuxa: masked email
|
||||||
// export dropped them: archived items (undelete) and spam
|
|
||||||
// training samples. Their indexes and id counters already
|
|
||||||
// travel in this family and in `data`, so they ride along.
|
|
||||||
SUBSPACE_DELETED_ITEMS,
|
|
||||||
SUBSPACE_SPAM_SAMPLES,
|
|
||||||
store::SUBSPACE_INBUXA, // inbuxa: the fork's own data (masked email, undelete, policies)
|
|
||||||
],
|
],
|
||||||
Family::Changelog => &[SUBSPACE_LOGS],
|
Family::Changelog => &[SUBSPACE_LOGS],
|
||||||
Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT],
|
Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT],
|
||||||
|
|||||||
@@ -54,13 +54,6 @@ Options:
|
|||||||
-o, --console Open the store console
|
-o, --console Open the store console
|
||||||
-h, --help Print help
|
-h, --help Print help
|
||||||
-V, --version Print version
|
-V, --version Print version
|
||||||
|
|
||||||
An export holds everything in the data and blob stores except short-lived
|
|
||||||
in-memory state (rate limits, locks, greylisting) and the full-text search
|
|
||||||
index, which belongs to one search backend. An import into an empty store
|
|
||||||
queues the index to be rebuilt when the server next starts. EXPORT_TYPES
|
|
||||||
limits an export to some of: data, registry, blob, changelog, queue, report,
|
|
||||||
telemetry, tasks.
|
|
||||||
"#
|
"#
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -240,9 +233,6 @@ impl BootManager {
|
|||||||
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
|
.parse_tcp_acceptors(&mut bootstrap, inner.clone())
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
// inbuxa: a reload isn't refused over objects that failed here
|
|
||||||
inner.build_server().record_build_errors(&bootstrap.errors);
|
|
||||||
|
|
||||||
BootManager {
|
BootManager {
|
||||||
inner,
|
inner,
|
||||||
bootstrap,
|
bootstrap,
|
||||||
@@ -266,10 +256,10 @@ impl BootManager {
|
|||||||
telemetry.enable();
|
telemetry.enable();
|
||||||
|
|
||||||
// Parse settings and restore
|
// Parse settings and restore
|
||||||
let core = Box::pin(Core::parse(&mut bootstrap, storage)).await;
|
Box::pin(Core::parse(&mut bootstrap, storage))
|
||||||
let imported = core.restore(path).await;
|
.await
|
||||||
// inbuxa: the search index isn't exported; rebuild it
|
.restore(path)
|
||||||
core.queue_reindex(&imported).await;
|
.await;
|
||||||
std::process::exit(0);
|
std::process::exit(0);
|
||||||
}
|
}
|
||||||
StoreOp::Console => {
|
StoreOp::Console => {
|
||||||
@@ -300,7 +290,6 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) {
|
|||||||
report_tx,
|
report_tx,
|
||||||
broadcast_tx: has_pubsub.then_some(broadcast_tx),
|
broadcast_tx: has_pubsub.then_some(broadcast_tx),
|
||||||
task_tx: Arc::new(Notify::new()),
|
task_tx: Arc::new(Notify::new()),
|
||||||
task_locks: Arc::new(crate::ipc::TaskLocks::default()),
|
|
||||||
train_task_controller: Arc::new(TrainTaskController::default()),
|
train_task_controller: Arc::new(TrainTaskController::default()),
|
||||||
},
|
},
|
||||||
IpcReceivers {
|
IpcReceivers {
|
||||||
|
|||||||
@@ -9,22 +9,15 @@
|
|||||||
use super::backup::MAGIC_MARKER;
|
use super::backup::MAGIC_MARKER;
|
||||||
use crate::{Core, DATABASE_SCHEMA_VERSION};
|
use crate::{Core, DATABASE_SCHEMA_VERSION};
|
||||||
use lz4_flex::frame::FrameDecoder;
|
use lz4_flex::frame::FrameDecoder;
|
||||||
use registry::{
|
use registry::schema::enums::CompressionAlgo;
|
||||||
schema::{
|
|
||||||
enums::{CompressionAlgo, TaskStoreMaintenanceType},
|
|
||||||
structs::{Task, TaskStatus, TaskStoreMaintenance},
|
|
||||||
},
|
|
||||||
types::EnumImpl,
|
|
||||||
};
|
|
||||||
use std::{
|
use std::{
|
||||||
fs::File,
|
fs::File,
|
||||||
io::{BufReader, ErrorKind, Read},
|
io::{BufReader, ErrorKind, Read},
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
};
|
};
|
||||||
use store::{
|
use store::{
|
||||||
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES,
|
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_QUOTA,
|
||||||
SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store,
|
SUBSPACE_REGISTRY_PK, Store, U32_LEN,
|
||||||
U32_LEN,
|
|
||||||
write::{
|
write::{
|
||||||
AnyClass, AnyKey, BatchBuilder, ValueClass,
|
AnyClass, AnyKey, BatchBuilder, ValueClass,
|
||||||
key::{DeserializeBigEndian, is_node_id_key},
|
key::{DeserializeBigEndian, is_node_id_key},
|
||||||
@@ -34,9 +27,7 @@ use types::{collection::Collection, field::Field};
|
|||||||
use utils::{UnwrapFailure, failed};
|
use utils::{UnwrapFailure, failed};
|
||||||
|
|
||||||
impl Core {
|
impl Core {
|
||||||
/// Imports an export into an empty store and returns the subspaces it
|
pub async fn restore(&self, src: PathBuf) {
|
||||||
/// wrote. inbuxa: the caller hands them to [`Core::queue_reindex`].
|
|
||||||
pub async fn restore(&self, src: PathBuf) -> Vec<u8> {
|
|
||||||
// Backup the core
|
// Backup the core
|
||||||
let paths = if src.is_dir() {
|
let paths = if src.is_dir() {
|
||||||
let mut paths = Vec::new();
|
let mut paths = Vec::new();
|
||||||
@@ -73,13 +64,6 @@ impl Core {
|
|||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut imported = paths
|
|
||||||
.iter()
|
|
||||||
.map(|path| KeyValueReader::new(path).subspace)
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
imported.sort_unstable();
|
|
||||||
imported.dedup();
|
|
||||||
|
|
||||||
let mut tasks = Vec::new();
|
let mut tasks = Vec::new();
|
||||||
for path in paths {
|
for path in paths {
|
||||||
let storage = self.storage.clone();
|
let storage = self.storage.clone();
|
||||||
@@ -92,54 +76,6 @@ impl Core {
|
|||||||
for task in tasks {
|
for task in tasks {
|
||||||
task.await.failed("Failed to wait for task");
|
task.await.failed("Failed to wait for task");
|
||||||
}
|
}
|
||||||
|
|
||||||
imported
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: an export never carries the full-text index. It is built by
|
|
||||||
/// and for one search backend (the SQL stores index into their own
|
|
||||||
/// tables, the key-value stores into a subspace, external engines keep it
|
|
||||||
/// themselves), so it would be wrong or unreadable after a move to
|
|
||||||
/// another one. Instead, an import queues the same reindex tasks an
|
|
||||||
/// administrator can queue by hand (`reindexAccounts` and
|
|
||||||
/// `reindexTelemetry` store maintenance), and the server rebuilds the
|
|
||||||
/// index for whatever search store it is configured with once it starts.
|
|
||||||
pub async fn queue_reindex(&self, imported: &[u8]) -> Vec<TaskStoreMaintenanceType> {
|
|
||||||
let mut queued = Vec::new();
|
|
||||||
if imported.contains(&SUBSPACE_PROPERTY) {
|
|
||||||
queued.push(TaskStoreMaintenanceType::ReindexAccounts);
|
|
||||||
}
|
|
||||||
if imported.contains(&SUBSPACE_TELEMETRY_SPAN) {
|
|
||||||
queued.push(TaskStoreMaintenanceType::ReindexTelemetry);
|
|
||||||
}
|
|
||||||
if queued.is_empty() {
|
|
||||||
return queued;
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
for maintenance_type in &queued {
|
|
||||||
batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance {
|
|
||||||
maintenance_type: *maintenance_type,
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
shard_index: None,
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
self.storage
|
|
||||||
.data
|
|
||||||
.write(batch.build_all())
|
|
||||||
.await
|
|
||||||
.failed("Failed to queue the reindex tasks");
|
|
||||||
|
|
||||||
println!(
|
|
||||||
"Queued {} to rebuild the search index; it runs when the server starts.",
|
|
||||||
queued
|
|
||||||
.iter()
|
|
||||||
.map(|t| t.as_str())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(" and ")
|
|
||||||
);
|
|
||||||
|
|
||||||
queued
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -189,22 +125,17 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) {
|
|||||||
}
|
}
|
||||||
SUBSPACE_COUNTER | SUBSPACE_QUOTA => {
|
SUBSPACE_COUNTER | SUBSPACE_QUOTA => {
|
||||||
while let Some((key, value)) = reader.next() {
|
while let Some((key, value)) = reader.next() {
|
||||||
let class = ValueClass::Any(AnyClass {
|
batch.add(
|
||||||
|
ValueClass::Any(AnyClass {
|
||||||
subspace: reader.subspace,
|
subspace: reader.subspace,
|
||||||
key,
|
key,
|
||||||
});
|
}),
|
||||||
let value = u64::from_le_bytes(
|
u64::from_le_bytes(
|
||||||
value
|
value
|
||||||
.try_into()
|
.try_into()
|
||||||
.expect("Failed to deserialize counter/quota"),
|
.expect("Failed to deserialize counter/quota"),
|
||||||
) as i64;
|
) as i64,
|
||||||
// inbuxa: the SQL stores add a negative amount with an UPDATE,
|
);
|
||||||
// which does nothing to a row that isn't there yet, so a
|
|
||||||
// negative counter vanished on import. Create the row first.
|
|
||||||
if value < 0 {
|
|
||||||
batch.add(class.clone(), 0);
|
|
||||||
}
|
|
||||||
batch.add(class, value);
|
|
||||||
if batch.is_large_batch() {
|
if batch.is_large_batch() {
|
||||||
store
|
store
|
||||||
.write(batch.build_all())
|
.write(batch.build_all())
|
||||||
|
|||||||
@@ -26,7 +26,6 @@ pub mod document;
|
|||||||
pub mod encryption;
|
pub mod encryption;
|
||||||
pub mod index;
|
pub mod index;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
pub mod ready; // inbuxa: readiness follows the data store
|
|
||||||
pub mod state;
|
pub mod state;
|
||||||
pub mod transaction;
|
pub mod transaction;
|
||||||
|
|
||||||
|
|||||||
@@ -1,83 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Readiness that reflects the data store.
|
|
||||||
//!
|
|
||||||
//! /healthz/ready used to answer 200 whenever a data store was configured,
|
|
||||||
//! so a load balancer kept sending traffic to a node through a database
|
|
||||||
//! outage. It now reads one key from the data store, with a short time
|
|
||||||
//! limit, and caches the answer for a couple of seconds so probes can't load
|
|
||||||
//! the database. Liveness stays 200: restarting a node doesn't bring its
|
|
||||||
//! database back, and an orchestrator that restarts on failed liveness would
|
|
||||||
//! otherwise restart every node at once.
|
|
||||||
|
|
||||||
use crate::Server;
|
|
||||||
use parking_lot::Mutex;
|
|
||||||
use std::{
|
|
||||||
sync::atomic::{AtomicBool, Ordering},
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{ValueKey, write::ValueClass};
|
|
||||||
|
|
||||||
/// How long a probe's answer is reused.
|
|
||||||
pub const READY_CACHE: Duration = Duration::from_secs(2);
|
|
||||||
/// How long a probe waits for the data store.
|
|
||||||
pub const READY_PROBE_TIMEOUT: Duration = Duration::from_secs(2);
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
pub struct StoreHealth {
|
|
||||||
last: Mutex<Option<(Instant, bool)>>,
|
|
||||||
probing: AtomicBool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Clears the probing flag even when the request is dropped mid-probe.
|
|
||||||
struct ProbeGuard<'x>(&'x AtomicBool);
|
|
||||||
|
|
||||||
impl Drop for ProbeGuard<'_> {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.0.store(false, Ordering::Release);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Server {
|
|
||||||
/// Whether the data store answers: a cached result younger than
|
|
||||||
/// READY_CACHE, or a fresh read bounded by READY_PROBE_TIMEOUT. While
|
|
||||||
/// one probe is running, other callers get the last answer.
|
|
||||||
pub async fn is_data_store_ready(&self) -> bool {
|
|
||||||
let store = &self.core.storage.data;
|
|
||||||
if store.is_none() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
let health = &self.inner.data.store_health;
|
|
||||||
let last = *health.last.lock();
|
|
||||||
if let Some((at, ready)) = last
|
|
||||||
&& at.elapsed() < READY_CACHE
|
|
||||||
{
|
|
||||||
return ready;
|
|
||||||
}
|
|
||||||
if health.probing.swap(true, Ordering::AcqRel) {
|
|
||||||
return last.is_none_or(|(_, ready)| ready);
|
|
||||||
}
|
|
||||||
let _guard = ProbeGuard(&health.probing);
|
|
||||||
|
|
||||||
let ready = tokio::time::timeout(
|
|
||||||
READY_PROBE_TIMEOUT,
|
|
||||||
store.get_value::<u64>(ValueKey::from(ValueClass::Property(0))),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.is_ok_and(|result| result.is_ok());
|
|
||||||
// Say so once per outage, not on every probe
|
|
||||||
if !ready && last.is_none_or(|(_, ready)| ready) {
|
|
||||||
trc::event!(
|
|
||||||
Store(trc::StoreEvent::UnexpectedError),
|
|
||||||
Details = "Readiness probe: the data store didn't answer",
|
|
||||||
Limit = READY_PROBE_TIMEOUT,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
*health.last.lock() = Some((Instant::now(), ready));
|
|
||||||
ready
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -14,26 +14,15 @@ pub mod webhooks;
|
|||||||
use tracers::log::spawn_log_tracer;
|
use tracers::log::spawn_log_tracer;
|
||||||
use tracers::otel::spawn_otel_tracer;
|
use tracers::otel::spawn_otel_tracer;
|
||||||
use tracers::stdout::spawn_console_tracer;
|
use tracers::stdout::spawn_console_tracer;
|
||||||
use ahash::AHashMap;
|
|
||||||
use parking_lot::Mutex;
|
|
||||||
use trc::{Collector, ipc::subscriber::SubscriberBuilder};
|
use trc::{Collector, ipc::subscriber::SubscriberBuilder};
|
||||||
use webhooks::spawn_webhook_tracer;
|
use webhooks::spawn_webhook_tracer;
|
||||||
|
|
||||||
use crate::config::telemetry::{Telemetry, TelemetrySubscriberType};
|
use crate::config::telemetry::{Telemetry, TelemetrySubscriberType};
|
||||||
|
|
||||||
/// inbuxa: the tracers this server started, by subscriber id, with the
|
|
||||||
/// settings each was built from. Live-tracing streams and other subscribers
|
|
||||||
/// registered elsewhere aren't listed, so a reload leaves them running.
|
|
||||||
static RUNNING_TRACERS: Mutex<Option<AHashMap<String, u64>>> = Mutex::new(None);
|
|
||||||
|
|
||||||
impl Telemetry {
|
impl Telemetry {
|
||||||
pub fn enable(self) {
|
pub fn enable(self) {
|
||||||
let mut running = RUNNING_TRACERS.lock();
|
|
||||||
let running = running.get_or_insert_with(AHashMap::new);
|
|
||||||
|
|
||||||
// Spawn tracers
|
// Spawn tracers
|
||||||
for tracer in self.tracers.subscribers {
|
for tracer in self.tracers.subscribers {
|
||||||
running.insert(tracer.id.clone(), tracer.settings);
|
|
||||||
tracer.typ.spawn(
|
tracer.typ.spawn(
|
||||||
SubscriberBuilder::new(tracer.id)
|
SubscriberBuilder::new(tracer.id)
|
||||||
.with_interests(tracer.interests)
|
.with_interests(tracer.interests)
|
||||||
@@ -48,39 +37,25 @@ impl Telemetry {
|
|||||||
Collector::reload();
|
Collector::reload();
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: upstream only refreshed the events, level and lossiness of a
|
|
||||||
// tracer that was already running, so a Log tracer moved to another
|
|
||||||
// path (or any tracer whose own settings changed) kept going as it was
|
|
||||||
// built until a restart, while the reload reported the change applied.
|
|
||||||
// A tracer whose settings changed is now started over: the new one is
|
|
||||||
// registered under the same id and the collector swaps it in at an
|
|
||||||
// event boundary, so no event is lost or written twice (see
|
|
||||||
// Update::RegisterSubscriber); the old one writes what it has queued
|
|
||||||
// and stops.
|
|
||||||
pub fn update(self) {
|
pub fn update(self) {
|
||||||
let mut running = RUNNING_TRACERS.lock();
|
|
||||||
let running = running.get_or_insert_with(AHashMap::new);
|
|
||||||
|
|
||||||
// Remove tracers that are no longer active
|
// Remove tracers that are no longer active
|
||||||
running.retain(|id, _| {
|
let active_subscribers = Collector::get_subscribers();
|
||||||
let keep = self
|
for subscribed_id in &active_subscribers {
|
||||||
|
if !self
|
||||||
.tracers
|
.tracers
|
||||||
.subscribers
|
.subscribers
|
||||||
.iter()
|
.iter()
|
||||||
.any(|tracer| tracer.id == *id);
|
.any(|tracer| tracer.id == *subscribed_id)
|
||||||
if !keep {
|
{
|
||||||
Collector::remove_subscriber(id.clone());
|
Collector::remove_subscriber(subscribed_id.clone());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
keep
|
|
||||||
});
|
|
||||||
|
|
||||||
// Start new tracers, start over those whose settings changed and
|
// Activate new tracers or update existing ones
|
||||||
// update the rest in place
|
|
||||||
for tracer in self.tracers.subscribers {
|
for tracer in self.tracers.subscribers {
|
||||||
if running.get(&tracer.id) == Some(&tracer.settings) {
|
if active_subscribers.contains(&tracer.id) {
|
||||||
Collector::update_subscriber(tracer.id, tracer.interests, tracer.lossy);
|
Collector::update_subscriber(tracer.id, tracer.interests, tracer.lossy);
|
||||||
} else {
|
} else {
|
||||||
running.insert(tracer.id.clone(), tracer.settings);
|
|
||||||
tracer.typ.spawn(
|
tracer.typ.spawn(
|
||||||
SubscriberBuilder::new(tracer.id)
|
SubscriberBuilder::new(tracer.id)
|
||||||
.with_interests(tracer.interests)
|
.with_interests(tracer.interests)
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::{path::PathBuf, time::SystemTime};
|
use std::{path::PathBuf, time::SystemTime};
|
||||||
@@ -17,27 +15,9 @@ use tokio::{
|
|||||||
};
|
};
|
||||||
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
use trc::{TelemetryEvent, ipc::subscriber::SubscriberBuilder, serializers::text::FmtWriter};
|
||||||
|
|
||||||
// inbuxa: when a Log tracer is started over on the same files (its rotation
|
|
||||||
// or format changed), the new one waits for the old one to write what it
|
|
||||||
// has queued, so their lines don't interleave. Keyed by path and prefix;
|
|
||||||
// each entry is the last tracer's "done" signal, sent when it ends.
|
|
||||||
type LogFileOwners = ahash::AHashMap<(String, String), tokio::sync::oneshot::Receiver<()>>;
|
|
||||||
static LOG_FILE_OWNERS: parking_lot::Mutex<Option<LogFileOwners>> = parking_lot::Mutex::new(None);
|
|
||||||
|
|
||||||
pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) {
|
pub(crate) fn spawn_log_tracer(builder: SubscriberBuilder, settings: LogTracer) {
|
||||||
let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>();
|
|
||||||
let previous = LOG_FILE_OWNERS
|
|
||||||
.lock()
|
|
||||||
.get_or_insert_with(Default::default)
|
|
||||||
.insert((settings.path.clone(), settings.prefix.clone()), done_rx);
|
|
||||||
let (_, mut rx) = builder.register();
|
let (_, mut rx) = builder.register();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
// Dropped when this tracer ends, however it ends
|
|
||||||
let _done = done_tx;
|
|
||||||
if let Some(previous) = previous {
|
|
||||||
let _ = previous.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(writer) = settings.build_writer().await {
|
if let Some(writer) = settings.build_writer().await {
|
||||||
let mut buf = FmtWriter::new(writer)
|
let mut buf = FmtWriter::new(writer)
|
||||||
.with_ansi(settings.ansi)
|
.with_ansi(settings.ansi)
|
||||||
|
|||||||
@@ -47,10 +47,6 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer
|
|||||||
let mut pending_spans = Vec::new();
|
let mut pending_spans = Vec::new();
|
||||||
|
|
||||||
let mut active_spans = AHashMap::new();
|
let mut active_spans = AHashMap::new();
|
||||||
let mut closing = false;
|
|
||||||
let started = std::time::SystemTime::now()
|
|
||||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
|
||||||
.map_or(0, |d| d.as_secs());
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
// Wait for the next event or timeout
|
// Wait for the next event or timeout
|
||||||
@@ -79,26 +75,12 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer
|
|||||||
events.iter().chain(std::iter::once(&event)),
|
events.iter().chain(std::iter::once(&event)),
|
||||||
&instrumentation,
|
&instrumentation,
|
||||||
));
|
));
|
||||||
} else if span.inner.timestamp < started {
|
|
||||||
// inbuxa: a span that was open when this
|
|
||||||
// tracer replaced another one (its settings
|
|
||||||
// changed) is exported with its end event
|
|
||||||
// rather than dropped
|
|
||||||
pending_spans.push(build_span_data(
|
|
||||||
span,
|
|
||||||
&event,
|
|
||||||
std::iter::once(&event),
|
|
||||||
&instrumentation,
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
// inbuxa: the tracer was removed or replaced; export
|
break;
|
||||||
// what is pending now rather than drop it
|
|
||||||
closing = true;
|
|
||||||
next_delivery = Instant::now();
|
|
||||||
}
|
}
|
||||||
Err(_) => (),
|
Err(_) => (),
|
||||||
}
|
}
|
||||||
@@ -149,9 +131,6 @@ pub(crate) fn spawn_otel_tracer(builder: SubscriberBuilder, mut otel: OtelTracer
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if closing {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER);
|
wakeup_time = next_retry.unwrap_or(LONG_1Y_SLUMBER);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer};
|
use crate::{LONG_1Y_SLUMBER, config::telemetry::WebhookTracer};
|
||||||
@@ -27,11 +25,6 @@ use trc::{
|
|||||||
|
|
||||||
pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) {
|
pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: WebhookTracer) {
|
||||||
let (tx, mut rx) = builder.register();
|
let (tx, mut rx) = builder.register();
|
||||||
// inbuxa: failed deliveries come back through a weak sender, so the
|
|
||||||
// channel closes when the collector drops this webhook (removed, or
|
|
||||||
// replaced after a settings change) and the task ends; upstream held a
|
|
||||||
// sender here and the task outlived its subscription
|
|
||||||
let tx = tx.downgrade();
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let settings = Arc::new(settings);
|
let settings = Arc::new(settings);
|
||||||
let mut wakeup_time = LONG_1Y_SLUMBER;
|
let mut wakeup_time = LONG_1Y_SLUMBER;
|
||||||
@@ -65,15 +58,6 @@ pub(crate) fn spawn_webhook_tracer(builder: SubscriberBuilder, settings: Webhook
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
// inbuxa: deliver what is pending rather than drop it
|
|
||||||
if !pending_events.is_empty() {
|
|
||||||
spawn_webhook_handler(
|
|
||||||
settings.clone(),
|
|
||||||
in_flight.clone(),
|
|
||||||
std::mem::take(&mut pending_events),
|
|
||||||
tx.clone(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(_) => (),
|
Err(_) => (),
|
||||||
@@ -118,7 +102,7 @@ fn spawn_webhook_handler(
|
|||||||
settings: Arc<WebhookTracer>,
|
settings: Arc<WebhookTracer>,
|
||||||
in_flight: Arc<AtomicBool>,
|
in_flight: Arc<AtomicBool>,
|
||||||
events: EventBatch,
|
events: EventBatch,
|
||||||
webhook_tx: mpsc::WeakSender<EventBatch>,
|
webhook_tx: mpsc::Sender<EventBatch>,
|
||||||
) {
|
) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
in_flight.store(true, Ordering::Relaxed);
|
in_flight.store(true, Ordering::Relaxed);
|
||||||
@@ -129,11 +113,7 @@ fn spawn_webhook_handler(
|
|||||||
if let Err(err) = post_webhook_events(&settings, &wrapper).await {
|
if let Err(err) = post_webhook_events(&settings, &wrapper).await {
|
||||||
trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err);
|
trc::event!(Telemetry(TelemetryEvent::WebhookError), Details = err);
|
||||||
|
|
||||||
let sent = match webhook_tx.upgrade() {
|
if webhook_tx.send(wrapper.events.into_inner()).await.is_err() {
|
||||||
Some(webhook_tx) => webhook_tx.send(wrapper.events.into_inner()).await.is_ok(),
|
|
||||||
None => false,
|
|
||||||
};
|
|
||||||
if !sent {
|
|
||||||
trc::event!(
|
trc::event!(
|
||||||
Server(ServerEvent::ThreadError),
|
Server(ServerEvent::ThreadError),
|
||||||
Details = "Failed to send failed webhook events back to main thread",
|
Details = "Failed to send failed webhook events back to main thread",
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ store = { path = "../store" }
|
|||||||
registry = { path = "../registry" }
|
registry = { path = "../registry" }
|
||||||
trc = { path = "../trc" }
|
trc = { path = "../trc" }
|
||||||
futures = { version = "0.3", optional = true }
|
futures = { version = "0.3", optional = true }
|
||||||
tokio = { version = "1.53", features = ["sync", "fs", "io-util", "rt", "time"] }
|
tokio = { version = "1.53", features = ["sync", "fs", "io-util"] }
|
||||||
async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true }
|
async-nats = { version = "0.50", default-features = false, features = ["server_2_10", "server_2_11", "aws-lc-rs"], optional = true }
|
||||||
zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
|
zenoh = { version = "1.10.0", default-features = false, features = ["auth_pubkey", "transport_multilink", "transport_compression", "transport_quic", "transport_tcp", "transport_tls", "transport_udp"], optional = true }
|
||||||
rdkafka = { version = "0.39", features = ["cmake-build"], optional = true }
|
rdkafka = { version = "0.39", features = ["cmake-build"], optional = true }
|
||||||
|
|||||||
@@ -2,22 +2,13 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::{
|
use std::sync::Arc;
|
||||||
sync::{
|
|
||||||
Arc,
|
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
},
|
|
||||||
time::Duration,
|
|
||||||
};
|
|
||||||
|
|
||||||
use crate::Coordinator;
|
use crate::Coordinator;
|
||||||
use async_nats::Client;
|
use async_nats::Client;
|
||||||
use registry::schema::structs::NatsCoordinator;
|
use registry::schema::structs::NatsCoordinator;
|
||||||
use trc::ClusterEvent;
|
|
||||||
|
|
||||||
pub mod pubsub;
|
pub mod pubsub;
|
||||||
|
|
||||||
@@ -56,116 +47,9 @@ impl NatsPubSub {
|
|||||||
opts = opts.token(credentials);
|
opts = opts.token(credentials);
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: connect in the background and keep trying, so a node that
|
|
||||||
// starts while NATS is down still joins the cluster once NATS is
|
|
||||||
// back, instead of running without a coordinator until restarted;
|
|
||||||
// and report the connection going and coming back
|
|
||||||
let reporter = Arc::new(Reporter::default());
|
|
||||||
opts = opts.retry_on_initial_connect().event_callback({
|
|
||||||
let reporter = reporter.clone();
|
|
||||||
move |event| {
|
|
||||||
let reporter = reporter.clone();
|
|
||||||
async move { reporter.report(event) }
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let connection_timeout = config.timeout_connection.into_inner();
|
|
||||||
|
|
||||||
async_nats::connect_with_options(config.addresses.into_inner(), opts)
|
async_nats::connect_with_options(config.addresses.into_inner(), opts)
|
||||||
.await
|
.await
|
||||||
.map(|client| {
|
.map(|client| Coordinator::Nats(Arc::new(NatsPubSub { client })))
|
||||||
reporter.watch_first_connection(client.clone(), connection_timeout);
|
|
||||||
Coordinator::Nats(Arc::new(NatsPubSub { client }))
|
|
||||||
})
|
|
||||||
.map_err(|err| format!("Failed to connect to Nats: {}", err))
|
.map_err(|err| format!("Failed to connect to Nats: {}", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: whether the client is connected to a NATS server right now.
|
|
||||||
pub fn is_connected(&self) -> bool {
|
|
||||||
matches!(
|
|
||||||
self.client.connection_state(),
|
|
||||||
async_nats::connection::State::Connected
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: reports the client's connection events as the server's own.
|
|
||||||
#[derive(Default)]
|
|
||||||
struct Reporter {
|
|
||||||
connected_once: AtomicBool,
|
|
||||||
// A failed attempt raises an error each time the client retries, every
|
|
||||||
// few seconds while NATS is down: report the first after each change
|
|
||||||
error_reported: AtomicBool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Reporter {
|
|
||||||
fn report(&self, event: async_nats::Event) {
|
|
||||||
match event {
|
|
||||||
async_nats::Event::Connected => {
|
|
||||||
self.connected_once.store(true, Ordering::Relaxed);
|
|
||||||
self.error_reported.store(false, Ordering::Relaxed);
|
|
||||||
trc::event!(Cluster(ClusterEvent::CoordinatorConnected), Type = "nats");
|
|
||||||
}
|
|
||||||
async_nats::Event::Disconnected => {
|
|
||||||
self.error_reported.store(false, Ordering::Relaxed);
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Connection lost; reconnecting in the background",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
async_nats::Event::Closed => {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Connection closed; no further attempts will be made",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
async_nats::Event::ClientError(async_nats::ClientError::MaxReconnects) => {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Gave up reconnecting (maxReconnects reached)",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
async_nats::Event::ClientError(err) => {
|
|
||||||
if !self.error_reported.swap(true, Ordering::Relaxed) {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Connection attempt failed; retrying",
|
|
||||||
Reason = err.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
event => {
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
Type = "nats",
|
|
||||||
Details = event.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The first connection is made in the background, so say so when it
|
|
||||||
/// hasn't been made within the connection timeout. The client keeps
|
|
||||||
/// trying, and reports the connection when it comes.
|
|
||||||
fn watch_first_connection(self: &Arc<Self>, client: Client, timeout: Duration) {
|
|
||||||
let reporter = self.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
tokio::time::sleep(timeout).await;
|
|
||||||
if !reporter.connected_once.load(Ordering::Relaxed)
|
|
||||||
&& !matches!(
|
|
||||||
client.connection_state(),
|
|
||||||
async_nats::connection::State::Connected
|
|
||||||
)
|
|
||||||
{
|
|
||||||
trc::event!(
|
|
||||||
Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
Type = "nats",
|
|
||||||
Details = "Not connected at startup; retrying in the background",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{Coordinator, Msg, PubSubStream};
|
use crate::{Coordinator, Msg, PubSubStream};
|
||||||
@@ -45,17 +43,6 @@ impl Coordinator {
|
|||||||
pub fn is_none(&self) -> bool {
|
pub fn is_none(&self) -> bool {
|
||||||
matches!(self, Coordinator::None)
|
matches!(self, Coordinator::None)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: whether the coordinator is connected right now, for the
|
|
||||||
/// backends that track it (NATS); `None` for the others and when no
|
|
||||||
/// coordinator is configured.
|
|
||||||
pub fn is_connected(&self) -> Option<bool> {
|
|
||||||
match self {
|
|
||||||
#[cfg(feature = "nats")]
|
|
||||||
Coordinator::Nats(store) => Some(store.is_connected()),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PubSubStream {
|
impl PubSubStream {
|
||||||
|
|||||||
@@ -553,10 +553,8 @@ impl ParseHttp for Server {
|
|||||||
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
|
return Ok(JsonProblemResponse(StatusCode::OK).into_http_response());
|
||||||
}
|
}
|
||||||
"ready" => {
|
"ready" => {
|
||||||
// inbuxa: ready only while the data store answers
|
|
||||||
// (a cached, time-limited read); liveness stays 200
|
|
||||||
return Ok(JsonProblemResponse({
|
return Ok(JsonProblemResponse({
|
||||||
if self.is_data_store_ready().await {
|
if !self.core.storage.data.is_none() {
|
||||||
StatusCode::OK
|
StatusCode::OK
|
||||||
} else {
|
} else {
|
||||||
StatusCode::SERVICE_UNAVAILABLE
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
@@ -564,27 +562,6 @@ impl ParseHttp for Server {
|
|||||||
})
|
})
|
||||||
.into_http_response());
|
.into_http_response());
|
||||||
}
|
}
|
||||||
// inbuxa: the cluster coordinator's connection, for
|
|
||||||
// monitoring. It stays out of live and ready on purpose:
|
|
||||||
// a node without its coordinator still serves mail, and
|
|
||||||
// failing those would have an orchestrator restart, or
|
|
||||||
// take out of service, every node at once when the
|
|
||||||
// coordinator goes down
|
|
||||||
"cluster" => {
|
|
||||||
let coordinator = &self.core.storage.coordinator;
|
|
||||||
let (status, state) = match coordinator.is_connected() {
|
|
||||||
Some(true) => (StatusCode::OK, "connected"),
|
|
||||||
Some(false) => (StatusCode::SERVICE_UNAVAILABLE, "disconnected"),
|
|
||||||
None if coordinator.is_none() => (StatusCode::OK, "none"),
|
|
||||||
None => (StatusCode::OK, "unknown"),
|
|
||||||
};
|
|
||||||
return Ok(http_proto::JsonResponse::with_status(
|
|
||||||
status,
|
|
||||||
serde_json::json!({ "coordinator": state }),
|
|
||||||
)
|
|
||||||
.no_cache()
|
|
||||||
.into_http_response());
|
|
||||||
}
|
|
||||||
_ => (),
|
_ => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::ahash_is_empty;
|
use super::ahash_is_empty;
|
||||||
@@ -73,23 +71,6 @@ pub struct SetResponse<T: JmapObject> {
|
|||||||
#[serde(rename = "notDestroyed")]
|
#[serde(rename = "notDestroyed")]
|
||||||
#[serde(skip_serializing_if = "VecMap::is_empty")]
|
#[serde(skip_serializing_if = "VecMap::is_empty")]
|
||||||
pub not_destroyed: VecMap<MaybeInvalid<Id>, SetError<T::Property>>,
|
pub not_destroyed: VecMap<MaybeInvalid<Id>, SetError<T::Property>>,
|
||||||
|
|
||||||
// inbuxa: on a registry write that changes the running settings, whether
|
|
||||||
// the server applied it
|
|
||||||
#[serde(rename = "x:settingsReload")]
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub settings_reload: Option<SettingsReload>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: the settings reload that followed a registry write.
|
|
||||||
#[derive(Debug, Clone, serde::Serialize)]
|
|
||||||
pub struct SettingsReload {
|
|
||||||
/// The running settings (here and, through the cluster, on every node)
|
|
||||||
/// include the write.
|
|
||||||
pub applied: bool,
|
|
||||||
/// Why they don't, when they don't.
|
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
|
||||||
pub description: Option<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'de, T: JmapObject> DeserializeArguments<'de> for SetRequest<'de, T> {
|
impl<'de, T: JmapObject> DeserializeArguments<'de> for SetRequest<'de, T> {
|
||||||
@@ -218,7 +199,6 @@ impl<T: JmapObject> SetResponse<T> {
|
|||||||
not_created: VecMap::new(),
|
not_created: VecMap::new(),
|
||||||
not_updated: VecMap::new(),
|
not_updated: VecMap::new(),
|
||||||
not_destroyed: VecMap::new(),
|
not_destroyed: VecMap::new(),
|
||||||
settings_reload: None,
|
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
Err(trc::JmapEvent::RequestTooLarge.into_err())
|
Err(trc::JmapEvent::RequestTooLarge.into_err())
|
||||||
|
|||||||
@@ -427,24 +427,9 @@ pub(crate) async fn trace_query(
|
|||||||
}
|
}
|
||||||
None => false,
|
None => false,
|
||||||
},
|
},
|
||||||
// The queue id column is an integer on every search backend, and
|
Property::QueueId => match value.as_str() {
|
||||||
// holds a trace's first queue id; the keywords carry all of them
|
|
||||||
Property::QueueId => match value
|
|
||||||
.as_str()
|
|
||||||
.and_then(|v| v.trim().parse::<u64>().ok())
|
|
||||||
.or_else(|| value.as_u64())
|
|
||||||
{
|
|
||||||
Some(queue_id) => {
|
Some(queue_id) => {
|
||||||
search.extend([
|
search.push(SearchFilter::eq(TracingSearchField::QueueId, queue_id.to_string()));
|
||||||
SearchFilter::Or,
|
|
||||||
SearchFilter::eq(TracingSearchField::QueueId, queue_id),
|
|
||||||
SearchFilter::has_text(
|
|
||||||
TracingSearchField::Keywords,
|
|
||||||
queue_id.to_string(),
|
|
||||||
nlp::language::Language::None,
|
|
||||||
),
|
|
||||||
SearchFilter::End,
|
|
||||||
]);
|
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
None => false,
|
None => false,
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::registry::mapping::{RegistrySetResponse, map_bootstrap_error};
|
use crate::registry::mapping::{RegistrySetResponse, map_bootstrap_error};
|
||||||
@@ -101,7 +99,7 @@ pub(crate) async fn action_set(
|
|||||||
} else {
|
} else {
|
||||||
set.response
|
set.response
|
||||||
.not_created
|
.not_created
|
||||||
.append(id, reload_refused(result.errors));
|
.append(id, map_bootstrap_error(result.errors));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Action::InvalidateCaches => {
|
Action::InvalidateCaches => {
|
||||||
@@ -575,14 +573,3 @@ async fn dmarc_troubleshoot(
|
|||||||
|
|
||||||
Some(request)
|
Some(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: a refused reload names the object that stopped it and says the
|
|
||||||
/// settings weren't applied; upstream passed on the first error's bare message
|
|
||||||
/// ("Invalid address: ..."), which read like a problem with the request.
|
|
||||||
fn reload_refused(errors: Vec<registry::types::error::Error>) -> SetError<Property> {
|
|
||||||
let description = format!(
|
|
||||||
"Settings were not reloaded. {}",
|
|
||||||
common::cache::reload::describe_reload_errors(&errors)
|
|
||||||
);
|
|
||||||
map_bootstrap_error(errors).with_description(description)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ use directory::core::secret::{hash_secret, is_password_hash};
|
|||||||
use http_proto::HttpSessionData;
|
use http_proto::HttpSessionData;
|
||||||
use jmap_proto::{
|
use jmap_proto::{
|
||||||
error::set::{SetError, SetErrorType},
|
error::set::{SetError, SetErrorType},
|
||||||
method::set::{SetRequest, SetResponse, SettingsReload},
|
method::set::{SetRequest, SetResponse},
|
||||||
object::registry::Registry,
|
object::registry::Registry,
|
||||||
references::resolve::ResolveCreatedReference,
|
references::resolve::ResolveCreatedReference,
|
||||||
request::{IntoValid, MaybeInvalid},
|
request::{IntoValid, MaybeInvalid},
|
||||||
@@ -931,28 +931,30 @@ impl RegistrySet for Server {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// inbuxa: a write to an object the running settings are built from
|
// inbuxa: DIR-17: a directory or the server default applies on the
|
||||||
// applies at once, here and on every node (DIR-17 did this for
|
// next request, here and on every node
|
||||||
// directories and the server default; now it covers every such object)
|
if matches!(
|
||||||
let mut result = result;
|
object_type,
|
||||||
if let Ok(response) = &mut result
|
ObjectType::Directory | ObjectType::Authentication
|
||||||
|
) && let Ok(response) = &result
|
||||||
&& (!response.created.is_empty()
|
&& (!response.created.is_empty()
|
||||||
|| !response.updated.is_empty()
|
|| !response.updated.is_empty()
|
||||||
|| !response.destroyed.is_empty())
|
|| !response.destroyed.is_empty())
|
||||||
&& let Some(reload) = self.reload_after_write(object_type).await
|
|
||||||
{
|
{
|
||||||
response.settings_reload = Some(match reload {
|
let change = common::ipc::RegistryChange::Reload(ObjectType::Directory);
|
||||||
Ok(()) => SettingsReload {
|
match Box::pin(self.reload_registry(change)).await {
|
||||||
applied: true,
|
Ok(reload) if !reload.has_errors() => {
|
||||||
description: None,
|
self.cluster_broadcast(common::ipc::BroadcastEvent::RegistryChange(change))
|
||||||
},
|
.await;
|
||||||
Err(reason) => SettingsReload {
|
}
|
||||||
applied: false,
|
Ok(_) => trc::event!(
|
||||||
description: Some(format!(
|
Registry(trc::RegistryEvent::BuildWarning),
|
||||||
"Saved, but the running settings were not reloaded. {reason}"
|
Details = "Settings didn't reload after a directory change",
|
||||||
)),
|
),
|
||||||
},
|
Err(err) => {
|
||||||
});
|
trc::error!(err.details("Failed to reload directories"));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -109,10 +109,6 @@ async fn main() -> std::io::Result<()> {
|
|||||||
// Wait for shutdown signal
|
// Wait for shutdown signal
|
||||||
wait_for_shutdown().await;
|
wait_for_shutdown().await;
|
||||||
|
|
||||||
// inbuxa: hand back the task locks this node holds, so other nodes can
|
|
||||||
// run those tasks now rather than when the locks expire
|
|
||||||
services::task_manager::lock::release_task_locks(&inner.build_server()).await;
|
|
||||||
|
|
||||||
// Shutdown collector
|
// Shutdown collector
|
||||||
Collector::shutdown();
|
Collector::shutdown();
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
|
|||||||
};
|
};
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut retry_count: u32 = 0;
|
let mut retry_count = 0;
|
||||||
|
|
||||||
trc::event!(Cluster(ClusterEvent::SubscriberStart));
|
trc::event!(Cluster(ClusterEvent::SubscriberStart));
|
||||||
|
|
||||||
@@ -53,7 +53,7 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
|
|||||||
);
|
);
|
||||||
|
|
||||||
match tokio::time::timeout(
|
match tokio::time::timeout(
|
||||||
subscribe_retry_delay(retry_count),
|
Duration::from_secs(1 << retry_count.max(6)),
|
||||||
shutdown_rx.changed(),
|
shutdown_rx.changed(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -62,7 +62,7 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
retry_count = retry_count.saturating_add(1);
|
retry_count += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -234,11 +234,6 @@ pub fn spawn_broadcast_subscriber(inner: Arc<Inner>, mut shutdown_rx: watch::Rec
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delay before the next subscribe attempt: 1 s, 2 s, 4 s ... capped at 64 s.
|
|
||||||
fn subscribe_retry_delay(retry_count: u32) -> Duration {
|
|
||||||
Duration::from_secs(1u64 << retry_count.min(6))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn log_event(event: &BroadcastEvent) -> trc::Value {
|
fn log_event(event: &BroadcastEvent) -> trc::Value {
|
||||||
match event {
|
match event {
|
||||||
BroadcastEvent::PushNotification(notification) => match notification {
|
BroadcastEvent::PushNotification(notification) => match notification {
|
||||||
@@ -301,19 +296,3 @@ fn log_event(event: &BroadcastEvent) -> trc::Value {
|
|||||||
BroadcastEvent::QueueRefresh => "QueueRefresh".into(),
|
BroadcastEvent::QueueRefresh => "QueueRefresh".into(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::subscribe_retry_delay;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn subscribe_retry_backoff_grows_then_caps() {
|
|
||||||
let schedule: Vec<u64> = (0..10)
|
|
||||||
.map(|n| subscribe_retry_delay(n).as_secs())
|
|
||||||
.collect();
|
|
||||||
assert_eq!(schedule, vec![1, 2, 4, 8, 16, 32, 64, 64, 64, 64]);
|
|
||||||
// No shift overflow at the top of the range.
|
|
||||||
assert_eq!(subscribe_retry_delay(u32::MAX), Duration::from_secs(64));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -91,15 +91,7 @@ impl SearchIndexTask for Server {
|
|||||||
build_contact_document(self, account_id, document_id).await
|
build_contact_document(self, account_id, document_id).await
|
||||||
}
|
}
|
||||||
IndexDocumentType::File => {
|
IndexDocumentType::File => {
|
||||||
// File indexing not implemented yet. inbuxa: still
|
// File indexing not implemented yet
|
||||||
// one result per task: update_tasks pairs them by
|
|
||||||
// position, and a missing one shifts every result
|
|
||||||
// after it onto the wrong task
|
|
||||||
results.push(IndexTaskResult {
|
|
||||||
task_type: TaskType::Insert,
|
|
||||||
index: task.document_type,
|
|
||||||
result: TaskResult::Ignored,
|
|
||||||
});
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -575,14 +567,20 @@ async fn build_contact_document(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// inbuxa: MON-16: a trace's search document, when trace search is on
|
// inbuxa: MON-16: a trace's search document, when trace search is on:
|
||||||
|
// its event types, queue ids, and addresses, their domains, hosts, IPs,
|
||||||
|
// message ids and account names as keywords
|
||||||
async fn build_tracing_span_document(
|
async fn build_tracing_span_document(
|
||||||
server: &Server,
|
server: &Server,
|
||||||
span_id: u64,
|
span_id: u64,
|
||||||
) -> trc::Result<Option<IndexDocument>> {
|
) -> trc::Result<Option<IndexDocument>> {
|
||||||
use common::telemetry::tracers::store::MaybeTrace;
|
use common::telemetry::tracers::store::MaybeTrace;
|
||||||
use registry::schema::structs::Search;
|
use registry::schema::{enums::SearchTracingField, structs::Search};
|
||||||
use store::write::{TelemetryClass, ValueClass};
|
use store::{
|
||||||
|
search::TracingSearchField,
|
||||||
|
write::{TelemetryClass, ValueClass},
|
||||||
|
};
|
||||||
|
use trc::Key;
|
||||||
|
|
||||||
let settings = server
|
let settings = server
|
||||||
.registry()
|
.registry()
|
||||||
@@ -592,6 +590,7 @@ async fn build_tracing_span_document(
|
|||||||
if !settings.index_telemetry {
|
if !settings.index_telemetry {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
let wants = |field: SearchTracingField| settings.index_tracing_fields.iter().any(|f| *f == field);
|
||||||
let Some(MaybeTrace(Some(trace))) = server
|
let Some(MaybeTrace(Some(trace))) = server
|
||||||
.tracing_store()
|
.tracing_store()
|
||||||
.get_value::<MaybeTrace>(ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(
|
.get_value::<MaybeTrace>(ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(
|
||||||
@@ -602,67 +601,23 @@ async fn build_tracing_span_document(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok(Some(trace_search_document(
|
|
||||||
span_id,
|
|
||||||
&trace,
|
|
||||||
&settings
|
|
||||||
.index_tracing_fields
|
|
||||||
.iter()
|
|
||||||
.copied()
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: MON-16: the search document for a stored trace.
|
|
||||||
///
|
|
||||||
/// The event type and queue id columns are integers on every search backend
|
|
||||||
/// (BIGINT on PostgreSQL and MySQL, long on Elasticsearch), and each holds a
|
|
||||||
/// single value per trace: the event type is the trace's opening event, the
|
|
||||||
/// one `x:Trace/query` filters on, and the queue id is the first queue id the
|
|
||||||
/// trace mentions. Every queue id also goes into the keywords, so a session
|
|
||||||
/// that queued several messages is found by any of them.
|
|
||||||
pub fn trace_search_document(
|
|
||||||
span_id: u64,
|
|
||||||
trace: ®istry::schema::structs::Trace,
|
|
||||||
fields: &[registry::schema::enums::SearchTracingField],
|
|
||||||
) -> IndexDocument {
|
|
||||||
use registry::schema::{enums::SearchTracingField, structs::TraceValue};
|
|
||||||
use store::search::TracingSearchField;
|
|
||||||
use trc::Key;
|
|
||||||
|
|
||||||
let wants = |field: SearchTracingField| fields.contains(&field);
|
|
||||||
let mut document = IndexDocument::new(SearchIndex::Tracing).with_id(span_id);
|
let mut document = IndexDocument::new(SearchIndex::Tracing).with_id(span_id);
|
||||||
if wants(SearchTracingField::EventType)
|
|
||||||
&& let Some(first) = trace.events.iter().next()
|
|
||||||
{
|
|
||||||
document.index_unsigned(TracingSearchField::EventType, first.event.to_id() as u64);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut seen = store::ahash::AHashSet::new();
|
let mut seen = store::ahash::AHashSet::new();
|
||||||
let mut queue_id_indexed = false;
|
|
||||||
for event in trace.events.iter() {
|
for event in trace.events.iter() {
|
||||||
|
if wants(SearchTracingField::EventType) && seen.insert(event.event.as_str().to_string()) {
|
||||||
|
document.index_keyword(TracingSearchField::EventType, event.event.as_str());
|
||||||
|
}
|
||||||
for kv in event.key_values.iter() {
|
for kv in event.key_values.iter() {
|
||||||
let text = match &kv.value {
|
let text = match &kv.value {
|
||||||
TraceValue::String(v) => v.value.clone(),
|
registry::schema::structs::TraceValue::String(v) => v.value.clone(),
|
||||||
TraceValue::UnsignedInt(v) => v.value.to_string(),
|
registry::schema::structs::TraceValue::UnsignedInt(v) => v.value.to_string(),
|
||||||
TraceValue::IpAddr(v) => v.value.to_string(),
|
registry::schema::structs::TraceValue::IpAddr(v) => v.value.to_string(),
|
||||||
_ => continue,
|
_ => continue,
|
||||||
};
|
};
|
||||||
match kv.key {
|
match kv.key {
|
||||||
Key::QueueId => {
|
Key::QueueId if wants(SearchTracingField::QueueId) => {
|
||||||
let Ok(queue_id) = text.parse::<u64>() else {
|
if seen.insert(format!("q:{text}")) {
|
||||||
continue;
|
document.index_keyword(TracingSearchField::QueueId, &text);
|
||||||
};
|
|
||||||
if wants(SearchTracingField::QueueId) && !queue_id_indexed {
|
|
||||||
document.index_unsigned(TracingSearchField::QueueId, queue_id);
|
|
||||||
queue_id_indexed = true;
|
|
||||||
}
|
|
||||||
if wants(SearchTracingField::Keywords) && seen.insert(format!("k:{text}")) {
|
|
||||||
document.index_text(
|
|
||||||
TracingSearchField::Keywords,
|
|
||||||
&text,
|
|
||||||
nlp::language::Language::None,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Key::From
|
Key::From
|
||||||
@@ -693,7 +648,7 @@ pub fn trace_search_document(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
document
|
Ok(Some(document))
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: UD-1, UD-4: archives a deleted file, event or contact noted at
|
// inbuxa: UD-1, UD-4: archives a deleted file, event or contact noted at
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::task_manager::*;
|
use crate::task_manager::*;
|
||||||
@@ -15,21 +13,13 @@ pub trait TaskLockManager: Sync + Send {
|
|||||||
|
|
||||||
impl TaskLockManager for Server {
|
impl TaskLockManager for Server {
|
||||||
async fn try_lock_task(&self, id: u64) -> bool {
|
async fn try_lock_task(&self, id: u64) -> bool {
|
||||||
// inbuxa: a node that is stopping claims nothing new
|
|
||||||
let locks = &self.inner.ipc.task_locks;
|
|
||||||
if locks.is_stopping() {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
match self
|
match self
|
||||||
.in_memory_store()
|
.in_memory_store()
|
||||||
.try_lock(KV_LOCK_TASK, &id.to_be_bytes(), locks.expiry())
|
.try_lock(KV_LOCK_TASK, &id.to_be_bytes(), DEFAULT_LOCK_EXPIRY)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
if result {
|
if !result {
|
||||||
locks.insert(id);
|
|
||||||
} else {
|
|
||||||
trc::event!(
|
trc::event!(
|
||||||
TaskManager(TaskManagerEvent::TaskLocked),
|
TaskManager(TaskManagerEvent::TaskLocked),
|
||||||
Id = id,
|
Id = id,
|
||||||
@@ -58,66 +48,5 @@ impl TaskLockManager for Server {
|
|||||||
.caused_by(trc::location!())
|
.caused_by(trc::location!())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
self.inner.ipc.task_locks.remove(id);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: on a graceful stop, stops claiming tasks and releases every task
|
|
||||||
/// lock this node holds, so the rest of the cluster can pick the tasks up at
|
|
||||||
/// once instead of after the lock expires. Returns how many were released.
|
|
||||||
pub async fn release_task_locks(server: &Server) -> usize {
|
|
||||||
let ids = server.inner.ipc.task_locks.stop();
|
|
||||||
for id in &ids {
|
|
||||||
if let Err(err) = server
|
|
||||||
.in_memory_store()
|
|
||||||
.remove_lock(KV_LOCK_TASK, &id.to_be_bytes())
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
trc::error!(
|
|
||||||
err.details("Failed to release task lock on shutdown")
|
|
||||||
.ctx(trc::Key::Id, *id)
|
|
||||||
.caused_by(trc::location!())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ids.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: renews the lease on every task this node is running, so it stays
|
|
||||||
/// claimed for as long as it runs while a node that dies loses its claims
|
|
||||||
/// within one lock lifetime. Returns how many leases were renewed and how
|
|
||||||
/// many were found lost (expired, perhaps taken by another node).
|
|
||||||
pub async fn renew_task_locks(server: &Server) -> (usize, usize) {
|
|
||||||
let locks = &server.inner.ipc.task_locks;
|
|
||||||
let expiry = locks.expiry();
|
|
||||||
let (mut renewed, mut lost) = (0, 0);
|
|
||||||
for id in locks.held_ids() {
|
|
||||||
match server
|
|
||||||
.in_memory_store()
|
|
||||||
.renew_lock(KV_LOCK_TASK, &id.to_be_bytes(), expiry)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(true) => renewed += 1,
|
|
||||||
Ok(false) => {
|
|
||||||
// Still held here as far as this node knows; the task
|
|
||||||
// finishes and its lock is removed as usual
|
|
||||||
if locks.is_held(id) {
|
|
||||||
lost += 1;
|
|
||||||
trc::event!(
|
|
||||||
TaskManager(TaskManagerEvent::TaskLocked),
|
|
||||||
Id = id,
|
|
||||||
Details = "Task lock expired while the task was running",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
trc::error!(
|
|
||||||
err.details("Failed to renew task lock")
|
|
||||||
.ctx(trc::Key::Id, id)
|
|
||||||
.caused_by(trc::location!())
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
(renewed, lost)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,18 +13,17 @@ use crate::task_manager::dkim::DkimManagementTask;
|
|||||||
use crate::task_manager::dns::DnsManagementTask;
|
use crate::task_manager::dns::DnsManagementTask;
|
||||||
use crate::task_manager::imip::SendImipTask;
|
use crate::task_manager::imip::SendImipTask;
|
||||||
use crate::task_manager::index::SearchIndexTask;
|
use crate::task_manager::index::SearchIndexTask;
|
||||||
use crate::task_manager::lock::{TaskLockManager, renew_task_locks};
|
use crate::task_manager::lock::TaskLockManager;
|
||||||
use crate::task_manager::maintenance::MaintenanceTask;
|
use crate::task_manager::maintenance::MaintenanceTask;
|
||||||
use crate::task_manager::merge_threads::MergeThreadsTask;
|
use crate::task_manager::merge_threads::MergeThreadsTask;
|
||||||
use crate::task_manager::report::{self, SubmitReportTask};
|
use crate::task_manager::report::{self, SubmitReportTask};
|
||||||
use crate::task_manager::restore_item::RestoreItemTask;
|
use crate::task_manager::restore_item::RestoreItemTask;
|
||||||
use crate::task_manager::spam_classifier::SpamFilterMaintenanceTask;
|
use crate::task_manager::spam_classifier::SpamFilterMaintenanceTask;
|
||||||
use crate::task_manager::{
|
use crate::task_manager::{
|
||||||
CLAIM_RECHECK_INTERVAL, Locked, QUEUE_REFRESH_INTERVAL, TaskDetails, TaskFailureType, TaskInfo,
|
DEFAULT_LOCK_EXPIRY, Locked, QUEUE_REFRESH_INTERVAL, TaskDetails, TaskFailureType, TaskInfo,
|
||||||
TaskJob, TaskManagerIpc, TaskResult,
|
TaskJob, TaskManagerIpc, TaskResult,
|
||||||
};
|
};
|
||||||
use common::BuildServer;
|
use common::BuildServer;
|
||||||
use common::config::network::ClusterRoles;
|
|
||||||
use common::config::server::{DEFAULT_TLS_TIMEOUT, ServerProtocol};
|
use common::config::server::{DEFAULT_TLS_TIMEOUT, ServerProtocol};
|
||||||
use common::network::limiter::ConcurrencyLimiter;
|
use common::network::limiter::ConcurrencyLimiter;
|
||||||
use common::network::{ServerInstance, TcpAcceptor};
|
use common::network::{ServerInstance, TcpAcceptor};
|
||||||
@@ -56,37 +55,24 @@ const PERPETUAL_RETRY_MIN_DELAY: u64 = 3600;
|
|||||||
const PERPETUAL_RETRY_MAX_DELAY: u64 = 21600;
|
const PERPETUAL_RETRY_MAX_DELAY: u64 = 21600;
|
||||||
|
|
||||||
pub fn spawn_task_manager(inner: Arc<Inner>) {
|
pub fn spawn_task_manager(inner: Arc<Inner>) {
|
||||||
// inbuxa: upstream didn't start the task manager on a node whose role
|
let is_clustered = {
|
||||||
// had no task types at boot, so adding one later did nothing until a
|
let server = inner.build_server();
|
||||||
// restart. It now always runs and reads the role on every scan and
|
let roles = &server.core.network.roles;
|
||||||
// before every job (task_enabled), so a role change applies at the next
|
|
||||||
// settings reload.
|
if !roles.account_maintenance
|
||||||
let is_clustered = inner.build_server().core.storage.coordinator.is_enabled();
|
&& !roles.store_maintenance
|
||||||
|
&& !roles.search_indexing
|
||||||
|
&& !roles.spam_training
|
||||||
|
&& !roles.task_manager
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
server.core.storage.coordinator.is_enabled()
|
||||||
|
};
|
||||||
|
|
||||||
trc::event!(TaskManager(TaskManagerEvent::ManagerStarted));
|
trc::event!(TaskManager(TaskManagerEvent::ManagerStarted));
|
||||||
|
|
||||||
// inbuxa: keep the leases of running tasks alive, every third of a lock
|
|
||||||
// lifetime, until the node stops
|
|
||||||
{
|
|
||||||
let inner = inner.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut renewed_at = Instant::now();
|
|
||||||
loop {
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
let locks = &inner.ipc.task_locks;
|
|
||||||
if locks.is_stopping() {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if renewed_at.elapsed() >= Duration::from_secs((locks.expiry() / 3).max(1)) {
|
|
||||||
renewed_at = Instant::now();
|
|
||||||
if locks.held() > 0 {
|
|
||||||
renew_task_locks(&inner.build_server()).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create dummy server instance for alarms
|
// Create dummy server instance for alarms
|
||||||
let server_instance = Arc::new(ServerInstance {
|
let server_instance = Arc::new(ServerInstance {
|
||||||
id: "_local".to_string(),
|
id: "_local".to_string(),
|
||||||
@@ -141,50 +127,72 @@ pub fn spawn_task_manager(inner: Arc<Inner>) {
|
|||||||
let server = inner.build_server();
|
let server = inner.build_server();
|
||||||
let batch_size = server.core.email.index_batch_size;
|
let batch_size = server.core.email.index_batch_size;
|
||||||
let mut batch = Vec::with_capacity(batch_size);
|
let mut batch = Vec::with_capacity(batch_size);
|
||||||
if let Some(task) = fetch_enabled_task(&server, job).await {
|
match server
|
||||||
batch.push(task);
|
.store()
|
||||||
|
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
||||||
|
TaskQueueClass::Task { id: job.id },
|
||||||
|
)))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Some(task)) => {
|
||||||
|
batch.push(TaskDetails { task, info: job });
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
trc::event!(
|
||||||
|
TaskManager(TaskManagerEvent::TaskIgnored),
|
||||||
|
Id = job.id,
|
||||||
|
Reason = "Task not found in store, likely already processed.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
trc::error!(
|
||||||
|
err.id(job.id)
|
||||||
|
.details("Failed to retrieve task details.")
|
||||||
|
.caused_by(trc::location!())
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
while batch.len() < batch_size {
|
while batch.len() < batch_size {
|
||||||
match rx.try_recv() {
|
match rx.try_recv() {
|
||||||
Ok(job) => {
|
Ok(job) => {
|
||||||
if let Some(task) = fetch_enabled_task(&server, job).await {
|
match server
|
||||||
batch.push(task);
|
.store()
|
||||||
|
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
||||||
|
TaskQueueClass::Task { id: job.id },
|
||||||
|
)))
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Some(task)) => {
|
||||||
|
batch.push(TaskDetails { task, info: job });
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
trc::event!(
|
||||||
|
TaskManager(TaskManagerEvent::TaskIgnored),
|
||||||
|
Id = job.id,
|
||||||
|
Reason = "Task not found in store, likely already processed.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
trc::error!(
|
||||||
|
err.id(job.id)
|
||||||
|
.details("Failed to retrieve task details.")
|
||||||
|
.caused_by(trc::location!())
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(_) => break,
|
Err(_) => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if batch.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dispatch. inbuxa: on a task of its own, so a panic
|
// Dispatch
|
||||||
// releases the batch's locks and leaves this worker
|
|
||||||
// running; a dead worker would keep claiming tasks it
|
|
||||||
// can never run
|
|
||||||
let mut refresh_queue = false;
|
let mut refresh_queue = false;
|
||||||
let ids = batch.iter().map(|task| task.info.id).collect::<Vec<_>>();
|
let results = server.index(&batch).await.into_iter().map(|r| {
|
||||||
let run = {
|
|
||||||
let server = server.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let results = server.index(&batch).await;
|
|
||||||
(batch, results)
|
|
||||||
})
|
|
||||||
};
|
|
||||||
match run.await {
|
|
||||||
Ok((mut batch, results)) => {
|
|
||||||
let results = results.into_iter().map(|r| {
|
|
||||||
refresh_queue |= r.result.is_retry();
|
refresh_queue |= r.result.is_retry();
|
||||||
r.result
|
r.result
|
||||||
});
|
});
|
||||||
update_tasks(&server, &mut batch, results).await;
|
update_tasks(&server, &mut batch, results).await;
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
worker_failed(&server, &ids, err).await;
|
|
||||||
refresh_queue = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if refresh_queue || rx.is_empty() {
|
if refresh_queue || rx.is_empty() {
|
||||||
server.notify_task_queue();
|
server.notify_task_queue();
|
||||||
@@ -198,32 +206,83 @@ pub fn spawn_task_manager(inner: Arc<Inner>) {
|
|||||||
let server = inner.build_server();
|
let server = inner.build_server();
|
||||||
let mut refresh_queue = false;
|
let mut refresh_queue = false;
|
||||||
|
|
||||||
if let Some(TaskDetails { task, info }) = fetch_enabled_task(&server, job).await
|
match server
|
||||||
|
.store()
|
||||||
|
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
||||||
|
TaskQueueClass::Task { id: job.id },
|
||||||
|
)))
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
// inbuxa: on a task of its own, as above
|
Ok(Some(task)) => {
|
||||||
let run = {
|
let result = match &task {
|
||||||
let server = server.clone();
|
Task::CalendarAlarmEmail(task) => {
|
||||||
let server_instance = server_instance.clone();
|
server.send_email_alarm(task, server_instance.clone()).await
|
||||||
tokio::spawn(async move {
|
}
|
||||||
let result = run_task(&server, &task, server_instance).await;
|
Task::CalendarAlarmNotification(task) => {
|
||||||
(task, result)
|
server.send_display_alarm(task).await
|
||||||
})
|
}
|
||||||
|
Task::CalendarItipMessage(task) => {
|
||||||
|
server.send_imip(task, server_instance.clone()).await
|
||||||
|
}
|
||||||
|
Task::MergeThreads(task) => server.merge_threads(task).await,
|
||||||
|
Task::DmarcReport(task) => {
|
||||||
|
server
|
||||||
|
.submit_report(report::ReportId::Dmarc(task.report_id.id()))
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
Task::TlsReport(task) => {
|
||||||
|
server
|
||||||
|
.submit_report(report::ReportId::Tls(task.report_id.id()))
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
Task::RestoreArchivedItem(task) => server.restore_item(task).await,
|
||||||
|
Task::DestroyAccount(task) => server.destroy_account(task).await,
|
||||||
|
Task::AccountMaintenance(task) => {
|
||||||
|
server.account_maintenance(task).await
|
||||||
|
}
|
||||||
|
Task::TenantMaintenance(task) => {
|
||||||
|
server.tenant_maintenance(task).await
|
||||||
|
}
|
||||||
|
Task::StoreMaintenance(task) => {
|
||||||
|
server.store_maintenance(task).await
|
||||||
|
}
|
||||||
|
Task::SpamFilterMaintenance(task) => {
|
||||||
|
Box::pin(server.spam_filter_maintenance(task)).await
|
||||||
|
}
|
||||||
|
Task::AcmeRenewal(task) => server.acme_management(task).await,
|
||||||
|
Task::DkimManagement(task_dkim_rotation) => {
|
||||||
|
server.dkim_management(task_dkim_rotation).await
|
||||||
|
}
|
||||||
|
Task::DnsManagement(task_dns_management) => {
|
||||||
|
server.dns_management(task_dns_management).await
|
||||||
|
}
|
||||||
|
Task::IndexDocument(_)
|
||||||
|
| Task::UnindexDocument(_)
|
||||||
|
| Task::IndexTrace(_) => unreachable!(),
|
||||||
};
|
};
|
||||||
match run.await {
|
|
||||||
Ok((task, result)) => {
|
|
||||||
refresh_queue = result.is_retry();
|
refresh_queue = result.is_retry();
|
||||||
|
|
||||||
update_tasks(
|
update_tasks(
|
||||||
&server,
|
&server,
|
||||||
&mut [TaskDetails { task, info }],
|
&mut [TaskDetails { task, info: job }],
|
||||||
vec![result],
|
vec![result],
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Ok(None) => {
|
||||||
worker_failed(&server, &[info.id], err).await;
|
trc::event!(
|
||||||
refresh_queue = true;
|
TaskManager(TaskManagerEvent::TaskIgnored),
|
||||||
|
Id = job.id,
|
||||||
|
Reason = "Task not found in store, likely already processed.",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
Err(err) => {
|
||||||
|
trc::error!(
|
||||||
|
err.id(job.id)
|
||||||
|
.details("Failed to retrieve task details.")
|
||||||
|
.caused_by(trc::location!())
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -262,24 +321,6 @@ pub(crate) trait TaskQueueManager: Sync + Send {
|
|||||||
|
|
||||||
impl TaskQueueManager for Server {
|
impl TaskQueueManager for Server {
|
||||||
async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration {
|
async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration {
|
||||||
// inbuxa: a node that is stopping has released its locks and claims
|
|
||||||
// nothing new
|
|
||||||
let task_locks = &self.inner.ipc.task_locks;
|
|
||||||
if task_locks.is_stopping() {
|
|
||||||
return Duration::from_secs(QUEUE_REFRESH_INTERVAL);
|
|
||||||
}
|
|
||||||
// inbuxa: with no task type enabled by this node's role there is
|
|
||||||
// nothing to claim; a settings reload wakes the manager when that
|
|
||||||
// changes
|
|
||||||
let roles = &self.core.network.roles;
|
|
||||||
if !(0..TaskType::COUNT as u16)
|
|
||||||
.filter_map(TaskType::from_id)
|
|
||||||
.any(|task_type| task_enabled(roles, task_type))
|
|
||||||
{
|
|
||||||
ipc.locked.clear();
|
|
||||||
return Duration::from_secs(QUEUE_REFRESH_INTERVAL);
|
|
||||||
}
|
|
||||||
let lock_expiry = task_locks.expiry();
|
|
||||||
let now_timestamp = now();
|
let now_timestamp = now();
|
||||||
let from_key = ValueKey::<ValueClass> {
|
let from_key = ValueKey::<ValueClass> {
|
||||||
account_id: 0,
|
account_id: 0,
|
||||||
@@ -302,6 +343,7 @@ impl TaskQueueManager for Server {
|
|||||||
let mut unreadable = Vec::new();
|
let mut unreadable = Vec::new();
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let mut next_event = None;
|
let mut next_event = None;
|
||||||
|
let roles = &self.core.network.roles;
|
||||||
ipc.revision += 1;
|
ipc.revision += 1;
|
||||||
let _ = self
|
let _ = self
|
||||||
.store()
|
.store()
|
||||||
@@ -328,13 +370,26 @@ impl TaskQueueManager for Server {
|
|||||||
});
|
});
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
};
|
};
|
||||||
// inbuxa: running here under a lease this node
|
let enabled = match task_type {
|
||||||
// renews; don't hand it to a worker again
|
TaskType::IndexDocument
|
||||||
if task_locks.is_held(task_id) {
|
| TaskType::UnindexDocument
|
||||||
return Ok(true);
|
| TaskType::IndexTrace => roles.search_indexing,
|
||||||
}
|
TaskType::AccountMaintenance
|
||||||
|
| TaskType::TenantMaintenance
|
||||||
let enabled = task_enabled(roles, task_type);
|
| TaskType::DestroyAccount => roles.account_maintenance,
|
||||||
|
TaskType::StoreMaintenance => roles.store_maintenance,
|
||||||
|
TaskType::SpamFilterMaintenance => roles.spam_training,
|
||||||
|
TaskType::CalendarAlarmEmail
|
||||||
|
| TaskType::CalendarAlarmNotification
|
||||||
|
| TaskType::CalendarItipMessage
|
||||||
|
| TaskType::MergeThreads
|
||||||
|
| TaskType::DmarcReport
|
||||||
|
| TaskType::TlsReport
|
||||||
|
| TaskType::RestoreArchivedItem
|
||||||
|
| TaskType::AcmeRenewal
|
||||||
|
| TaskType::DkimManagement
|
||||||
|
| TaskType::DnsManagement => true,
|
||||||
|
};
|
||||||
|
|
||||||
if !enabled {
|
if !enabled {
|
||||||
trc::event!(
|
trc::event!(
|
||||||
@@ -351,7 +406,9 @@ impl TaskQueueManager for Server {
|
|||||||
let locked = entry.get_mut();
|
let locked = entry.get_mut();
|
||||||
if locked.expires <= now || locked.due < task_due {
|
if locked.expires <= now || locked.due < task_due {
|
||||||
locked.expires = Instant::now()
|
locked.expires = Instant::now()
|
||||||
+ std::time::Duration::from_secs(lock_expiry + 1);
|
+ std::time::Duration::from_secs(
|
||||||
|
DEFAULT_LOCK_EXPIRY + 1,
|
||||||
|
);
|
||||||
locked.due = task_due;
|
locked.due = task_due;
|
||||||
tasks.push((
|
tasks.push((
|
||||||
TaskJob {
|
TaskJob {
|
||||||
@@ -367,7 +424,9 @@ impl TaskQueueManager for Server {
|
|||||||
Entry::Vacant(entry) => {
|
Entry::Vacant(entry) => {
|
||||||
entry.insert(Locked {
|
entry.insert(Locked {
|
||||||
expires: Instant::now()
|
expires: Instant::now()
|
||||||
+ std::time::Duration::from_secs(lock_expiry + 1),
|
+ std::time::Duration::from_secs(
|
||||||
|
DEFAULT_LOCK_EXPIRY + 1,
|
||||||
|
),
|
||||||
due: task_due,
|
due: task_due,
|
||||||
revision: ipc.revision,
|
revision: ipc.revision,
|
||||||
});
|
});
|
||||||
@@ -423,26 +482,12 @@ impl TaskQueueManager for Server {
|
|||||||
let tx = &ipc.txs[task_type_idx as usize];
|
let tx = &ipc.txs[task_type_idx as usize];
|
||||||
|
|
||||||
if tx.capacity() > 0 {
|
if tx.capacity() > 0 {
|
||||||
let id = task_job.id;
|
if self.try_lock_task(task_job.id).await && tx.send(task_job).await.is_err() {
|
||||||
if !self.try_lock_task(id).await {
|
|
||||||
// inbuxa: another node holds the task. Look again after a
|
|
||||||
// short while rather than a full lock lifetime from now:
|
|
||||||
// the holder may have claimed it after this scan began,
|
|
||||||
// or run on a clock ahead of this one, and waiting the
|
|
||||||
// whole lifetime again would leave the task stuck for
|
|
||||||
// another hour past its lock if that holder died
|
|
||||||
if let Some(locked) = ipc.locked.get_mut(&id) {
|
|
||||||
locked.expires =
|
|
||||||
Instant::now() + Duration::from_secs(claim_recheck_interval(lock_expiry));
|
|
||||||
}
|
|
||||||
} else if tx.send(task_job).await.is_err() {
|
|
||||||
trc::event!(
|
trc::event!(
|
||||||
Server(trc::ServerEvent::ThreadError),
|
Server(trc::ServerEvent::ThreadError),
|
||||||
Details = "Error sending task.",
|
Details = "Error sending task.",
|
||||||
CausedBy = trc::location!()
|
CausedBy = trc::location!()
|
||||||
);
|
);
|
||||||
// inbuxa: nothing will run it here, so don't hold it
|
|
||||||
self.remove_index_lock(id).await;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// If the channel is full, release the lock so it can be picked up in the next iteration
|
// If the channel is full, release the lock so it can be picked up in the next iteration
|
||||||
@@ -454,178 +499,9 @@ impl TaskQueueManager for Server {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
ipc.locked
|
ipc.locked
|
||||||
.retain(|_, locked| locked.expires > now && locked.revision == ipc.revision);
|
.retain(|_, locked| locked.expires > now && locked.revision == ipc.revision);
|
||||||
let sleep_for = Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| {
|
Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| {
|
||||||
timestamp.saturating_sub(store::write::now())
|
timestamp.saturating_sub(store::write::now())
|
||||||
}));
|
}))
|
||||||
|
|
||||||
// inbuxa: wake up when a claim held elsewhere is due to be tried
|
|
||||||
// again, rather than only on the next task or refresh
|
|
||||||
ipc.locked
|
|
||||||
.values()
|
|
||||||
.map(|locked| locked.expires.saturating_duration_since(now))
|
|
||||||
.min()
|
|
||||||
.map_or(sleep_for, |recheck| sleep_for.min(recheck.max(Duration::from_secs(1))))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: whether this node's cluster role lets it run a task type. Upstream
|
|
||||||
/// checked the dedicated roles (search indexing, account and store
|
|
||||||
/// maintenance, spam training) and let every node with a task manager run
|
|
||||||
/// the rest, whatever its taskQueueProcessing setting. Every task type now
|
|
||||||
/// answers to one ClusterTaskType:
|
|
||||||
///
|
|
||||||
/// - IndexDocument, UnindexDocument, IndexTrace: searchIndexing
|
|
||||||
/// - AccountMaintenance, TenantMaintenance, DestroyAccount: accountMaintenance
|
|
||||||
/// - StoreMaintenance: storeMaintenance
|
|
||||||
/// - SpamFilterMaintenance: spamClassifierTraining
|
|
||||||
/// - DmarcReport, TlsReport: outboundMta. They build and send reports to
|
|
||||||
/// other domains (TLS reports can go straight to an HTTPS endpoint), which
|
|
||||||
/// is the outbound MTA's business.
|
|
||||||
/// - CalendarAlarmEmail, CalendarAlarmNotification, CalendarItipMessage,
|
|
||||||
/// MergeThreads, RestoreArchivedItem, AcmeRenewal, DkimManagement,
|
|
||||||
/// DnsManagement: taskQueueProcessing, the role for queue tasks with no
|
|
||||||
/// role of their own.
|
|
||||||
///
|
|
||||||
/// A node that may not run a task leaves it unclaimed, so a node that may
|
|
||||||
/// picks it up.
|
|
||||||
pub fn task_enabled(roles: &ClusterRoles, task_type: TaskType) -> bool {
|
|
||||||
match task_type {
|
|
||||||
TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace => {
|
|
||||||
roles.search_indexing
|
|
||||||
}
|
|
||||||
TaskType::AccountMaintenance | TaskType::TenantMaintenance | TaskType::DestroyAccount => {
|
|
||||||
roles.account_maintenance
|
|
||||||
}
|
|
||||||
TaskType::StoreMaintenance => roles.store_maintenance,
|
|
||||||
TaskType::SpamFilterMaintenance => roles.spam_training,
|
|
||||||
TaskType::DmarcReport | TaskType::TlsReport => roles.outbound_mta,
|
|
||||||
TaskType::CalendarAlarmEmail
|
|
||||||
| TaskType::CalendarAlarmNotification
|
|
||||||
| TaskType::CalendarItipMessage
|
|
||||||
| TaskType::MergeThreads
|
|
||||||
| TaskType::RestoreArchivedItem
|
|
||||||
| TaskType::AcmeRenewal
|
|
||||||
| TaskType::DkimManagement
|
|
||||||
| TaskType::DnsManagement => roles.task_manager,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn run_task(
|
|
||||||
server: &Server,
|
|
||||||
task: &Task,
|
|
||||||
server_instance: Arc<ServerInstance>,
|
|
||||||
) -> TaskResult {
|
|
||||||
match task {
|
|
||||||
Task::CalendarAlarmEmail(task) => {
|
|
||||||
server.send_email_alarm(task, server_instance.clone()).await
|
|
||||||
}
|
|
||||||
Task::CalendarAlarmNotification(task) => {
|
|
||||||
server.send_display_alarm(task).await
|
|
||||||
}
|
|
||||||
Task::CalendarItipMessage(task) => {
|
|
||||||
server.send_imip(task, server_instance.clone()).await
|
|
||||||
}
|
|
||||||
Task::MergeThreads(task) => server.merge_threads(task).await,
|
|
||||||
Task::DmarcReport(task) => {
|
|
||||||
server
|
|
||||||
.submit_report(report::ReportId::Dmarc(task.report_id.id()))
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
Task::TlsReport(task) => {
|
|
||||||
server
|
|
||||||
.submit_report(report::ReportId::Tls(task.report_id.id()))
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
Task::RestoreArchivedItem(task) => server.restore_item(task).await,
|
|
||||||
Task::DestroyAccount(task) => server.destroy_account(task).await,
|
|
||||||
Task::AccountMaintenance(task) => {
|
|
||||||
server.account_maintenance(task).await
|
|
||||||
}
|
|
||||||
Task::TenantMaintenance(task) => {
|
|
||||||
server.tenant_maintenance(task).await
|
|
||||||
}
|
|
||||||
Task::StoreMaintenance(task) => {
|
|
||||||
server.store_maintenance(task).await
|
|
||||||
}
|
|
||||||
Task::SpamFilterMaintenance(task) => {
|
|
||||||
Box::pin(server.spam_filter_maintenance(task)).await
|
|
||||||
}
|
|
||||||
Task::AcmeRenewal(task) => server.acme_management(task).await,
|
|
||||||
Task::DkimManagement(task_dkim_rotation) => {
|
|
||||||
server.dkim_management(task_dkim_rotation).await
|
|
||||||
}
|
|
||||||
Task::DnsManagement(task_dns_management) => {
|
|
||||||
server.dns_management(task_dns_management).await
|
|
||||||
}
|
|
||||||
Task::IndexDocument(_)
|
|
||||||
| Task::UnindexDocument(_)
|
|
||||||
| Task::IndexTrace(_) => unreachable!(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: reads a claimed task when this node's role still allows its type.
|
|
||||||
/// The role may have changed since the task was claimed (a settings reload in
|
|
||||||
/// between); the claim is then handed back at once for a node that may run
|
|
||||||
/// it, rather than held until the lease runs out.
|
|
||||||
async fn fetch_enabled_task(server: &Server, job: TaskJob) -> Option<TaskDetails> {
|
|
||||||
if task_enabled(&server.core.network.roles, job.typ) {
|
|
||||||
fetch_task(server, job).await
|
|
||||||
} else {
|
|
||||||
trc::event!(
|
|
||||||
TaskManager(TaskManagerEvent::TaskIgnored),
|
|
||||||
Id = job.id,
|
|
||||||
Details = job.typ.as_str(),
|
|
||||||
Reason = "Task type was disabled by cluster roles after it was claimed.",
|
|
||||||
);
|
|
||||||
server.remove_index_lock(job.id).await;
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reads a claimed task. When it is gone or can't be read, the claim is
|
|
||||||
/// released: inbuxa: holding it would block the task, everywhere, until
|
|
||||||
/// the lock expired.
|
|
||||||
async fn fetch_task(server: &Server, job: TaskJob) -> Option<TaskDetails> {
|
|
||||||
match server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Task {
|
|
||||||
id: job.id,
|
|
||||||
})))
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(Some(task)) => Some(TaskDetails { task, info: job }),
|
|
||||||
Ok(None) => {
|
|
||||||
trc::event!(
|
|
||||||
TaskManager(TaskManagerEvent::TaskIgnored),
|
|
||||||
Id = job.id,
|
|
||||||
Reason = "Task not found in store, likely already processed.",
|
|
||||||
);
|
|
||||||
server.remove_index_lock(job.id).await;
|
|
||||||
None
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
trc::error!(
|
|
||||||
err.id(job.id)
|
|
||||||
.details("Failed to retrieve task details.")
|
|
||||||
.caused_by(trc::location!())
|
|
||||||
);
|
|
||||||
server.remove_index_lock(job.id).await;
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: a task panicked: its locks are released so it runs again, here or
|
|
||||||
/// on another node, and the worker carries on.
|
|
||||||
async fn worker_failed(server: &Server, ids: &[u64], err: tokio::task::JoinError) {
|
|
||||||
trc::event!(
|
|
||||||
Server(trc::ServerEvent::ThreadError),
|
|
||||||
Details = "Task worker failed",
|
|
||||||
Reason = err.to_string(),
|
|
||||||
CausedBy = trc::location!()
|
|
||||||
);
|
|
||||||
for id in ids {
|
|
||||||
server.remove_index_lock(*id).await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -756,13 +632,6 @@ async fn update_tasks(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: how long to wait before trying again to claim a task another node
|
|
||||||
/// holds: a twelfth of the lock lifetime, so five minutes for the one-hour
|
|
||||||
/// lock, never more than that and never under a second.
|
|
||||||
pub(crate) fn claim_recheck_interval(lock_expiry: u64) -> u64 {
|
|
||||||
(lock_expiry / 12).clamp(1, CLAIM_RECHECK_INTERVAL)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn perpetual_retry_time(typ: TaskType, attempt: u64) -> Option<u64> {
|
pub fn perpetual_retry_time(typ: TaskType, attempt: u64) -> Option<u64> {
|
||||||
matches!(
|
matches!(
|
||||||
typ,
|
typ,
|
||||||
|
|||||||
@@ -35,9 +35,7 @@ pub mod scheduler;
|
|||||||
pub mod spam_classifier;
|
pub mod spam_classifier;
|
||||||
|
|
||||||
const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes
|
const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes
|
||||||
// inbuxa: the lock lifetime (one hour) lives in common::ipc::TaskLocks, per
|
const DEFAULT_LOCK_EXPIRY: u64 = 60 * 60; // 1 hour
|
||||||
// server, so a graceful stop can release the locks and the tests can shorten it
|
|
||||||
const CLAIM_RECHECK_INTERVAL: u64 = 60 * 5; // 5 minutes
|
|
||||||
|
|
||||||
pub(crate) struct TaskManagerIpc {
|
pub(crate) struct TaskManagerIpc {
|
||||||
txs: [mpsc::Sender<TaskJob>; TaskType::COUNT],
|
txs: [mpsc::Sender<TaskJob>; TaskType::COUNT],
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use common::config::smtp::session::Milter;
|
use common::config::smtp::session::Milter;
|
||||||
@@ -27,19 +25,7 @@ impl MilterClient<TcpStream> {
|
|||||||
pub async fn connect(config: &Milter, session_id: u64) -> Result<Self> {
|
pub async fn connect(config: &Milter, session_id: u64) -> Result<Self> {
|
||||||
tokio::time::timeout(config.timeout_command, async {
|
tokio::time::timeout(config.timeout_command, async {
|
||||||
let mut last_err = Error::Disconnected;
|
let mut last_err = Error::Disconnected;
|
||||||
// inbuxa: a hostname is resolved here, per connection, rather
|
for addr in &config.addrs {
|
||||||
// than while the settings are built
|
|
||||||
let resolved;
|
|
||||||
let addrs = if config.addrs.is_empty() {
|
|
||||||
resolved = tokio::net::lookup_host((config.hostname.as_str(), config.port))
|
|
||||||
.await
|
|
||||||
.map_err(Error::Io)?
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
&resolved
|
|
||||||
} else {
|
|
||||||
&config.addrs
|
|
||||||
};
|
|
||||||
for addr in addrs {
|
|
||||||
match TcpStream::connect(addr).await {
|
match TcpStream::connect(addr).await {
|
||||||
Ok(stream) => {
|
Ok(stream) => {
|
||||||
return Ok(MilterClient {
|
return Ok(MilterClient {
|
||||||
|
|||||||
+1
-10
@@ -44,16 +44,7 @@ impl StartQueueManager for BootManager {
|
|||||||
impl SpawnQueueManager for IpcReceivers {
|
impl SpawnQueueManager for IpcReceivers {
|
||||||
fn spawn_queue_manager(&mut self, inner: Arc<Inner>) {
|
fn spawn_queue_manager(&mut self, inner: Arc<Inner>) {
|
||||||
let core = inner.shared_core.load();
|
let core = inner.shared_core.load();
|
||||||
// inbuxa: upstream started these only when the node's role included
|
if !core.storage.registry.is_recovery_mode() && core.network.roles.outbound_mta {
|
||||||
// outboundMta at boot, so turning the role on later did nothing and
|
|
||||||
// turning it off left them delivering until a restart. They now run
|
|
||||||
// on every node: the queue follows the role live (see Queue::start),
|
|
||||||
// and the report scheduler records DMARC and TLS results on every
|
|
||||||
// node, whatever its role (see reporting/scheduler.rs). This also
|
|
||||||
// drains the queue channel on nodes
|
|
||||||
// without the role, where every queued message's refresh used to sit
|
|
||||||
// in a channel nobody read until it filled and queueing blocked.
|
|
||||||
if !core.storage.registry.is_recovery_mode() {
|
|
||||||
// Spawn queue manager
|
// Spawn queue manager
|
||||||
self.queue_rx.take().unwrap().spawn(inner.clone());
|
self.queue_rx.take().unwrap().spawn(inner.clone());
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{Message, QueueId, Status, spool::SmtpSpool};
|
use super::{Message, QueueId, Status, spool::SmtpSpool};
|
||||||
@@ -41,9 +39,6 @@ pub struct Queue {
|
|||||||
pub urgent_refresh: bool,
|
pub urgent_refresh: bool,
|
||||||
pub last_scan: Instant,
|
pub last_scan: Instant,
|
||||||
pub last_full_scan: Instant,
|
pub last_full_scan: Instant,
|
||||||
/// inbuxa: whether this node's role included outboundMta when last
|
|
||||||
/// checked (None before the first check)
|
|
||||||
pub role_enabled: Option<bool>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -72,9 +67,6 @@ impl SpawnQueue for mpsc::Receiver<QueueEvent> {
|
|||||||
const BACK_PRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(60);
|
const BACK_PRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(60);
|
||||||
const MIN_SCAN_INTERVAL: Duration = Duration::from_millis(100);
|
const MIN_SCAN_INTERVAL: Duration = Duration::from_millis(100);
|
||||||
const FULL_SCAN_INTERVAL: Duration = Duration::from_secs(QUEUE_REFRESH / 2);
|
const FULL_SCAN_INTERVAL: Duration = Duration::from_secs(QUEUE_REFRESH / 2);
|
||||||
/// inbuxa: how often a node without the outbound MTA role looks at its role
|
|
||||||
/// again when nothing else wakes it (a settings reload does)
|
|
||||||
const ROLE_RECHECK_INTERVAL: Duration = Duration::from_secs(30);
|
|
||||||
|
|
||||||
impl Queue {
|
impl Queue {
|
||||||
pub fn new(core: Arc<Inner>, rx: mpsc::Receiver<QueueEvent>) -> Self {
|
pub fn new(core: Arc<Inner>, rx: mpsc::Receiver<QueueEvent>) -> Self {
|
||||||
@@ -95,7 +87,6 @@ impl Queue {
|
|||||||
urgent_refresh: false,
|
urgent_refresh: false,
|
||||||
last_scan: now.checked_sub(MIN_SCAN_INTERVAL).unwrap_or(now),
|
last_scan: now.checked_sub(MIN_SCAN_INTERVAL).unwrap_or(now),
|
||||||
last_full_scan: now,
|
last_full_scan: now,
|
||||||
role_enabled: None,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,27 +123,6 @@ impl Queue {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: follow the node's role live. Without outboundMta the
|
|
||||||
// queue claims nothing new; deliveries already running finish
|
|
||||||
// and report back as usual, releasing their locks. When the role
|
|
||||||
// comes back, the whole queue is scanned at once.
|
|
||||||
let role_enabled = self.core.shared_core.load().network.roles.outbound_mta;
|
|
||||||
if self.role_enabled.replace(role_enabled) == Some(false) && role_enabled {
|
|
||||||
trc::event!(
|
|
||||||
Queue(trc::QueueEvent::Started),
|
|
||||||
Details = "This node's cluster role now includes outboundMta",
|
|
||||||
);
|
|
||||||
self.scan_from = 0;
|
|
||||||
self.pending_refresh = true;
|
|
||||||
self.urgent_refresh = true;
|
|
||||||
}
|
|
||||||
if !role_enabled {
|
|
||||||
self.pending_refresh = false;
|
|
||||||
self.urgent_refresh = false;
|
|
||||||
self.next_refresh = Instant::now() + ROLE_RECHECK_INTERVAL;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
self.pending_refresh |= refresh_queue;
|
self.pending_refresh |= refresh_queue;
|
||||||
if !self.pending_refresh && self.next_refresh > Instant::now() {
|
if !self.pending_refresh && self.next_refresh > Instant::now() {
|
||||||
continue;
|
continue;
|
||||||
|
|||||||
@@ -2,12 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::AggregateTimestamp;
|
use super::AggregateTimestamp;
|
||||||
use super::shared::{MAX_WRITE_RETRIES, Revisioned, write_retry_pause};
|
|
||||||
use crate::{
|
use crate::{
|
||||||
core::Session,
|
core::Session,
|
||||||
queue::RecipientDomain,
|
queue::RecipientDomain,
|
||||||
@@ -352,27 +349,18 @@ impl DmarcReporting for Server {
|
|||||||
let object_id = ObjectType::DmarcInternalReport.to_id();
|
let object_id = ObjectType::DmarcInternalReport.to_id();
|
||||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
||||||
|
|
||||||
// Delete report. inbuxa: only the version read here, so a record
|
let Some(report) = self
|
||||||
// another node appends meanwhile is sent with it rather than lost
|
|
||||||
let mut attempt = 0;
|
|
||||||
let report = loop {
|
|
||||||
let Some(Revisioned {
|
|
||||||
revision,
|
|
||||||
value: report,
|
|
||||||
}) = self
|
|
||||||
.store()
|
.store()
|
||||||
.get_value::<Revisioned<DmarcInternalReport>>(ValueKey::from(key.clone()))
|
.get_value::<DmarcInternalReport>(ValueKey::from(key.clone()))
|
||||||
.await
|
.await
|
||||||
.caused_by(trc::location!())?
|
.caused_by(trc::location!())?
|
||||||
else {
|
else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Delete report
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
batch
|
batch.clear(key).clear(RegistryClass::PrimaryKey {
|
||||||
.assert_value(key.clone(), AssertValue::Hash(revision))
|
|
||||||
.clear(key.clone())
|
|
||||||
.clear(RegistryClass::PrimaryKey {
|
|
||||||
object_id: object_id.into(),
|
object_id: object_id.into(),
|
||||||
index_id: Property::Domain.to_id(),
|
index_id: Property::Domain.to_id(),
|
||||||
key: KeySerializer::new(report.domain.len() + U64_LEN)
|
key: KeySerializer::new(report.domain.len() + U64_LEN)
|
||||||
@@ -380,15 +368,10 @@ impl DmarcReporting for Server {
|
|||||||
.write(report.policy_identifier)
|
.write(report.policy_identifier)
|
||||||
.finalize(),
|
.finalize(),
|
||||||
});
|
});
|
||||||
match self.store().write(batch.build_all()).await {
|
self.store()
|
||||||
Ok(_) => break report,
|
.write(batch.build_all())
|
||||||
Err(err) if err.is_assertion_failure() && attempt < MAX_WRITE_RETRIES => {
|
.await
|
||||||
attempt += 1;
|
.caused_by(trc::location!())?;
|
||||||
write_retry_pause(attempt).await;
|
|
||||||
}
|
|
||||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let span_id = self.inner.data.span_id_gen.generate();
|
let span_id = self.inner.data.span_id_gen.generate();
|
||||||
let event_from = report.report.date_range_begin.timestamp() as u64;
|
let event_from = report.report.date_range_begin.timestamp() as u64;
|
||||||
@@ -693,11 +676,8 @@ impl DmarcReporting for Server {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
// inbuxa: another node appended first; try again
|
if err.is_assertion_failure() && rety_count < 3 {
|
||||||
// after a short pause
|
|
||||||
if err.is_assertion_failure() && rety_count < MAX_WRITE_RETRIES {
|
|
||||||
rety_count += 1;
|
rety_count += 1;
|
||||||
write_retry_pause(rety_count).await;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
trc::error!(
|
trc::error!(
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use common::config::smtp::report::AggregateFrequency;
|
use common::config::smtp::report::AggregateFrequency;
|
||||||
@@ -17,7 +15,6 @@ pub mod inbound;
|
|||||||
pub mod index;
|
pub mod index;
|
||||||
pub mod scheduler;
|
pub mod scheduler;
|
||||||
pub mod send;
|
pub mod send;
|
||||||
pub mod shared; // inbuxa: reports written by every node
|
|
||||||
pub mod spf;
|
pub mod spf;
|
||||||
pub mod tls;
|
pub mod tls;
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{dmarc::DmarcReporting, tls::TlsReporting};
|
use super::{dmarc::DmarcReporting, tls::TlsReporting};
|
||||||
@@ -20,17 +18,6 @@ impl SpawnReport for mpsc::Receiver<ReportingEvent> {
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
while let Some(event) = self.recv().await {
|
while let Some(event) = self.recv().await {
|
||||||
let server = inner.build_server();
|
let server = inner.build_server();
|
||||||
// inbuxa: every node records what it received, whatever its
|
|
||||||
// role. An aggregate report covers all of a domain's mail,
|
|
||||||
// whichever node took it, and recording is a store write
|
|
||||||
// that nodes already share: the report's primary key is
|
|
||||||
// versioned, so concurrent appends from several nodes retry
|
|
||||||
// rather than overwrite. Only building and sending the
|
|
||||||
// report (the DmarcReport and TlsReport tasks) belongs to
|
|
||||||
// the outbound MTA; the task manager keeps those to nodes
|
|
||||||
// with that role. Upstream ran this only on outbound MTA
|
|
||||||
// nodes, so mail received anywhere else never reached a
|
|
||||||
// report.
|
|
||||||
match event {
|
match event {
|
||||||
ReportingEvent::Dmarc(event) => server.schedule_dmarc(event).await,
|
ReportingEvent::Dmarc(event) => server.schedule_dmarc(event).await,
|
||||||
ReportingEvent::Tls(event) => server.schedule_tls(event).await,
|
ReportingEvent::Tls(event) => server.schedule_tls(event).await,
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! inbuxa: internal DMARC and TLS reports are shared by every node. Any node
|
|
||||||
//! that receives mail appends to them, so several nodes can write one report
|
|
||||||
//! at once, and the node that sends it may do so while another is appending.
|
|
||||||
//! Appends already guard the report's versioned primary key and retry when
|
|
||||||
//! another writer got there first; these helpers give those retries room and
|
|
||||||
//! let the sender delete exactly the report it read.
|
|
||||||
|
|
||||||
use rand::RngExt;
|
|
||||||
use std::time::Duration;
|
|
||||||
use store::{Deserialize, xxhash_rust::xxh3::xxh3_64};
|
|
||||||
|
|
||||||
/// How many times a report write that lost to another writer is retried.
|
|
||||||
/// Upstream retried three times, when only outbound MTA nodes wrote.
|
|
||||||
pub(crate) const MAX_WRITE_RETRIES: u32 = 10;
|
|
||||||
|
|
||||||
/// A short random pause, longer on each attempt, before retrying a report
|
|
||||||
/// write that lost to another node, so the writers spread out instead of
|
|
||||||
/// colliding again.
|
|
||||||
pub(crate) async fn write_retry_pause(attempt: u32) {
|
|
||||||
let ms = rand::rng().random_range(5..=25u64) * u64::from(attempt.max(1));
|
|
||||||
tokio::time::sleep(Duration::from_millis(ms)).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A stored value with the hash of the bytes it was read from, for
|
|
||||||
/// `AssertValue::Hash`: a write asserting it fails if anyone changed the
|
|
||||||
/// value since.
|
|
||||||
pub(crate) struct Revisioned<T> {
|
|
||||||
pub revision: u64,
|
|
||||||
pub value: T,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<T: Deserialize> Deserialize for Revisioned<T> {
|
|
||||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
|
||||||
Ok(Revisioned {
|
|
||||||
revision: xxh3_64(bytes),
|
|
||||||
value: T::deserialize(bytes)?,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,12 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::AggregateTimestamp;
|
use super::AggregateTimestamp;
|
||||||
use super::shared::{MAX_WRITE_RETRIES, Revisioned, write_retry_pause};
|
|
||||||
use crate::{
|
use crate::{
|
||||||
queue::RecipientDomain,
|
queue::RecipientDomain,
|
||||||
reporting::{index::InternalReportIndex, send::MtaReportSend},
|
reporting::{index::InternalReportIndex, send::MtaReportSend},
|
||||||
@@ -73,40 +70,28 @@ impl TlsReporting for Server {
|
|||||||
let object_id = ObjectType::TlsInternalReport.to_id();
|
let object_id = ObjectType::TlsInternalReport.to_id();
|
||||||
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
let key = ValueClass::Registry(RegistryClass::Item { object_id, item_id });
|
||||||
|
|
||||||
// Delete report. inbuxa: only the version read here, so a result
|
let Some(report) = self
|
||||||
// another node appends meanwhile is sent with it rather than lost
|
|
||||||
let mut attempt = 0;
|
|
||||||
let report = loop {
|
|
||||||
let Some(Revisioned {
|
|
||||||
revision,
|
|
||||||
value: report,
|
|
||||||
}) = self
|
|
||||||
.store()
|
.store()
|
||||||
.get_value::<Revisioned<TlsInternalReport>>(ValueKey::from(key.clone()))
|
.get_value::<TlsInternalReport>(ValueKey::from(key.clone()))
|
||||||
.await
|
.await
|
||||||
.caused_by(trc::location!())?
|
.caused_by(trc::location!())?
|
||||||
else {
|
else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Delete report
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
batch
|
batch.clear(key).clear(RegistryClass::PrimaryKey {
|
||||||
.assert_value(key.clone(), AssertValue::Hash(revision))
|
|
||||||
.clear(key.clone())
|
|
||||||
.clear(RegistryClass::PrimaryKey {
|
|
||||||
object_id: object_id.into(),
|
object_id: object_id.into(),
|
||||||
index_id: Property::Domain.to_id(),
|
index_id: Property::Domain.to_id(),
|
||||||
key: report.domain.as_bytes().to_vec(),
|
key: report.domain.as_bytes().to_vec(),
|
||||||
});
|
});
|
||||||
match self.core.storage.data.write(batch.build_all()).await {
|
self.core
|
||||||
Ok(_) => break report,
|
.storage
|
||||||
Err(err) if err.is_assertion_failure() && attempt < MAX_WRITE_RETRIES => {
|
.data
|
||||||
attempt += 1;
|
.write(batch.build_all())
|
||||||
write_retry_pause(attempt).await;
|
.await
|
||||||
}
|
.caused_by(trc::location!())?;
|
||||||
Err(err) => return Err(err.caused_by(trc::location!())),
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let domain_name = report.domain.as_str();
|
let domain_name = report.domain.as_str();
|
||||||
let event_from = report.report.date_range_start.timestamp() as u64;
|
let event_from = report.report.date_range_start.timestamp() as u64;
|
||||||
@@ -492,11 +477,8 @@ impl TlsReporting for Server {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
// inbuxa: another node appended first; try again
|
if err.is_assertion_failure() && rety_count < 3 {
|
||||||
// after a short pause
|
|
||||||
if err.is_assertion_failure() && rety_count < MAX_WRITE_RETRIES {
|
|
||||||
rety_count += 1;
|
rety_count += 1;
|
||||||
write_retry_pause(rety_count).await;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
trc::error!(
|
trc::error!(
|
||||||
|
|||||||
@@ -48,21 +48,16 @@ pub(crate) async fn pyzor_check(
|
|||||||
// Send message to address. inbuxa: in tests, a fixed table answers
|
// Send message to address. inbuxa: in tests, a fixed table answers
|
||||||
// instead of a public server (test_response).
|
// instead of a public server (test_response).
|
||||||
#[cfg(not(feature = "test_mode"))]
|
#[cfg(not(feature = "test_mode"))]
|
||||||
let response = match tokio::time::timeout(config.timeout, config.address()).await {
|
let response = pyzor_send_message(config.address, config.timeout, &request).await;
|
||||||
Ok(Ok(address)) => pyzor_send_message(address, config.timeout, &request).await,
|
|
||||||
Ok(Err(err)) => Err(err),
|
|
||||||
Err(_) => Err(std::io::Error::new(
|
|
||||||
std::io::ErrorKind::TimedOut,
|
|
||||||
"Timed out resolving the Pyzor server",
|
|
||||||
)),
|
|
||||||
};
|
|
||||||
#[cfg(feature = "test_mode")]
|
#[cfg(feature = "test_mode")]
|
||||||
let response = std::io::Result::Ok(test_response(&request));
|
let response = std::io::Result::Ok(test_response(&request));
|
||||||
|
|
||||||
response.map(Into::into).map_err(|err| {
|
response
|
||||||
|
.map(Into::into)
|
||||||
|
.map_err(|err| {
|
||||||
trc::SpamEvent::PyzorError
|
trc::SpamEvent::PyzorError
|
||||||
.into_err()
|
.into_err()
|
||||||
.ctx(trc::Key::Url, format!("{}:{}", config.host, config.port))
|
.ctx(trc::Key::Url, config.address.to_string())
|
||||||
.reason(err)
|
.reason(err)
|
||||||
.details("Pyzor failed")
|
.details("Pyzor failed")
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -30,9 +30,6 @@ pub mod s3;
|
|||||||
pub mod sqlite;
|
pub mod sqlite;
|
||||||
// inbuxa: scale-out storage (sharded stores)
|
// inbuxa: scale-out storage (sharded stores)
|
||||||
pub mod scaleout;
|
pub mod scaleout;
|
||||||
// inbuxa: client-side SQL query limits
|
|
||||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
|
||||||
pub mod query_timeout;
|
|
||||||
|
|
||||||
|
|
||||||
pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize;
|
pub const MAX_TOKEN_LENGTH: usize = (u8::MAX >> 1) as usize;
|
||||||
|
|||||||
@@ -2,15 +2,13 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::ops::Range;
|
use std::ops::Range;
|
||||||
|
|
||||||
use mysql_async::prelude::Queryable;
|
use mysql_async::prelude::Queryable;
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, into_error};
|
use super::{MysqlStore, into_error};
|
||||||
|
|
||||||
impl MysqlStore {
|
impl MysqlStore {
|
||||||
pub(crate) async fn get_blob(
|
pub(crate) async fn get_blob(
|
||||||
@@ -18,9 +16,7 @@ impl MysqlStore {
|
|||||||
key: &[u8],
|
key: &[u8],
|
||||||
range: Range<usize>,
|
range: Range<usize>,
|
||||||
) -> trc::Result<Option<Vec<u8>>> {
|
) -> trc::Result<Option<Vec<u8>>> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prep("SELECT v FROM t WHERE k = ?")
|
.prep("SELECT v FROM t WHERE k = ?")
|
||||||
.await
|
.await
|
||||||
@@ -40,15 +36,10 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)")
|
.prep("INSERT INTO t (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)")
|
||||||
.await
|
.await
|
||||||
@@ -57,15 +48,10 @@ impl MysqlStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
.map(|_| ())
|
.map(|_| ())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prep("DELETE FROM t WHERE k = ?")
|
.prep("DELETE FROM t WHERE k = ?")
|
||||||
.await
|
.await
|
||||||
@@ -74,8 +60,5 @@ impl MysqlStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
.map(|hits| hits.affected_rows() > 0)
|
.map(|hits| hits.affected_rows() > 0)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,15 +2,13 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use mysql_async::{Params, Row, prelude::Queryable};
|
use mysql_async::{Params, Row, prelude::Queryable};
|
||||||
|
|
||||||
use crate::{IntoRows, QueryResult, QueryType, Value};
|
use crate::{IntoRows, QueryResult, QueryType, Value};
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, into_error};
|
use super::{MysqlStore, into_error};
|
||||||
|
|
||||||
impl MysqlStore {
|
impl MysqlStore {
|
||||||
pub(crate) async fn sql_query<T: QueryResult>(
|
pub(crate) async fn sql_query<T: QueryResult>(
|
||||||
@@ -18,9 +16,7 @@ impl MysqlStore {
|
|||||||
query: &str,
|
query: &str,
|
||||||
params: &[Value<'_>],
|
params: &[Value<'_>],
|
||||||
) -> trc::Result<T> {
|
) -> trc::Result<T> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prep(query).await.map_err(into_error)?;
|
let s = conn.prep(query).await.map_err(into_error)?;
|
||||||
let params = Params::Positional(params.iter().map(Into::into).collect());
|
let params = Params::Positional(params.iter().map(Into::into).collect());
|
||||||
|
|
||||||
@@ -42,9 +38,6 @@ impl MysqlStore {
|
|||||||
.await
|
.await
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, into_error};
|
use super::{MysqlStore, into_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
backend::mysql::MysqlSearchField,
|
backend::mysql::MysqlSearchField,
|
||||||
search::{
|
search::{
|
||||||
@@ -32,9 +32,6 @@ impl MysqlStore {
|
|||||||
.max_allowed_packet(config.max_allowed_packet.map(|v| v as usize))
|
.max_allowed_packet(config.max_allowed_packet.map(|v| v as usize))
|
||||||
.wait_timeout(config.timeout.map(|t| t.as_secs() as usize))
|
.wait_timeout(config.timeout.map(|t| t.as_secs() as usize))
|
||||||
.client_found_rows(true)
|
.client_found_rows(true)
|
||||||
// inbuxa: notice a server that went away without closing the
|
|
||||||
// connection in minutes, not the system default of two hours
|
|
||||||
.tcp_keepalive(Some(super::POOL_KEEPALIVE_IDLE))
|
|
||||||
.tcp_port(config.port as u16);
|
.tcp_port(config.port as u16);
|
||||||
|
|
||||||
if config.use_tls {
|
if config.use_tls {
|
||||||
@@ -72,7 +69,6 @@ impl MysqlStore {
|
|||||||
.db_name(Some(replica.database.clone()))
|
.db_name(Some(replica.database.clone()))
|
||||||
.tcp_port(replica.port as u16),
|
.tcp_port(replica.port as u16),
|
||||||
),
|
),
|
||||||
timeouts: Default::default(),
|
|
||||||
})),
|
})),
|
||||||
replica.host,
|
replica.host,
|
||||||
replica.port as u16,
|
replica.port as u16,
|
||||||
@@ -82,7 +78,6 @@ impl MysqlStore {
|
|||||||
|
|
||||||
let primary = Store::MySQL(Arc::new(MysqlStore {
|
let primary = Store::MySQL(Arc::new(MysqlStore {
|
||||||
conn_pool: Pool::new(opts),
|
conn_pool: Pool::new(opts),
|
||||||
timeouts: Default::default(),
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// ST-1: no replicas, no change
|
// ST-1: no replicas, no change
|
||||||
@@ -100,9 +95,8 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for table in [
|
for table in [
|
||||||
SUBSPACE_ACL,
|
SUBSPACE_ACL,
|
||||||
SUBSPACE_TASK_QUEUE,
|
SUBSPACE_TASK_QUEUE,
|
||||||
@@ -172,15 +166,11 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
create_search_tables::<EmailSearchField>(&mut conn).await?;
|
create_search_tables::<EmailSearchField>(&mut conn).await?;
|
||||||
create_search_tables::<CalendarSearchField>(&mut conn).await?;
|
create_search_tables::<CalendarSearchField>(&mut conn).await?;
|
||||||
create_search_tables::<ContactSearchField>(&mut conn).await?;
|
create_search_tables::<ContactSearchField>(&mut conn).await?;
|
||||||
@@ -188,9 +178,6 @@ impl MysqlStore {
|
|||||||
create_search_tables::<TracingSearchField>(&mut conn).await?;
|
create_search_tables::<TracingSearchField>(&mut conn).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::backend::query_timeout::QueryTimeouts;
|
|
||||||
use crate::{
|
use crate::{
|
||||||
search::{
|
search::{
|
||||||
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
||||||
@@ -15,7 +14,7 @@ use crate::{
|
|||||||
write::SearchIndex,
|
write::SearchIndex,
|
||||||
};
|
};
|
||||||
use mysql_async::Pool;
|
use mysql_async::Pool;
|
||||||
use std::{fmt::Display, time::Duration};
|
use std::fmt::Display;
|
||||||
|
|
||||||
pub mod blob;
|
pub mod blob;
|
||||||
pub mod lookup;
|
pub mod lookup;
|
||||||
@@ -26,72 +25,6 @@ pub mod write;
|
|||||||
|
|
||||||
pub struct MysqlStore {
|
pub struct MysqlStore {
|
||||||
pub(crate) conn_pool: Pool,
|
pub(crate) conn_pool: Pool,
|
||||||
/// inbuxa: client-side query limits (see backend::query_timeout)
|
|
||||||
pub(crate) timeouts: QueryTimeouts,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: how long a request waits for a pooled connection (including
|
|
||||||
/// opening one). mysql_async's pool has no wait timeout, so upstream waited
|
|
||||||
/// forever when the server stopped answering.
|
|
||||||
pub(crate) const POOL_WAIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
|
|
||||||
/// inbuxa: idle time before TCP keepalive probes start.
|
|
||||||
pub(crate) const POOL_KEEPALIVE_IDLE: std::time::Duration = std::time::Duration::from_secs(60);
|
|
||||||
|
|
||||||
impl MysqlStore {
|
|
||||||
/// inbuxa: a pooled connection, or an error once POOL_WAIT_TIMEOUT has
|
|
||||||
/// passed without one.
|
|
||||||
pub(crate) async fn conn(&self) -> trc::Result<mysql_async::Conn> {
|
|
||||||
pool_conn(&self.conn_pool, POOL_WAIT_TIMEOUT).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn pool_conn(
|
|
||||||
pool: &Pool,
|
|
||||||
wait: std::time::Duration,
|
|
||||||
) -> trc::Result<mysql_async::Conn> {
|
|
||||||
match tokio::time::timeout(wait, pool.get_conn()).await {
|
|
||||||
Ok(result) => result.map_err(into_error),
|
|
||||||
Err(_) => Err(trc::StoreEvent::MysqlError
|
|
||||||
.reason("Timed out waiting for a database connection")
|
|
||||||
.details(format!("No connection within {} s", wait.as_secs()))),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: the error for an operation that ran past its time limit.
|
|
||||||
pub(crate) fn query_timeout_error(limit: Duration) -> trc::Error {
|
|
||||||
trc::StoreEvent::MysqlError
|
|
||||||
.reason("Query timed out")
|
|
||||||
.details(format!(
|
|
||||||
"No answer from the database within {} s",
|
|
||||||
limit.as_secs()
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: ends an operation run on `conn` under `limit`. When it ran out,
|
|
||||||
/// the connection is closed rather than returned to the pool: a query may
|
|
||||||
/// still be in flight on it, or a transaction open. Conn::disconnect marks
|
|
||||||
/// the connection closed before it sends anything, so even when the server
|
|
||||||
/// doesn't answer and the attempt is dropped, the pool discards it instead
|
|
||||||
/// of waiting to clean it up.
|
|
||||||
pub(crate) fn bounded<T>(
|
|
||||||
conn: mysql_async::Conn,
|
|
||||||
result: Result<trc::Result<T>, tokio::time::error::Elapsed>,
|
|
||||||
limit: Duration,
|
|
||||||
) -> trc::Result<T> {
|
|
||||||
match result {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(_) => {
|
|
||||||
discard(conn);
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: closes a connection whose state is unknown (see bounded).
|
|
||||||
pub(crate) fn discard(conn: mysql_async::Conn) {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let _ = tokio::time::timeout(Duration::from_secs(1), conn.disconnect()).await;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{MysqlStore, bounded, discard, into_error, is_timeout_error, query_timeout_error};
|
use super::{MysqlStore, into_error, is_timeout_error};
|
||||||
use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass};
|
use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass};
|
||||||
use futures::TryStreamExt;
|
use futures::TryStreamExt;
|
||||||
use mysql_async::{Row, prelude::Queryable};
|
use mysql_async::{Row, prelude::Queryable};
|
||||||
@@ -16,9 +14,7 @@ impl MysqlStore {
|
|||||||
where
|
where
|
||||||
U: Deserialize + 'static,
|
U: Deserialize + 'static,
|
||||||
{
|
{
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prep(format!(
|
.prep(format!(
|
||||||
"SELECT v FROM {} WHERE k = ?",
|
"SELECT v FROM {} WHERE k = ?",
|
||||||
@@ -37,15 +33,10 @@ impl MysqlStore {
|
|||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prep(format!(
|
.prep(format!(
|
||||||
"SELECT 1 FROM {} WHERE k = ?",
|
"SELECT 1 FROM {} WHERE k = ?",
|
||||||
@@ -58,9 +49,6 @@ impl MysqlStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
.map(|r| r.is_some())
|
.map(|r| r.is_some())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn iterate<T: Key>(
|
pub(crate) async fn iterate<T: Key>(
|
||||||
@@ -68,20 +56,18 @@ impl MysqlStore {
|
|||||||
params: IterateParams<T>,
|
params: IterateParams<T>,
|
||||||
mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result<bool> + Sync + Send,
|
mut cb: impl for<'x> FnMut(&'x [u8], &'x [u8]) -> trc::Result<bool> + Sync + Send,
|
||||||
) -> trc::Result<()> {
|
) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let table = char::from(params.begin.subspace());
|
let table = char::from(params.begin.subspace());
|
||||||
let begin = params.begin.serialize(0);
|
let begin = params.begin.serialize(0);
|
||||||
let end = params.end.serialize(0);
|
let end = params.end.serialize(0);
|
||||||
let keys = if params.values { "k, v" } else { "k" };
|
let keys = if params.values { "k, v" } else { "k" };
|
||||||
|
|
||||||
// inbuxa: a scan may run for hours, so the query limit bounds each
|
let s = conn
|
||||||
// wait for the database (preparing, the query starting, the next
|
.prep(&match (params.first, params.ascending) {
|
||||||
// row) rather than the scan. A wait that runs out closes the
|
|
||||||
// connection.
|
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let query = match (params.first, params.ascending) {
|
|
||||||
(true, true) => {
|
(true, true) => {
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1")
|
format!(
|
||||||
|
"SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k ASC LIMIT 1"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
(true, false) => {
|
(true, false) => {
|
||||||
format!(
|
format!(
|
||||||
@@ -94,16 +80,10 @@ impl MysqlStore {
|
|||||||
(false, false) => {
|
(false, false) => {
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC")
|
format!("SELECT {keys} FROM {table} WHERE k >= ? AND k <= ? ORDER BY k DESC")
|
||||||
}
|
}
|
||||||
};
|
})
|
||||||
let s = match tokio::time::timeout(limit, conn.prep(&query)).await {
|
.await
|
||||||
Ok(s) => s.map_err(into_error)?,
|
.map_err(into_error)?;
|
||||||
Err(_) => {
|
|
||||||
discard(conn);
|
|
||||||
return Err(query_timeout_error(limit));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let mut from = begin;
|
let mut from = begin;
|
||||||
let mut stalled = false;
|
|
||||||
let mut to = end;
|
let mut to = end;
|
||||||
let mut resume_key = None;
|
let mut resume_key = None;
|
||||||
|
|
||||||
@@ -112,26 +92,13 @@ impl MysqlStore {
|
|||||||
let mut timed_out = false;
|
let mut timed_out = false;
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut rows = match tokio::time::timeout(
|
let mut rows = conn
|
||||||
limit,
|
.exec_stream::<Row, _, _>(&s, (from.clone(), to.clone()))
|
||||||
conn.exec_stream::<Row, _, _>(&s, (from.clone(), to.clone())),
|
|
||||||
)
|
|
||||||
.await
|
.await
|
||||||
{
|
.map_err(into_error)?;
|
||||||
Ok(rows) => rows.map_err(into_error)?,
|
|
||||||
// Leaves the scan loop for the timeout below
|
|
||||||
Err(_) => break,
|
|
||||||
};
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let next = match tokio::time::timeout(limit, rows.try_next()).await {
|
match rows.try_next().await {
|
||||||
Ok(next) => next,
|
|
||||||
Err(_) => {
|
|
||||||
stalled = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
match next {
|
|
||||||
Ok(Some(mut row)) => {
|
Ok(Some(mut row)) => {
|
||||||
let value = if params.values {
|
let value = if params.values {
|
||||||
row.take_opt::<Vec<u8>, _>(1)
|
row.take_opt::<Vec<u8>, _>(1)
|
||||||
@@ -167,10 +134,6 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if stalled {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
match last_key {
|
match last_key {
|
||||||
Some(last_key) if timed_out => {
|
Some(last_key) if timed_out => {
|
||||||
if params.ascending {
|
if params.ascending {
|
||||||
@@ -183,9 +146,6 @@ impl MysqlStore {
|
|||||||
_ => return Ok(()),
|
_ => return Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
discard(conn);
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn get_counter(
|
pub(crate) async fn get_counter(
|
||||||
@@ -195,9 +155,7 @@ impl MysqlStore {
|
|||||||
let key = key.into();
|
let key = key.into();
|
||||||
let table = char::from(key.subspace());
|
let table = char::from(key.subspace());
|
||||||
let key = key.serialize(0);
|
let key = key.serialize(0);
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prep(format!("SELECT v FROM {table} WHERE k = ?"))
|
.prep(format!("SELECT v FROM {table} WHERE k = ?"))
|
||||||
.await
|
.await
|
||||||
@@ -207,8 +165,5 @@ impl MysqlStore {
|
|||||||
Ok(None) => Ok(0),
|
Ok(None) => Ok(0),
|
||||||
Err(e) => Err(into_error(e)),
|
Err(e) => Err(into_error(e)),
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,16 +2,14 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
backend::{
|
backend::{
|
||||||
MAX_TOKEN_LENGTH,
|
MAX_TOKEN_LENGTH,
|
||||||
mysql::{
|
mysql::{
|
||||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, bounded,
|
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlSearchField, MysqlStore, into_error,
|
||||||
into_error, is_timeout_error,
|
is_timeout_error,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
search::{
|
search::{
|
||||||
@@ -21,14 +19,12 @@ use crate::{
|
|||||||
write::SearchIndex,
|
write::SearchIndex,
|
||||||
};
|
};
|
||||||
use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable};
|
use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable};
|
||||||
use nlp::{language::Language, tokenizers::word::WordTokenizer};
|
use nlp::tokenizers::word::WordTokenizer;
|
||||||
use std::fmt::Write;
|
use std::fmt::Write;
|
||||||
|
|
||||||
impl MysqlStore {
|
impl MysqlStore {
|
||||||
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let mut tx_opts = TxOpts::default();
|
let mut tx_opts = TxOpts::default();
|
||||||
tx_opts
|
tx_opts
|
||||||
.with_consistent_snapshot(false)
|
.with_consistent_snapshot(false)
|
||||||
@@ -80,9 +76,6 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
trx.commit().await.map_err(into_error)
|
trx.commit().await.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn query<R: SearchDocumentId>(
|
pub async fn query<R: SearchDocumentId>(
|
||||||
@@ -101,18 +94,13 @@ impl MysqlStore {
|
|||||||
build_sort(&mut query, sort);
|
build_sort(&mut query, sort);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prep(query).await.map_err(into_error)?;
|
let s = conn.prep(query).await.map_err(into_error)?;
|
||||||
|
|
||||||
conn.exec::<i64, _, _>(s, params)
|
conn.exec::<i64, _, _>(s, params)
|
||||||
.await
|
.await
|
||||||
.map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect())
|
.map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect())
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||||
@@ -120,9 +108,7 @@ impl MysqlStore {
|
|||||||
let mut query = format!("DELETE FROM {table} ");
|
let mut query = format!("DELETE FROM {table} ");
|
||||||
let params = build_filter(&mut query, &filter.filters);
|
let params = build_filter(&mut query, &filter.filters);
|
||||||
|
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prep(&query).await.map_err(into_error)?;
|
let s = conn.prep(&query).await.map_err(into_error)?;
|
||||||
|
|
||||||
match conn.exec_drop(s, params.clone()).await {
|
match conn.exec_drop(s, params.clone()).await {
|
||||||
@@ -149,9 +135,7 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
deleted += affected;
|
deleted += affected;
|
||||||
}
|
}
|
||||||
Err(err)
|
Err(err) if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE => {
|
||||||
if is_timeout_error(&err) && chunk_size > MIN_DELETE_CHUNK_SIZE =>
|
|
||||||
{
|
|
||||||
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
chunk_size = (chunk_size / 2).max(MIN_DELETE_CHUNK_SIZE);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -159,26 +143,9 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: InnoDB's default full-text stopword list
|
|
||||||
// (INFORMATION_SCHEMA.INNODB_FT_DEFAULT_STOPWORD) and innodb_ft_min_token_size
|
|
||||||
// default; words outside these are not in a FULLTEXT index.
|
|
||||||
const FT_STOPWORDS: &[&str] = &[
|
|
||||||
"a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", "from", "how",
|
|
||||||
"i", "in", "is", "it", "la", "of", "on", "or", "that", "the", "this", "to", "was", "what",
|
|
||||||
"when", "where", "who", "will", "with", "und", "www",
|
|
||||||
];
|
|
||||||
const FT_MIN_TOKEN_SIZE: usize = 3;
|
|
||||||
|
|
||||||
fn is_ft_indexed(word: &str) -> bool {
|
|
||||||
word.chars().count() >= FT_MIN_TOKEN_SIZE && !FT_STOPWORDS.contains(&word)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec<Value> {
|
fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec<Value> {
|
||||||
if filters.is_empty() {
|
if filters.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -204,77 +171,30 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec<Value> {
|
|||||||
|
|
||||||
if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains)
|
if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains)
|
||||||
{
|
{
|
||||||
let (value, mode, unindexed) = match (value, op) {
|
let (value, mode) = match (value, op) {
|
||||||
(SearchValue::Text { value, .. }, SearchOperator::Equal) => (
|
(SearchValue::Text { value, .. }, SearchOperator::Equal) => {
|
||||||
Value::Bytes(format!("{value:?}").into_bytes()),
|
(Value::Bytes(format!("{value:?}").into_bytes()), "BOOLEAN")
|
||||||
"BOOLEAN",
|
}
|
||||||
Vec::new(),
|
(SearchValue::Text { value, .. }, ..) => {
|
||||||
),
|
|
||||||
(SearchValue::Text { value, language }, ..) => {
|
|
||||||
let mut text_query = String::with_capacity(value.len() + 1);
|
let mut text_query = String::with_capacity(value.len() + 1);
|
||||||
let mut unindexed = Vec::new();
|
|
||||||
|
|
||||||
for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) {
|
for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) {
|
||||||
// inbuxa: InnoDB never indexes stopwords ("com",
|
|
||||||
// "de", "www", ...) or words under
|
|
||||||
// innodb_ft_min_token_size, and a required
|
|
||||||
// (+word) term it has not indexed matches no row,
|
|
||||||
// so "example.com" or "[email protected]" found
|
|
||||||
// nothing. Such words are matched with a
|
|
||||||
// word-boundary REGEXP instead.
|
|
||||||
if is_ft_indexed(&item.word) {
|
|
||||||
if !text_query.is_empty() {
|
if !text_query.is_empty() {
|
||||||
text_query.push(' ');
|
text_query.push(' ');
|
||||||
}
|
}
|
||||||
text_query.push('+');
|
text_query.push('+');
|
||||||
text_query.push_str(&item.word);
|
text_query.push_str(&item.word);
|
||||||
} else {
|
|
||||||
unindexed.push(item.word);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// For language text (bodies, subjects) the unindexed
|
(Value::Bytes(text_query.into_bytes()), "BOOLEAN")
|
||||||
// words are noise words and only checked when nothing
|
|
||||||
// else is left to match; keyword text (addresses,
|
|
||||||
// contact fields) checks every word, as the other
|
|
||||||
// backends do.
|
|
||||||
if !text_query.is_empty() && !matches!(language, Language::None) {
|
|
||||||
unindexed.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
(Value::Bytes(text_query.into_bytes()), "BOOLEAN", unindexed)
|
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
debug_assert!(false, "Invalid search value for text field");
|
debug_assert!(false, "Invalid search value for text field");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if unindexed.is_empty() {
|
let _ = write!(query, "MATCH({}) AGAINST(? IN {mode} MODE)", field.column());
|
||||||
let _ =
|
|
||||||
write!(query, "MATCH({}) AGAINST(? IN {mode} MODE)", field.column());
|
|
||||||
values.push(value);
|
values.push(value);
|
||||||
} else {
|
|
||||||
query.push('(');
|
|
||||||
let is_empty = matches!(&value, Value::Bytes(v) if v.is_empty());
|
|
||||||
if !is_empty {
|
|
||||||
let _ = write!(
|
|
||||||
query,
|
|
||||||
"MATCH({}) AGAINST(? IN {mode} MODE) AND ",
|
|
||||||
field.column()
|
|
||||||
);
|
|
||||||
values.push(value);
|
|
||||||
}
|
|
||||||
for (i, word) in unindexed.iter().enumerate() {
|
|
||||||
if i > 0 {
|
|
||||||
query.push_str(" AND ");
|
|
||||||
}
|
|
||||||
let _ = write!(query, "{} REGEXP ?", field.column());
|
|
||||||
values.push(Value::Bytes(
|
|
||||||
format!("(^|[^[:alnum:]]){word}([^[:alnum:]]|$)").into_bytes(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
query.push(')');
|
|
||||||
}
|
|
||||||
} else if let SearchValue::KeyValues(kv) = value {
|
} else if let SearchValue::KeyValues(kv) = value {
|
||||||
let (key, value) = kv.iter().next().unwrap();
|
let (key, value) = kv.iter().next().unwrap();
|
||||||
|
|
||||||
|
|||||||
@@ -2,13 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{
|
use super::{DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, into_error, is_timeout_error};
|
||||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, MysqlStore, bounded, into_error, is_timeout_error,
|
|
||||||
};
|
|
||||||
use crate::{
|
use crate::{
|
||||||
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
||||||
SUBSPACE_REGISTRY_IDX,
|
SUBSPACE_REGISTRY_IDX,
|
||||||
@@ -33,9 +29,8 @@ impl MysqlStore {
|
|||||||
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let mut retry_count = 0;
|
let mut retry_count = 0;
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
loop {
|
loop {
|
||||||
let err = match self.write_trx(&mut conn, &mut batch).await {
|
let err = match self.write_trx(&mut conn, &mut batch).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
@@ -70,9 +65,6 @@ impl MysqlStore {
|
|||||||
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
||||||
retry_count += 1;
|
retry_count += 1;
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn write_trx(
|
async fn write_trx(
|
||||||
@@ -390,23 +382,16 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||||
purge_table(&mut conn, char::from(subspace)).await?;
|
purge_table(&mut conn, char::from(subspace)).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
||||||
let mut conn = self.conn().await?;
|
let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let table = char::from(from.subspace());
|
let table = char::from(from.subspace());
|
||||||
let mut from = from.serialize(0);
|
let mut from = from.serialize(0);
|
||||||
let to = to.serialize(0);
|
let to = to.serialize(0);
|
||||||
@@ -463,9 +448,6 @@ impl MysqlStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,15 +2,13 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::ops::Range;
|
use std::ops::Range;
|
||||||
|
|
||||||
use crate::backend::postgres::into_pool_error;
|
use crate::backend::postgres::into_pool_error;
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error};
|
use super::{PostgresStore, into_error};
|
||||||
|
|
||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub(crate) async fn get_blob(
|
pub(crate) async fn get_blob(
|
||||||
@@ -19,8 +17,6 @@ impl PostgresStore {
|
|||||||
range: Range<usize>,
|
range: Range<usize>,
|
||||||
) -> trc::Result<Option<Vec<u8>>> {
|
) -> trc::Result<Option<Vec<u8>>> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached("SELECT v FROM t WHERE k = $1")
|
.prepare_cached("SELECT v FROM t WHERE k = $1")
|
||||||
.await
|
.await
|
||||||
@@ -43,15 +39,10 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
pub(crate) async fn put_blob(&self, key: &[u8], data: &[u8]) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached(
|
.prepare_cached(
|
||||||
"INSERT INTO t (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v",
|
"INSERT INTO t (k, v) VALUES ($1, $2) ON CONFLICT (k) DO UPDATE SET v = EXCLUDED.v",
|
||||||
@@ -62,15 +53,10 @@ impl PostgresStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
.map(|_| ())
|
.map(|_| ())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
pub(crate) async fn delete_blob(&self, key: &[u8]) -> trc::Result<bool> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached("DELETE FROM t WHERE k = $1")
|
.prepare_cached("DELETE FROM t WHERE k = $1")
|
||||||
.await
|
.await
|
||||||
@@ -79,8 +65,5 @@ impl PostgresStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
.map(|hits| hits > 0)
|
.map(|hits| hits > 0)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{QueryResult, QueryType, backend::postgres::into_pool_error};
|
use crate::{QueryResult, QueryType, backend::postgres::into_pool_error};
|
||||||
@@ -14,7 +12,7 @@ use tokio_postgres::types::{FromSql, ToSql, Type};
|
|||||||
|
|
||||||
use crate::IntoRows;
|
use crate::IntoRows;
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error};
|
use super::{PostgresStore, into_error};
|
||||||
|
|
||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub(crate) async fn sql_query<T: QueryResult>(
|
pub(crate) async fn sql_query<T: QueryResult>(
|
||||||
@@ -23,8 +21,6 @@ impl PostgresStore {
|
|||||||
params_: &[crate::Value<'_>],
|
params_: &[crate::Value<'_>],
|
||||||
) -> trc::Result<T> {
|
) -> trc::Result<T> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prepare_cached(query).await.map_err(into_error)?;
|
let s = conn.prepare_cached(query).await.map_err(into_error)?;
|
||||||
let params = params_
|
let params = params_
|
||||||
.iter()
|
.iter()
|
||||||
@@ -52,9 +48,6 @@ impl PostgresStore {
|
|||||||
.await
|
.await
|
||||||
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
.map_or_else(|e| Err(into_error(e)), |r| Ok(T::from_query_all(r))),
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error};
|
use super::{PostgresStore, into_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
backend::postgres::{
|
backend::postgres::{
|
||||||
PsqlSearchField, into_pool_error,
|
PsqlSearchField, into_pool_error,
|
||||||
@@ -22,34 +22,11 @@ use crate::{
|
|||||||
use ::registry::schema::{enums::PostgreSqlRecyclingMethod, structs};
|
use ::registry::schema::{enums::PostgreSqlRecyclingMethod, structs};
|
||||||
use ahash::AHashSet;
|
use ahash::AHashSet;
|
||||||
use deadpool_postgres::{
|
use deadpool_postgres::{
|
||||||
Config, ManagerConfig, Object, Pool, PoolConfig, RecyclingMethod, Runtime, Timeouts,
|
Config, ManagerConfig, Object, Pool, PoolConfig, RecyclingMethod, Runtime,
|
||||||
};
|
};
|
||||||
use std::time::Duration;
|
|
||||||
use tokio_postgres::NoTls;
|
use tokio_postgres::NoTls;
|
||||||
use utils::tls::rustls_client_config;
|
use utils::tls::rustls_client_config;
|
||||||
|
|
||||||
/// inbuxa: how long a request waits for a pooled connection.
|
|
||||||
pub(crate) const POOL_WAIT_TIMEOUT: Duration = Duration::from_secs(30);
|
|
||||||
/// inbuxa: how long opening a connection may take when the store sets no
|
|
||||||
/// timeout of its own.
|
|
||||||
pub(crate) const POOL_CREATE_TIMEOUT: Duration = Duration::from_secs(15);
|
|
||||||
/// inbuxa: how long checking a pooled connection before reuse may take.
|
|
||||||
pub(crate) const POOL_RECYCLE_TIMEOUT: Duration = Duration::from_secs(10);
|
|
||||||
/// inbuxa: idle time before TCP keepalive probes start.
|
|
||||||
pub(crate) const POOL_KEEPALIVE_IDLE: Duration = Duration::from_secs(60);
|
|
||||||
|
|
||||||
/// inbuxa: the pool's timeouts. Opening a connection is bounded by the
|
|
||||||
/// store's own timeout when it has one; waiting for one covers at least that
|
|
||||||
/// long, so a slow connect isn't cut short by the wait.
|
|
||||||
pub(crate) fn pool_timeouts(connect_timeout: Option<Duration>) -> Timeouts {
|
|
||||||
let create = connect_timeout.unwrap_or(POOL_CREATE_TIMEOUT);
|
|
||||||
Timeouts {
|
|
||||||
wait: POOL_WAIT_TIMEOUT.max(create).into(),
|
|
||||||
create: create.into(),
|
|
||||||
recycle: POOL_RECYCLE_TIMEOUT.into(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub async fn open(config: structs::PostgreSqlStore) -> Result<Store, String> {
|
pub async fn open(config: structs::PostgreSqlStore) -> Result<Store, String> {
|
||||||
// inbuxa: ST-15: where the primary is, to tell a replica from it
|
// inbuxa: ST-15: where the primary is, to tell a replica from it
|
||||||
@@ -69,20 +46,9 @@ impl PostgresStore {
|
|||||||
PostgreSqlRecyclingMethod::Clean => RecyclingMethod::Clean,
|
PostgreSqlRecyclingMethod::Clean => RecyclingMethod::Clean,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
// inbuxa: upstream set no pool timeouts, so a request waited for a
|
if let Some(max_conn) = config.pool_max_connections {
|
||||||
// free connection, or for one to be made or recycled, for as long as
|
cfg.pool = PoolConfig::new(max_conn as usize).into();
|
||||||
// it took: forever when the server stopped answering. A worker now
|
}
|
||||||
// gets an error instead and the task or request is retried.
|
|
||||||
let mut pool = config
|
|
||||||
.pool_max_connections
|
|
||||||
.map(|max_conn| PoolConfig::new(max_conn as usize))
|
|
||||||
.unwrap_or_default();
|
|
||||||
pool.timeouts = pool_timeouts(cfg.connect_timeout);
|
|
||||||
cfg.pool = pool.into();
|
|
||||||
// Notice a server that went away without closing the connection in
|
|
||||||
// minutes rather than the system default of two hours
|
|
||||||
cfg.keepalives = true.into();
|
|
||||||
cfg.keepalives_idle = POOL_KEEPALIVE_IDLE.into();
|
|
||||||
|
|
||||||
let primary_pool = if config.use_tls {
|
let primary_pool = if config.use_tls {
|
||||||
cfg.create_pool(
|
cfg.create_pool(
|
||||||
@@ -119,7 +85,6 @@ impl PostgresStore {
|
|||||||
Store::PostgreSQL(Arc::new(PostgresStore {
|
Store::PostgreSQL(Arc::new(PostgresStore {
|
||||||
conn_pool: pool,
|
conn_pool: pool,
|
||||||
ts_configs: ts_configs.clone(),
|
ts_configs: ts_configs.clone(),
|
||||||
timeouts: Default::default(),
|
|
||||||
})),
|
})),
|
||||||
replica.host,
|
replica.host,
|
||||||
replica.port as u16,
|
replica.port as u16,
|
||||||
@@ -130,7 +95,6 @@ impl PostgresStore {
|
|||||||
let primary = Store::PostgreSQL(Arc::new(PostgresStore {
|
let primary = Store::PostgreSQL(Arc::new(PostgresStore {
|
||||||
conn_pool: primary_pool,
|
conn_pool: primary_pool,
|
||||||
ts_configs,
|
ts_configs,
|
||||||
timeouts: Default::default(),
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// ST-1: no replicas, no change
|
// ST-1: no replicas, no change
|
||||||
@@ -149,8 +113,7 @@ impl PostgresStore {
|
|||||||
|
|
||||||
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_storage_tables(&self) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for table in [
|
for table in [
|
||||||
SUBSPACE_ACL,
|
SUBSPACE_ACL,
|
||||||
SUBSPACE_TASK_QUEUE,
|
SUBSPACE_TASK_QUEUE,
|
||||||
@@ -216,15 +179,11 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
pub(crate) async fn create_search_tables(&self) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
create_search_tables::<EmailSearchField>(&conn).await?;
|
create_search_tables::<EmailSearchField>(&conn).await?;
|
||||||
create_search_tables::<CalendarSearchField>(&conn).await?;
|
create_search_tables::<CalendarSearchField>(&conn).await?;
|
||||||
create_search_tables::<ContactSearchField>(&conn).await?;
|
create_search_tables::<ContactSearchField>(&conn).await?;
|
||||||
@@ -232,9 +191,6 @@ impl PostgresStore {
|
|||||||
create_search_tables::<TracingSearchField>(&conn).await?;
|
create_search_tables::<TracingSearchField>(&conn).await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,21 +231,12 @@ async fn create_search_tables<T: SearchableField + PsqlSearchField + 'static>(
|
|||||||
for field in T::all_fields() {
|
for field in T::all_fields() {
|
||||||
if field.is_text() || field.is_json() {
|
if field.is_text() || field.is_json() {
|
||||||
let column_name = field.column();
|
let column_name = field.column();
|
||||||
// inbuxa: with GIN's default fastupdate=on, new entries wait in
|
|
||||||
// an unindexed pending list that every search scans in full
|
|
||||||
// until a VACUUM (or 4 MB of backlog) merges it. On a mailbox
|
|
||||||
// taking steady mail that list never drains and searches slow
|
|
||||||
// from milliseconds to hundreds of them. Pay the index update
|
|
||||||
// at insert time instead.
|
|
||||||
let index_name = format!("gin_{table_name}_{column_name}");
|
|
||||||
let create_index_query = format!(
|
let create_index_query = format!(
|
||||||
"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} USING GIN({column_name}) WITH (fastupdate = off)",
|
"CREATE INDEX IF NOT EXISTS gin_{table_name}_{column_name} ON {table_name} USING GIN({column_name})",
|
||||||
);
|
);
|
||||||
conn.execute(&create_index_query, &[])
|
conn.execute(&create_index_query, &[])
|
||||||
.await
|
.await
|
||||||
.map_err(into_error)?;
|
.map_err(into_error)?;
|
||||||
// Indexes made before this change keep fastupdate=on
|
|
||||||
disable_gin_fastupdate(conn, &index_name).await;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if field.is_indexed() {
|
if field.is_indexed() {
|
||||||
@@ -306,69 +253,6 @@ async fn create_search_tables<T: SearchableField + PsqlSearchField + 'static>(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: turns fastupdate off on a GIN index made with the default and
|
|
||||||
/// merges the pending list it has built up. Idempotent: an index that already
|
|
||||||
/// has the option is left alone, so this costs one catalog read per index at
|
|
||||||
/// startup. A failure is logged and startup goes on, since search still works,
|
|
||||||
/// only slower.
|
|
||||||
async fn disable_gin_fastupdate(conn: &Object, index_name: &str) {
|
|
||||||
if let Err(err) = try_disable_gin_fastupdate(conn, index_name).await {
|
|
||||||
trc::event!(
|
|
||||||
Store(trc::StoreEvent::PostgresqlError),
|
|
||||||
Details = format!("Failed to turn off fastupdate on search index {index_name}"),
|
|
||||||
Reason = err.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn try_disable_gin_fastupdate(conn: &Object, index_name: &str) -> trc::Result<()> {
|
|
||||||
let options = conn
|
|
||||||
.query_opt(
|
|
||||||
"SELECT COALESCE(reloptions, '{}')::text[] FROM pg_class WHERE oid = to_regclass($1)",
|
|
||||||
&[&index_name],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?
|
|
||||||
.map(|row| row.try_get::<_, Vec<String>>(0))
|
|
||||||
.transpose()
|
|
||||||
.map_err(into_error)?;
|
|
||||||
let Some(options) = options else {
|
|
||||||
return Ok(());
|
|
||||||
};
|
|
||||||
if gin_fastupdate_is_off(&options) {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
// SET (fastupdate) takes a SHARE UPDATE EXCLUSIVE lock, which doesn't
|
|
||||||
// block reads or writes. Turning it off stops new entries going to the
|
|
||||||
// pending list but doesn't flush the entries already there.
|
|
||||||
conn.execute(
|
|
||||||
&format!("ALTER INDEX {index_name} SET (fastupdate = off)"),
|
|
||||||
&[],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
conn.query_one(
|
|
||||||
"SELECT gin_clean_pending_list($1::text::regclass)",
|
|
||||||
&[&index_name],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(into_error)?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether a relation's reloptions turn GIN's fastupdate off.
|
|
||||||
fn gin_fastupdate_is_off(options: &[String]) -> bool {
|
|
||||||
options.iter().any(|option| {
|
|
||||||
option.split_once('=').is_some_and(|(name, value)| {
|
|
||||||
name.trim().eq_ignore_ascii_case("fastupdate")
|
|
||||||
&& matches!(
|
|
||||||
value.trim().to_ascii_lowercase().as_str(),
|
|
||||||
"off" | "false" | "no" | "0" | "f" | "n"
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> {
|
async fn discover_ts_configs(pool: &Pool) -> AHashSet<&'static str> {
|
||||||
let mut ts_configs = AHashSet::from_iter([PG_FALLBACK_LANG, PG_UNSTEMMED_LANG]);
|
let mut ts_configs = AHashSet::from_iter([PG_FALLBACK_LANG, PG_UNSTEMMED_LANG]);
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@
|
|||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::backend::query_timeout::QueryTimeouts;
|
|
||||||
use crate::{
|
use crate::{
|
||||||
search::{
|
search::{
|
||||||
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
CalendarSearchField, ContactSearchField, EmailSearchField, FileSearchField, SearchField,
|
||||||
@@ -15,8 +14,7 @@ use crate::{
|
|||||||
write::SearchIndex,
|
write::SearchIndex,
|
||||||
};
|
};
|
||||||
use ahash::AHashSet;
|
use ahash::AHashSet;
|
||||||
use deadpool_postgres::{Object, Pool};
|
use deadpool_postgres::Pool;
|
||||||
use std::time::Duration;
|
|
||||||
use tokio_postgres::error::SqlState;
|
use tokio_postgres::error::SqlState;
|
||||||
|
|
||||||
pub mod blob;
|
pub mod blob;
|
||||||
@@ -30,8 +28,6 @@ pub mod write;
|
|||||||
pub struct PostgresStore {
|
pub struct PostgresStore {
|
||||||
pub(crate) conn_pool: Pool,
|
pub(crate) conn_pool: Pool,
|
||||||
pub(crate) ts_configs: AHashSet<&'static str>,
|
pub(crate) ts_configs: AHashSet<&'static str>,
|
||||||
/// inbuxa: client-side query limits (see backend::query_timeout)
|
|
||||||
pub(crate) timeouts: QueryTimeouts,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
@@ -76,34 +72,6 @@ pub(crate) fn is_timeout_error(err: &tokio_postgres::Error) -> bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: the error for an operation that ran past its time limit.
|
|
||||||
pub(crate) fn query_timeout_error(limit: Duration) -> trc::Error {
|
|
||||||
trc::StoreEvent::PostgresqlError
|
|
||||||
.reason("Query timed out")
|
|
||||||
.details(format!(
|
|
||||||
"No answer from the database within {} s",
|
|
||||||
limit.as_secs()
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// inbuxa: ends an operation run on `conn` under `limit`. When it ran out,
|
|
||||||
/// the connection is taken out of the pool and closed: a query may still be
|
|
||||||
/// in flight on it, or a transaction open, so it can't be handed to the
|
|
||||||
/// next caller.
|
|
||||||
pub(crate) fn bounded<T>(
|
|
||||||
conn: Object,
|
|
||||||
result: Result<trc::Result<T>, tokio::time::error::Elapsed>,
|
|
||||||
limit: Duration,
|
|
||||||
) -> trc::Result<T> {
|
|
||||||
match result {
|
|
||||||
Ok(result) => result,
|
|
||||||
Err(_) => {
|
|
||||||
drop(Object::take(conn));
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
pub(crate) fn into_pool_error(err: deadpool_postgres::PoolError) -> trc::Error {
|
pub(crate) fn into_pool_error(err: deadpool_postgres::PoolError) -> trc::Error {
|
||||||
match err {
|
match err {
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error, is_timeout_error, query_timeout_error};
|
use super::{PostgresStore, into_error, is_timeout_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
Deserialize, IterateParams, Key, ValueKey, backend::postgres::into_pool_error,
|
Deserialize, IterateParams, Key, ValueKey, backend::postgres::into_pool_error,
|
||||||
write::ValueClass,
|
write::ValueClass,
|
||||||
@@ -19,8 +17,6 @@ impl PostgresStore {
|
|||||||
U: Deserialize + 'static,
|
U: Deserialize + 'static,
|
||||||
{
|
{
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached(&format!(
|
.prepare_cached(&format!(
|
||||||
"SELECT v FROM {} WHERE k = $1",
|
"SELECT v FROM {} WHERE k = $1",
|
||||||
@@ -39,15 +35,10 @@ impl PostgresStore {
|
|||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
pub(crate) async fn key_exists(&self, key: impl Key) -> trc::Result<bool> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached(&format!(
|
.prepare_cached(&format!(
|
||||||
"SELECT 1 FROM {} WHERE k = $1",
|
"SELECT 1 FROM {} WHERE k = $1",
|
||||||
@@ -60,9 +51,6 @@ impl PostgresStore {
|
|||||||
.await
|
.await
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
.map(|r| r.is_some())
|
.map(|r| r.is_some())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn iterate<T: Key>(
|
pub(crate) async fn iterate<T: Key>(
|
||||||
@@ -76,12 +64,8 @@ impl PostgresStore {
|
|||||||
let end = params.end.serialize(0);
|
let end = params.end.serialize(0);
|
||||||
let keys = if params.values { "k, v" } else { "k" };
|
let keys = if params.values { "k, v" } else { "k" };
|
||||||
|
|
||||||
// inbuxa: a scan may run for hours, so the query limit bounds each
|
let s = conn
|
||||||
// wait for the database (preparing, the query starting, the next
|
.prepare_cached(&match (params.first, params.ascending) {
|
||||||
// row) rather than the scan. A wait that runs out closes the
|
|
||||||
// connection.
|
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let query = match (params.first, params.ascending) {
|
|
||||||
(true, true) => {
|
(true, true) => {
|
||||||
format!(
|
format!(
|
||||||
"SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC LIMIT 1"
|
"SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k ASC LIMIT 1"
|
||||||
@@ -98,43 +82,26 @@ impl PostgresStore {
|
|||||||
(false, false) => {
|
(false, false) => {
|
||||||
format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC")
|
format!("SELECT {keys} FROM {table} WHERE k >= $1 AND k <= $2 ORDER BY k DESC")
|
||||||
}
|
}
|
||||||
};
|
})
|
||||||
let s = match tokio::time::timeout(limit, conn.prepare_cached(&query)).await {
|
.await.map_err(into_error)?;
|
||||||
Ok(s) => s.map_err(into_error)?,
|
|
||||||
Err(_) => {
|
|
||||||
drop(deadpool_postgres::Object::take(conn));
|
|
||||||
return Err(query_timeout_error(limit));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let mut from = begin;
|
let mut from = begin;
|
||||||
let mut to = end;
|
let mut to = end;
|
||||||
let mut resume_key: Option<Vec<u8>> = None;
|
let mut resume_key: Option<Vec<u8>> = None;
|
||||||
|
|
||||||
let mut stalled = false;
|
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let mut last_key = None;
|
let mut last_key = None;
|
||||||
let mut timed_out = false;
|
let mut timed_out = false;
|
||||||
|
|
||||||
{
|
{
|
||||||
let rows =
|
let rows = conn
|
||||||
match tokio::time::timeout(limit, conn.query_raw(&s, &[&from, &to])).await {
|
.query_raw(&s, &[&from, &to])
|
||||||
Ok(rows) => rows.map_err(into_error)?,
|
.await
|
||||||
// Leaves the scan loop for the timeout below
|
.map_err(into_error)?;
|
||||||
Err(_) => break,
|
|
||||||
};
|
|
||||||
|
|
||||||
pin_mut!(rows);
|
pin_mut!(rows);
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let next = match tokio::time::timeout(limit, rows.try_next()).await {
|
match rows.try_next().await {
|
||||||
Ok(next) => next,
|
|
||||||
Err(_) => {
|
|
||||||
stalled = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
match next {
|
|
||||||
Ok(Some(row)) => {
|
Ok(Some(row)) => {
|
||||||
let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?;
|
let key = row.try_get::<_, &[u8]>(0).map_err(into_error)?;
|
||||||
let value = if params.values {
|
let value = if params.values {
|
||||||
@@ -165,10 +132,6 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if stalled {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
match last_key {
|
match last_key {
|
||||||
Some(last_key) if timed_out => {
|
Some(last_key) if timed_out => {
|
||||||
if params.ascending {
|
if params.ascending {
|
||||||
@@ -181,9 +144,6 @@ impl PostgresStore {
|
|||||||
_ => return Ok(()),
|
_ => return Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
drop(deadpool_postgres::Object::take(conn));
|
|
||||||
Err(query_timeout_error(limit))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn get_counter(
|
pub(crate) async fn get_counter(
|
||||||
@@ -195,8 +155,6 @@ impl PostgresStore {
|
|||||||
let key = key.serialize(0);
|
let key = key.serialize(0);
|
||||||
|
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached(&format!("SELECT v FROM {table} WHERE k = $1"))
|
.prepare_cached(&format!("SELECT v FROM {table} WHERE k = $1"))
|
||||||
.await
|
.await
|
||||||
@@ -206,8 +164,5 @@ impl PostgresStore {
|
|||||||
Ok(None) => Ok(0),
|
Ok(None) => Ok(0),
|
||||||
Err(e) => Err(into_error(e)),
|
Err(e) => Err(into_error(e)),
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,17 +2,12 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
backend::{
|
backend::postgres::{
|
||||||
MAX_TOKEN_LENGTH,
|
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, PostgresStore, PsqlSearchField, into_error,
|
||||||
postgres::{
|
into_pool_error, is_timeout_error,
|
||||||
DELETE_CHUNK_SIZE, MIN_DELETE_CHUNK_SIZE, PostgresStore, PsqlSearchField, bounded,
|
|
||||||
into_error, into_pool_error, is_timeout_error,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
search::{
|
search::{
|
||||||
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
|
IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator,
|
||||||
@@ -20,7 +15,7 @@ use crate::{
|
|||||||
},
|
},
|
||||||
write::SearchIndex,
|
write::SearchIndex,
|
||||||
};
|
};
|
||||||
use nlp::{language::Language, tokenizers::space::SpaceTokenizer};
|
use nlp::language::Language;
|
||||||
use std::fmt::Write;
|
use std::fmt::Write;
|
||||||
use tokio_postgres::{
|
use tokio_postgres::{
|
||||||
IsolationLevel,
|
IsolationLevel,
|
||||||
@@ -36,8 +31,6 @@ impl PostgresStore {
|
|||||||
|
|
||||||
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
pub async fn index(&self, documents: Vec<IndexDocument>) -> trc::Result<()> {
|
||||||
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let trx = conn
|
let trx = conn
|
||||||
.build_transaction()
|
.build_transaction()
|
||||||
.isolation_level(IsolationLevel::ReadCommitted)
|
.isolation_level(IsolationLevel::ReadCommitted)
|
||||||
@@ -50,24 +43,6 @@ impl PostgresStore {
|
|||||||
let primary_keys = index.primary_keys();
|
let primary_keys = index.primary_keys();
|
||||||
let all_fields = index.all_fields();
|
let all_fields = index.all_fields();
|
||||||
let fields = document.fields;
|
let fields = document.fields;
|
||||||
// inbuxa: keyword text (addresses, contact fields, ...) is split into
|
|
||||||
// words before it reaches the text parser, see keyword_terms();
|
|
||||||
// language text gets the words inside its URLs, host names and
|
|
||||||
// file names added, see url_terms().
|
|
||||||
let keywords = primary_keys
|
|
||||||
.iter()
|
|
||||||
.chain(all_fields)
|
|
||||||
.map(|field| match fields.get(field) {
|
|
||||||
Some(SearchValue::Text {
|
|
||||||
value,
|
|
||||||
language: Language::None,
|
|
||||||
}) if field.is_text() => Some(keyword_terms(value)),
|
|
||||||
Some(SearchValue::Text { value, .. }) if field.is_text() => {
|
|
||||||
url_terms(value)
|
|
||||||
}
|
|
||||||
_ => None,
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let mut values = Vec::with_capacity(fields.len() + 2);
|
let mut values = Vec::with_capacity(fields.len() + 2);
|
||||||
let mut query = format!("INSERT INTO {} (", index.psql_table());
|
let mut query = format!("INSERT INTO {} (", index.psql_table());
|
||||||
|
|
||||||
@@ -92,27 +67,14 @@ impl PostgresStore {
|
|||||||
|
|
||||||
if let Some(value) = fields.get(field) {
|
if let Some(value) = fields.get(field) {
|
||||||
let value_ref = format!("${}", values.len() + 1);
|
let value_ref = format!("${}", values.len() + 1);
|
||||||
let (text_len, language) =
|
let (text_len, language) = if let SearchValue::Text { value, language } = value
|
||||||
if let SearchValue::Text { value, language } = value {
|
{
|
||||||
(value.len(), self.ts_config(language))
|
(value.len(), self.ts_config(language))
|
||||||
} else {
|
} else {
|
||||||
(0, PG_UNSTEMMED_LANG)
|
(0, PG_UNSTEMMED_LANG)
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(keywords) = &keywords[i] {
|
if field.is_text() {
|
||||||
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
|
||||||
values.push(keywords as &(dyn ToSql + Sync));
|
|
||||||
if field.sort_column().is_some() {
|
|
||||||
let value_ref = format!("${}", values.len() + 1);
|
|
||||||
if text_len > 255 {
|
|
||||||
let _ = write!(&mut query, ",left({value_ref},255)");
|
|
||||||
} else {
|
|
||||||
let _ = write!(&mut query, ",{value_ref}");
|
|
||||||
}
|
|
||||||
values.push(value as &(dyn ToSql + Sync));
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
} else if field.is_text() {
|
|
||||||
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})");
|
||||||
} else if text_len > 512 {
|
} else if text_len > 512 {
|
||||||
query.push_str("left(");
|
query.push_str("left(");
|
||||||
@@ -162,9 +124,6 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
trx.commit().await.map_err(into_error)
|
trx.commit().await.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn query<R: SearchDocumentId>(
|
pub async fn query<R: SearchDocumentId>(
|
||||||
@@ -175,13 +134,10 @@ impl PostgresStore {
|
|||||||
) -> trc::Result<Vec<R>> {
|
) -> trc::Result<Vec<R>> {
|
||||||
let mut query = format!("SELECT {} FROM {}", R::field().column(), index.psql_table());
|
let mut query = format!("SELECT {} FROM {}", R::field().column(), index.psql_table());
|
||||||
let params = self.build_filter(&mut query, filters);
|
let params = self.build_filter(&mut query, filters);
|
||||||
let params = params.iter().map(SqlParam::as_sql).collect::<Vec<_>>();
|
|
||||||
if !sort.is_empty() {
|
if !sort.is_empty() {
|
||||||
build_sort(&mut query, sort);
|
build_sort(&mut query, sort);
|
||||||
}
|
}
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn.prepare_cached(&query).await.map_err(into_error)?;
|
let s = conn.prepare_cached(&query).await.map_err(into_error)?;
|
||||||
|
|
||||||
conn.query(&s, params.as_slice())
|
conn.query(&s, params.as_slice())
|
||||||
@@ -192,9 +148,6 @@ impl PostgresStore {
|
|||||||
.collect::<Result<Vec<R>, _>>()
|
.collect::<Result<Vec<R>, _>>()
|
||||||
})
|
})
|
||||||
.map_err(into_error)
|
.map_err(into_error)
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
|
||||||
@@ -202,10 +155,7 @@ impl PostgresStore {
|
|||||||
let table = filter.index.psql_table();
|
let table = filter.index.psql_table();
|
||||||
let mut where_clause = String::new();
|
let mut where_clause = String::new();
|
||||||
let params = self.build_filter(&mut where_clause, &filter.filters);
|
let params = self.build_filter(&mut where_clause, &filter.filters);
|
||||||
let params = params.iter().map(SqlParam::as_sql).collect::<Vec<_>>();
|
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let s = conn
|
let s = conn
|
||||||
.prepare_cached(&format!("DELETE FROM {table}{where_clause}"))
|
.prepare_cached(&format!("DELETE FROM {table}{where_clause}"))
|
||||||
.await
|
.await
|
||||||
@@ -240,16 +190,13 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn build_filter<'x>(
|
fn build_filter<'x>(
|
||||||
&self,
|
&self,
|
||||||
query: &mut String,
|
query: &mut String,
|
||||||
filters: &'x [SearchFilter],
|
filters: &'x [SearchFilter],
|
||||||
) -> Vec<SqlParam<'x>> {
|
) -> Vec<&'x (dyn ToSql + Sync)> {
|
||||||
if filters.is_empty() {
|
if filters.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
@@ -290,54 +237,28 @@ impl PostgresStore {
|
|||||||
|
|
||||||
if matches!(language, Language::None) {
|
if matches!(language, Language::None) {
|
||||||
let _ = write!(query, "@@ {method}('{config}', ${value_pos})");
|
let _ = write!(query, "@@ {method}('{config}', ${value_pos})");
|
||||||
if let SearchValue::Text { value, .. } = value {
|
|
||||||
values.push(SqlParam::Owned(keyword_terms(value)));
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// inbuxa: a query word written as a URL, host,
|
|
||||||
// file or hyphenated word also matches as its word
|
|
||||||
// parts, which url_terms() indexes
|
|
||||||
let parts = match value {
|
|
||||||
SearchValue::Text { value, .. } => query_url_terms(value),
|
|
||||||
_ => None,
|
|
||||||
};
|
|
||||||
let parts_pos = value_pos + 1;
|
|
||||||
let _ = write!(query, "@@ ({method}('{config}', ${value_pos})");
|
let _ = write!(query, "@@ ({method}('{config}', ${value_pos})");
|
||||||
if parts.is_some() {
|
|
||||||
let _ = write!(query, " || {method}('{config}', ${parts_pos})");
|
|
||||||
}
|
|
||||||
for fallback in [PG_FALLBACK_LANG, PG_UNSTEMMED_LANG] {
|
for fallback in [PG_FALLBACK_LANG, PG_UNSTEMMED_LANG] {
|
||||||
if fallback != config && self.ts_configs.contains(fallback) {
|
if fallback != config && self.ts_configs.contains(fallback) {
|
||||||
let _ =
|
let _ =
|
||||||
write!(query, " || {method}('{fallback}', ${value_pos})");
|
write!(query, " || {method}('{fallback}', ${value_pos})");
|
||||||
if parts.is_some() {
|
|
||||||
let _ = write!(
|
|
||||||
query,
|
|
||||||
" || {method}('{fallback}', ${parts_pos})"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
query.push(')');
|
query.push(')');
|
||||||
values.push(SqlParam::Ref(value));
|
|
||||||
if let Some(parts) = parts {
|
|
||||||
values.push(SqlParam::Owned(parts));
|
|
||||||
}
|
}
|
||||||
continue;
|
values.push(value as &(dyn ToSql + Sync));
|
||||||
}
|
|
||||||
values.push(SqlParam::Ref(value));
|
|
||||||
} else if let SearchValue::KeyValues(kv) = value {
|
} else if let SearchValue::KeyValues(kv) = value {
|
||||||
query.push_str(field.column());
|
query.push_str(field.column());
|
||||||
query.push(' ');
|
query.push(' ');
|
||||||
|
|
||||||
let (key, value) = kv.iter().next().unwrap();
|
let (key, value) = kv.iter().next().unwrap();
|
||||||
values.push(SqlParam::Ref(key));
|
values.push(key as &(dyn ToSql + Sync));
|
||||||
|
|
||||||
if !value.is_empty() {
|
if !value.is_empty() {
|
||||||
let _ = write!(query, "->> ${value_pos} ");
|
let _ = write!(query, "->> ${value_pos} ");
|
||||||
op.write_pqsql(query, values.len() + 1);
|
op.write_pqsql(query, values.len() + 1);
|
||||||
values.push(SqlParam::Ref(value));
|
values.push(value as &(dyn ToSql + Sync));
|
||||||
} else {
|
} else {
|
||||||
let _ = write!(query, " ? ${value_pos}");
|
let _ = write!(query, " ? ${value_pos}");
|
||||||
}
|
}
|
||||||
@@ -346,7 +267,7 @@ impl PostgresStore {
|
|||||||
query.push(' ');
|
query.push(' ');
|
||||||
|
|
||||||
op.write_pqsql(query, value_pos);
|
op.write_pqsql(query, value_pos);
|
||||||
values.push(SqlParam::Ref(value));
|
values.push(value as &(dyn ToSql + Sync));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
SearchFilter::And | SearchFilter::Or => {
|
SearchFilter::And | SearchFilter::Or => {
|
||||||
@@ -400,107 +321,6 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: PostgreSQL's text parser keeps "[email protected]" (and host names,
|
|
||||||
// URLs, file paths, ...) as a single token, so a search for "user" or
|
|
||||||
// "example.com" never matched an address. Keyword text is split into words the
|
|
||||||
// same way the built-in index splits it (SpaceTokenizer: lowercase runs of
|
|
||||||
// alphanumerics) on both the indexing and the query side, so a full address,
|
|
||||||
// its local part, its domain and the display-name words all match, as they do
|
|
||||||
// on the other backends.
|
|
||||||
pub(crate) fn keyword_terms(value: &str) -> String {
|
|
||||||
let mut terms = String::with_capacity(value.len());
|
|
||||||
for token in SpaceTokenizer::new(value, MAX_TOKEN_LENGTH) {
|
|
||||||
if !terms.is_empty() {
|
|
||||||
terms.push(' ');
|
|
||||||
}
|
|
||||||
terms.push_str(&token);
|
|
||||||
}
|
|
||||||
terms
|
|
||||||
}
|
|
||||||
|
|
||||||
// inbuxa: in language text (subject, body, attachments) PostgreSQL's parser
|
|
||||||
// keeps a URL, a host name, a path or a file name as tokens of its own:
|
|
||||||
// "https://x.example/shipping-support/" gives a url, a host and a url_path,
|
|
||||||
// "invoice-2024.pdf" a file, so a body search for "shipping" or "invoice"
|
|
||||||
// missed messages where the word appears only there, while the built-in index
|
|
||||||
// splits them into words. The text is indexed as it was, followed by the word
|
|
||||||
// parts of each such token (SpaceTokenizer, as keyword_terms() splits), so
|
|
||||||
// they go through the same configuration and stemming as the words around
|
|
||||||
// them. On sample mail the text vector grows by about 15% for a newsletter
|
|
||||||
// full of tracking links and 30% for a short order notice with three links.
|
|
||||||
// Plain words, and words that only carry punctuation ("end.", "(see"),
|
|
||||||
// add nothing; hyphenated words are already split by the parser. Returns None
|
|
||||||
// when there is nothing to add, so most text is indexed exactly as before.
|
|
||||||
/// Characters that join the parts of a URL, host, path, address or file name.
|
|
||||||
const URL_SEPARATORS: [char; 13] = [
|
|
||||||
'/', '.', '@', ':', '?', '=', '&', '#', '_', '%', '+', '~', '\\',
|
|
||||||
];
|
|
||||||
|
|
||||||
pub(crate) fn url_terms(value: &str) -> Option<String> {
|
|
||||||
let mut terms = String::new();
|
|
||||||
// Each word is added once: a phrase search still finds the first URL it
|
|
||||||
// is in, and a newsletter's hundred tracking links don't add a hundred
|
|
||||||
// positions for "utm" and "campaign"
|
|
||||||
let mut seen = std::collections::HashSet::new();
|
|
||||||
for token in value.split(|c: char| {
|
|
||||||
c.is_whitespace() || matches!(c, '<' | '>' | '"' | '(' | ')' | '[' | ']' | '{' | '}')
|
|
||||||
}) {
|
|
||||||
let token = token.trim_matches(|c: char| !c.is_alphanumeric());
|
|
||||||
if token.contains(URL_SEPARATORS) {
|
|
||||||
for word in SpaceTokenizer::new(token, MAX_TOKEN_LENGTH) {
|
|
||||||
if !seen.insert(word.clone()) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if terms.is_empty() {
|
|
||||||
terms.reserve(value.len() + 64);
|
|
||||||
terms.push_str(value);
|
|
||||||
terms.push('\n');
|
|
||||||
} else {
|
|
||||||
terms.push(' ');
|
|
||||||
}
|
|
||||||
terms.push_str(&word);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
(!terms.is_empty()).then_some(terms)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The query side of url_terms(): each query word that is a URL, host, file
|
|
||||||
/// name or hyphenated word replaced by its word parts, or None when there is
|
|
||||||
/// none. It is searched in addition to the query as written, so documents
|
|
||||||
/// indexed before url_terms() still match as they did.
|
|
||||||
pub(crate) fn query_url_terms(value: &str) -> Option<String> {
|
|
||||||
let mut terms = String::with_capacity(value.len());
|
|
||||||
let mut changed = false;
|
|
||||||
for token in value.split_whitespace() {
|
|
||||||
let word = token.trim_matches(|c: char| !c.is_alphanumeric());
|
|
||||||
if !terms.is_empty() {
|
|
||||||
terms.push(' ');
|
|
||||||
}
|
|
||||||
if word.contains(URL_SEPARATORS) || word.contains('-') {
|
|
||||||
changed = true;
|
|
||||||
terms.push_str(&keyword_terms(word));
|
|
||||||
} else {
|
|
||||||
terms.push_str(token);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
changed.then_some(terms)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) enum SqlParam<'x> {
|
|
||||||
Ref(&'x (dyn ToSql + Sync)),
|
|
||||||
Owned(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SqlParam<'_> {
|
|
||||||
fn as_sql(&self) -> &(dyn ToSql + Sync) {
|
|
||||||
match self {
|
|
||||||
SqlParam::Ref(value) => *value,
|
|
||||||
SqlParam::Owned(value) => value,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_sort(query: &mut String, sort: &[SearchComparator]) {
|
fn build_sort(query: &mut String, sort: &[SearchComparator]) {
|
||||||
query.push_str(" ORDER BY ");
|
query.push_str(" ORDER BY ");
|
||||||
for (i, comparator) in sort.iter().enumerate() {
|
for (i, comparator) in sort.iter().enumerate() {
|
||||||
|
|||||||
@@ -2,11 +2,9 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{PostgresStore, bounded, into_error, is_timeout_error};
|
use super::{PostgresStore, into_error, is_timeout_error};
|
||||||
use crate::{
|
use crate::{
|
||||||
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA,
|
||||||
SUBSPACE_REGISTRY_IDX,
|
SUBSPACE_REGISTRY_IDX,
|
||||||
@@ -32,8 +30,6 @@ enum CommitError {
|
|||||||
impl PostgresStore {
|
impl PostgresStore {
|
||||||
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result<AssignedIds> {
|
||||||
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let mut conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.query;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let mut retry_count = 0;
|
let mut retry_count = 0;
|
||||||
|
|
||||||
@@ -76,9 +72,6 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn write_trx(
|
async fn write_trx(
|
||||||
@@ -400,22 +393,16 @@ impl PostgresStore {
|
|||||||
|
|
||||||
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
pub(crate) async fn purge_store(&self) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
for subspace in [SUBSPACE_QUOTA, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER] {
|
||||||
purge_table(&conn, char::from(subspace)).await?;
|
purge_table(&conn, char::from(subspace)).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
pub(crate) async fn delete_range(&self, from: impl Key, to: impl Key) -> trc::Result<()> {
|
||||||
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
let conn = self.conn_pool.get().await.map_err(into_pool_error)?;
|
||||||
let limit = self.timeouts.maintenance;
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
let table = char::from(from.subspace());
|
let table = char::from(from.subspace());
|
||||||
let mut from = from.serialize(0);
|
let mut from = from.serialize(0);
|
||||||
let to = to.serialize(0);
|
let to = to.serialize(0);
|
||||||
@@ -472,9 +459,6 @@ impl PostgresStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
|
||||||
.await;
|
|
||||||
bounded(conn, result, limit)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Client-side limits on SQL queries.
|
|
||||||
//!
|
|
||||||
//! The pool timeouts bound getting a connection, not using one. A database
|
|
||||||
//! that stops answering while the TCP connection stays up (a paused
|
|
||||||
//! container, a hung server whose kernel still acknowledges keepalives)
|
|
||||||
//! left a query on a checked-out connection waiting for as long as it took.
|
|
||||||
//! A server-side statement_timeout can't help there: the server that would
|
|
||||||
//! enforce it is the one not answering. So each operation on a PostgreSQL
|
|
||||||
//! or MySQL connection runs under a time limit here, and a connection whose
|
|
||||||
//! operation ran out is closed rather than put back in the pool, since its
|
|
||||||
//! protocol state is unknown.
|
|
||||||
//!
|
|
||||||
//! Two limits:
|
|
||||||
//! - `query`, two minutes, for request-path work: reads, writes, blob
|
|
||||||
//! transfers, search queries and document indexing. Those take
|
|
||||||
//! milliseconds; two minutes leaves room for a large blob over a slow
|
|
||||||
//! link and still ends a hang.
|
|
||||||
//! - `maintenance`, thirty minutes, for work that legitimately runs long in
|
|
||||||
//! one statement: range deletes (account removal, purges), unindexing,
|
|
||||||
//! and creating tables and indexes at startup.
|
|
||||||
//!
|
|
||||||
//! Iterating over a range (exports, reindexing, maintenance scans) can run
|
|
||||||
//! for hours, so there the `query` limit applies to each wait for the next
|
|
||||||
//! row instead of the whole scan.
|
|
||||||
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub struct QueryTimeouts {
|
|
||||||
pub query: Duration,
|
|
||||||
pub maintenance: Duration,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl QueryTimeouts {
|
|
||||||
pub const QUERY: Duration = Duration::from_secs(120);
|
|
||||||
pub const MAINTENANCE: Duration = Duration::from_secs(30 * 60);
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for QueryTimeouts {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self {
|
|
||||||
query: Self::QUERY,
|
|
||||||
maintenance: Self::MAINTENANCE,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "test_mode")]
|
|
||||||
impl crate::Store {
|
|
||||||
/// Sets the query limits of a SQL store that was just built (tests only:
|
|
||||||
/// the limits aren't configurable).
|
|
||||||
pub fn with_query_timeouts(self, timeouts: QueryTimeouts) -> Self {
|
|
||||||
match self {
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
crate::Store::PostgreSQL(mut store) => {
|
|
||||||
std::sync::Arc::get_mut(&mut store)
|
|
||||||
.expect("store already shared")
|
|
||||||
.timeouts = timeouts;
|
|
||||||
crate::Store::PostgreSQL(store)
|
|
||||||
}
|
|
||||||
#[cfg(feature = "mysql")]
|
|
||||||
crate::Store::MySQL(mut store) => {
|
|
||||||
std::sync::Arc::get_mut(&mut store)
|
|
||||||
.expect("store already shared")
|
|
||||||
.timeouts = timeouts;
|
|
||||||
crate::Store::MySQL(store)
|
|
||||||
}
|
|
||||||
store => store,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use super::{RedisPool, RedisStore, into_error};
|
use super::{RedisPool, RedisStore, into_error};
|
||||||
@@ -81,30 +79,6 @@ impl RedisStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: see InMemoryStore::renew_lock
|
|
||||||
pub async fn renew_lock(&self, key: &[u8], expires: u64) -> trc::Result<bool> {
|
|
||||||
match &self.pool {
|
|
||||||
RedisPool::Single(pool) => {
|
|
||||||
with_conn(pool, async |conn| {
|
|
||||||
Self::renew_lock_(conn, key, expires).await
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
RedisPool::Cluster(pool) => {
|
|
||||||
with_conn(pool, async |conn| {
|
|
||||||
Self::renew_lock_(conn, key, expires).await
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
RedisPool::Sentinel(pool) => {
|
|
||||||
with_conn(pool, async |conn| {
|
|
||||||
Self::renew_lock_(conn, key, expires).await
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn key_delete(&self, key: &[u8]) -> trc::Result<()> {
|
pub async fn key_delete(&self, key: &[u8]) -> trc::Result<()> {
|
||||||
match &self.pool {
|
match &self.pool {
|
||||||
RedisPool::Single(pool) => {
|
RedisPool::Single(pool) => {
|
||||||
@@ -252,22 +226,6 @@ impl RedisStore {
|
|||||||
.map(|reply| reply.is_some())
|
.map(|reply| reply.is_some())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn renew_lock_(
|
|
||||||
conn: &mut impl AsyncCommands,
|
|
||||||
key: &[u8],
|
|
||||||
expires: u64,
|
|
||||||
) -> RedisResult<bool> {
|
|
||||||
redis::cmd("SET")
|
|
||||||
.arg(key)
|
|
||||||
.arg(now() + expires)
|
|
||||||
.arg("XX")
|
|
||||||
.arg("EX")
|
|
||||||
.arg(expires as i64)
|
|
||||||
.query_async::<Option<String>>(conn)
|
|
||||||
.await
|
|
||||||
.map(|reply| reply.is_some())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn key_delete_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<()> {
|
async fn key_delete_(conn: &mut impl AsyncCommands, key: &[u8]) -> RedisResult<()> {
|
||||||
conn.del(key).await
|
conn.del(key).await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
@@ -25,14 +23,6 @@ use utils::snowflake::MAX_NODE_ID;
|
|||||||
|
|
||||||
const STALE_NODE_TIMEOUT: u64 = 60 * 60; // 1 hour
|
const STALE_NODE_TIMEOUT: u64 = 60 * 60; // 1 hour
|
||||||
const DEAD_NODE_TIMEOUT: u64 = 60 * 60 * 24; // 24 hours
|
const DEAD_NODE_TIMEOUT: u64 = 60 * 60 * 24; // 24 hours
|
||||||
|
|
||||||
// INBUXA: every node renews its lease once a minute, so the lease doubles as
|
|
||||||
// a heartbeat. A node not heard from in three minutes is reported Stale, which
|
|
||||||
// is what Cluster Health on the dashboard counts. Taking over a lease still
|
|
||||||
// needs the full hour of silence, so a node that is slow rather than gone
|
|
||||||
// never loses its id to another host.
|
|
||||||
const HEARTBEAT_INTERVAL: u64 = 60; // 1 minute
|
|
||||||
const UNRESPONSIVE_NODE_TIMEOUT: u64 = 3 * HEARTBEAT_INTERVAL;
|
|
||||||
const MAX_LEASE_RETRIES: u32 = 5;
|
const MAX_LEASE_RETRIES: u32 = 5;
|
||||||
|
|
||||||
struct NodeSlot {
|
struct NodeSlot {
|
||||||
@@ -106,7 +96,7 @@ impl RegistryStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn refresh_node_id_interval(&self) -> Duration {
|
pub fn refresh_node_id_interval(&self) -> Duration {
|
||||||
Duration::from_secs(HEARTBEAT_INTERVAL)
|
Duration::from_secs(STALE_NODE_TIMEOUT / 2)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn cluster_node_list(&self) -> trc::Result<Vec<ClusterNode>> {
|
pub async fn cluster_node_list(&self) -> trc::Result<Vec<ClusterNode>> {
|
||||||
@@ -299,10 +289,6 @@ impl NodeSlot {
|
|||||||
self.elapsed > DEAD_NODE_TIMEOUT
|
self.elapsed > DEAD_NODE_TIMEOUT
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_responsive(&self) -> bool {
|
|
||||||
self.elapsed <= UNRESPONSIVE_NODE_TIMEOUT
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_assignable(&self) -> bool {
|
fn is_assignable(&self) -> bool {
|
||||||
self.node_id <= MAX_NODE_ID
|
self.node_id <= MAX_NODE_ID
|
||||||
}
|
}
|
||||||
@@ -310,10 +296,10 @@ impl NodeSlot {
|
|||||||
fn status(&self) -> ClusterNodeStatus {
|
fn status(&self) -> ClusterNodeStatus {
|
||||||
if self.is_dead() {
|
if self.is_dead() {
|
||||||
ClusterNodeStatus::Inactive
|
ClusterNodeStatus::Inactive
|
||||||
} else if self.is_responsive() {
|
} else if self.is_stale() {
|
||||||
ClusterNodeStatus::Active
|
|
||||||
} else {
|
|
||||||
ClusterNodeStatus::Stale
|
ClusterNodeStatus::Stale
|
||||||
|
} else {
|
||||||
|
ClusterNodeStatus::Active
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -328,42 +314,3 @@ impl From<NodeSlot> for ClusterNode {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
fn slot(elapsed: u64) -> NodeSlot {
|
|
||||||
NodeSlot {
|
|
||||||
node_id: 1,
|
|
||||||
hostname: "mx2.example.org".into(),
|
|
||||||
last_renewal: 0,
|
|
||||||
elapsed,
|
|
||||||
hash: 0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn status_follows_the_heartbeat() {
|
|
||||||
assert_eq!(slot(0).status(), ClusterNodeStatus::Active);
|
|
||||||
assert_eq!(slot(UNRESPONSIVE_NODE_TIMEOUT).status(), ClusterNodeStatus::Active);
|
|
||||||
assert_eq!(slot(UNRESPONSIVE_NODE_TIMEOUT + 1).status(), ClusterNodeStatus::Stale);
|
|
||||||
assert_eq!(slot(DEAD_NODE_TIMEOUT).status(), ClusterNodeStatus::Stale);
|
|
||||||
assert_eq!(slot(DEAD_NODE_TIMEOUT + 1).status(), ClusterNodeStatus::Inactive);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_silent_node_keeps_its_id_for_an_hour() {
|
|
||||||
// Reported Stale after three minutes, but not free to take over.
|
|
||||||
let quiet = slot(UNRESPONSIVE_NODE_TIMEOUT + 1);
|
|
||||||
assert_eq!(quiet.status(), ClusterNodeStatus::Stale);
|
|
||||||
assert!(!quiet.is_stale());
|
|
||||||
assert!(slot(STALE_NODE_TIMEOUT + 1).is_stale());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn several_renewals_fit_before_a_node_looks_unresponsive() {
|
|
||||||
assert!(UNRESPONSIVE_NODE_TIMEOUT >= 3 * HEARTBEAT_INTERVAL);
|
|
||||||
assert!(HEARTBEAT_INTERVAL * 2 < STALE_NODE_TIMEOUT);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -401,57 +401,6 @@ impl InMemoryStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// inbuxa: extends a lock this node holds to `duration` seconds from now.
|
|
||||||
/// Returns false when the lock is gone or has expired: it may have been
|
|
||||||
/// taken by someone else since, so it is left alone.
|
|
||||||
pub async fn renew_lock(&self, prefix: u8, key: &[u8], duration: u64) -> trc::Result<bool> {
|
|
||||||
match self {
|
|
||||||
InMemoryStore::Store(store) => {
|
|
||||||
let key = KeyValue::<()>::build_key(prefix, key);
|
|
||||||
let key = ValueClass::InMemory(InMemoryClass::Key(key));
|
|
||||||
let Some(lock_expiry) = store
|
|
||||||
.get_value::<u64>(ValueKey::from(key.clone()))
|
|
||||||
.await
|
|
||||||
.caused_by(trc::location!())?
|
|
||||||
else {
|
|
||||||
return Ok(false);
|
|
||||||
};
|
|
||||||
let now = now();
|
|
||||||
if lock_expiry <= now {
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch.assert_value(key.clone(), AssertValue::U64(lock_expiry));
|
|
||||||
batch.set(key, (now + duration).serialize());
|
|
||||||
match store.write(batch.build_all()).await {
|
|
||||||
Ok(_) => Ok(true),
|
|
||||||
Err(err) if err.is_assertion_failure() => Ok(false),
|
|
||||||
Err(err) => Err(err
|
|
||||||
.details("Failed to renew lock.")
|
|
||||||
.caused_by(trc::location!())),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
InMemoryStore::Sharded(store) => {
|
|
||||||
Box::pin(
|
|
||||||
store
|
|
||||||
.member(&KeyValue::<()>::build_key(prefix, key))
|
|
||||||
.renew_lock(prefix, key, duration),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
#[cfg(feature = "redis")]
|
|
||||||
InMemoryStore::Redis(store) => {
|
|
||||||
store
|
|
||||||
.renew_lock(&KeyValue::<()>::build_key(prefix, key), duration)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
InMemoryStore::Static(_) | InMemoryStore::Http(_) => {
|
|
||||||
Err(trc::StoreEvent::NotSupported.into_err())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn remove_lock(&self, prefix: u8, key: &[u8]) -> trc::Result<()> {
|
pub async fn remove_lock(&self, prefix: u8, key: &[u8]) -> trc::Result<()> {
|
||||||
self.key_delete(KeyValue::<()>::build_key(prefix, key))
|
self.key_delete(KeyValue::<()>::build_key(prefix, key))
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -10,9 +10,8 @@
|
|||||||
|
|
||||||
// inbuxa: 637 to 641 are the fork's SCIM events (SCIM-54); 642 is
|
// inbuxa: 637 to 641 are the fork's SCIM events (SCIM-54); 642 is
|
||||||
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
|
// auth.legacy-protocol-refused (legacy-protocols LP-6); 643 is
|
||||||
// security.legacy-protocols-changed (LP-8); 644 to 646 are the cluster
|
// security.legacy-protocols-changed (LP-8)
|
||||||
// coordinator's connection events
|
pub const TOTAL_EVENT_COUNT: usize = 644;
|
||||||
pub const TOTAL_EVENT_COUNT: usize = 647;
|
|
||||||
pub const TOTAL_METRIC_COUNT: usize = 369;
|
pub const TOTAL_METRIC_COUNT: usize = 369;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
@@ -151,10 +150,6 @@ pub enum ClusterEvent {
|
|||||||
MessageSkipped = 47,
|
MessageSkipped = 47,
|
||||||
MessageInvalid = 49,
|
MessageInvalid = 49,
|
||||||
NodeIdRenewed = 275,
|
NodeIdRenewed = 275,
|
||||||
// inbuxa: the coordinator's connection
|
|
||||||
CoordinatorConnected = 644,
|
|
||||||
CoordinatorDisconnected = 645,
|
|
||||||
CoordinatorError = 646,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
|
|||||||
@@ -81,10 +81,6 @@ impl EventType {
|
|||||||
b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped),
|
b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped),
|
||||||
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
|
b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid),
|
||||||
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
b"cluster.node-id-renewed" => EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
||||||
// inbuxa: coordinator connection
|
|
||||||
b"cluster.coordinator-connected" => EventType::Cluster(ClusterEvent::CoordinatorConnected),
|
|
||||||
b"cluster.coordinator-disconnected" => EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
b"cluster.coordinator-error" => EventType::Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess),
|
b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess),
|
||||||
b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure),
|
b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure),
|
||||||
b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound),
|
b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound),
|
||||||
@@ -746,14 +742,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped",
|
EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped",
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
|
EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid",
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
|
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "cluster.node-id-renewed",
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => {
|
|
||||||
"cluster.coordinator-connected"
|
|
||||||
}
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => {
|
|
||||||
"cluster.coordinator-disconnected"
|
|
||||||
}
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => "cluster.coordinator-error",
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success",
|
EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success",
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure",
|
EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure",
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found",
|
EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found",
|
||||||
@@ -1536,10 +1524,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped) => 47,
|
EventType::Cluster(ClusterEvent::MessageSkipped) => 47,
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
|
EventType::Cluster(ClusterEvent::MessageInvalid) => 49,
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
|
EventType::Cluster(ClusterEvent::NodeIdRenewed) => 275,
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => 644,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => 645,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => 646,
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => 67,
|
EventType::Dane(DaneEvent::AuthenticationSuccess) => 67,
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure) => 66,
|
EventType::Dane(DaneEvent::AuthenticationFailure) => 66,
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound) => 69,
|
EventType::Dane(DaneEvent::NoCertificatesFound) => 69,
|
||||||
@@ -2192,10 +2176,6 @@ impl EventType {
|
|||||||
47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)),
|
47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)),
|
||||||
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
|
49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)),
|
||||||
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
|
275 => Some(EventType::Cluster(ClusterEvent::NodeIdRenewed)),
|
||||||
// inbuxa: coordinator connection
|
|
||||||
644 => Some(EventType::Cluster(ClusterEvent::CoordinatorConnected)),
|
|
||||||
645 => Some(EventType::Cluster(ClusterEvent::CoordinatorDisconnected)),
|
|
||||||
646 => Some(EventType::Cluster(ClusterEvent::CoordinatorError)),
|
|
||||||
67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)),
|
67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)),
|
||||||
66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)),
|
66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)),
|
||||||
69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)),
|
69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)),
|
||||||
@@ -3134,10 +3114,6 @@ impl EventType {
|
|||||||
EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn,
|
EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn,
|
||||||
EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn,
|
EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn,
|
||||||
EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn,
|
EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn,
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => Level::Info,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => Level::Warn,
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => Level::Warn,
|
|
||||||
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
|
EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn,
|
||||||
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
|
EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn,
|
||||||
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
|
EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn,
|
||||||
@@ -3268,10 +3244,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped",
|
EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped",
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message",
|
EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message",
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "Node ID renewed",
|
EventType::Cluster(ClusterEvent::NodeIdRenewed) => "Node ID renewed",
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected) => "Coordinator connected",
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected) => "Coordinator unavailable",
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError) => "Coordinator error",
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
|
EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful",
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
|
EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed",
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
|
EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE",
|
||||||
@@ -4350,10 +4322,6 @@ impl EventType {
|
|||||||
EventType::Cluster(ClusterEvent::MessageSkipped),
|
EventType::Cluster(ClusterEvent::MessageSkipped),
|
||||||
EventType::Cluster(ClusterEvent::MessageInvalid),
|
EventType::Cluster(ClusterEvent::MessageInvalid),
|
||||||
EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
EventType::Cluster(ClusterEvent::NodeIdRenewed),
|
||||||
// inbuxa: coordinator connection
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorConnected),
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorDisconnected),
|
|
||||||
EventType::Cluster(ClusterEvent::CoordinatorError),
|
|
||||||
EventType::Dane(DaneEvent::AuthenticationSuccess),
|
EventType::Dane(DaneEvent::AuthenticationSuccess),
|
||||||
EventType::Dane(DaneEvent::AuthenticationFailure),
|
EventType::Dane(DaneEvent::AuthenticationFailure),
|
||||||
EventType::Dane(DaneEvent::NoCertificatesFound),
|
EventType::Dane(DaneEvent::NoCertificatesFound),
|
||||||
|
|||||||
@@ -245,28 +245,10 @@ impl Collector {
|
|||||||
Update::RegisterReceiver { receiver } => {
|
Update::RegisterReceiver { receiver } => {
|
||||||
self.receivers.push(receiver);
|
self.receivers.push(receiver);
|
||||||
}
|
}
|
||||||
Update::RegisterSubscriber { mut subscriber } => {
|
Update::RegisterSubscriber { subscriber } => {
|
||||||
// inbuxa: a subscriber registered under the id of a
|
|
||||||
// running one replaces it (a tracer whose settings
|
|
||||||
// changed). Every event collected so far went to the old
|
|
||||||
// one, every later event goes to the new one: the old
|
|
||||||
// one's batch is sent first (anything its full channel
|
|
||||||
// can't take moves over, rather than being dropped), and
|
|
||||||
// dropping it closes its channel, so its task writes
|
|
||||||
// what is queued and ends.
|
|
||||||
if let Some(old) = self.subscribers.iter_mut().find(|s| s.id == subscriber.id) {
|
|
||||||
let _ = old.send_batch();
|
|
||||||
if !old.batch.is_empty() {
|
|
||||||
let mut batch = std::mem::take(&mut old.batch);
|
|
||||||
batch.append(&mut subscriber.batch);
|
|
||||||
subscriber.batch = batch;
|
|
||||||
}
|
|
||||||
*old = subscriber;
|
|
||||||
} else {
|
|
||||||
ACTIVE_SUBSCRIBERS.lock().push(subscriber.id.clone());
|
ACTIVE_SUBSCRIBERS.lock().push(subscriber.id.clone());
|
||||||
self.subscribers.push(subscriber);
|
self.subscribers.push(subscriber);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
Update::UnregisterSubscriber { id } => {
|
Update::UnregisterSubscriber { id } => {
|
||||||
ACTIVE_SUBSCRIBERS.lock().retain(|s| s != &id);
|
ACTIVE_SUBSCRIBERS.lock().retain(|s| s != &id);
|
||||||
self.subscribers.retain(|s| s.id != id);
|
self.subscribers.retain(|s| s.id != id);
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -107,9 +105,6 @@ impl SubscriberBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Registers the subscriber with the collector. inbuxa: one registered
|
|
||||||
/// under the id of a running subscriber replaces it, handing over at an
|
|
||||||
/// event boundary; the old one's channel then closes.
|
|
||||||
pub fn register(self) -> (mpsc::Sender<EventBatch>, mpsc::Receiver<EventBatch>) {
|
pub fn register(self) -> (mpsc::Sender<EventBatch>, mpsc::Receiver<EventBatch>) {
|
||||||
let (tx, rx) = mpsc::channel(8192);
|
let (tx, rx) = mpsc::channel(8192);
|
||||||
|
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
|
|||||||
#[macro_export]
|
#[macro_export]
|
||||||
macro_rules! brand_version {
|
macro_rules! brand_version {
|
||||||
() => {
|
() => {
|
||||||
"2026.9.25.1"
|
"2026.9.24.4"
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -212,15 +212,10 @@ unchanged.
|
|||||||
- **MON-16.** With `indexTelemetry` on, storing a trace schedules an
|
- **MON-16.** With `indexTelemetry` on, storing a trace schedules an
|
||||||
`IndexTrace` task. The task builds one document for `SearchIndex::Tracing`
|
`IndexTrace` task. The task builds one document for `SearchIndex::Tracing`
|
||||||
with the fields named in `indexTracingFields`:
|
with the fields named in `indexTracingFields`:
|
||||||
- `eventType`: the trace's opening event, as its numeric id;
|
- `eventType`: every event type in the trace;
|
||||||
- `queueId`: the first `queueId` value, as an integer;
|
- `queueId`: every `queueId` value;
|
||||||
- `keywords`: every address in `from` and `to`, each address's domain, every
|
- `keywords`: every address in `from` and `to`, each address's domain, every
|
||||||
`domain`, `hostname`, `remoteIp`, `messageId` and `accountName` value,
|
`domain`, `hostname`, `remoteIp`, `messageId` and `accountName` value.
|
||||||
and every `queueId` value.
|
|
||||||
The event type and queue id are single integer columns on every search
|
|
||||||
backend (BIGINT on PostgreSQL and MySQL), so the `queueId` filter matches
|
|
||||||
the column or any queue id in the keywords, and a session that queued
|
|
||||||
several messages is found by each of them.
|
|
||||||
So searching `example.org` finds every trace to or from that domain, as the
|
So searching `example.org` finds every trace to or from that domain, as the
|
||||||
upstream suite expects. With `indexTelemetry` off nothing is indexed, and
|
upstream suite expects. With `indexTelemetry` off nothing is indexed, and
|
||||||
the `text` and `queueId` filters are refused (see "Interfaces").
|
the `text` and `queueId` filters are refused (see "Interfaces").
|
||||||
|
|||||||
Binary file not shown.
@@ -1 +1 @@
|
|||||||
XFI3xuKC_rH1KZyaVBF0uTIiRDXRqyYboijquiGz2eg
|
VbnFuwCOTBh0s2T-NuRhb2JaJr8Jl5s3LgXv4Pv2sTg
|
||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
@@ -13,7 +11,6 @@ use crate::{
|
|||||||
server::TestServerBuilder,
|
server::TestServerBuilder,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
use common::BuildServer;
|
|
||||||
use imap_proto::ResponseType;
|
use imap_proto::ResponseType;
|
||||||
use registry::{
|
use registry::{
|
||||||
schema::{
|
schema::{
|
||||||
@@ -21,8 +18,7 @@ use registry::{
|
|||||||
prelude::{ObjectType, Property, SocketAddr},
|
prelude::{ObjectType, Property, SocketAddr},
|
||||||
structs::{
|
structs::{
|
||||||
ClusterListenerGroup, ClusterListenerGroupProperties, ClusterRole, ClusterTaskGroup,
|
ClusterListenerGroup, ClusterListenerGroupProperties, ClusterRole, ClusterTaskGroup,
|
||||||
Coordinator, Imap, MtaDeliverySchedule, MtaVirtualQueue, NatsCoordinator,
|
Coordinator, Imap, NatsCoordinator, NetworkListener, RedisStore,
|
||||||
NetworkListener, RedisStore,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
types::map::Map,
|
types::map::Map,
|
||||||
@@ -213,45 +209,6 @@ pub async fn cluster_tests() {
|
|||||||
Some("John Doe")
|
Some("John Doe")
|
||||||
);
|
);
|
||||||
|
|
||||||
// inbuxa: a settings write applies on every node, no ReloadSettings
|
|
||||||
let queue_id = admin
|
|
||||||
.registry_create_object(MtaVirtualQueue {
|
|
||||||
name: "clusterq".into(),
|
|
||||||
threads_per_node: 1,
|
|
||||||
description: None,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
admin
|
|
||||||
.registry_create_object(MtaDeliverySchedule {
|
|
||||||
name: "cluster-autoreload".into(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
for (node_id, test) in servers.iter().enumerate() {
|
|
||||||
let started = std::time::Instant::now();
|
|
||||||
while !test
|
|
||||||
.server
|
|
||||||
.inner
|
|
||||||
.build_server()
|
|
||||||
.core
|
|
||||||
.smtp
|
|
||||||
.queue
|
|
||||||
.queue_strategy
|
|
||||||
.contains_key("cluster-autoreload")
|
|
||||||
{
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < std::time::Duration::from_secs(5),
|
|
||||||
"node {node_id} didn't pick up the new delivery schedule"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
|
||||||
}
|
|
||||||
println!(
|
|
||||||
"Node {node_id} has the new delivery schedule after {} ms",
|
|
||||||
started.elapsed().as_millis()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run IMAP idle tests across nodes
|
// Run IMAP idle tests across nodes
|
||||||
let mut node1_client = imap_client("[email protected]", "this is john's secret", 1).await;
|
let mut node1_client = imap_client("[email protected]", "this is john's secret", 1).await;
|
||||||
let mut node2_client = imap_client("[email protected]", "this is john's secret", 2).await;
|
let mut node2_client = imap_client("[email protected]", "this is john's secret", 2).await;
|
||||||
|
|||||||
@@ -1,145 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! A node that starts while its NATS coordinator is down joins the cluster
|
|
||||||
//! once NATS comes up, without a restart, and reports the coordinator's
|
|
||||||
//! connection on `/healthz/cluster` as it goes and comes back.
|
|
||||||
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
|
||||||
use coordinator::Coordinator;
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::NetworkListenerProtocol,
|
|
||||||
structs::{Coordinator as CoordinatorSetting, NatsCoordinator},
|
|
||||||
},
|
|
||||||
types::map::Map,
|
|
||||||
};
|
|
||||||
use serde_json::{Value, json};
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
use testcontainers::{
|
|
||||||
GenericImage, ImageExt, core::IntoContainerPort, core::WaitFor, runners::AsyncRunner,
|
|
||||||
};
|
|
||||||
|
|
||||||
const HTTP_PORT: u16 = 11_310;
|
|
||||||
const TOPIC: &str = "inbuxa-coordinator-test";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn coordinator_reconnect_tests() {
|
|
||||||
println!("Running coordinator reconnect tests...");
|
|
||||||
|
|
||||||
// A port with no NATS server behind it, yet
|
|
||||||
let nats_port = std::net::TcpListener::bind("127.0.0.1:0")
|
|
||||||
.unwrap()
|
|
||||||
.local_addr()
|
|
||||||
.unwrap()
|
|
||||||
.port();
|
|
||||||
let config = NatsCoordinator {
|
|
||||||
addresses: Map::new(vec![format!("127.0.0.1:{nats_port}")]),
|
|
||||||
use_tls: false,
|
|
||||||
timeout_connection: 1_000u64.into(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
// 1. The node starts, without a build error, while NATS is down, and
|
|
||||||
// says so
|
|
||||||
let test = TestServerBuilder::new("coordinator_reconnect_tests")
|
|
||||||
.await
|
|
||||||
.with_object(CoordinatorSetting::Nats(config.clone()))
|
|
||||||
.await
|
|
||||||
.with_listener(NetworkListenerProtocol::Http, "http", HTTP_PORT, true)
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
let coordinator = test.server.core.storage.coordinator.clone();
|
|
||||||
assert!(
|
|
||||||
coordinator.is_enabled(),
|
|
||||||
"a coordinator, though not connected"
|
|
||||||
);
|
|
||||||
assert_eq!(coordinator.is_connected(), Some(false));
|
|
||||||
assert_eq!(
|
|
||||||
cluster_health().await,
|
|
||||||
(503, json!({"coordinator": "disconnected"}))
|
|
||||||
);
|
|
||||||
|
|
||||||
// A subscription made now, as the broadcast subscriber makes it at
|
|
||||||
// startup, has to work once NATS is up
|
|
||||||
let mut stream = coordinator.subscribe(TOPIC).await.unwrap();
|
|
||||||
|
|
||||||
// 2. NATS comes up: the node connects on its own
|
|
||||||
let nats = GenericImage::new("nats", "latest")
|
|
||||||
.with_wait_for(WaitFor::message_on_stderr("Server is ready"))
|
|
||||||
.with_mapped_port(nats_port, 4222.tcp())
|
|
||||||
.start()
|
|
||||||
.await
|
|
||||||
.expect("Failed to start NATS container");
|
|
||||||
wait_for_health(200, "connected").await;
|
|
||||||
let other_node = coordinator::backend::nats::NatsPubSub::open(config.clone())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
wait_until_connected(&other_node).await;
|
|
||||||
round_trip(&other_node, &mut stream, b"after startup").await;
|
|
||||||
|
|
||||||
// 3. NATS goes away: the node reports it; and it comes back: the node
|
|
||||||
// reconnects and the same subscription carries on
|
|
||||||
nats.stop().await.unwrap();
|
|
||||||
wait_for_health(503, "disconnected").await;
|
|
||||||
nats.start().await.unwrap();
|
|
||||||
wait_for_health(200, "connected").await;
|
|
||||||
wait_until_connected(&other_node).await;
|
|
||||||
round_trip(&other_node, &mut stream, b"after reconnect").await;
|
|
||||||
|
|
||||||
drop(nats);
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn cluster_health() -> (u16, Value) {
|
|
||||||
let response = reqwest::Client::builder()
|
|
||||||
.danger_accept_invalid_certs(true)
|
|
||||||
.timeout(Duration::from_secs(5))
|
|
||||||
.build()
|
|
||||||
.unwrap()
|
|
||||||
.get(format!("https://127.0.0.1:{HTTP_PORT}/healthz/cluster"))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let status = response.status().as_u16();
|
|
||||||
(status, response.json().await.unwrap())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for_health(status: u16, state: &str) {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let health = cluster_health().await;
|
|
||||||
if health == (status, json!({"coordinator": state})) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(30),
|
|
||||||
"expected {status} {state}, still {health:?}"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_until_connected(coordinator: &Coordinator) {
|
|
||||||
let started = Instant::now();
|
|
||||||
while coordinator.is_connected() != Some(true) {
|
|
||||||
assert!(started.elapsed() < Duration::from_secs(30), "not connected");
|
|
||||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Another node publishes; this one's subscription receives it.
|
|
||||||
async fn round_trip(from: &Coordinator, stream: &mut coordinator::PubSubStream, payload: &[u8]) {
|
|
||||||
from.publish(TOPIC, payload.to_vec()).await.unwrap();
|
|
||||||
let message = tokio::time::timeout(Duration::from_secs(10), stream.next())
|
|
||||||
.await
|
|
||||||
.expect("no message within 10 seconds")
|
|
||||||
.expect("subscription ended");
|
|
||||||
assert_eq!(message.payload(), payload);
|
|
||||||
}
|
|
||||||
@@ -1,317 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! DMARC results recorded on a node without outboundMta reach the aggregate
|
|
||||||
//! report, which a node with outboundMta builds and sends. Before, a front
|
|
||||||
//! node's results were dropped (or, before live roles, left in a channel
|
|
||||||
//! nobody read), so the report covered only the mail the outbound nodes
|
|
||||||
//! received. Also checks that nodes appending to one report at once lose
|
|
||||||
//! nothing. Needs a store the nodes can share (STORE=PostgreSql or MySql).
|
|
||||||
|
|
||||||
use crate::{smtp::inbound::TestMessage, utils::server::TestServerBuilder};
|
|
||||||
use common::{Server, config::smtp::report::AggregateFrequency, ipc::DmarcEvent};
|
|
||||||
use mail_auth::{
|
|
||||||
common::parse::TxtRecordParser,
|
|
||||||
dmarc::Dmarc,
|
|
||||||
report::{ActionDisposition, DmarcResult, Record, Report},
|
|
||||||
};
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::ClusterTaskType,
|
|
||||||
prelude::{ObjectType, Property},
|
|
||||||
structs::{
|
|
||||||
ClusterListenerGroup, ClusterRole, ClusterTaskGroup, ClusterTaskGroupProperties,
|
|
||||||
DmarcInternalReport, DmarcReportSettings, Expression, Task, TaskDmarcReport,
|
|
||||||
TaskStatus,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
types::{EnumImpl, map::Map},
|
|
||||||
};
|
|
||||||
use smtp::reporting::{dmarc::DmarcReporting, send::MtaReportSend};
|
|
||||||
use std::{
|
|
||||||
collections::BTreeSet,
|
|
||||||
net::IpAddr,
|
|
||||||
sync::Arc,
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{
|
|
||||||
ValueKey,
|
|
||||||
registry::{RegistryFilter, RegistryFilterValue, RegistryQuery},
|
|
||||||
write::{BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, now},
|
|
||||||
};
|
|
||||||
use types::id::Id;
|
|
||||||
|
|
||||||
const FRONT_ROLE: &str = "front_reports_front";
|
|
||||||
const MTA_ROLE: &str = "front_reports_mta";
|
|
||||||
const DOMAIN: &str = "front-reports.example";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn front_node_report_tests() {
|
|
||||||
if matches!(
|
|
||||||
std::env::var("STORE").as_deref(),
|
|
||||||
Ok("RocksDb" | "Sqlite") | Err(_)
|
|
||||||
) {
|
|
||||||
println!("Skipping front node report tests: they need a store the nodes can share.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!(
|
|
||||||
"Running front node report tests on {}...",
|
|
||||||
std::env::var("STORE").unwrap_or_default()
|
|
||||||
);
|
|
||||||
|
|
||||||
// A front role without outboundMta, an MTA role with it
|
|
||||||
let seed = TestServerBuilder::new("front_reports_seed").await;
|
|
||||||
seed.insert_object(role(FRONT_ROLE, &[ClusterTaskType::PushNotifications]))
|
|
||||||
.await;
|
|
||||||
seed.insert_object(role(MTA_ROLE, &[ClusterTaskType::OutboundMta]))
|
|
||||||
.await;
|
|
||||||
seed.insert_object(DmarcReportSettings {
|
|
||||||
aggregate_max_report_size: Expression {
|
|
||||||
else_: "1048576".into(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let seed = seed.disable_services().build().await;
|
|
||||||
|
|
||||||
// The front node receives mail from two sources: the events its SMTP
|
|
||||||
// sessions hand the report scheduler
|
|
||||||
let front = TestServerBuilder::new_with_role(
|
|
||||||
"front_reports_front",
|
|
||||||
"front.front-reports.example".into(),
|
|
||||||
Some(FRONT_ROLE.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await;
|
|
||||||
let front_server = front.server.clone();
|
|
||||||
assert!(!front_server.core.network.roles.outbound_mta);
|
|
||||||
for ip in ["192.0.2.1", "192.0.2.2"] {
|
|
||||||
front_server.schedule_report(event(ip)).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Both are recorded in the shared report. Upstream, and main after live
|
|
||||||
// roles, left the front node's results out
|
|
||||||
let report_id = wait_for_report(&front_server, 2).await;
|
|
||||||
|
|
||||||
// Make the report due now. The front node leaves it alone: building and
|
|
||||||
// sending it is the outbound MTA's
|
|
||||||
move_task(&front_server, report_id, TaskStatus::now()).await;
|
|
||||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
|
||||||
front_server.notify_task_queue();
|
|
||||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
||||||
assert!(
|
|
||||||
task_exists(&front_server, report_id).await,
|
|
||||||
"the front node ran the report task"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Several writers append to the report at once, from both nodes: none
|
|
||||||
// of their records is lost. The report waits in the future meanwhile, or
|
|
||||||
// the MTA node would send it as soon as it starts
|
|
||||||
move_task(
|
|
||||||
&front_server,
|
|
||||||
report_id,
|
|
||||||
TaskStatus::at(now() as i64 + 3600),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let mut mta = TestServerBuilder::new_with_role(
|
|
||||||
"front_reports_mta",
|
|
||||||
"mta.front-reports.example".into(),
|
|
||||||
Some(MTA_ROLE.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.capture_queue()
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await;
|
|
||||||
let mta_server = mta.server.clone();
|
|
||||||
assert!(mta_server.core.network.roles.outbound_mta);
|
|
||||||
|
|
||||||
let concurrent: Vec<String> = (10..18).map(|n| format!("192.0.2.{n}")).collect();
|
|
||||||
let mut handles = Vec::new();
|
|
||||||
for (n, ip) in concurrent.iter().enumerate() {
|
|
||||||
let server = if n % 2 == 0 {
|
|
||||||
front_server.clone()
|
|
||||||
} else {
|
|
||||||
mta_server.clone()
|
|
||||||
};
|
|
||||||
let ip = ip.clone();
|
|
||||||
handles.push(tokio::spawn(async move {
|
|
||||||
server.schedule_dmarc(Box::new(event(&ip))).await;
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
for handle in handles {
|
|
||||||
handle.await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Due again, the MTA node sends the report with every record in it
|
|
||||||
move_task(&mta_server, report_id, TaskStatus::now()).await;
|
|
||||||
let message = mta.expect_message().await;
|
|
||||||
let report =
|
|
||||||
Report::parse_rfc5322(message.read_message(&mta).await.as_bytes(), usize::MAX).unwrap();
|
|
||||||
assert_eq!(report.domain(), DOMAIN);
|
|
||||||
let sent: BTreeSet<IpAddr> = report
|
|
||||||
.records()
|
|
||||||
.iter()
|
|
||||||
.map(|r| r.source_ip().unwrap())
|
|
||||||
.collect();
|
|
||||||
let expected: BTreeSet<IpAddr> = ["192.0.2.1", "192.0.2.2"]
|
|
||||||
.into_iter()
|
|
||||||
.map(String::from)
|
|
||||||
.chain(concurrent)
|
|
||||||
.map(|ip| ip.parse().unwrap())
|
|
||||||
.collect();
|
|
||||||
assert_eq!(sent, expected);
|
|
||||||
wait_for(Duration::from_secs(20), "report task to finish", || async {
|
|
||||||
!task_exists(&mta_server, report_id).await
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
assert!(reports(&mta_server).await.is_empty());
|
|
||||||
|
|
||||||
if seed.is_reset() {
|
|
||||||
seed.temp_dir.delete();
|
|
||||||
front.temp_dir.delete();
|
|
||||||
mta.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn role(name: &str, tasks: &[ClusterTaskType]) -> ClusterRole {
|
|
||||||
ClusterRole {
|
|
||||||
name: name.into(),
|
|
||||||
description: None,
|
|
||||||
listeners: ClusterListenerGroup::EnableAll,
|
|
||||||
tasks: ClusterTaskGroup::EnableSome(ClusterTaskGroupProperties {
|
|
||||||
task_types: Map::new(tasks.to_vec()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn event(ip: &str) -> DmarcEvent {
|
|
||||||
DmarcEvent {
|
|
||||||
domain: DOMAIN.to_string(),
|
|
||||||
report_record: Record::new()
|
|
||||||
.with_source_ip(ip.parse().unwrap())
|
|
||||||
.with_action_disposition(ActionDisposition::Pass)
|
|
||||||
.with_dmarc_dkim_result(DmarcResult::Pass)
|
|
||||||
.with_dmarc_spf_result(DmarcResult::Pass)
|
|
||||||
.with_envelope_from("sender.example")
|
|
||||||
.with_header_from("sender.example"),
|
|
||||||
dmarc_record: Arc::new(
|
|
||||||
Dmarc::parse(format!("v=DMARC1; p=reject; rua=mailto:reports@{DOMAIN}").as_bytes())
|
|
||||||
.unwrap(),
|
|
||||||
),
|
|
||||||
interval: AggregateFrequency::Daily,
|
|
||||||
span_id: 0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reports(server: &Server) -> Vec<(u64, DmarcInternalReport)> {
|
|
||||||
let ids = server
|
|
||||||
.registry()
|
|
||||||
.query::<Vec<Id>>(RegistryQuery::new(ObjectType::DmarcInternalReport).filter(
|
|
||||||
RegistryFilter::greater_than(
|
|
||||||
Property::Domain,
|
|
||||||
RegistryFilterValue::Bytes(vec![]),
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let mut reports = Vec::new();
|
|
||||||
for id in ids {
|
|
||||||
if let Some(report) = server
|
|
||||||
.store()
|
|
||||||
.get_value::<DmarcInternalReport>(ValueKey::from(ValueClass::Registry(
|
|
||||||
RegistryClass::Item {
|
|
||||||
object_id: ObjectType::DmarcInternalReport.to_id(),
|
|
||||||
item_id: id.id(),
|
|
||||||
},
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
{
|
|
||||||
reports.push((id.id(), report));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reports
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for the report for `DOMAIN` to hold `records` records; returns its id.
|
|
||||||
async fn wait_for_report(server: &Server, records: usize) -> u64 {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let found = reports(server)
|
|
||||||
.await
|
|
||||||
.into_iter()
|
|
||||||
.find(|(_, report)| report.domain == DOMAIN);
|
|
||||||
if let Some((id, report)) = &found
|
|
||||||
&& report.report.records.len() == records
|
|
||||||
{
|
|
||||||
return *id;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(10),
|
|
||||||
"no report with {records} records for {DOMAIN}: {found:?}"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reschedules the report's task.
|
|
||||||
async fn move_task(server: &Server, id: u64, status: TaskStatus) {
|
|
||||||
let task = server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.expect("report task missing");
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.clear(ValueClass::TaskQueue(TaskQueueClass::Due {
|
|
||||||
id,
|
|
||||||
due: task.due_timestamp(),
|
|
||||||
}))
|
|
||||||
.schedule_task_with_id(
|
|
||||||
id,
|
|
||||||
Task::DmarcReport(TaskDmarcReport {
|
|
||||||
report_id: id.into(),
|
|
||||||
status,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server.notify_task_queue();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn task_exists(server: &Server, id: u64) -> bool {
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.is_some()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for<F, Fut>(within: Duration, what: &str, mut check: F)
|
|
||||||
where
|
|
||||||
F: FnMut() -> Fut,
|
|
||||||
Fut: Future<Output = bool>,
|
|
||||||
{
|
|
||||||
let started = Instant::now();
|
|
||||||
while !check().await {
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < within,
|
|
||||||
"still waiting for the {what} after {:?}",
|
|
||||||
started.elapsed()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,297 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! A node follows edits to its cluster role without a restart: outbound
|
|
||||||
//! delivery and report tasks start when the role gains outboundMta and stop
|
|
||||||
//! when it loses it. Upstream decided at boot whether the queue, report and
|
|
||||||
//! task managers ran at all. Needs a store the seed and the node can share
|
|
||||||
//! (STORE=PostgreSql or MySql).
|
|
||||||
|
|
||||||
use crate::utils::server::{TestServer, TestServerBuilder};
|
|
||||||
use common::Server;
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::ClusterTaskType,
|
|
||||||
prelude::{Object, ObjectType},
|
|
||||||
structs::{
|
|
||||||
ClusterListenerGroup, ClusterRole, ClusterTaskGroup, ClusterTaskGroupProperties, Task,
|
|
||||||
TaskStatus, TaskTlsReport,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
types::{id::ObjectId, map::Map},
|
|
||||||
};
|
|
||||||
use smtp::{
|
|
||||||
queue::{Message, Status},
|
|
||||||
reporting::send::MtaReportSend,
|
|
||||||
};
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
use store::{
|
|
||||||
Deserialize, IterateParams, ValueKey,
|
|
||||||
registry::write::{RegistryWrite, RegistryWriteResult},
|
|
||||||
write::{AlignedBytes, Archive, BatchBuilder, QueueClass, TaskQueueClass, ValueClass},
|
|
||||||
};
|
|
||||||
use types::id::Id;
|
|
||||||
use utils::snowflake::SnowflakeIdGenerator;
|
|
||||||
|
|
||||||
const BUSY_ROLE: &str = "live_role_busy";
|
|
||||||
const IDLE_ROLE: &str = "live_role_idle";
|
|
||||||
const WITH: &[ClusterTaskType] = &[
|
|
||||||
ClusterTaskType::PushNotifications,
|
|
||||||
ClusterTaskType::OutboundMta,
|
|
||||||
];
|
|
||||||
const WITHOUT: &[ClusterTaskType] = &[ClusterTaskType::PushNotifications];
|
|
||||||
const RCPT_DOMAIN: &str = "live-role.invalid";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn live_role_tests() {
|
|
||||||
if matches!(
|
|
||||||
std::env::var("STORE").as_deref(),
|
|
||||||
Ok("RocksDb" | "Sqlite") | Err(_)
|
|
||||||
) {
|
|
||||||
println!("Skipping live role tests: they need a store the nodes can share.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!(
|
|
||||||
"Running live role tests on {}...",
|
|
||||||
std::env::var("STORE").unwrap_or_default()
|
|
||||||
);
|
|
||||||
|
|
||||||
// Two roles: one with outboundMta, one with no task type at all
|
|
||||||
let seed = TestServerBuilder::new("live_roles_seed").await;
|
|
||||||
let busy_id = seed.insert_object(role(BUSY_ROLE, WITH)).await;
|
|
||||||
let idle_id = seed.insert_object(role(IDLE_ROLE, WITHOUT)).await;
|
|
||||||
let seed = seed.disable_services().build().await;
|
|
||||||
let registry = seed.server.clone();
|
|
||||||
|
|
||||||
// 1. The rehearsal case: a node started with outboundMta has it taken
|
|
||||||
// away. Upstream kept delivering, report messages included, until a
|
|
||||||
// restart.
|
|
||||||
let node = start_node("live_roles_busy", BUSY_ROLE).await;
|
|
||||||
let server = node.server.clone();
|
|
||||||
assert!(server.core.network.roles.outbound_mta);
|
|
||||||
let (msg, task) = queue_work(&server, "busy-before").await;
|
|
||||||
assert_runs(&server, &msg, task).await;
|
|
||||||
|
|
||||||
set_role(®istry, &server, busy_id, role(BUSY_ROLE, WITHOUT)).await;
|
|
||||||
let (msg, task) = queue_work(&server, "busy-off").await;
|
|
||||||
assert_idle(&server, &msg, task).await;
|
|
||||||
|
|
||||||
// Given back, it takes up the work left waiting
|
|
||||||
set_role(®istry, &server, busy_id, role(BUSY_ROLE, WITH)).await;
|
|
||||||
assert_runs(&server, &msg, task).await;
|
|
||||||
|
|
||||||
// Off again, so it leaves the next node's work alone
|
|
||||||
set_role(®istry, &server, busy_id, role(BUSY_ROLE, WITHOUT)).await;
|
|
||||||
|
|
||||||
// 2. A node started with no task type at all gains outboundMta.
|
|
||||||
// Upstream never started its queue, report or task manager, so the
|
|
||||||
// role did nothing until a restart.
|
|
||||||
let node2 = start_node("live_roles_idle", IDLE_ROLE).await;
|
|
||||||
let server2 = node2.server.clone();
|
|
||||||
assert!(!server2.core.network.roles.outbound_mta);
|
|
||||||
let (msg, task) = queue_work(&server2, "idle-off").await;
|
|
||||||
assert_idle(&server2, &msg, task).await;
|
|
||||||
set_role(®istry, &server2, idle_id, role(IDLE_ROLE, WITH)).await;
|
|
||||||
assert_runs(&server2, &msg, task).await;
|
|
||||||
set_role(®istry, &server2, idle_id, role(IDLE_ROLE, WITHOUT)).await;
|
|
||||||
|
|
||||||
if seed.is_reset() {
|
|
||||||
seed.temp_dir.delete();
|
|
||||||
node.temp_dir.delete();
|
|
||||||
node2.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn start_node(name: &str, role: &str) -> TestServer {
|
|
||||||
TestServerBuilder::new_with_role(
|
|
||||||
name,
|
|
||||||
format!("{name}.example.com").replace('_', "-"),
|
|
||||||
Some(role.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Neither the message nor the report task is touched.
|
|
||||||
async fn assert_idle(server: &Server, msg: &str, task: u64) {
|
|
||||||
tokio::time::sleep(Duration::from_secs(4)).await;
|
|
||||||
server.notify_task_queue();
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
assert!(
|
|
||||||
!attempted(server, msg).await,
|
|
||||||
"delivery attempted without outboundMta"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
is_pending(server, task).await,
|
|
||||||
"report task claimed without outboundMta"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Delivery of the message is attempted and the report task runs.
|
|
||||||
async fn assert_runs(server: &Server, msg: &str, task: u64) {
|
|
||||||
wait_for(Duration::from_secs(20), "message delivery attempt", || {
|
|
||||||
attempted(server, msg)
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
wait_for(Duration::from_secs(20), "report task to run", || async {
|
|
||||||
!is_pending(server, task).await
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
fn role(name: &str, tasks: &[ClusterTaskType]) -> ClusterRole {
|
|
||||||
ClusterRole {
|
|
||||||
name: name.into(),
|
|
||||||
description: None,
|
|
||||||
listeners: ClusterListenerGroup::EnableAll,
|
|
||||||
tasks: ClusterTaskGroup::EnableSome(ClusterTaskGroupProperties {
|
|
||||||
task_types: Map::new(tasks.to_vec()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Stores a new version of a role and reloads the node's settings, as a
|
|
||||||
/// JMAP write to the role does.
|
|
||||||
async fn set_role(registry: &Server, node: &Server, id: Id, new: ClusterRole) {
|
|
||||||
let enabled = matches!(&new.tasks, ClusterTaskGroup::EnableSome(group)
|
|
||||||
if group.task_types.iter().any(|t| *t == ClusterTaskType::OutboundMta));
|
|
||||||
let old = registry
|
|
||||||
.registry()
|
|
||||||
.get(ObjectId::new(ObjectType::ClusterRole, id))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.expect("role not found");
|
|
||||||
let new = Object::from(new);
|
|
||||||
let result = registry
|
|
||||||
.registry()
|
|
||||||
.write(RegistryWrite::update(id, &new, &old))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(
|
|
||||||
matches!(result, RegistryWriteResult::Success(_)),
|
|
||||||
"role update refused"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
node.reload_after_write(ObjectType::ClusterRole).await,
|
|
||||||
Some(Ok(()))
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
node.inner.shared_core.load().network.roles.outbound_mta,
|
|
||||||
enabled
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Queues a message to an unreachable domain and schedules a TLS report
|
|
||||||
/// task, both due now. Returns the recipient's local part and the task id.
|
|
||||||
async fn queue_work(server: &Server, name: &str) -> (String, u64) {
|
|
||||||
let local = format!("{name}-{}", SnowflakeIdGenerator::global_id().unwrap());
|
|
||||||
let rcpt = format!("{local}@{RCPT_DOMAIN}");
|
|
||||||
server
|
|
||||||
.send_autogenerated(
|
|
||||||
"[email protected]",
|
|
||||||
[rcpt.as_str()].into_iter(),
|
|
||||||
format!(
|
|
||||||
"From: [email protected]\r\nTo: {rcpt}\r\n\
|
|
||||||
Subject: live role test\r\n\r\nTest\r\n"
|
|
||||||
)
|
|
||||||
.into_bytes(),
|
|
||||||
None,
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert!(
|
|
||||||
queued_recipient(server, &rcpt).await.is_some(),
|
|
||||||
"message to {rcpt} was not queued"
|
|
||||||
);
|
|
||||||
|
|
||||||
let task = SnowflakeIdGenerator::global_id().unwrap();
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch.schedule_task_with_id(
|
|
||||||
task,
|
|
||||||
Task::TlsReport(TaskTlsReport {
|
|
||||||
report_id: u64::MAX.into(),
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server.notify_task_queue();
|
|
||||||
|
|
||||||
(rcpt, task)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Whether delivery to `rcpt` was tried: the message is gone, or its
|
|
||||||
/// recipient is no longer scheduled or has a retry count.
|
|
||||||
async fn attempted(server: &Server, rcpt: &str) -> bool {
|
|
||||||
match queued_recipient(server, rcpt).await {
|
|
||||||
None => true,
|
|
||||||
Some((status_scheduled, retries)) => !status_scheduled || retries > 0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The queued recipient `rcpt`: whether it is still scheduled, and how many
|
|
||||||
/// times delivery was retried.
|
|
||||||
async fn queued_recipient(server: &Server, rcpt: &str) -> Option<(bool, u32)> {
|
|
||||||
let mut found = None;
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.iterate(
|
|
||||||
IterateParams::new(
|
|
||||||
ValueKey::from(ValueClass::Queue(QueueClass::Message(0))),
|
|
||||||
ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))),
|
|
||||||
),
|
|
||||||
|_, value| {
|
|
||||||
let message = <Archive<AlignedBytes> as Deserialize>::deserialize(value)?
|
|
||||||
.deserialize::<Message>()?;
|
|
||||||
if let Some(recipient) = message
|
|
||||||
.recipients
|
|
||||||
.iter()
|
|
||||||
.find(|recipient| recipient.address.as_ref() == rcpt)
|
|
||||||
{
|
|
||||||
found = Some((
|
|
||||||
matches!(recipient.status, Status::Scheduled),
|
|
||||||
recipient.retry.inner,
|
|
||||||
));
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
Ok(true)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
found
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn is_pending(server: &Server, id: u64) -> bool {
|
|
||||||
matches!(
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.map(|task| task.status().clone()),
|
|
||||||
Some(TaskStatus::Pending(_))
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for<F, Fut>(within: Duration, what: &str, mut check: F)
|
|
||||||
where
|
|
||||||
F: FnMut() -> Fut,
|
|
||||||
Fut: Future<Output = bool>,
|
|
||||||
{
|
|
||||||
let started = Instant::now();
|
|
||||||
while !check().await {
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < within,
|
|
||||||
"still waiting for the {what} after {:?}",
|
|
||||||
started.elapsed()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,14 +2,7 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
pub mod broadcast;
|
pub mod broadcast;
|
||||||
pub mod front_reports; // inbuxa: every node records DMARC and TLS results
|
|
||||||
pub mod live_roles; // inbuxa: role edits apply without a restart
|
|
||||||
#[cfg(feature = "nats")]
|
|
||||||
pub mod coordinator; // inbuxa: coordinator reconnects
|
|
||||||
pub mod stress;
|
pub mod stress;
|
||||||
pub mod task_roles; // inbuxa: task types follow cluster roles
|
|
||||||
|
|||||||
@@ -1,218 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Two task managers with different cluster roles over one shared store:
|
|
||||||
//! each runs only the task types its role allows, and a task one node may
|
|
||||||
//! not run is left for the node that may. Needs a store both nodes can open
|
|
||||||
//! (STORE=PostgreSql or MySql).
|
|
||||||
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
|
||||||
use common::Server;
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::{ClusterTaskType, IndexDocumentType},
|
|
||||||
structs::{
|
|
||||||
ClusterListenerGroup, ClusterRole, ClusterTaskGroup, ClusterTaskGroupProperties, Task,
|
|
||||||
TaskDnsManagement, TaskIndexDocument, TaskStatus, TaskTlsReport,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
types::map::Map,
|
|
||||||
};
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
use store::{
|
|
||||||
ValueKey,
|
|
||||||
write::{BatchBuilder, TaskQueueClass, ValueClass},
|
|
||||||
};
|
|
||||||
use utils::snowflake::SnowflakeIdGenerator;
|
|
||||||
|
|
||||||
const QUEUE_ROLE: &str = "tasks_queue";
|
|
||||||
const INDEX_MTA_ROLE: &str = "tasks_index_mta";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn task_role_tests() {
|
|
||||||
if matches!(
|
|
||||||
std::env::var("STORE").as_deref(),
|
|
||||||
Ok("RocksDb" | "Sqlite") | Err(_)
|
|
||||||
) {
|
|
||||||
println!("Skipping task role tests: they need a store both nodes can open.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!(
|
|
||||||
"Running task role tests on {}...",
|
|
||||||
std::env::var("STORE").unwrap_or_default()
|
|
||||||
);
|
|
||||||
|
|
||||||
// The roles, stored by a node that runs no services of its own (a node
|
|
||||||
// looks its role up when it starts)
|
|
||||||
let seed = TestServerBuilder::new("task_roles_seed")
|
|
||||||
.await
|
|
||||||
.with_object(role(QUEUE_ROLE, &[ClusterTaskType::TaskQueueProcessing]))
|
|
||||||
.await
|
|
||||||
.with_object(role(
|
|
||||||
INDEX_MTA_ROLE,
|
|
||||||
&[
|
|
||||||
ClusterTaskType::SearchIndexing,
|
|
||||||
ClusterTaskType::OutboundMta,
|
|
||||||
],
|
|
||||||
))
|
|
||||||
.await
|
|
||||||
.disable_services()
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
// Node A runs queue tasks (taskQueueProcessing) only
|
|
||||||
let node_a = TestServerBuilder::new_with_role(
|
|
||||||
"task_roles_a",
|
|
||||||
"node-a.example.com".into(),
|
|
||||||
Some(QUEUE_ROLE.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await;
|
|
||||||
let server_a = node_a.server.clone();
|
|
||||||
let roles = &server_a.core.network.roles;
|
|
||||||
assert!(roles.task_manager && !roles.search_indexing && !roles.outbound_mta);
|
|
||||||
|
|
||||||
// A DNS task (taskQueueProcessing), an unindex task (searchIndexing) and
|
|
||||||
// a TLS report (outboundMta), all due now
|
|
||||||
let [dns, unindex, report] = new_task_ids();
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.schedule_task_with_id(
|
|
||||||
dns,
|
|
||||||
Task::DnsManagement(TaskDnsManagement {
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.schedule_task_with_id(
|
|
||||||
unindex,
|
|
||||||
Task::UnindexDocument(TaskIndexDocument {
|
|
||||||
account_id: 0u32.into(),
|
|
||||||
document_id: u32::MAX.into(),
|
|
||||||
document_type: IndexDocumentType::File,
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.schedule_task_with_id(
|
|
||||||
report,
|
|
||||||
Task::TlsReport(TaskTlsReport {
|
|
||||||
report_id: u64::MAX.into(),
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
server_a.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server_a.notify_task_queue();
|
|
||||||
|
|
||||||
// Node A runs the DNS task and leaves the other two alone. Upstream ran
|
|
||||||
// the TLS report here too: report tasks ran on any node with a task
|
|
||||||
// manager.
|
|
||||||
wait_until_run(&server_a, &[dns], Duration::from_secs(20)).await;
|
|
||||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
|
||||||
server_a.notify_task_queue();
|
|
||||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
|
||||||
assert!(
|
|
||||||
is_pending(&server_a, unindex).await,
|
|
||||||
"unindex ran on node A"
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
is_pending(&server_a, report).await,
|
|
||||||
"TLS report ran on node A"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Node B (search indexing and outbound MTA) comes up and picks up what
|
|
||||||
// node A left
|
|
||||||
let node_b = TestServerBuilder::new_with_role(
|
|
||||||
"task_roles_b",
|
|
||||||
"node-b.example.com".into(),
|
|
||||||
Some(INDEX_MTA_ROLE.into()),
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.build_with_opts(false)
|
|
||||||
.await;
|
|
||||||
let server_b = node_b.server.clone();
|
|
||||||
let roles = &server_b.core.network.roles;
|
|
||||||
assert!(!roles.task_manager && roles.search_indexing && roles.outbound_mta);
|
|
||||||
server_b.notify_task_queue();
|
|
||||||
wait_until_run(&server_b, &[unindex, report], Duration::from_secs(20)).await;
|
|
||||||
|
|
||||||
// A queue task scheduled now still runs, on node A: node B may not
|
|
||||||
// claim it
|
|
||||||
let [dns] = new_task_ids();
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch.schedule_task_with_id(
|
|
||||||
dns,
|
|
||||||
Task::DnsManagement(TaskDnsManagement {
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
server_b.store().write(batch.build_all()).await.unwrap();
|
|
||||||
server_b.notify_task_queue();
|
|
||||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
|
||||||
assert!(is_pending(&server_b, dns).await, "DNS task ran on node B");
|
|
||||||
server_a.notify_task_queue();
|
|
||||||
wait_until_run(&server_a, &[dns], Duration::from_secs(20)).await;
|
|
||||||
|
|
||||||
if seed.is_reset() {
|
|
||||||
seed.temp_dir.delete();
|
|
||||||
node_a.temp_dir.delete();
|
|
||||||
node_b.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn role(name: &str, tasks: &[ClusterTaskType]) -> ClusterRole {
|
|
||||||
ClusterRole {
|
|
||||||
name: name.into(),
|
|
||||||
description: None,
|
|
||||||
listeners: ClusterListenerGroup::EnableAll,
|
|
||||||
tasks: ClusterTaskGroup::EnableSome(ClusterTaskGroupProperties {
|
|
||||||
task_types: Map::new(tasks.to_vec()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn new_task_ids<const N: usize>() -> [u64; N] {
|
|
||||||
std::array::from_fn(|_| SnowflakeIdGenerator::global_id().unwrap())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Still due and never run: present, and pending.
|
|
||||||
async fn is_pending(server: &Server, id: u64) -> bool {
|
|
||||||
matches!(
|
|
||||||
server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.map(|task| task.status().clone()),
|
|
||||||
Some(TaskStatus::Pending(_))
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_until_run(server: &Server, ids: &[u64], within: Duration) {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let mut left = 0;
|
|
||||||
for id in ids {
|
|
||||||
if is_pending(server, *id).await {
|
|
||||||
left += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if left == 0 {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < within,
|
|
||||||
"{left} task(s) still pending after {:?}",
|
|
||||||
started.elapsed()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
@@ -77,7 +75,7 @@ async fn milter_session() {
|
|||||||
else_: "true".into(),
|
else_: "true".into(),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
hostname: "localhost".into(), // inbuxa: resolved when the session connects
|
hostname: "127.0.0.1".into(),
|
||||||
port: 9332,
|
port: 9332,
|
||||||
use_tls: false,
|
use_tls: false,
|
||||||
stages: Map::new(vec![MtaStage::Data]),
|
stages: Map::new(vec![MtaStage::Data]),
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::utils::server::TestServer;
|
use crate::utils::server::TestServer;
|
||||||
@@ -42,25 +40,17 @@ pub mod vrfy;
|
|||||||
const EVENT_TIMEOUT: Duration = Duration::from_secs(5);
|
const EVENT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
impl TestServer {
|
impl TestServer {
|
||||||
// inbuxa: registry writes reload the settings, and each reload sends the
|
|
||||||
// queue a ReloadSettings; read_event, try_read_event and assert_no_events
|
|
||||||
// pass over those (expect_reload_settings still waits for one)
|
|
||||||
pub async fn read_event(&mut self) -> QueueEvent {
|
pub async fn read_event(&mut self) -> QueueEvent {
|
||||||
while let Some(event) = self.queue_events.pop_front() {
|
if let Some(event) = self.queue_events.pop_front() {
|
||||||
if !event.is_reload_settings() {
|
|
||||||
return event;
|
return event;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
loop {
|
|
||||||
match tokio::time::timeout(EVENT_TIMEOUT, self.queue_rx.recv()).await {
|
match tokio::time::timeout(EVENT_TIMEOUT, self.queue_rx.recv()).await {
|
||||||
Ok(Some(event)) if event.is_reload_settings() => (),
|
Ok(Some(event)) => event,
|
||||||
Ok(Some(event)) => return event,
|
|
||||||
Ok(None) => panic!("Channel closed."),
|
Ok(None) => panic!("Channel closed."),
|
||||||
Err(_) => panic!("No queue event received."),
|
Err(_) => panic!("No queue event received."),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn read_event_matching(
|
pub async fn read_event_matching(
|
||||||
&mut self,
|
&mut self,
|
||||||
@@ -88,41 +78,28 @@ impl TestServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn try_read_event(&mut self) -> Option<QueueEvent> {
|
pub async fn try_read_event(&mut self) -> Option<QueueEvent> {
|
||||||
while let Some(event) = self.queue_events.pop_front() {
|
if let Some(event) = self.queue_events.pop_front() {
|
||||||
if !event.is_reload_settings() {
|
|
||||||
return Some(event);
|
return Some(event);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
loop {
|
|
||||||
match tokio::time::timeout(EVENT_TIMEOUT, self.queue_rx.recv()).await {
|
match tokio::time::timeout(EVENT_TIMEOUT, self.queue_rx.recv()).await {
|
||||||
Ok(Some(event)) if event.is_reload_settings() => (),
|
Ok(Some(event)) => Some(event),
|
||||||
Ok(Some(event)) => return Some(event),
|
|
||||||
Ok(None) => panic!("Channel closed."),
|
Ok(None) => panic!("Channel closed."),
|
||||||
Err(_) => return None,
|
Err(_) => None,
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn assert_no_events(&mut self) {
|
pub fn assert_no_events(&mut self) {
|
||||||
if let Some(event) = self
|
if let Some(event) = self.queue_events.pop_front() {
|
||||||
.queue_events
|
|
||||||
.iter()
|
|
||||||
.find(|event| !event.is_reload_settings())
|
|
||||||
{
|
|
||||||
panic!("Expected empty queue but got {event:?}");
|
panic!("Expected empty queue but got {event:?}");
|
||||||
}
|
}
|
||||||
self.queue_events.clear();
|
|
||||||
|
|
||||||
loop {
|
|
||||||
match self.queue_rx.try_recv() {
|
match self.queue_rx.try_recv() {
|
||||||
Ok(event) if event.is_reload_settings() => (),
|
Err(TryRecvError::Empty) => (),
|
||||||
Err(TryRecvError::Empty) => break,
|
|
||||||
Ok(event) => panic!("Expected empty queue but got {event:?}"),
|
Ok(event) => panic!("Expected empty queue but got {event:?}"),
|
||||||
Err(err) => panic!("Queue error: {err:?}"),
|
Err(err) => panic!("Queue error: {err:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn assert_queue_is_empty(&self) {
|
pub async fn assert_queue_is_empty(&self) {
|
||||||
assert_eq!(self.read_queued_messages().await, vec![]);
|
assert_eq!(self.read_queued_messages().await, vec![]);
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::utils::{
|
use crate::utils::{
|
||||||
@@ -11,22 +9,14 @@ use crate::utils::{
|
|||||||
server::TestServer,
|
server::TestServer,
|
||||||
temp_dir::TempDir,
|
temp_dir::TempDir,
|
||||||
};
|
};
|
||||||
use ::registry::schema::{
|
use ::registry::schema::enums::CompressionAlgo;
|
||||||
enums::{CompressionAlgo, TaskStoreMaintenanceType},
|
|
||||||
prelude::ObjectType,
|
|
||||||
structs::Task,
|
|
||||||
};
|
|
||||||
use ahash::AHashSet;
|
use ahash::AHashSet;
|
||||||
use common::{
|
use common::{DATABASE_SCHEMA_VERSION, manager::backup::BackupParams};
|
||||||
DATABASE_SCHEMA_VERSION,
|
|
||||||
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, backup::BackupParams},
|
|
||||||
};
|
|
||||||
use store::{
|
use store::{
|
||||||
rand,
|
rand,
|
||||||
write::{
|
write::{
|
||||||
AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, Operation, QueueClass, QueueEvent,
|
AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, Operation, QueueClass, QueueEvent,
|
||||||
RegistryClass, TaskQueueClass, ValueClass,
|
RegistryClass, ValueClass, key::KeySerializer,
|
||||||
key::{DeserializeBigEndian, KeySerializer},
|
|
||||||
},
|
},
|
||||||
*,
|
*,
|
||||||
};
|
};
|
||||||
@@ -177,50 +167,6 @@ pub async fn test(test: &TestServer) {
|
|||||||
}
|
}
|
||||||
db.write(batch.build_all()).await.unwrap();
|
db.write(batch.build_all()).await.unwrap();
|
||||||
|
|
||||||
// inbuxa: registry objects kept outside the registry subspace (archived
|
|
||||||
// items for undelete, spam training samples, directory entries) and the
|
|
||||||
// fork's own subspace. Exports used to leave the first two behind.
|
|
||||||
println!("Creating archived items, spam samples and fork data...");
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
for item_id in [1u64, 2, 3] {
|
|
||||||
for object in [
|
|
||||||
ObjectType::ArchivedItem,
|
|
||||||
ObjectType::SpamTrainingSample,
|
|
||||||
ObjectType::Account,
|
|
||||||
] {
|
|
||||||
batch.set(
|
|
||||||
ValueClass::Registry(RegistryClass::Item {
|
|
||||||
object_id: object as u16,
|
|
||||||
item_id,
|
|
||||||
}),
|
|
||||||
random_bytes(item_id as usize * 64),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
batch.set(
|
|
||||||
ValueClass::Any(AnyClass {
|
|
||||||
subspace: SUBSPACE_INBUXA,
|
|
||||||
key: [b'U', b'x']
|
|
||||||
.into_iter()
|
|
||||||
.chain(item_id.to_be_bytes())
|
|
||||||
.collect(),
|
|
||||||
}),
|
|
||||||
random_bytes(32),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
db.write(batch.build_all()).await.unwrap();
|
|
||||||
|
|
||||||
// inbuxa: the trained spam classifier lives in blobs with fixed names
|
|
||||||
let mut named_blobs = Vec::new();
|
|
||||||
for key in [SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY] {
|
|
||||||
let data = random_bytes(4096);
|
|
||||||
test.server
|
|
||||||
.blob_store()
|
|
||||||
.put_blob(key, &data, CompressionAlgo::Lz4)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
named_blobs.push((key, data));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create directory data
|
// Create directory data
|
||||||
println!("Creating directory data...");
|
println!("Creating directory data...");
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
@@ -239,17 +185,6 @@ pub async fn test(test: &TestServer) {
|
|||||||
println!("Calculating store hash...");
|
println!("Calculating store hash...");
|
||||||
let snapshot = Snapshot::new(&db).await;
|
let snapshot = Snapshot::new(&db).await;
|
||||||
assert!(!snapshot.keys.is_empty(), "Store hash counts are empty",);
|
assert!(!snapshot.keys.is_empty(), "Store hash counts are empty",);
|
||||||
for subspace in [
|
|
||||||
SUBSPACE_DELETED_ITEMS,
|
|
||||||
SUBSPACE_SPAM_SAMPLES,
|
|
||||||
SUBSPACE_INBUXA,
|
|
||||||
] {
|
|
||||||
assert!(
|
|
||||||
snapshot.keys.iter().any(|k| k.subspace == subspace),
|
|
||||||
"No test data in subspace {}",
|
|
||||||
char::from(subspace)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Export store
|
// Export store
|
||||||
println!("Exporting store...");
|
println!("Exporting store...");
|
||||||
@@ -275,188 +210,22 @@ pub async fn test(test: &TestServer) {
|
|||||||
.finalize(),
|
.finalize(),
|
||||||
);
|
);
|
||||||
db.write(batch.build_all()).await.unwrap();
|
db.write(batch.build_all()).await.unwrap();
|
||||||
for (key, _) in &named_blobs {
|
test.server.core.restore(temp_dir.path.clone()).await;
|
||||||
test.server.blob_store().delete_blob(key).await.unwrap();
|
|
||||||
}
|
|
||||||
let imported = test.server.core.restore(temp_dir.path.clone()).await;
|
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
batch.clear(ValueClass::NodeId(0));
|
batch.clear(ValueClass::NodeId(0));
|
||||||
db.write(batch.build_all()).await.unwrap();
|
db.write(batch.build_all()).await.unwrap();
|
||||||
for subspace in [
|
|
||||||
SUBSPACE_DELETED_ITEMS,
|
|
||||||
SUBSPACE_SPAM_SAMPLES,
|
|
||||||
SUBSPACE_INBUXA,
|
|
||||||
] {
|
|
||||||
assert!(
|
|
||||||
imported.contains(&subspace),
|
|
||||||
"Subspace {} was not exported",
|
|
||||||
char::from(subspace)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify hash
|
// Verify hash
|
||||||
print!("Verifying store hash...");
|
print!("Verifying store hash...");
|
||||||
snapshot.assert_is_eq(&Snapshot::new(&db).await);
|
snapshot.assert_is_eq(&Snapshot::new(&db).await);
|
||||||
assert_named_blobs(test.server.blob_store(), &named_blobs).await;
|
|
||||||
println!(" GREAT SUCCESS!");
|
println!(" GREAT SUCCESS!");
|
||||||
|
|
||||||
// inbuxa: import the same export into a fresh store of another backend,
|
|
||||||
// the way a move from one database to another does it
|
|
||||||
#[cfg(all(feature = "rocks", feature = "sqlite"))]
|
|
||||||
cross_backend(test, &db, &temp_dir, &named_blobs).await;
|
|
||||||
|
|
||||||
// Destroy store
|
// Destroy store
|
||||||
for (key, _) in &named_blobs {
|
|
||||||
test.server.blob_store().delete_blob(key).await.unwrap();
|
|
||||||
}
|
|
||||||
store_destroy(&db).await;
|
store_destroy(&db).await;
|
||||||
store_assert_is_empty(&db, db.clone().into(), true).await;
|
store_assert_is_empty(&db, db.clone().into(), true).await;
|
||||||
temp_dir.delete();
|
temp_dir.delete();
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(feature = "rocks", feature = "sqlite"))]
|
|
||||||
async fn cross_backend(
|
|
||||||
test: &TestServer,
|
|
||||||
source: &Store,
|
|
||||||
export: &TempDir,
|
|
||||||
named_blobs: &[(&[u8], Vec<u8>)],
|
|
||||||
) {
|
|
||||||
let source_type = std::env::var("STORE").unwrap();
|
|
||||||
let target_type = if source_type.eq_ignore_ascii_case("sqlite") {
|
|
||||||
"RocksDb"
|
|
||||||
} else {
|
|
||||||
"Sqlite"
|
|
||||||
};
|
|
||||||
println!("Importing the export into a fresh {target_type} store...");
|
|
||||||
|
|
||||||
let target_dir = TempDir::new("art_vandelay_cross_backend", true);
|
|
||||||
let target = Store::build(
|
|
||||||
crate::utils::storage::build_data_store(target_type, &target_dir.path.to_string_lossy())
|
|
||||||
.await,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
target.create_tables().await.unwrap();
|
|
||||||
store_destroy(&target).await;
|
|
||||||
|
|
||||||
let mut core = test.server.core.as_ref().clone();
|
|
||||||
core.storage.data = target.clone();
|
|
||||||
core.storage.blob = target.clone().into();
|
|
||||||
let imported = core.restore(export.path.clone()).await;
|
|
||||||
|
|
||||||
// Counters are stored differently by the SQL and key-value backends, so
|
|
||||||
// compare their keys here and their values through the counter API.
|
|
||||||
print!("Verifying {target_type} store hash...");
|
|
||||||
Snapshot::new_portable(source)
|
|
||||||
.await
|
|
||||||
.assert_is_eq(&Snapshot::new_portable(&target).await);
|
|
||||||
for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA] {
|
|
||||||
let mut keys = Vec::new();
|
|
||||||
source
|
|
||||||
.iterate(
|
|
||||||
IterateParams::new(
|
|
||||||
AnyKey {
|
|
||||||
subspace,
|
|
||||||
key: vec![0u8],
|
|
||||||
},
|
|
||||||
AnyKey {
|
|
||||||
subspace,
|
|
||||||
key: vec![u8::MAX; 10],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.no_values(),
|
|
||||||
|key, _| {
|
|
||||||
keys.push(key.to_vec());
|
|
||||||
Ok(true)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
for key in keys {
|
|
||||||
let class = || {
|
|
||||||
ValueClass::Any(AnyClass {
|
|
||||||
subspace,
|
|
||||||
key: key.clone(),
|
|
||||||
})
|
|
||||||
};
|
|
||||||
assert_eq!(
|
|
||||||
source.get_counter(class()).await.unwrap(),
|
|
||||||
target.get_counter(class()).await.unwrap(),
|
|
||||||
"Counter mismatch in {} for {key:?}",
|
|
||||||
char::from(subspace)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert_named_blobs(&core.storage.blob, named_blobs).await;
|
|
||||||
println!(" GREAT SUCCESS!");
|
|
||||||
|
|
||||||
// The search index isn't exported; the import queues its rebuild
|
|
||||||
let queued = core.queue_reindex(&imported).await;
|
|
||||||
let expected = [
|
|
||||||
TaskStoreMaintenanceType::ReindexAccounts,
|
|
||||||
TaskStoreMaintenanceType::ReindexTelemetry,
|
|
||||||
];
|
|
||||||
assert_eq!(queued, expected);
|
|
||||||
let mut task_ids = Vec::new();
|
|
||||||
target
|
|
||||||
.iterate(
|
|
||||||
IterateParams::new(
|
|
||||||
AnyKey {
|
|
||||||
subspace: SUBSPACE_TASK_QUEUE,
|
|
||||||
key: vec![0u8],
|
|
||||||
},
|
|
||||||
AnyKey {
|
|
||||||
subspace: SUBSPACE_TASK_QUEUE,
|
|
||||||
key: vec![u8::MAX; 20],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.no_values(),
|
|
||||||
|key, _| {
|
|
||||||
if key.deserialize_be_u64(0)? == 0 {
|
|
||||||
task_ids.push(key.deserialize_be_u64(U64_LEN)?);
|
|
||||||
}
|
|
||||||
Ok(true)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let mut found = Vec::new();
|
|
||||||
for id in task_ids {
|
|
||||||
match target
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
{
|
|
||||||
Some(Task::StoreMaintenance(task)) => found.push(task.maintenance_type),
|
|
||||||
other => panic!("Unexpected task {other:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
found.sort_by_key(|t| *t as u16);
|
|
||||||
assert_eq!(found, expected, "Queued tasks don't match");
|
|
||||||
|
|
||||||
store_destroy(&target).await;
|
|
||||||
drop(core);
|
|
||||||
drop(target);
|
|
||||||
target_dir.delete();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn assert_named_blobs(blob_store: &BlobStore, named_blobs: &[(&[u8], Vec<u8>)]) {
|
|
||||||
for (key, data) in named_blobs {
|
|
||||||
assert_eq!(
|
|
||||||
blob_store
|
|
||||||
.get_blob(key, 0..usize::MAX)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.as_ref(),
|
|
||||||
Some(data),
|
|
||||||
"Blob {} was not restored",
|
|
||||||
String::from_utf8_lossy(key)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq)]
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
struct Snapshot {
|
struct Snapshot {
|
||||||
keys: AHashSet<KeyValue>,
|
keys: AHashSet<KeyValue>,
|
||||||
@@ -471,19 +240,7 @@ struct KeyValue {
|
|||||||
|
|
||||||
impl Snapshot {
|
impl Snapshot {
|
||||||
async fn new(db: &Store) -> Self {
|
async fn new(db: &Store) -> Self {
|
||||||
Self::build(db, !db.is_sql(), true).await
|
let is_sql = db.is_sql();
|
||||||
}
|
|
||||||
|
|
||||||
/// Comparable across backends: no counter values, which the SQL and
|
|
||||||
/// key-value stores encode differently, and no blobs, which only live in
|
|
||||||
/// the data store when it doubles as the blob store.
|
|
||||||
#[cfg(all(feature = "rocks", feature = "sqlite"))]
|
|
||||||
async fn new_portable(db: &Store) -> Self {
|
|
||||||
Self::build(db, false, false).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn build(db: &Store, counter_values: bool, with_blobs: bool) -> Self {
|
|
||||||
let is_sql = !counter_values;
|
|
||||||
|
|
||||||
let mut keys = AHashSet::new();
|
let mut keys = AHashSet::new();
|
||||||
|
|
||||||
@@ -508,12 +265,7 @@ impl Snapshot {
|
|||||||
(SUBSPACE_QUOTA, !is_sql),
|
(SUBSPACE_QUOTA, !is_sql),
|
||||||
(SUBSPACE_REPORT_OUT, true),
|
(SUBSPACE_REPORT_OUT, true),
|
||||||
(SUBSPACE_REPORT_IN, true),
|
(SUBSPACE_REPORT_IN, true),
|
||||||
(SUBSPACE_DIRECTORY, true),
|
|
||||||
(SUBSPACE_INBUXA, true),
|
|
||||||
] {
|
] {
|
||||||
if subspace == SUBSPACE_BLOBS && !with_blobs {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let from_key = AnyKey {
|
let from_key = AnyKey {
|
||||||
subspace,
|
subspace,
|
||||||
key: vec![0u8],
|
key: vec![0u8],
|
||||||
|
|||||||
@@ -10,8 +10,6 @@ pub mod blob;
|
|||||||
pub mod import_export;
|
pub mod import_export;
|
||||||
pub mod lookup;
|
pub mod lookup;
|
||||||
pub mod ops;
|
pub mod ops;
|
||||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
|
||||||
pub mod pool_timeout; // inbuxa: SQL pools give up instead of hanging
|
|
||||||
pub mod query;
|
pub mod query;
|
||||||
pub mod registry;
|
pub mod registry;
|
||||||
#[cfg(feature = "postgres")]
|
#[cfg(feature = "postgres")]
|
||||||
@@ -21,11 +19,8 @@ pub mod replica_mysql; // inbuxa: read replicas on MySQL
|
|||||||
#[cfg(all(feature = "postgres", feature = "redis"))]
|
#[cfg(all(feature = "postgres", feature = "redis"))]
|
||||||
pub mod replica_cluster; // inbuxa: read replicas across nodes
|
pub mod replica_cluster; // inbuxa: read replicas across nodes
|
||||||
pub mod scaleout; // inbuxa: scale-out storage
|
pub mod scaleout; // inbuxa: scale-out storage
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
pub mod search_gin; // inbuxa: GIN indexes without a pending list
|
|
||||||
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
#[cfg(any(feature = "postgres", feature = "mysql"))]
|
||||||
pub mod sql_timeout;
|
pub mod sql_timeout;
|
||||||
pub mod task_locks; // inbuxa: task locks across nodes
|
|
||||||
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
use crate::utils::server::TestServerBuilder;
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
|
|||||||
@@ -1,376 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! A database that accepts connections and then says nothing (a hung or
|
|
||||||
//! half-dead server, a black-holed failover) gives a worker an error within
|
|
||||||
//! the pool's timeouts. Upstream's pools had none, so the worker waited for
|
|
||||||
//! good. No database is needed: a local listener that never answers plays
|
|
||||||
//! the server.
|
|
||||||
//!
|
|
||||||
//! inbuxa: the same for a database that stops answering while connections
|
|
||||||
//! are already open (a paused container): a query on a checked-out
|
|
||||||
//! connection ends within the query limit, the store works again once the
|
|
||||||
//! database is back, and /healthz/ready says 503 in between while
|
|
||||||
//! /healthz/live stays 200. These need the local test databases; a proxy
|
|
||||||
//! that can stop forwarding plays the pause.
|
|
||||||
|
|
||||||
use registry::schema::structs::DataStore;
|
|
||||||
use std::{
|
|
||||||
sync::{
|
|
||||||
Arc,
|
|
||||||
atomic::{AtomicBool, Ordering},
|
|
||||||
},
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use store::{
|
|
||||||
IterateParams, Store, ValueKey,
|
|
||||||
backend::query_timeout::QueryTimeouts,
|
|
||||||
write::{BatchBuilder, ValueClass},
|
|
||||||
};
|
|
||||||
use tokio::{
|
|
||||||
io::{AsyncReadExt, AsyncWriteExt},
|
|
||||||
net::{TcpListener, TcpStream},
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Accepts connections on a local port and never sends a byte.
|
|
||||||
async fn silent_server() -> u16 {
|
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let port = listener.local_addr().unwrap().port();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
let mut held = Vec::new();
|
|
||||||
while let Ok((socket, _)) = listener.accept().await {
|
|
||||||
held.push(socket);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
port
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Builds the store and reads a key; both must end, with an error for the
|
|
||||||
/// read, well within `limit`.
|
|
||||||
async fn assert_times_out(data_store: DataStore, limit: Duration) {
|
|
||||||
let started = Instant::now();
|
|
||||||
let result = tokio::time::timeout(limit, async {
|
|
||||||
match Store::build(data_store).await {
|
|
||||||
Ok(store) => store
|
|
||||||
.get_value::<u64>(ValueKey::from(ValueClass::Property(0)))
|
|
||||||
.await
|
|
||||||
.map(|_| ())
|
|
||||||
.map_err(|err| err.to_string()),
|
|
||||||
Err(err) => Err(err.to_string()),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let elapsed = started.elapsed();
|
|
||||||
match result {
|
|
||||||
Ok(Err(err)) => println!("Got {err} after {elapsed:?}"),
|
|
||||||
Ok(Ok(())) => panic!("a silent server answered?"),
|
|
||||||
Err(_) => panic!("still waiting for a connection after {elapsed:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_pool_timeout() {
|
|
||||||
use registry::schema::structs::PostgreSqlStore;
|
|
||||||
|
|
||||||
let port = silent_server().await;
|
|
||||||
println!("Running PostgreSQL pool timeout test...");
|
|
||||||
// The store's own timeout bounds opening a connection, handshake
|
|
||||||
// included (tokio-postgres's connect_timeout covers only the TCP connect)
|
|
||||||
assert_times_out(
|
|
||||||
DataStore::PostgreSql(PostgreSqlStore {
|
|
||||||
host: "127.0.0.1".into(),
|
|
||||||
port: port as u64,
|
|
||||||
database: "none".into(),
|
|
||||||
timeout: Some(Duration::from_secs(2).into()),
|
|
||||||
use_tls: false,
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
Duration::from_secs(20),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "mysql")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn mysql_pool_timeout() {
|
|
||||||
use registry::schema::structs::MySqlStore;
|
|
||||||
|
|
||||||
let port = silent_server().await;
|
|
||||||
println!("Running MySQL pool timeout test...");
|
|
||||||
// mysql_async has no pool timeout; the store waits 30 s for a connection
|
|
||||||
assert_times_out(
|
|
||||||
DataStore::MySql(MySqlStore {
|
|
||||||
host: "127.0.0.1".into(),
|
|
||||||
port: port as u64,
|
|
||||||
database: "none".into(),
|
|
||||||
use_tls: false,
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
Duration::from_secs(60),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A TCP proxy to a local port that can stop forwarding, in both
|
|
||||||
/// directions, while keeping every connection open: a paused server whose
|
|
||||||
/// kernel still keeps the connections up.
|
|
||||||
struct PausableProxy {
|
|
||||||
port: u16,
|
|
||||||
paused: Arc<AtomicBool>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl PausableProxy {
|
|
||||||
async fn start(upstream: u16) -> Self {
|
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
||||||
let port = listener.local_addr().unwrap().port();
|
|
||||||
let paused = Arc::new(AtomicBool::new(false));
|
|
||||||
let paused_ = paused.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
while let Ok((client, _)) = listener.accept().await {
|
|
||||||
let Ok(server) = TcpStream::connect(("127.0.0.1", upstream)).await else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let (client_rx, client_tx) = client.into_split();
|
|
||||||
let (server_rx, server_tx) = server.into_split();
|
|
||||||
tokio::spawn(forward(client_rx, server_tx, paused_.clone()));
|
|
||||||
tokio::spawn(forward(server_rx, client_tx, paused_.clone()));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
PausableProxy { port, paused }
|
|
||||||
}
|
|
||||||
|
|
||||||
fn pause(&self, paused: bool) {
|
|
||||||
self.paused.store(paused, Ordering::SeqCst);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn forward(
|
|
||||||
mut from: tokio::net::tcp::OwnedReadHalf,
|
|
||||||
mut to: tokio::net::tcp::OwnedWriteHalf,
|
|
||||||
paused: Arc<AtomicBool>,
|
|
||||||
) {
|
|
||||||
let mut buf = vec![0u8; 16384];
|
|
||||||
loop {
|
|
||||||
while paused.load(Ordering::SeqCst) {
|
|
||||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
||||||
}
|
|
||||||
let n = match from.read(&mut buf).await {
|
|
||||||
Ok(0) | Err(_) => return,
|
|
||||||
Ok(n) => n,
|
|
||||||
};
|
|
||||||
// Hold what arrived while paused until the pause ends
|
|
||||||
while paused.load(Ordering::SeqCst) {
|
|
||||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
||||||
}
|
|
||||||
if to.write_all(&buf[..n]).await.is_err() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const TEST_LIMITS: QueryTimeouts = QueryTimeouts {
|
|
||||||
query: Duration::from_secs(2),
|
|
||||||
maintenance: Duration::from_secs(3),
|
|
||||||
};
|
|
||||||
|
|
||||||
/// Opens `connections` pooled connections at once, so the operations that
|
|
||||||
/// follow find one idle and check it out.
|
|
||||||
async fn warm(store: &Store, connections: usize) {
|
|
||||||
let reads = (0..connections).map(|_| async {
|
|
||||||
store
|
|
||||||
.get_value::<u64>(ValueKey::from(ValueClass::Property(0)))
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
});
|
|
||||||
futures::future::join_all(reads).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// With the database paused, reads, scans and writes on connections the
|
|
||||||
/// pool already holds end in an error within the query limit; once it is
|
|
||||||
/// back, the store works again.
|
|
||||||
async fn assert_queries_time_out(store: Store, proxy: &PausableProxy) {
|
|
||||||
store.create_tables().await.unwrap();
|
|
||||||
warm(&store, 4).await;
|
|
||||||
// mysql_async resets a connection on its way back to the pool; let
|
|
||||||
// those finish, or the connections are stuck in the reset when the
|
|
||||||
// pause starts and the pool's own wait timeout answers instead
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
proxy.pause(true);
|
|
||||||
|
|
||||||
let key = || ValueKey::from(ValueClass::Property(0));
|
|
||||||
let limit = TEST_LIMITS.query;
|
|
||||||
for (what, op) in [("read", 0), ("scan", 1), ("write", 2)] {
|
|
||||||
let started = Instant::now();
|
|
||||||
let result = tokio::time::timeout(Duration::from_secs(20), async {
|
|
||||||
match op {
|
|
||||||
0 => store.get_value::<u64>(key()).await.map(|_| ()),
|
|
||||||
1 => {
|
|
||||||
store
|
|
||||||
.iterate(
|
|
||||||
IterateParams::new(
|
|
||||||
ValueKey::from(ValueClass::Property(0)),
|
|
||||||
ValueKey::from(ValueClass::Property(u8::MAX)),
|
|
||||||
),
|
|
||||||
|_, _| Ok(true),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
batch
|
|
||||||
.with_account_id(u32::MAX - 7)
|
|
||||||
.with_collection(types::collection::Collection::Email)
|
|
||||||
.with_document(0)
|
|
||||||
.set(ValueClass::Property(0), 1u64.to_be_bytes().to_vec());
|
|
||||||
store.write(batch.build_all()).await.map(|_| ())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
let elapsed = started.elapsed();
|
|
||||||
match result {
|
|
||||||
Ok(Err(err)) => {
|
|
||||||
let err = format!("{err:?}");
|
|
||||||
println!("Paused database, {what}: {err} after {elapsed:?}");
|
|
||||||
assert!(err.contains("Query timed out"), "{what}: {err}");
|
|
||||||
assert!(
|
|
||||||
elapsed >= limit && elapsed < limit * 3,
|
|
||||||
"{what} ended after {elapsed:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(Ok(())) => panic!("{what} succeeded against a paused database"),
|
|
||||||
Err(_) => panic!("{what} still waiting after {elapsed:?}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
proxy.pause(false);
|
|
||||||
tokio::time::timeout(Duration::from_secs(20), store.get_value::<u64>(key()))
|
|
||||||
.await
|
|
||||||
.expect("still waiting after the database came back")
|
|
||||||
.expect("the store didn't recover");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_query_timeout() {
|
|
||||||
println!("Running PostgreSQL query timeout test...");
|
|
||||||
let DataStore::PostgreSql(mut config) =
|
|
||||||
crate::utils::storage::build_data_store("PostgreSql", "").await
|
|
||||||
else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
let proxy = PausableProxy::start(config.port as u16).await;
|
|
||||||
config.host = "127.0.0.1".into();
|
|
||||||
config.port = proxy.port as u64;
|
|
||||||
// New connections through the paused proxy give up as quickly
|
|
||||||
config.timeout = Some(TEST_LIMITS.query.into());
|
|
||||||
let store = Store::build(DataStore::PostgreSql(config))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.with_query_timeouts(TEST_LIMITS);
|
|
||||||
assert_queries_time_out(store, &proxy).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "mysql")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn mysql_query_timeout() {
|
|
||||||
println!("Running MySQL query timeout test...");
|
|
||||||
let DataStore::MySql(mut config) = crate::utils::storage::build_data_store("MySql", "").await
|
|
||||||
else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
let proxy = PausableProxy::start(config.port as u16).await;
|
|
||||||
config.host = "127.0.0.1".into();
|
|
||||||
config.port = proxy.port as u64;
|
|
||||||
let store = Store::build(DataStore::MySql(config))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.with_query_timeouts(TEST_LIMITS);
|
|
||||||
assert_queries_time_out(store, &proxy).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// /healthz/ready follows the data store; /healthz/live doesn't.
|
|
||||||
#[cfg(feature = "postgres")]
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_readiness() {
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
|
||||||
use registry::schema::enums::NetworkListenerProtocol;
|
|
||||||
|
|
||||||
const HTTP_PORT: u16 = 11_320;
|
|
||||||
if std::env::var("STORE").as_deref() != Ok("PostgreSql") {
|
|
||||||
println!("Skipping the readiness test: it runs with STORE=PostgreSql.");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!("Running readiness test...");
|
|
||||||
|
|
||||||
let test = TestServerBuilder::new("postgres_readiness")
|
|
||||||
.await
|
|
||||||
.with_listener(NetworkListenerProtocol::Http, "http", HTTP_PORT, true)
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
// Point the running node's data store at the database through the proxy
|
|
||||||
let DataStore::PostgreSql(mut config) =
|
|
||||||
crate::utils::storage::build_data_store("PostgreSql", "").await
|
|
||||||
else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
let proxy = PausableProxy::start(config.port as u16).await;
|
|
||||||
config.host = "127.0.0.1".into();
|
|
||||||
config.port = proxy.port as u64;
|
|
||||||
config.timeout = Some(TEST_LIMITS.query.into());
|
|
||||||
let store = Store::build(DataStore::PostgreSql(config))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.with_query_timeouts(TEST_LIMITS);
|
|
||||||
let inner = &test.server.inner;
|
|
||||||
let mut core = inner.shared_core.load_full().as_ref().clone();
|
|
||||||
core.storage.data = store;
|
|
||||||
inner.shared_core.store(Arc::new(core));
|
|
||||||
|
|
||||||
let health = |path: &'static str| async move {
|
|
||||||
reqwest::Client::builder()
|
|
||||||
.danger_accept_invalid_certs(true)
|
|
||||||
.timeout(Duration::from_secs(10))
|
|
||||||
.build()
|
|
||||||
.unwrap()
|
|
||||||
.get(format!("https://127.0.0.1:{HTTP_PORT}/healthz/{path}"))
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.status()
|
|
||||||
.as_u16()
|
|
||||||
};
|
|
||||||
let wait_for = |path: &'static str, status: u16| async move {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let got = health(path).await;
|
|
||||||
if got == status {
|
|
||||||
println!("/healthz/{path}: {got} after {:?}", started.elapsed());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(20),
|
|
||||||
"/healthz/{path} still {got}, expected {status}"
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
wait_for("ready", 200).await;
|
|
||||||
proxy.pause(true);
|
|
||||||
wait_for("ready", 503).await;
|
|
||||||
assert_eq!(health("live").await, 200);
|
|
||||||
proxy.pause(false);
|
|
||||||
wait_for("ready", 200).await;
|
|
||||||
assert_eq!(health("live").await, 200);
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,8 +2,6 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
*
|
|
||||||
* Modified by Coffey Labs in 2026 for INBUXA.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::{store::deflate_test_resource, utils::server::TestServer};
|
use crate::{store::deflate_test_resource, utils::server::TestServer};
|
||||||
@@ -124,20 +122,6 @@ pub async fn test(test: &TestServer) {
|
|||||||
println!("Running global id filtering tests...");
|
println!("Running global id filtering tests...");
|
||||||
test_global(store.clone()).await;
|
test_global(store.clone()).await;
|
||||||
|
|
||||||
// inbuxa: trace documents as the index task builds them
|
|
||||||
println!("Running trace document tests...");
|
|
||||||
test_trace_documents(store.clone()).await;
|
|
||||||
|
|
||||||
// inbuxa: address fields match by full address, local part, domain and
|
|
||||||
// display name on every backend
|
|
||||||
println!("Running address search tests...");
|
|
||||||
test_address_search(store.clone()).await;
|
|
||||||
|
|
||||||
// inbuxa: words inside URLs, host names and file names in body text
|
|
||||||
// are found on every backend
|
|
||||||
println!("Running URL word search tests...");
|
|
||||||
test_url_word_search(store.clone()).await;
|
|
||||||
|
|
||||||
// Large document insert test
|
// Large document insert test
|
||||||
println!("Running large document insert tests...");
|
println!("Running large document insert tests...");
|
||||||
let mut large_text = String::with_capacity(20 * 1024 * 1024);
|
let mut large_text = String::with_capacity(20 * 1024 * 1024);
|
||||||
@@ -825,388 +809,3 @@ async fn test_global(store: SearchStore) {
|
|||||||
AHashSet::from_iter([3, 4, 5])
|
AHashSet::from_iter([3, 4, 5])
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// inbuxa: MON-16: documents built by the index task from stored traces go
|
|
||||||
// into every search backend (the SQL backends type etyp and qid as BIGINT)
|
|
||||||
// and are found again by queue id and keyword.
|
|
||||||
async fn test_trace_documents(store: SearchStore) {
|
|
||||||
use registry::schema::{
|
|
||||||
enums::SearchTracingField,
|
|
||||||
structs::{
|
|
||||||
Trace, TraceEvent, TraceKeyValue, TraceValue, TraceValueString,
|
|
||||||
TraceValueUnsignedInt,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
use services::task_manager::index::trace_search_document;
|
|
||||||
use trc::{DeliveryEvent, EventType, Key, SmtpEvent};
|
|
||||||
|
|
||||||
let kv_u = |key: Key, value: u64| TraceKeyValue {
|
|
||||||
key,
|
|
||||||
value: TraceValue::UnsignedInt(TraceValueUnsignedInt { value }),
|
|
||||||
};
|
|
||||||
let kv_s = |key: Key, value: &str| TraceKeyValue {
|
|
||||||
key,
|
|
||||||
value: TraceValue::String(TraceValueString {
|
|
||||||
value: value.to_string(),
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
let event = |event: EventType, key_values: Vec<TraceKeyValue>| TraceEvent {
|
|
||||||
event,
|
|
||||||
key_values: key_values.into(),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
let fields = [
|
|
||||||
SearchTracingField::EventType,
|
|
||||||
SearchTracingField::QueueId,
|
|
||||||
SearchTracingField::Keywords,
|
|
||||||
];
|
|
||||||
|
|
||||||
// An SMTP session that queued two messages, and a delivery attempt
|
|
||||||
let session = Trace {
|
|
||||||
events: vec![
|
|
||||||
event(
|
|
||||||
EventType::Smtp(SmtpEvent::ConnectionStart),
|
|
||||||
vec![kv_s(Key::RemoteIp, "192.0.2.7")],
|
|
||||||
),
|
|
||||||
event(
|
|
||||||
EventType::Smtp(SmtpEvent::MailFrom),
|
|
||||||
vec![kv_s(Key::From, "[email protected]")],
|
|
||||||
),
|
|
||||||
event(
|
|
||||||
EventType::Smtp(SmtpEvent::RcptTo),
|
|
||||||
vec![kv_u(Key::QueueId, 9_000_000_001), kv_s(Key::To, "[email protected]")],
|
|
||||||
),
|
|
||||||
event(
|
|
||||||
EventType::Smtp(SmtpEvent::RcptTo),
|
|
||||||
vec![kv_u(Key::QueueId, 9_000_000_002)],
|
|
||||||
),
|
|
||||||
]
|
|
||||||
.into(),
|
|
||||||
};
|
|
||||||
let delivery = Trace {
|
|
||||||
events: vec![event(
|
|
||||||
EventType::Delivery(DeliveryEvent::AttemptStart),
|
|
||||||
vec![kv_u(Key::QueueId, 9_000_000_003), kv_s(Key::Hostname, "relay.example.net")],
|
|
||||||
)]
|
|
||||||
.into(),
|
|
||||||
};
|
|
||||||
let documents = vec![
|
|
||||||
trace_search_document(100, &session, &fields),
|
|
||||||
trace_search_document(101, &delivery, &fields),
|
|
||||||
];
|
|
||||||
assert!(
|
|
||||||
documents
|
|
||||||
.iter()
|
|
||||||
.all(|d| d.has_field(&SearchField::Tracing(TracingSearchField::QueueId))
|
|
||||||
&& d.has_field(&SearchField::Tracing(TracingSearchField::EventType))),
|
|
||||||
"trace documents carry a queue id and an event type"
|
|
||||||
);
|
|
||||||
store.index(documents).await.unwrap();
|
|
||||||
if let SearchStore::ElasticSearch(store) = &store {
|
|
||||||
store.refresh_index(SearchIndex::Tracing).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
let query = |filters: Vec<SearchFilter>| {
|
|
||||||
let store = store.clone();
|
|
||||||
async move {
|
|
||||||
store
|
|
||||||
.query_global(
|
|
||||||
SearchQuery::new(SearchIndex::Tracing)
|
|
||||||
.with_filter(SearchFilter::ge(SearchField::Id, 100u64))
|
|
||||||
.with_filters(filters),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.into_iter()
|
|
||||||
.collect::<AHashSet<_>>()
|
|
||||||
}
|
|
||||||
};
|
|
||||||
// By queue id, the way x:Trace/query asks: the queue id column, or any
|
|
||||||
// queue id in the keywords
|
|
||||||
let by_queue_id = |queue_id: u64| {
|
|
||||||
vec![
|
|
||||||
SearchFilter::Or,
|
|
||||||
SearchFilter::eq(TracingSearchField::QueueId, queue_id),
|
|
||||||
SearchFilter::has_text(
|
|
||||||
TracingSearchField::Keywords,
|
|
||||||
queue_id.to_string(),
|
|
||||||
Language::None,
|
|
||||||
),
|
|
||||||
SearchFilter::End,
|
|
||||||
]
|
|
||||||
};
|
|
||||||
assert_eq!(query(by_queue_id(9_000_000_001)).await, AHashSet::from_iter([100]));
|
|
||||||
assert_eq!(query(by_queue_id(9_000_000_002)).await, AHashSet::from_iter([100]));
|
|
||||||
assert_eq!(query(by_queue_id(9_000_000_003)).await, AHashSet::from_iter([101]));
|
|
||||||
assert_eq!(query(by_queue_id(9_000_000_004)).await, AHashSet::new());
|
|
||||||
assert_eq!(
|
|
||||||
query(vec![SearchFilter::eq(TracingSearchField::QueueId, 9_000_000_003u64)]).await,
|
|
||||||
AHashSet::from_iter([101])
|
|
||||||
);
|
|
||||||
// By opening event type
|
|
||||||
assert_eq!(
|
|
||||||
query(vec![SearchFilter::eq(
|
|
||||||
TracingSearchField::EventType,
|
|
||||||
EventType::Delivery(DeliveryEvent::AttemptStart).to_id() as u64,
|
|
||||||
)])
|
|
||||||
.await,
|
|
||||||
AHashSet::from_iter([101])
|
|
||||||
);
|
|
||||||
// By keyword: an address, lowercased, and its domain
|
|
||||||
assert_eq!(
|
|
||||||
query(vec![SearchFilter::has_text(
|
|
||||||
TracingSearchField::Keywords,
|
|
||||||
"example.org",
|
|
||||||
Language::None,
|
|
||||||
)])
|
|
||||||
.await,
|
|
||||||
AHashSet::from_iter([100])
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
query(vec![SearchFilter::has_text(
|
|
||||||
TracingSearchField::Keywords,
|
|
||||||
"relay.example.net",
|
|
||||||
Language::None,
|
|
||||||
)])
|
|
||||||
.await,
|
|
||||||
AHashSet::from_iter([101])
|
|
||||||
);
|
|
||||||
|
|
||||||
for id in [100u64, 101] {
|
|
||||||
store
|
|
||||||
.unindex(
|
|
||||||
SearchQuery::new(SearchIndex::Tracing)
|
|
||||||
.with_filter(SearchFilter::eq(SearchField::Id, id)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// inbuxa: the message indexer passes each display name and each address of
|
|
||||||
// From/To/Cc/Bcc as keyword text (Language::None). The built-in index splits
|
|
||||||
// that text into words, so an address is found by its full form, its local
|
|
||||||
// part, its domain or a display-name word; PostgreSQL kept the whole address
|
|
||||||
// as one token and MySQL dropped stopwords such as "com" and words under three
|
|
||||||
// characters. The expected results below are the built-in (RocksDB/SQLite)
|
|
||||||
// results and must be the same on every backend.
|
|
||||||
async fn test_address_search(store: SearchStore) {
|
|
||||||
const ACCOUNT_ID: u32 = 7;
|
|
||||||
let messages: [[&[(&str, &str)]; 4]; 5] = [
|
|
||||||
// From, To, Cc, Bcc
|
|
||||||
[
|
|
||||||
&[("Amazon.com", "[email protected]")],
|
|
||||||
&[("Jane Doe", "[email protected]")],
|
|
||||||
&[],
|
|
||||||
&[],
|
|
||||||
],
|
|
||||||
[
|
|
||||||
&[("", "[email protected]")],
|
|
||||||
&[("", "[email protected]")],
|
|
||||||
&[("Jane Doe", "[email protected]")],
|
|
||||||
&[],
|
|
||||||
],
|
|
||||||
[
|
|
||||||
&[("GitHub", "[email protected]")],
|
|
||||||
&[("Jo Li", "[email protected]")],
|
|
||||||
&[],
|
|
||||||
&[("Audit", "[email protected]")],
|
|
||||||
],
|
|
||||||
[
|
|
||||||
&[("Jane Doe", "[email protected]")],
|
|
||||||
&[("Amazon Web Services", "[email protected]")],
|
|
||||||
&[("Bob", "[email protected]")],
|
|
||||||
&[("", "[email protected]")],
|
|
||||||
],
|
|
||||||
[
|
|
||||||
&[("Newsletter", "[email protected]")],
|
|
||||||
&[("", "[email protected]")],
|
|
||||||
&[],
|
|
||||||
&[],
|
|
||||||
],
|
|
||||||
];
|
|
||||||
let fields = [
|
|
||||||
EmailSearchField::From,
|
|
||||||
EmailSearchField::To,
|
|
||||||
EmailSearchField::Cc,
|
|
||||||
EmailSearchField::Bcc,
|
|
||||||
];
|
|
||||||
|
|
||||||
let mut documents = Vec::new();
|
|
||||||
let mut mask = RoaringBitmap::new();
|
|
||||||
for (document_id, message) in messages.iter().enumerate() {
|
|
||||||
let mut document = IndexDocument::new(SearchIndex::Email)
|
|
||||||
.with_account_id(ACCOUNT_ID)
|
|
||||||
.with_document_id(document_id as u32);
|
|
||||||
for (field, addresses) in fields.iter().zip(message.iter()) {
|
|
||||||
for (name, address) in addresses.iter() {
|
|
||||||
if !name.is_empty() {
|
|
||||||
document.index_text(field.clone(), name, Language::None);
|
|
||||||
}
|
|
||||||
document.index_text(field.clone(), address, Language::None);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
document.index_unsigned(EmailSearchField::ReceivedAt, document_id as u64);
|
|
||||||
documents.push(document);
|
|
||||||
mask.insert(document_id as u32);
|
|
||||||
}
|
|
||||||
store.index(documents).await.unwrap();
|
|
||||||
if let SearchStore::ElasticSearch(store) = &store {
|
|
||||||
store.refresh_index(SearchIndex::Email).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (field, text, expected) in [
|
|
||||||
// full address
|
|
||||||
(EmailSearchField::From, "[email protected]", vec![0u32]),
|
|
||||||
(EmailSearchField::To, "[email protected]", vec![0]),
|
|
||||||
(EmailSearchField::Cc, "[email protected]", vec![1]),
|
|
||||||
(EmailSearchField::Bcc, "[email protected]", vec![3]),
|
|
||||||
(EmailSearchField::To, "[email protected]", vec![1, 2]),
|
|
||||||
// local part
|
|
||||||
(EmailSearchField::From, "noreply", vec![0, 2]),
|
|
||||||
(EmailSearchField::To, "jo", vec![1, 2]),
|
|
||||||
(EmailSearchField::Cc, "bob", vec![3]),
|
|
||||||
(EmailSearchField::Bcc, "audit", vec![2]),
|
|
||||||
// domain
|
|
||||||
(EmailSearchField::From, "amazon.com", vec![0, 1]),
|
|
||||||
(EmailSearchField::From, "amazon", vec![0, 1]),
|
|
||||||
(EmailSearchField::To, "example.org", vec![0, 4]),
|
|
||||||
(EmailSearchField::To, "io.de", vec![1, 2]),
|
|
||||||
(EmailSearchField::Cc, "example.net", vec![3]),
|
|
||||||
(EmailSearchField::Bcc, "example.org", vec![2]),
|
|
||||||
(EmailSearchField::From, "www.example.com", vec![4]),
|
|
||||||
(EmailSearchField::From, "com", vec![0, 1, 2, 4]),
|
|
||||||
// display name
|
|
||||||
(EmailSearchField::From, "Jane", vec![3]),
|
|
||||||
(EmailSearchField::From, "jane doe", vec![3]),
|
|
||||||
(EmailSearchField::To, "Web Services", vec![3]),
|
|
||||||
(EmailSearchField::To, "Li", vec![2]),
|
|
||||||
(EmailSearchField::Cc, "Doe", vec![1]),
|
|
||||||
(EmailSearchField::Bcc, "Audit", vec![2]),
|
|
||||||
// hyphenated local part
|
|
||||||
(EmailSearchField::From, "shipment-tracking", vec![1]),
|
|
||||||
(EmailSearchField::From, "tracking", vec![1]),
|
|
||||||
// no match
|
|
||||||
(EmailSearchField::From, "amazon.org", vec![]),
|
|
||||||
(EmailSearchField::To, "noreply", vec![]),
|
|
||||||
(EmailSearchField::Bcc, "jane", vec![]),
|
|
||||||
] {
|
|
||||||
let ids = store
|
|
||||||
.query_account(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filters(vec![
|
|
||||||
SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID),
|
|
||||||
SearchFilter::has_keyword(field.clone(), text),
|
|
||||||
])
|
|
||||||
.with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt))
|
|
||||||
.with_mask(mask.clone()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(ids, expected, "{field:?} {text:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
// TEXT-style search across all address fields
|
|
||||||
let ids = store
|
|
||||||
.query_account(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filters(vec![
|
|
||||||
SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID),
|
|
||||||
SearchFilter::Or,
|
|
||||||
SearchFilter::has_keyword(EmailSearchField::From, "example.org"),
|
|
||||||
SearchFilter::has_keyword(EmailSearchField::To, "example.org"),
|
|
||||||
SearchFilter::has_keyword(EmailSearchField::Cc, "example.org"),
|
|
||||||
SearchFilter::has_keyword(EmailSearchField::Bcc, "example.org"),
|
|
||||||
SearchFilter::End,
|
|
||||||
])
|
|
||||||
.with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt))
|
|
||||||
.with_mask(mask.clone()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(ids, vec![0, 1, 2, 3, 4]);
|
|
||||||
|
|
||||||
store
|
|
||||||
.unindex(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filter(SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn test_url_word_search(store: SearchStore) {
|
|
||||||
const ACCOUNT_ID: u32 = 8;
|
|
||||||
let bodies = [
|
|
||||||
"Track your parcel here: https://x.example/shipping-support/ and reply.",
|
|
||||||
"Reset it at https://mail.example.com/login/?password=reset&user=jane now.",
|
|
||||||
"Attached is invoice-2024.pdf for your records.",
|
|
||||||
"Shipping was fast, thanks again.",
|
|
||||||
"Nothing to see at www.example.org/about-us, really.",
|
|
||||||
];
|
|
||||||
|
|
||||||
let mut documents = Vec::new();
|
|
||||||
let mut mask = RoaringBitmap::new();
|
|
||||||
for (document_id, body) in bodies.iter().enumerate() {
|
|
||||||
let mut document = IndexDocument::new(SearchIndex::Email)
|
|
||||||
.with_account_id(ACCOUNT_ID)
|
|
||||||
.with_document_id(document_id as u32);
|
|
||||||
document.index_text(EmailSearchField::Body, body, Language::English);
|
|
||||||
document.index_unsigned(EmailSearchField::ReceivedAt, document_id as u64);
|
|
||||||
documents.push(document);
|
|
||||||
mask.insert(document_id as u32);
|
|
||||||
}
|
|
||||||
store.index(documents).await.unwrap();
|
|
||||||
if let SearchStore::ElasticSearch(store) = &store {
|
|
||||||
store.refresh_index(SearchIndex::Email).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (text, expected) in [
|
|
||||||
// only inside a URL path, a query string or a file name
|
|
||||||
("shipping", vec![0u32, 3]),
|
|
||||||
("support", vec![0]),
|
|
||||||
("password", vec![1]),
|
|
||||||
("login", vec![1]),
|
|
||||||
("jane", vec![1]),
|
|
||||||
("invoice", vec![2]),
|
|
||||||
("pdf", vec![2]),
|
|
||||||
("2024", vec![2]),
|
|
||||||
// host names
|
|
||||||
("example", vec![0, 1, 4]),
|
|
||||||
("mail", vec![1]),
|
|
||||||
// written as they appear
|
|
||||||
("https://x.example/shipping-support/", vec![0]),
|
|
||||||
("shipping-support", vec![0]),
|
|
||||||
("invoice-2024.pdf", vec![2]),
|
|
||||||
("mail.example.com", vec![1]),
|
|
||||||
// plain words are unaffected
|
|
||||||
("parcel", vec![0]),
|
|
||||||
("records", vec![2]),
|
|
||||||
("thanks", vec![3]),
|
|
||||||
// no match
|
|
||||||
("billing", vec![]),
|
|
||||||
("example.net", vec![]),
|
|
||||||
] {
|
|
||||||
let ids = store
|
|
||||||
.query_account(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filters(vec![
|
|
||||||
SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID),
|
|
||||||
SearchFilter::has_english_text(EmailSearchField::Body, text),
|
|
||||||
])
|
|
||||||
.with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt))
|
|
||||||
.with_mask(mask.clone()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(ids, expected, "Body {text:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
store
|
|
||||||
.unindex(
|
|
||||||
SearchQuery::new(SearchIndex::Email)
|
|
||||||
.with_filter(SearchFilter::eq(SearchField::AccountId, ACCOUNT_ID)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,159 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! PostgreSQL full-text GIN indexes are built with fastupdate off, and an
|
|
||||||
//! index made earlier with the default is switched over at startup. With
|
|
||||||
//! fastupdate on, new entries wait in a pending list that every search scans
|
|
||||||
//! in full until VACUUM merges it.
|
|
||||||
|
|
||||||
use crate::utils::storage::build_data_store;
|
|
||||||
use registry::schema::structs::DataStore;
|
|
||||||
use store::{Rows, SearchStore, Store};
|
|
||||||
|
|
||||||
const SCHEMA: &str = "gin_fastupdate_test";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn postgres_gin_fastupdate() {
|
|
||||||
println!("Running PostgreSQL GIN fastupdate test...");
|
|
||||||
|
|
||||||
// Work in a schema of our own so the shared search tables are untouched
|
|
||||||
let admin = Store::build(build_data_store("PostgreSql", "").await)
|
|
||||||
.await
|
|
||||||
.expect("Failed to connect to PostgreSQL");
|
|
||||||
for query in [
|
|
||||||
format!("DROP SCHEMA IF EXISTS {SCHEMA} CASCADE"),
|
|
||||||
format!("CREATE SCHEMA {SCHEMA}"),
|
|
||||||
] {
|
|
||||||
admin.sql_query::<usize>(&query, vec![]).await.unwrap();
|
|
||||||
}
|
|
||||||
let DataStore::PostgreSql(mut config) = build_data_store("PostgreSql", "").await else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
config.options = Some(format!("-c search_path={SCHEMA}"));
|
|
||||||
let store = Store::build(DataStore::PostgreSql(config))
|
|
||||||
.await
|
|
||||||
.expect("Failed to connect to PostgreSQL");
|
|
||||||
let search = SearchStore::Store(store.clone());
|
|
||||||
|
|
||||||
// A fresh schema
|
|
||||||
search.create_indexes().await.unwrap();
|
|
||||||
let indexes = gin_indexes(&admin).await;
|
|
||||||
assert!(
|
|
||||||
indexes.len() >= 4,
|
|
||||||
"expected the search GIN indexes, found {indexes:?}"
|
|
||||||
);
|
|
||||||
for (name, options) in &indexes {
|
|
||||||
assert!(
|
|
||||||
options.contains("fastupdate=off"),
|
|
||||||
"fresh index {name} has options {options:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// A schema from before the change: the same indexes, made with the
|
|
||||||
// default fastupdate=on, and a pending list with something in it
|
|
||||||
for (name, _) in &indexes {
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>(
|
|
||||||
&format!("ALTER INDEX {SCHEMA}.{name} RESET (fastupdate)"),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
for (name, options) in gin_indexes(&admin).await {
|
|
||||||
assert!(
|
|
||||||
!options.contains("fastupdate"),
|
|
||||||
"index {name} still has options {options:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>(
|
|
||||||
&format!(
|
|
||||||
"INSERT INTO {SCHEMA}.s_email (accid, docid, subj, body) \
|
|
||||||
SELECT 1, n, to_tsvector('simple', 'pending subject ' || n), \
|
|
||||||
to_tsvector('simple', 'pending body text ' || n) \
|
|
||||||
FROM generate_series(1, 500) n"
|
|
||||||
),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(
|
|
||||||
pending_tuples(&admin, "gin_s_email_body").await > 0,
|
|
||||||
"no pending list to merge"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Startup on the existing schema switches every index over and merges
|
|
||||||
// what was pending
|
|
||||||
search.create_indexes().await.unwrap();
|
|
||||||
for (name, options) in gin_indexes(&admin).await {
|
|
||||||
assert!(
|
|
||||||
options.contains("fastupdate=off"),
|
|
||||||
"existing index {name} has options {options:?} after startup"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert_eq!(pending_tuples(&admin, "gin_s_email_body").await, 0);
|
|
||||||
|
|
||||||
// And a second startup changes nothing
|
|
||||||
search.create_indexes().await.unwrap();
|
|
||||||
for (name, options) in gin_indexes(&admin).await {
|
|
||||||
assert!(options.contains("fastupdate=off"), "{name}: {options:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>(&format!("DROP SCHEMA {SCHEMA} CASCADE"), vec![])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The GIN indexes in the test schema with their reloptions.
|
|
||||||
async fn gin_indexes(admin: &Store) -> Vec<(String, String)> {
|
|
||||||
admin
|
|
||||||
.sql_query::<Rows>(
|
|
||||||
&format!(
|
|
||||||
"SELECT c.relname::text, COALESCE(array_to_string(c.reloptions, ','), '') \
|
|
||||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace \
|
|
||||||
JOIN pg_am a ON a.oid = c.relam \
|
|
||||||
WHERE n.nspname = '{SCHEMA}' AND c.relkind = 'i' AND a.amname = 'gin' \
|
|
||||||
ORDER BY 1"
|
|
||||||
),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.rows
|
|
||||||
.into_iter()
|
|
||||||
.map(|row| {
|
|
||||||
let mut values = row.values.into_iter();
|
|
||||||
(
|
|
||||||
values.next().unwrap().to_str().into_owned(),
|
|
||||||
values.next().unwrap().to_str().into_owned(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Tuples waiting in a GIN index's pending list (pgstattuple is a contrib
|
|
||||||
/// extension the test database has).
|
|
||||||
async fn pending_tuples(admin: &Store, index: &str) -> i64 {
|
|
||||||
admin
|
|
||||||
.sql_query::<usize>("CREATE EXTENSION IF NOT EXISTS pgstattuple", vec![])
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
admin
|
|
||||||
.sql_query::<Rows>(
|
|
||||||
&format!("SELECT pending_tuples FROM pgstatginindex('{SCHEMA}.{index}'::regclass)"),
|
|
||||||
vec![],
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.rows
|
|
||||||
.into_iter()
|
|
||||||
.next()
|
|
||||||
.and_then(|row| row.values.into_iter().next())
|
|
||||||
.map(|value| value.to_str().parse::<i64>().unwrap())
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
@@ -1,210 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
//! Task locks across nodes: tasks claimed by a node that then disappears
|
|
||||||
//! run elsewhere once its locks expire, and a node that stops gracefully
|
|
||||||
//! hands its locks back at once. The other node is played by writing its
|
|
||||||
//! locks straight into the shared in-memory store, as a node that claimed
|
|
||||||
//! the tasks and died leaves them.
|
|
||||||
|
|
||||||
use crate::utils::server::TestServerBuilder;
|
|
||||||
use common::{KV_LOCK_TASK, Server};
|
|
||||||
use registry::schema::{
|
|
||||||
enums::IndexDocumentType,
|
|
||||||
structs::{Task, TaskIndexDocument, TaskStatus},
|
|
||||||
};
|
|
||||||
use services::task_manager::lock::{TaskLockManager, release_task_locks};
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
use store::{
|
|
||||||
ValueKey,
|
|
||||||
write::{BatchBuilder, TaskQueueClass, ValueClass},
|
|
||||||
};
|
|
||||||
use utils::snowflake::SnowflakeIdGenerator;
|
|
||||||
|
|
||||||
// Short enough for a test, long enough that the recheck interval (a twelfth
|
|
||||||
// of it) is well below it
|
|
||||||
const LOCK_EXPIRY: u64 = 12;
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn task_lock_tests() {
|
|
||||||
let test = TestServerBuilder::new("task_lock_tests")
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
let server = test.server.clone();
|
|
||||||
println!(
|
|
||||||
"Running task lock tests on {}...",
|
|
||||||
std::env::var("STORE").unwrap_or_default()
|
|
||||||
);
|
|
||||||
server.inner.ipc.task_locks.set_expiry(LOCK_EXPIRY);
|
|
||||||
|
|
||||||
// 1. Another node claimed the tasks and died. Its locks block them until
|
|
||||||
// they expire; then this node runs them, without waiting for anything
|
|
||||||
// else to wake it
|
|
||||||
let ids = new_task_ids(4);
|
|
||||||
for id in &ids {
|
|
||||||
assert!(foreign_lock(&server, *id, LOCK_EXPIRY).await);
|
|
||||||
}
|
|
||||||
schedule(&server, &ids).await;
|
|
||||||
let started = Instant::now();
|
|
||||||
server.notify_task_queue();
|
|
||||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
|
||||||
assert_eq!(pending(&server, &ids).await, ids.len(), "held by the other node");
|
|
||||||
wait_until_done(&server, &ids, Duration::from_secs(LOCK_EXPIRY + 10)).await;
|
|
||||||
let elapsed = started.elapsed();
|
|
||||||
assert!(
|
|
||||||
elapsed >= Duration::from_secs(LOCK_EXPIRY - 2),
|
|
||||||
"ran before the other node's locks expired: {elapsed:?}"
|
|
||||||
);
|
|
||||||
|
|
||||||
// 2. The other node's locks outlive what this node expects: claimed just
|
|
||||||
// after this node looked, or by a node whose clock runs ahead. This node
|
|
||||||
// keeps checking at the recheck interval, so the tasks run soon after
|
|
||||||
// those locks expire, not a whole lock lifetime later
|
|
||||||
let held_for = LOCK_EXPIRY + LOCK_EXPIRY / 2;
|
|
||||||
let ids = new_task_ids(4);
|
|
||||||
for id in &ids {
|
|
||||||
assert!(foreign_lock(&server, *id, held_for).await);
|
|
||||||
}
|
|
||||||
schedule(&server, &ids).await;
|
|
||||||
let started = Instant::now();
|
|
||||||
server.notify_task_queue();
|
|
||||||
wait_until_done(&server, &ids, Duration::from_secs(held_for + 8)).await;
|
|
||||||
let elapsed = started.elapsed();
|
|
||||||
assert!(
|
|
||||||
elapsed >= Duration::from_secs(held_for - 2),
|
|
||||||
"ran before the other node's locks expired: {elapsed:?}"
|
|
||||||
);
|
|
||||||
|
|
||||||
// 3. A task that runs longer than a lock lifetime keeps its claim: the
|
|
||||||
// task manager renews the lease while this node holds it, and the claim
|
|
||||||
// ends when the task does. (Before, a lock simply lasted an hour.)
|
|
||||||
let [id] = new_task_ids(1)[..] else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
assert!(server.try_lock_task(id).await, "claim {id}");
|
|
||||||
tokio::time::sleep(Duration::from_secs(LOCK_EXPIRY + LOCK_EXPIRY / 2)).await;
|
|
||||||
assert!(
|
|
||||||
!foreign_lock(&server, id, LOCK_EXPIRY).await,
|
|
||||||
"lease lapsed while the task ran"
|
|
||||||
);
|
|
||||||
server.remove_index_lock(id).await;
|
|
||||||
assert!(
|
|
||||||
foreign_lock(&server, id, LOCK_EXPIRY).await,
|
|
||||||
"released when the task ended"
|
|
||||||
);
|
|
||||||
let _ = server
|
|
||||||
.in_memory_store()
|
|
||||||
.remove_lock(KV_LOCK_TASK, &id.to_be_bytes())
|
|
||||||
.await;
|
|
||||||
assert!(
|
|
||||||
common::ipc::TaskLocks::DEFAULT_EXPIRY <= 5 * 60,
|
|
||||||
"a dead node's tasks wait no more than a few minutes"
|
|
||||||
);
|
|
||||||
|
|
||||||
// 4. A graceful stop releases the locks this node holds: another node
|
|
||||||
// can claim those tasks at once, and this one claims nothing more
|
|
||||||
let ids = new_task_ids(3);
|
|
||||||
for id in &ids {
|
|
||||||
assert!(server.try_lock_task(*id).await, "claim {id}");
|
|
||||||
}
|
|
||||||
assert_eq!(server.inner.ipc.task_locks.held(), ids.len());
|
|
||||||
for id in &ids {
|
|
||||||
assert!(
|
|
||||||
!foreign_lock(&server, *id, LOCK_EXPIRY).await,
|
|
||||||
"held while this node runs"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert_eq!(release_task_locks(&server).await, ids.len());
|
|
||||||
assert_eq!(server.inner.ipc.task_locks.held(), 0);
|
|
||||||
for id in &ids {
|
|
||||||
assert!(
|
|
||||||
foreign_lock(&server, *id, LOCK_EXPIRY).await,
|
|
||||||
"released on stop: {id}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let [id] = new_task_ids(1)[..] else {
|
|
||||||
unreachable!()
|
|
||||||
};
|
|
||||||
assert!(!server.try_lock_task(id).await, "a stopping node claims nothing");
|
|
||||||
|
|
||||||
for id in ids {
|
|
||||||
let _ = server
|
|
||||||
.in_memory_store()
|
|
||||||
.remove_lock(KV_LOCK_TASK, &id.to_be_bytes())
|
|
||||||
.await;
|
|
||||||
}
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn new_task_ids(count: usize) -> Vec<u64> {
|
|
||||||
(0..count)
|
|
||||||
.map(|_| SnowflakeIdGenerator::global_id().unwrap())
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The other node's claim on a task, as its task manager takes it.
|
|
||||||
async fn foreign_lock(server: &Server, id: u64, seconds: u64) -> bool {
|
|
||||||
server
|
|
||||||
.in_memory_store()
|
|
||||||
.try_lock(KV_LOCK_TASK, &id.to_be_bytes(), seconds)
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Unindex tasks for files that don't exist: files aren't search-indexed and
|
|
||||||
/// there is no undelete note, so running one only drops it from the queue.
|
|
||||||
async fn schedule(server: &Server, ids: &[u64]) {
|
|
||||||
let mut batch = BatchBuilder::new();
|
|
||||||
for (n, id) in ids.iter().enumerate() {
|
|
||||||
batch.schedule_task_with_id(
|
|
||||||
*id,
|
|
||||||
Task::UnindexDocument(TaskIndexDocument {
|
|
||||||
account_id: 0u32.into(),
|
|
||||||
document_id: (u32::MAX - n as u32).into(),
|
|
||||||
document_type: IndexDocumentType::File,
|
|
||||||
status: TaskStatus::now(),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
server.store().write(batch.build_all()).await.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn pending(server: &Server, ids: &[u64]) -> usize {
|
|
||||||
let mut count = 0;
|
|
||||||
for id in ids {
|
|
||||||
if server
|
|
||||||
.store()
|
|
||||||
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
|
||||||
TaskQueueClass::Task { id: *id },
|
|
||||||
)))
|
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.is_some()
|
|
||||||
{
|
|
||||||
count += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
count
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_until_done(server: &Server, ids: &[u64], within: Duration) {
|
|
||||||
let started = Instant::now();
|
|
||||||
loop {
|
|
||||||
let left = pending(server, ids).await;
|
|
||||||
if left == 0 {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < within,
|
|
||||||
"{left} task(s) still pending after {:?}",
|
|
||||||
started.elapsed()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,272 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
// inbuxa: a registry write to an object the running settings are built from
|
|
||||||
// applies without an x:Action ReloadSettings, and the set response says so.
|
|
||||||
|
|
||||||
use crate::utils::{
|
|
||||||
jmap::JmapResponse,
|
|
||||||
server::{TestServer, TestServerBuilder},
|
|
||||||
};
|
|
||||||
use common::BuildServer;
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::TracingLevel,
|
|
||||||
prelude::ObjectType,
|
|
||||||
structs::{
|
|
||||||
AllowedIp, CertificateManagement, DkimManagement, DnsManagement, Domain, Expression,
|
|
||||||
MtaDeliverySchedule, MtaStageAuth, MtaVirtualQueue, Tracer, TracerStdout,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
types::ipmask::IpAddrOrMask,
|
|
||||||
};
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn settings_reload_tests() {
|
|
||||||
let mut test = TestServerBuilder::new("settings_reload_tests")
|
|
||||||
.await
|
|
||||||
.with_default_listeners()
|
|
||||||
.await
|
|
||||||
.with_object(MtaStageAuth {
|
|
||||||
require: Expression {
|
|
||||||
else_: "false".to_string(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let admin = test
|
|
||||||
.create_user_account(
|
|
||||||
"admin",
|
|
||||||
"[email protected]",
|
|
||||||
"these_pretzels_are_making_me_thirsty",
|
|
||||||
&[],
|
|
||||||
"Admin",
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
test.account("admin")
|
|
||||||
.assign_roles_to_account(admin.id(), &["user", "system"])
|
|
||||||
.await;
|
|
||||||
test.insert_account(admin);
|
|
||||||
|
|
||||||
test_write_applies(&test).await;
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn test_write_applies(test: &TestServer) {
|
|
||||||
println!("Running settings reload after registry writes...");
|
|
||||||
let admin = test.account("[email protected]");
|
|
||||||
|
|
||||||
// A delivery schedule is in use as soon as it is saved
|
|
||||||
let response = admin
|
|
||||||
.registry_create([MtaVirtualQueue {
|
|
||||||
name: "autorld".into(),
|
|
||||||
threads_per_node: 2,
|
|
||||||
description: None,
|
|
||||||
}])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
let queue_id = response.created_id(0);
|
|
||||||
assert!(!has_schedule(test, "autoreload-schedule"));
|
|
||||||
let response = admin
|
|
||||||
.registry_create([MtaDeliverySchedule {
|
|
||||||
name: "autoreload-schedule".into(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
}])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
assert!(has_schedule(test, "autoreload-schedule"));
|
|
||||||
|
|
||||||
// Destroyed, it's gone at once too
|
|
||||||
let schedule_id = response.created_id(0);
|
|
||||||
let response = admin
|
|
||||||
.registry_destroy(ObjectType::MtaDeliverySchedule, [schedule_id])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
assert!(!has_schedule(test, "autoreload-schedule"));
|
|
||||||
|
|
||||||
// Concurrent writes all end up in the running settings
|
|
||||||
let names = (0..8)
|
|
||||||
.map(|i| format!("autoreload-{i}"))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let mut writes = Vec::new();
|
|
||||||
for name in &names {
|
|
||||||
writes.push(admin.registry_create([MtaDeliverySchedule {
|
|
||||||
name: name.clone(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
}]));
|
|
||||||
}
|
|
||||||
let mut schedule_ids = Vec::new();
|
|
||||||
for response in futures::future::join_all(writes).await {
|
|
||||||
assert_applied(&response);
|
|
||||||
schedule_ids.push(response.created_id(0));
|
|
||||||
}
|
|
||||||
for name in &names {
|
|
||||||
assert!(has_schedule(test, name), "{name} missing");
|
|
||||||
}
|
|
||||||
// A burst of separate requests shares a reload or two: each arrives
|
|
||||||
// tens of milliseconds after the last, so none overlaps a running
|
|
||||||
// reload, and the reload waits for writes to settle instead
|
|
||||||
let reloads = test.server.inner.data.settings_reload.reloads();
|
|
||||||
let started = std::time::Instant::now();
|
|
||||||
let burst = (0..10)
|
|
||||||
.map(|i| format!("autoreload-burst-{i}"))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let mut writes = Vec::new();
|
|
||||||
for name in &burst {
|
|
||||||
writes.push(admin.registry_create([MtaDeliverySchedule {
|
|
||||||
name: name.clone(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
}]));
|
|
||||||
}
|
|
||||||
for response in futures::future::join_all(writes).await {
|
|
||||||
assert_applied(&response);
|
|
||||||
schedule_ids.push(response.created_id(0));
|
|
||||||
}
|
|
||||||
let burst_reloads = test.server.inner.data.settings_reload.reloads() - reloads;
|
|
||||||
println!(
|
|
||||||
"10 concurrent writes: {burst_reloads} reload(s), {} ms",
|
|
||||||
started.elapsed().as_millis()
|
|
||||||
);
|
|
||||||
assert!(
|
|
||||||
(1..=2).contains(&burst_reloads),
|
|
||||||
"{burst_reloads} reloads for 10 concurrent writes"
|
|
||||||
);
|
|
||||||
for name in &burst {
|
|
||||||
assert!(has_schedule(test, name), "{name} missing");
|
|
||||||
}
|
|
||||||
|
|
||||||
// A single write still reloads promptly
|
|
||||||
let reloads = test.server.inner.data.settings_reload.reloads();
|
|
||||||
let started = std::time::Instant::now();
|
|
||||||
let response = admin
|
|
||||||
.registry_create([MtaDeliverySchedule {
|
|
||||||
name: "autoreload-single".into(),
|
|
||||||
queue_id,
|
|
||||||
..Default::default()
|
|
||||||
}])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
schedule_ids.push(response.created_id(0));
|
|
||||||
println!("1 write: {} ms", started.elapsed().as_millis());
|
|
||||||
assert_eq!(
|
|
||||||
test.server.inner.data.settings_reload.reloads() - reloads,
|
|
||||||
1
|
|
||||||
);
|
|
||||||
assert!(has_schedule(test, "autoreload-single"));
|
|
||||||
|
|
||||||
// Several objects in one request: one reload
|
|
||||||
let response = admin
|
|
||||||
.registry_destroy(ObjectType::MtaDeliverySchedule, schedule_ids.iter())
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
for name in names.iter().chain(&burst) {
|
|
||||||
assert!(!has_schedule(test, name), "{name} still present");
|
|
||||||
}
|
|
||||||
|
|
||||||
// A write whose reload fails is stored, and the response says the
|
|
||||||
// settings weren't reloaded: only one console tracer is allowed.
|
|
||||||
let response = admin
|
|
||||||
.registry_create([
|
|
||||||
Tracer::Stdout(TracerStdout {
|
|
||||||
enable: true,
|
|
||||||
level: TracingLevel::Error,
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
Tracer::Stdout(TracerStdout {
|
|
||||||
enable: true,
|
|
||||||
level: TracingLevel::Error,
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
])
|
|
||||||
.await;
|
|
||||||
let reload = settings_reload(&response).expect("x:settingsReload missing");
|
|
||||||
assert_eq!(reload["applied"], Value::Bool(false), "{response:?}");
|
|
||||||
let description = reload["description"].as_str().unwrap_or_default();
|
|
||||||
assert!(
|
|
||||||
description.starts_with("Saved, but the running settings were not reloaded. ")
|
|
||||||
&& description.contains("Only one console tracer is allowed"),
|
|
||||||
"{description}"
|
|
||||||
);
|
|
||||||
let tracer_ids = [response.created_id(0), response.created_id(1)];
|
|
||||||
let response = admin
|
|
||||||
.registry_destroy(ObjectType::Tracer, tracer_ids.iter())
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
|
|
||||||
// An allowed IP is live as soon as it is saved, and gone once
|
|
||||||
// destroyed. It lives in the core's security settings, which the
|
|
||||||
// blocked-IP reload it used to get doesn't rebuild.
|
|
||||||
let ip: std::net::IpAddr = "198.51.100.7".parse().unwrap();
|
|
||||||
assert!(!is_allowed(test, ip));
|
|
||||||
let response = admin
|
|
||||||
.registry_create([AllowedIp {
|
|
||||||
address: IpAddrOrMask::from_ip(ip),
|
|
||||||
reason: Some("autoreload".into()),
|
|
||||||
..Default::default()
|
|
||||||
}])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
assert!(
|
|
||||||
is_allowed(test, ip),
|
|
||||||
"allowed IP not in the running settings"
|
|
||||||
);
|
|
||||||
let allowed_id = response.created_id(0);
|
|
||||||
let response = admin
|
|
||||||
.registry_destroy(ObjectType::AllowedIp, [allowed_id])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
assert!(!is_allowed(test, ip), "destroyed allowed IP still live");
|
|
||||||
|
|
||||||
// Data that isn't part of the running settings doesn't reload them
|
|
||||||
let response = admin
|
|
||||||
.registry_create([Domain {
|
|
||||||
name: "autoreload.example.org".into(),
|
|
||||||
certificate_management: CertificateManagement::Manual,
|
|
||||||
dns_management: DnsManagement::Manual,
|
|
||||||
dkim_management: DkimManagement::Manual,
|
|
||||||
..Default::default()
|
|
||||||
}])
|
|
||||||
.await;
|
|
||||||
assert!(settings_reload(&response).is_none(), "{response:?}");
|
|
||||||
}
|
|
||||||
|
|
||||||
fn settings_reload(response: &JmapResponse) -> Option<&Value> {
|
|
||||||
response.pointer("/methodResponses/0/1/x:settingsReload")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn assert_applied(response: &JmapResponse) {
|
|
||||||
assert_eq!(
|
|
||||||
settings_reload(response),
|
|
||||||
Some(&serde_json::json!({"applied": true})),
|
|
||||||
"{response:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn has_schedule(test: &TestServer, name: &str) -> bool {
|
|
||||||
test.server
|
|
||||||
.inner
|
|
||||||
.build_server()
|
|
||||||
.core
|
|
||||||
.smtp
|
|
||||||
.queue
|
|
||||||
.queue_strategy
|
|
||||||
.contains_key(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn is_allowed(test: &TestServer, ip: std::net::IpAddr) -> bool {
|
|
||||||
test.server.inner.build_server().is_ip_allowed(ip)
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,6 @@ pub mod authentication;
|
|||||||
pub mod ai;
|
pub mod ai;
|
||||||
pub mod ai_calibration;
|
pub mod ai_calibration;
|
||||||
pub mod authorization;
|
pub mod authorization;
|
||||||
pub mod auto_reload; // inbuxa: registry writes apply at once
|
|
||||||
pub mod branding;
|
pub mod branding;
|
||||||
pub mod crypto;
|
pub mod crypto;
|
||||||
pub mod delivery;
|
pub mod delivery;
|
||||||
@@ -21,10 +20,8 @@ pub mod monitoring;
|
|||||||
pub mod oidc;
|
pub mod oidc;
|
||||||
pub mod purge;
|
pub mod purge;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
pub mod reload; // inbuxa: reloads and build errors
|
|
||||||
pub mod security;
|
pub mod security;
|
||||||
pub mod task;
|
pub mod task;
|
||||||
pub mod tracer_reload; // inbuxa: tracers start over when their settings change
|
|
||||||
pub mod tenant;
|
pub mod tenant;
|
||||||
pub mod undelete;
|
pub mod undelete;
|
||||||
|
|
||||||
|
|||||||
@@ -148,73 +148,6 @@ pub async fn test(test: &mut TestServer) {
|
|||||||
"test 9: to"
|
"test 9: to"
|
||||||
);
|
);
|
||||||
|
|
||||||
// MON-16: the queueId filter finds the traces that name a queue id (the
|
|
||||||
// session that queued the message and its delivery attempt) through the
|
|
||||||
// search index, given as a string or a number (the index column is an
|
|
||||||
// integer)
|
|
||||||
fn queue_ids(value: &Value, out: &mut Vec<u64>) {
|
|
||||||
match value {
|
|
||||||
Value::Object(map) => {
|
|
||||||
if map.get("key").and_then(|k| k.as_str()) == Some("queueId")
|
|
||||||
&& let Some(id) = map
|
|
||||||
.get("value")
|
|
||||||
.and_then(|v| v.get("value").unwrap_or(v).as_u64())
|
|
||||||
{
|
|
||||||
out.push(id);
|
|
||||||
}
|
|
||||||
map.values().for_each(|v| queue_ids(v, out));
|
|
||||||
}
|
|
||||||
Value::Array(list) => list.iter().for_each(|v| queue_ids(v, out)),
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let with_ids = traces
|
|
||||||
.iter()
|
|
||||||
.map(|t| {
|
|
||||||
let mut ids = Vec::new();
|
|
||||||
queue_ids(t, &mut ids);
|
|
||||||
(t["id"].as_str().unwrap().to_string(), ids)
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let queue_id = with_ids
|
|
||||||
.iter()
|
|
||||||
.find_map(|(_, ids)| ids.first().copied())
|
|
||||||
.expect("MON-16: a trace with a queue id");
|
|
||||||
let mut expected = with_ids
|
|
||||||
.iter()
|
|
||||||
.filter(|(_, ids)| ids.contains(&queue_id))
|
|
||||||
.map(|(id, _)| id.clone())
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
expected.sort();
|
|
||||||
for filter in [json!(queue_id.to_string()), json!(queue_id)] {
|
|
||||||
let response = admin
|
|
||||||
.jmap_method_call("x:Trace/query", json!({"filter": {"queueId": filter}}))
|
|
||||||
.await;
|
|
||||||
let mut found = response
|
|
||||||
.0
|
|
||||||
.pointer("/methodResponses/0/1/ids")
|
|
||||||
.and_then(|ids| ids.as_array())
|
|
||||||
.map(|ids| {
|
|
||||||
ids.iter()
|
|
||||||
.filter_map(|id| id.as_str().map(str::to_string))
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
})
|
|
||||||
.unwrap_or_default();
|
|
||||||
found.sort();
|
|
||||||
assert_eq!(found, expected, "MON-16: queueId {filter}: {response:?}");
|
|
||||||
}
|
|
||||||
let response = admin
|
|
||||||
.jmap_method_call(
|
|
||||||
"x:Trace/query",
|
|
||||||
json!({"filter": {"queueId": (queue_id ^ 0x5a5a_5a5a).to_string()}}),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert_eq!(
|
|
||||||
response.0.pointer("/methodResponses/0/1/ids"),
|
|
||||||
Some(&json!([])),
|
|
||||||
"MON-16: an unknown queue id"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Acceptance test 24: destroy removes a trace; create is refused
|
// Acceptance test 24: destroy removes a trace; create is refused
|
||||||
let trace_id = traces[0]["id"].as_str().unwrap().to_string();
|
let trace_id = traces[0]["id"].as_str().unwrap().to_string();
|
||||||
let response = admin
|
let response = admin
|
||||||
|
|||||||
@@ -1,213 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
// inbuxa: a settings reload isn't held back by a DNS lookup, or by objects
|
|
||||||
// that already failed when the running settings were built; an error in an
|
|
||||||
// object that built then still refuses it, and says which object.
|
|
||||||
|
|
||||||
use crate::utils::server::{TestServer, TestServerBuilder};
|
|
||||||
use common::{BuildServer, config::mailstore::spamfilter::PyzorConfig, ipc::RegistryChange};
|
|
||||||
use registry::schema::{
|
|
||||||
enums::TracingLevel,
|
|
||||||
prelude::{ObjectType, Property},
|
|
||||||
structs::{Action, Expression, MtaStageAuth, SpamPyzor, Tracer, TracerStdout},
|
|
||||||
};
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn reload_tests() {
|
|
||||||
let mut test = TestServerBuilder::new("reload_tests")
|
|
||||||
.await
|
|
||||||
.with_default_listeners()
|
|
||||||
.await
|
|
||||||
.with_object(MtaStageAuth {
|
|
||||||
require: Expression {
|
|
||||||
else_: "false".to_string(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let admin = test
|
|
||||||
.create_user_account(
|
|
||||||
"admin",
|
|
||||||
"[email protected]",
|
|
||||||
"these_pretzels_are_making_me_thirsty",
|
|
||||||
&[],
|
|
||||||
"Admin",
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
test.account("admin")
|
|
||||||
.assign_roles_to_account(admin.id(), &["user", "system"])
|
|
||||||
.await;
|
|
||||||
test.insert_account(admin);
|
|
||||||
|
|
||||||
test_unresolvable_pyzor(&test).await;
|
|
||||||
test_build_errors(&test).await;
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn test_unresolvable_pyzor(test: &TestServer) {
|
|
||||||
println!("Running reload with an unresolvable Pyzor host...");
|
|
||||||
let admin = test.account("[email protected]");
|
|
||||||
|
|
||||||
// Upstream resolved the host while building the settings and refused the
|
|
||||||
// reload when that failed.
|
|
||||||
admin
|
|
||||||
.registry_update_setting(
|
|
||||||
SpamPyzor {
|
|
||||||
enable: true,
|
|
||||||
host: "pyzor.invalid".into(),
|
|
||||||
port: 24441,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&[Property::Enable, Property::Host, Property::Port],
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
admin.reload_settings().await;
|
|
||||||
|
|
||||||
let pyzor = running_pyzor(test);
|
|
||||||
assert_eq!(pyzor.host, "pyzor.invalid");
|
|
||||||
assert_eq!(pyzor.port, 24441);
|
|
||||||
assert!(pyzor.address().await.is_err());
|
|
||||||
|
|
||||||
// An IP address needs no lookup
|
|
||||||
admin
|
|
||||||
.registry_update_setting(
|
|
||||||
SpamPyzor {
|
|
||||||
host: "192.0.2.1".into(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&[Property::Host],
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
admin.reload_settings().await;
|
|
||||||
assert_eq!(
|
|
||||||
running_pyzor(test).address().await.unwrap().to_string(),
|
|
||||||
"192.0.2.1:24441"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn test_build_errors(test: &TestServer) {
|
|
||||||
println!("Running reload with build errors...");
|
|
||||||
let admin = test.account("[email protected]");
|
|
||||||
let pyzor_ratio = running_pyzor(test).ratio;
|
|
||||||
assert_ne!(pyzor_ratio, 0.25);
|
|
||||||
|
|
||||||
// Two console tracers: only one is allowed, so the build of one of them
|
|
||||||
// fails. Neither existed when the running settings were built.
|
|
||||||
let mut tracer_ids = Vec::new();
|
|
||||||
for _ in 0..2 {
|
|
||||||
tracer_ids.push(
|
|
||||||
admin
|
|
||||||
.registry_create_object(Tracer::Stdout(TracerStdout {
|
|
||||||
enable: true,
|
|
||||||
level: TracingLevel::Error,
|
|
||||||
..Default::default()
|
|
||||||
}))
|
|
||||||
.await,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
admin
|
|
||||||
.registry_update_setting(
|
|
||||||
SpamPyzor {
|
|
||||||
ratio: 0.25.into(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
&[Property::Ratio],
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
// A new error refuses the reload and names the object
|
|
||||||
let err = admin
|
|
||||||
.registry_create_object_expect_err(Action::ReloadSettings)
|
|
||||||
.await;
|
|
||||||
let description = err.description.clone().unwrap_or_default();
|
|
||||||
assert!(
|
|
||||||
description.starts_with("Settings were not reloaded. ")
|
|
||||||
&& description.contains("Tracer")
|
|
||||||
&& description.contains("Only one console tracer is allowed"),
|
|
||||||
"{err:?}"
|
|
||||||
);
|
|
||||||
assert_eq!(running_pyzor(test).ratio, pyzor_ratio);
|
|
||||||
|
|
||||||
// Had the running settings been built with that tracer failing, as a
|
|
||||||
// restart now would, the same error doesn't hold the reload back.
|
|
||||||
let result = Box::pin(
|
|
||||||
test.server
|
|
||||||
.reload_registry(RegistryChange::Reload(ObjectType::DataStore)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(!result.replaced_core);
|
|
||||||
assert_eq!(result.errors.len(), 1, "{:?}", result.errors);
|
|
||||||
test.server.record_build_errors(&result.errors);
|
|
||||||
|
|
||||||
admin.reload_settings().await;
|
|
||||||
assert_eq!(running_pyzor(test).ratio, 0.25);
|
|
||||||
let result = Box::pin(
|
|
||||||
test.server
|
|
||||||
.reload_registry(RegistryChange::Reload(ObjectType::DataStore)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(result.replaced_core);
|
|
||||||
assert!(result.errors.is_empty());
|
|
||||||
assert_eq!(result.known_errors.len(), 1);
|
|
||||||
|
|
||||||
// Once fixed, the object is no longer known to fail, so a new error
|
|
||||||
// there refuses the reload again.
|
|
||||||
admin
|
|
||||||
.registry_destroy(ObjectType::Tracer, tracer_ids.iter())
|
|
||||||
.await
|
|
||||||
.assert_destroyed(&tracer_ids);
|
|
||||||
admin.reload_settings().await;
|
|
||||||
let result = Box::pin(
|
|
||||||
test.server
|
|
||||||
.reload_registry(RegistryChange::Reload(ObjectType::DataStore)),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(result.replaced_core);
|
|
||||||
assert!(result.errors.is_empty() && result.known_errors.is_empty());
|
|
||||||
|
|
||||||
for _ in 0..2 {
|
|
||||||
tracer_ids.push(
|
|
||||||
admin
|
|
||||||
.registry_create_object(Tracer::Stdout(TracerStdout {
|
|
||||||
enable: true,
|
|
||||||
level: TracingLevel::Error,
|
|
||||||
..Default::default()
|
|
||||||
}))
|
|
||||||
.await,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
admin
|
|
||||||
.registry_create_object_expect_err(Action::ReloadSettings)
|
|
||||||
.await;
|
|
||||||
let tracer_ids = tracer_ids.split_off(2);
|
|
||||||
admin
|
|
||||||
.registry_destroy(ObjectType::Tracer, tracer_ids.iter())
|
|
||||||
.await
|
|
||||||
.assert_destroyed(&tracer_ids);
|
|
||||||
admin.reload_settings().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
fn running_pyzor(test: &TestServer) -> PyzorConfig {
|
|
||||||
test.server
|
|
||||||
.inner
|
|
||||||
.build_server()
|
|
||||||
.core
|
|
||||||
.spam
|
|
||||||
.pyzor
|
|
||||||
.clone()
|
|
||||||
.expect("Pyzor enabled")
|
|
||||||
}
|
|
||||||
@@ -1,204 +0,0 @@
|
|||||||
/*
|
|
||||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
|
||||||
*
|
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
|
||||||
*/
|
|
||||||
|
|
||||||
// inbuxa: a tracer whose own settings change is started over by the reload
|
|
||||||
// that follows the write: a Log tracer moved to another directory writes
|
|
||||||
// there from then on, and no event is lost or written twice on the way.
|
|
||||||
|
|
||||||
use crate::utils::{
|
|
||||||
jmap::JmapResponse,
|
|
||||||
server::{TestServer, TestServerBuilder},
|
|
||||||
};
|
|
||||||
use registry::{
|
|
||||||
schema::{
|
|
||||||
enums::{EventPolicy, LogRotateFrequency, TracingLevel},
|
|
||||||
prelude::ObjectType,
|
|
||||||
structs::{Expression, MtaStageAuth, Tracer, TracerLog},
|
|
||||||
},
|
|
||||||
types::map::Map,
|
|
||||||
};
|
|
||||||
use serde_json::json;
|
|
||||||
use std::{
|
|
||||||
path::{Path, PathBuf},
|
|
||||||
time::{Duration, Instant},
|
|
||||||
};
|
|
||||||
use trc::{EventType, ServerEvent};
|
|
||||||
|
|
||||||
const PREFIX: &str = "tracer-reload";
|
|
||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
|
||||||
pub async fn tracer_reload_tests() {
|
|
||||||
let mut test = TestServerBuilder::new("tracer_reload_tests")
|
|
||||||
.await
|
|
||||||
.with_default_listeners()
|
|
||||||
.await
|
|
||||||
.with_object(MtaStageAuth {
|
|
||||||
require: Expression {
|
|
||||||
else_: "false".to_string(),
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
..Default::default()
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.build()
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let admin = test
|
|
||||||
.create_user_account(
|
|
||||||
"admin",
|
|
||||||
"[email protected]",
|
|
||||||
"these_pretzels_are_making_me_thirsty",
|
|
||||||
&[],
|
|
||||||
"Admin",
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
test.account("admin")
|
|
||||||
.assign_roles_to_account(admin.id(), &["user", "system"])
|
|
||||||
.await;
|
|
||||||
test.insert_account(admin);
|
|
||||||
|
|
||||||
test_log_tracer_moves(&test).await;
|
|
||||||
|
|
||||||
if test.is_reset() {
|
|
||||||
test.temp_dir.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn test_log_tracer_moves(test: &TestServer) {
|
|
||||||
println!("Running Log tracer path change...");
|
|
||||||
let admin = test.account("[email protected]");
|
|
||||||
let old_dir = test.temp_dir.path.join("tracer-old");
|
|
||||||
let new_dir = test.temp_dir.path.join("tracer-new");
|
|
||||||
for dir in [&old_dir, &new_dir] {
|
|
||||||
let _ = std::fs::remove_dir_all(dir);
|
|
||||||
std::fs::create_dir_all(dir).unwrap();
|
|
||||||
}
|
|
||||||
let old_file = old_dir.join(PREFIX);
|
|
||||||
let new_file = new_dir.join(PREFIX);
|
|
||||||
|
|
||||||
// A Log tracer for one event type, written to the old directory
|
|
||||||
let response = admin
|
|
||||||
.registry_create([Tracer::Log(TracerLog {
|
|
||||||
path: old_dir.to_string_lossy().into_owned(),
|
|
||||||
prefix: PREFIX.into(),
|
|
||||||
rotate: LogRotateFrequency::Never,
|
|
||||||
ansi: false,
|
|
||||||
multiline: false,
|
|
||||||
enable: true,
|
|
||||||
level: TracingLevel::Trace,
|
|
||||||
lossy: false,
|
|
||||||
events: Map::new(vec![EventType::Server(ServerEvent::Licensing)]),
|
|
||||||
events_policy: EventPolicy::Include,
|
|
||||||
})])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
let tracer_id = response.created_id(0);
|
|
||||||
emit("marker-before");
|
|
||||||
wait_for(&old_file, "marker-before").await;
|
|
||||||
|
|
||||||
// Events keep coming while the path changes
|
|
||||||
let stream = tokio::spawn(async {
|
|
||||||
for i in 0..2000u32 {
|
|
||||||
emit(&format!("seq-{i:05}-end"));
|
|
||||||
if i % 50 == 0 {
|
|
||||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
|
||||||
let response = admin
|
|
||||||
.registry_update(
|
|
||||||
ObjectType::Tracer,
|
|
||||||
[(tracer_id, json!({"path": new_dir.to_string_lossy()}))],
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
stream.await.unwrap();
|
|
||||||
|
|
||||||
// Once the reload has run, events go to the new file only
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
emit("marker-after");
|
|
||||||
wait_for(&new_file, "marker-after").await;
|
|
||||||
wait_for(&new_file, "seq-01999-end").await;
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
let old = read(&old_file);
|
|
||||||
let new = read(&new_file);
|
|
||||||
assert!(!old.contains("marker-after"), "old file still written to");
|
|
||||||
assert!(!new.contains("marker-before"));
|
|
||||||
|
|
||||||
// Every event written once, in one file or the other
|
|
||||||
let old_seq = count_seq(&old);
|
|
||||||
let new_seq = count_seq(&new);
|
|
||||||
println!(
|
|
||||||
"{} events in the old file, {} in the new one",
|
|
||||||
old_seq.iter().filter(|c| **c > 0).count(),
|
|
||||||
new_seq.iter().filter(|c| **c > 0).count()
|
|
||||||
);
|
|
||||||
for i in 0..2000 {
|
|
||||||
assert_eq!(
|
|
||||||
old_seq[i] + new_seq[i],
|
|
||||||
1,
|
|
||||||
"seq-{i:05} written {} + {} times",
|
|
||||||
old_seq[i],
|
|
||||||
new_seq[i]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
new_seq.iter().any(|c| *c > 0),
|
|
||||||
"no event of the stream reached the new file"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Removing the tracer stops it
|
|
||||||
let response = admin
|
|
||||||
.registry_destroy(ObjectType::Tracer, [tracer_id])
|
|
||||||
.await;
|
|
||||||
assert_applied(&response);
|
|
||||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
|
||||||
emit("marker-removed");
|
|
||||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
|
||||||
assert!(!read(&new_file).contains("marker-removed"));
|
|
||||||
assert!(!read(&old_file).contains("marker-removed"));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn emit(marker: &str) {
|
|
||||||
trc::event!(Server(ServerEvent::Licensing), Details = marker.to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
fn read(path: &Path) -> String {
|
|
||||||
std::fs::read_to_string(path).unwrap_or_default()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn count_seq(text: &str) -> Vec<u32> {
|
|
||||||
let mut counts = vec![0u32; 2000];
|
|
||||||
for part in text.split("seq-").skip(1) {
|
|
||||||
if let Some(n) = part.get(..5).and_then(|n| n.parse::<usize>().ok())
|
|
||||||
&& part[5..].starts_with("-end")
|
|
||||||
{
|
|
||||||
counts[n] += 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
counts
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn wait_for(path: &PathBuf, marker: &str) {
|
|
||||||
let started = Instant::now();
|
|
||||||
while !read(path).contains(marker) {
|
|
||||||
assert!(
|
|
||||||
started.elapsed() < Duration::from_secs(10),
|
|
||||||
"{marker} not in {}",
|
|
||||||
path.display()
|
|
||||||
);
|
|
||||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn assert_applied(response: &JmapResponse) {
|
|
||||||
assert_eq!(
|
|
||||||
response.pointer("/methodResponses/0/1/x:settingsReload"),
|
|
||||||
Some(&json!({"applied": true})),
|
|
||||||
"{response:?}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user