Compare commits
12
Commits
v2026.9.24
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
499e4d7810 | ||
|
|
212cd77cd3 | ||
|
|
7735780807 | ||
|
|
e223f7d327 | ||
|
|
30df055e39 | ||
|
|
5393c4405a | ||
|
|
cc532b914c | ||
|
|
c09eff2214 | ||
|
|
eba4c7a32e | ||
|
|
17426f6d60 | ||
|
|
d7c9416713 | ||
|
|
238079da66 |
+8
-2
@@ -1,10 +1,16 @@
|
|||||||
// Ignore everything
|
# Ignore everything
|
||||||
*
|
*
|
||||||
|
|
||||||
// Allow what is needed
|
# Allow what is needed
|
||||||
!crates
|
!crates
|
||||||
!tests
|
!tests
|
||||||
!resources
|
!resources
|
||||||
|
|
||||||
|
# The patched dependency Cargo.toml's [patch.crates-io] points at. Without
|
||||||
|
# it the build context has no vendor/, and `cargo chef cook` fails on
|
||||||
|
# "failed to load source for dependency sieve-rs" -- which CI cannot see,
|
||||||
|
# because CI builds from a checkout and only the image build has a context.
|
||||||
|
!vendor
|
||||||
|
|
||||||
!Cargo.lock
|
!Cargo.lock
|
||||||
!Cargo.toml
|
!Cargo.toml
|
||||||
|
|||||||
@@ -35,6 +35,11 @@ jobs:
|
|||||||
- run: python3 tools/fork/name-check.py
|
- run: python3 tools/fork/name-check.py
|
||||||
- if: always()
|
- if: always()
|
||||||
run: python3 tools/fork/notice-check.py
|
run: python3 tools/fork/notice-check.py
|
||||||
|
# Cargo can patch a dependency to a directory in this repository, and
|
||||||
|
# the image builds from a context .dockerignore prunes to almost
|
||||||
|
# nothing. CI never sees the difference; a release does.
|
||||||
|
- if: always()
|
||||||
|
run: python3 tools/fork/context-check.py
|
||||||
|
|
||||||
build:
|
build:
|
||||||
# Either runner (host1 or host2): the build needs no docker socket.
|
# Either runner (host1 or host2): the build needs no docker socket.
|
||||||
|
|||||||
@@ -12,6 +12,9 @@
|
|||||||
# An issue is opened once per release: an existing one with the same title,
|
# An issue is opened once per release: an existing one with the same title,
|
||||||
# open or closed, stops a second.
|
# open or closed, stops a second.
|
||||||
#
|
#
|
||||||
|
# It also watches spam-filter, whose rules the server bundles
|
||||||
|
# (resources/spam-filter/), and opens an issue for a newer release.
|
||||||
|
#
|
||||||
# Daily 06:17 UTC; run it by hand with workflow_dispatch.
|
# Daily 06:17 UTC; run it by hand with workflow_dispatch.
|
||||||
name: upstream-watch
|
name: upstream-watch
|
||||||
|
|
||||||
@@ -64,7 +67,7 @@ jobs:
|
|||||||
and key(r["tag_name"]) > key(base)),
|
and key(r["tag_name"]) > key(base)),
|
||||||
key=lambda r: key(r["tag_name"]))
|
key=lambda r: key(r["tag_name"]))
|
||||||
if not newer:
|
if not newer:
|
||||||
print(f"Up to date: {base} is the newest upstream release."); sys.exit(0)
|
print(f"Up to date: {base} is the newest upstream release.")
|
||||||
|
|
||||||
# Titles and bodies stay free of the upstream project's name, as the
|
# Titles and bodies stay free of the upstream project's name, as the
|
||||||
# rest of the fork's user-visible text does.
|
# rest of the fork's user-visible text does.
|
||||||
@@ -85,4 +88,35 @@ jobs:
|
|||||||
"add any new third-party notices to `THIRD-PARTY.md`, then merge `upstream` into `main`.")
|
"add any new third-party notices to `THIRD-PARTY.md`, then merge `upstream` into `main`.")
|
||||||
issue = call("POST", f"{api}/issues", {"title": title, "body": body})
|
issue = call("POST", f"{api}/issues", {"title": title, "body": body})
|
||||||
print(f"{tag}: opened #{issue['number']}.")
|
print(f"{tag}: opened #{issue['number']}.")
|
||||||
|
|
||||||
|
# The spam filter rules bundled with the server (resources/spam-filter/):
|
||||||
|
# an issue when spam-filter publishes a newer release than the one
|
||||||
|
# BUNDLED_SPAM_RULES_VERSION names on main.
|
||||||
|
src = call("GET", f"{api}/contents/crates/common/src/manager/spam_rules.rs?ref=main")
|
||||||
|
import base64
|
||||||
|
text = base64.b64decode(src["content"]).decode()
|
||||||
|
m = re.search(r'BUNDLED_SPAM_RULES_VERSION: &str = "(\d+\.\d+\.\d+)"', text)
|
||||||
|
if not m:
|
||||||
|
print("Can't read BUNDLED_SPAM_RULES_VERSION from spam_rules.rs", file=sys.stderr); sys.exit(1)
|
||||||
|
bundled = "v" + m.group(1)
|
||||||
|
rels = call("GET", "https://api.github.com/repos/stalwartlabs/spam-filter/releases?per_page=30", token=None)
|
||||||
|
newer = sorted((r for r in rels
|
||||||
|
if not r["draft"] and not r["prerelease"] and SEMVER.match(r["tag_name"])
|
||||||
|
and key(r["tag_name"]) > key(bundled)),
|
||||||
|
key=lambda r: key(r["tag_name"]))
|
||||||
|
if not newer:
|
||||||
|
print(f"Up to date: the bundled spam rules are {bundled}, the newest release."); sys.exit(0)
|
||||||
|
latest = newer[-1]
|
||||||
|
tag = latest["tag_name"]
|
||||||
|
title = f"Update the bundled spam rules to {tag}"
|
||||||
|
existing = {i["title"] for i in call("GET", f"{api}/issues?state=all&type=issues&q=bundled+spam+rules&limit=50")}
|
||||||
|
if title in existing:
|
||||||
|
print(f"spam rules {tag}: issue already exists."); sys.exit(0)
|
||||||
|
body = (f"spam-filter published {tag} on {latest['published_at'][:10]}. "
|
||||||
|
f"The server bundles {bundled}.\n\n"
|
||||||
|
"Update it as resources/spam-filter/README.md describes: take the rules file "
|
||||||
|
f"from the {tag} release (by tag, not `latest`), set BUNDLED_SPAM_RULES_VERSION, "
|
||||||
|
"and run the antispam test.")
|
||||||
|
issue = call("POST", f"{api}/issues", {"title": title, "body": body})
|
||||||
|
print(f"spam rules {tag}: opened #{issue['number']}.")
|
||||||
PY
|
PY
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
|
|||||||
g++-x86-64-linux-gnu binutils-x86-64-linux-gnu
|
g++-x86-64-linux-gnu binutils-x86-64-linux-gnu
|
||||||
RUN rustup target add "$(cat /target.txt)"
|
RUN rustup target add "$(cat /target.txt)"
|
||||||
COPY --from=planner /recipe.json /recipe.json
|
COPY --from=planner /recipe.json /recipe.json
|
||||||
|
# inbuxa: [patch.crates-io] points sieve-rs at vendor/, and the recipe only
|
||||||
|
# carries the workspace's own manifests, so cooking the dependencies needs the
|
||||||
|
# vendored crate itself (the context allows it since #27; this puts it here).
|
||||||
|
COPY vendor/ vendor/
|
||||||
RUN RUSTFLAGS="$(cat /flags.txt)" cargo chef cook --target "$(cat /target.txt)" --release --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats" --recipe-path /recipe.json
|
RUN RUSTFLAGS="$(cat /flags.txt)" cargo chef cook --target "$(cat /target.txt)" --release --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats" --recipe-path /recipe.json
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN RUSTFLAGS="$(cat /flags.txt)" cargo build --target "$(cat /target.txt)" --release -p inbuxa --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"
|
RUN RUSTFLAGS="$(cat /flags.txt)" cargo build --target "$(cat /target.txt)" --release -p inbuxa --no-default-features --features "sqlite postgres mysql rocks s3 redis azure nats"
|
||||||
|
|||||||
+1
-1
@@ -24,7 +24,7 @@ carry their own license files.
|
|||||||
| `crates/common/src/network/acme/directory.rs`, `crates/common/src/network/acme/jose.rs`, `crates/common/src/network/acme/order.rs` | [rustls-acme](https://github.com/FlorianUekermann/rustls-acme) (MIT or Apache-2.0) | Copyright (c) Florian Uekermann |
|
| `crates/common/src/network/acme/directory.rs`, `crates/common/src/network/acme/jose.rs`, `crates/common/src/network/acme/order.rs` | [rustls-acme](https://github.com/FlorianUekermann/rustls-acme) (MIT or Apache-2.0) | Copyright (c) Florian Uekermann |
|
||||||
| `crates/types/src/id.rs` | [crockford](https://github.com/archer884/crockford) (MIT or Apache-2.0) | Copyright (c) 2017 J/A <archer884@gmail.com> |
|
| `crates/types/src/id.rs` | [crockford](https://github.com/archer884/crockford) (MIT or Apache-2.0) | Copyright (c) 2017 J/A <archer884@gmail.com> |
|
||||||
| `crates/nlp/src/tokenizers/types.rs` | test cases from [linkify](https://github.com/robinst/linkify) (MIT or Apache-2.0) | Copyright (c) 2017 Robin Stocker |
|
| `crates/nlp/src/tokenizers/types.rs` | test cases from [linkify](https://github.com/robinst/linkify) (MIT or Apache-2.0) | Copyright (c) 2017 Robin Stocker |
|
||||||
| `tests/resources/smtp/antispam/spam-filter-rules.json.gz` | the published rules of [spam-filter](https://github.com/stalwartlabs/spam-filter) v3.0.2, unmodified, for the spam filter's tests (MIT or Apache-2.0) | Copyright (C) 2024, Stalwart Labs LLC |
|
| `resources/spam-filter/spam-filter-rules.json.gz` | the published rules of [spam-filter](https://github.com/stalwartlabs/spam-filter) v3.0.2, unmodified, built into the server as its default spam rules (MIT or Apache-2.0) | Copyright (C) 2024, Stalwart Labs LLC |
|
||||||
|
|
||||||
Each notice above applies with this permission notice:
|
Each notice above applies with this permission notice:
|
||||||
|
|
||||||
|
|||||||
@@ -243,7 +243,8 @@ impl SpamFilterConfig {
|
|||||||
spam_threshold: spam.score_spam.into_inner() as f32,
|
spam_threshold: spam.score_spam.into_inner() as f32,
|
||||||
},
|
},
|
||||||
grey_list_expiry: spam.greylist_for.map(|d| d.into_inner().as_secs()),
|
grey_list_expiry: spam.greylist_for.map(|d| d.into_inner().as_secs()),
|
||||||
spam_rules_url: spam.spam_filter_rules_url,
|
// inbuxa: unset, empty or upstream's old default means the bundled rules
|
||||||
|
spam_rules_url: crate::manager::spam_rules::rules_url(spam.spam_filter_rules_url),
|
||||||
url_client: utils::http::http_client_builder(true)
|
url_client: utils::http::http_client_builder(true)
|
||||||
.pool_max_idle_per_host(0)
|
.pool_max_idle_per_host(0)
|
||||||
.redirect(reqwest::redirect::Policy::none())
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
|
|||||||
@@ -23,6 +23,13 @@ use utils::{UnwrapFailure, codec::leb128::Leb128_};
|
|||||||
|
|
||||||
pub(super) const MAGIC_MARKER: u8 = 123;
|
pub(super) const MAGIC_MARKER: u8 = 123;
|
||||||
|
|
||||||
|
// inbuxa: blobs kept under a fixed name instead of a content hash. Nothing
|
||||||
|
// links to them, so the export names them outright.
|
||||||
|
const NAMED_BLOBS: &[&[u8]] = &[
|
||||||
|
crate::manager::SPAM_CLASSIFIER_KEY,
|
||||||
|
crate::manager::SPAM_TRAINER_KEY,
|
||||||
|
];
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
|
||||||
pub(super) enum Family {
|
pub(super) enum Family {
|
||||||
Data = 0,
|
Data = 0,
|
||||||
@@ -143,15 +150,21 @@ impl Core {
|
|||||||
.await
|
.await
|
||||||
.failed("Failed to iterate over data store");
|
.failed("Failed to iterate over data store");
|
||||||
|
|
||||||
for hash in blobs {
|
// inbuxa: the trained spam classifier and its trainer state are
|
||||||
|
// blobs stored under fixed names with no blob link, so the walk
|
||||||
|
// over links above never reaches them.
|
||||||
|
let named = NAMED_BLOBS.iter().map(|key| key.to_vec());
|
||||||
|
for key in blobs
|
||||||
|
.into_iter()
|
||||||
|
.map(|hash| hash.as_slice().to_vec())
|
||||||
|
.chain(named)
|
||||||
|
{
|
||||||
if let Some(blob) = blob_store
|
if let Some(blob) = blob_store
|
||||||
.get_blob(hash.as_slice(), 0..usize::MAX)
|
.get_blob(&key, 0..usize::MAX)
|
||||||
.await
|
.await
|
||||||
.failed("Failed to get blob")
|
.failed("Failed to get blob")
|
||||||
{
|
{
|
||||||
writer
|
writer.send((key, blob)).failed("Failed to send key");
|
||||||
.send((hash.as_slice().to_vec(), blob))
|
|
||||||
.failed("Failed to send key");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
@@ -323,7 +336,13 @@ impl Family {
|
|||||||
SUBSPACE_REGISTRY_IDX,
|
SUBSPACE_REGISTRY_IDX,
|
||||||
SUBSPACE_REGISTRY_PK,
|
SUBSPACE_REGISTRY_PK,
|
||||||
SUBSPACE_DIRECTORY,
|
SUBSPACE_DIRECTORY,
|
||||||
store::SUBSPACE_INBUXA, // inbuxa: masked email
|
// inbuxa: registry objects the upstream list left out, so an
|
||||||
|
// export dropped them: archived items (undelete) and spam
|
||||||
|
// training samples. Their indexes and id counters already
|
||||||
|
// travel in this family and in `data`, so they ride along.
|
||||||
|
SUBSPACE_DELETED_ITEMS,
|
||||||
|
SUBSPACE_SPAM_SAMPLES,
|
||||||
|
store::SUBSPACE_INBUXA, // inbuxa: the fork's own data (masked email, undelete, policies)
|
||||||
],
|
],
|
||||||
Family::Changelog => &[SUBSPACE_LOGS],
|
Family::Changelog => &[SUBSPACE_LOGS],
|
||||||
Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT],
|
Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT],
|
||||||
|
|||||||
@@ -54,6 +54,13 @@ Options:
|
|||||||
-o, --console Open the store console
|
-o, --console Open the store console
|
||||||
-h, --help Print help
|
-h, --help Print help
|
||||||
-V, --version Print version
|
-V, --version Print version
|
||||||
|
|
||||||
|
An export holds everything in the data and blob stores except short-lived
|
||||||
|
in-memory state (rate limits, locks, greylisting) and the full-text search
|
||||||
|
index, which belongs to one search backend. An import into an empty store
|
||||||
|
queues the index to be rebuilt when the server next starts. EXPORT_TYPES
|
||||||
|
limits an export to some of: data, registry, blob, changelog, queue, report,
|
||||||
|
telemetry, tasks.
|
||||||
"#
|
"#
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -256,10 +263,10 @@ impl BootManager {
|
|||||||
telemetry.enable();
|
telemetry.enable();
|
||||||
|
|
||||||
// Parse settings and restore
|
// Parse settings and restore
|
||||||
Box::pin(Core::parse(&mut bootstrap, storage))
|
let core = Box::pin(Core::parse(&mut bootstrap, storage)).await;
|
||||||
.await
|
let imported = core.restore(path).await;
|
||||||
.restore(path)
|
// inbuxa: the search index isn't exported; rebuild it
|
||||||
.await;
|
core.queue_reindex(&imported).await;
|
||||||
std::process::exit(0);
|
std::process::exit(0);
|
||||||
}
|
}
|
||||||
StoreOp::Console => {
|
StoreOp::Console => {
|
||||||
|
|||||||
@@ -530,13 +530,22 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
|
|||||||
use store::write::BatchBuilder;
|
use store::write::BatchBuilder;
|
||||||
use types::id::Id;
|
use types::id::Id;
|
||||||
|
|
||||||
if bp.registry.count_object(ObjectType::SpamRule).await? == 0
|
// inbuxa: rules are always to hand, since a copy ships with the server
|
||||||
&& bp
|
// (spam_rules). They load on first boot, and again when the bundled
|
||||||
.registry
|
// version differs from the one last loaded, which only adds what's
|
||||||
|
// missing: new tags and rules, never a changed score.
|
||||||
|
let rules_url = super::spam_rules::rules_url(
|
||||||
|
bp.registry
|
||||||
.object::<SpamSettings>(Id::singleton())
|
.object::<SpamSettings>(Id::singleton())
|
||||||
.await?
|
.await?
|
||||||
.is_none_or(|spam| spam.spam_filter_rules_url.is_some())
|
.and_then(|spam| spam.spam_filter_rules_url),
|
||||||
{
|
);
|
||||||
|
let bundled_is_new = rules_url.is_none()
|
||||||
|
&& super::spam_rules::applied_version(&bp.data_store)
|
||||||
|
.await?
|
||||||
|
.as_deref()
|
||||||
|
!= Some(super::spam_rules::BUNDLED_SPAM_RULES_VERSION);
|
||||||
|
if bp.registry.count_object(ObjectType::SpamRule).await? == 0 || bundled_is_new {
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
batch.schedule_task(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
|
batch.schedule_task(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
|
||||||
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
|
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ pub mod console;
|
|||||||
pub mod defaults;
|
pub mod defaults;
|
||||||
pub mod first_party;
|
pub mod first_party;
|
||||||
pub mod restore;
|
pub mod restore;
|
||||||
|
pub mod spam_rules; // inbuxa: rules bundled with the server
|
||||||
|
|
||||||
pub const SPAM_TRAINER_KEY: &[u8] = "INBUXA_SPAM_TRAIN_DATA.lz4".as_bytes();
|
pub const SPAM_TRAINER_KEY: &[u8] = "INBUXA_SPAM_TRAIN_DATA.lz4".as_bytes();
|
||||||
pub const SPAM_CLASSIFIER_KEY: &[u8] = "INBUXA_SPAM_CLASSIFIER_MODEL.lz4".as_bytes();
|
pub const SPAM_CLASSIFIER_KEY: &[u8] = "INBUXA_SPAM_CLASSIFIER_MODEL.lz4".as_bytes();
|
||||||
|
|||||||
@@ -9,15 +9,22 @@
|
|||||||
use super::backup::MAGIC_MARKER;
|
use super::backup::MAGIC_MARKER;
|
||||||
use crate::{Core, DATABASE_SCHEMA_VERSION};
|
use crate::{Core, DATABASE_SCHEMA_VERSION};
|
||||||
use lz4_flex::frame::FrameDecoder;
|
use lz4_flex::frame::FrameDecoder;
|
||||||
use registry::schema::enums::CompressionAlgo;
|
use registry::{
|
||||||
|
schema::{
|
||||||
|
enums::{CompressionAlgo, TaskStoreMaintenanceType},
|
||||||
|
structs::{Task, TaskStatus, TaskStoreMaintenance},
|
||||||
|
},
|
||||||
|
types::EnumImpl,
|
||||||
|
};
|
||||||
use std::{
|
use std::{
|
||||||
fs::File,
|
fs::File,
|
||||||
io::{BufReader, ErrorKind, Read},
|
io::{BufReader, ErrorKind, Read},
|
||||||
path::{Path, PathBuf},
|
path::{Path, PathBuf},
|
||||||
};
|
};
|
||||||
use store::{
|
use store::{
|
||||||
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_QUOTA,
|
BlobStore, IterateParams, SUBSPACE_BLOBS, SUBSPACE_COUNTER, SUBSPACE_INDEXES,
|
||||||
SUBSPACE_REGISTRY_PK, Store, U32_LEN,
|
SUBSPACE_PROPERTY, SUBSPACE_QUOTA, SUBSPACE_REGISTRY_PK, SUBSPACE_TELEMETRY_SPAN, Store,
|
||||||
|
U32_LEN,
|
||||||
write::{
|
write::{
|
||||||
AnyClass, AnyKey, BatchBuilder, ValueClass,
|
AnyClass, AnyKey, BatchBuilder, ValueClass,
|
||||||
key::{DeserializeBigEndian, is_node_id_key},
|
key::{DeserializeBigEndian, is_node_id_key},
|
||||||
@@ -27,7 +34,9 @@ use types::{collection::Collection, field::Field};
|
|||||||
use utils::{UnwrapFailure, failed};
|
use utils::{UnwrapFailure, failed};
|
||||||
|
|
||||||
impl Core {
|
impl Core {
|
||||||
pub async fn restore(&self, src: PathBuf) {
|
/// Imports an export into an empty store and returns the subspaces it
|
||||||
|
/// wrote. inbuxa: the caller hands them to [`Core::queue_reindex`].
|
||||||
|
pub async fn restore(&self, src: PathBuf) -> Vec<u8> {
|
||||||
// Backup the core
|
// Backup the core
|
||||||
let paths = if src.is_dir() {
|
let paths = if src.is_dir() {
|
||||||
let mut paths = Vec::new();
|
let mut paths = Vec::new();
|
||||||
@@ -64,6 +73,13 @@ impl Core {
|
|||||||
std::process::exit(1);
|
std::process::exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut imported = paths
|
||||||
|
.iter()
|
||||||
|
.map(|path| KeyValueReader::new(path).subspace)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
imported.sort_unstable();
|
||||||
|
imported.dedup();
|
||||||
|
|
||||||
let mut tasks = Vec::new();
|
let mut tasks = Vec::new();
|
||||||
for path in paths {
|
for path in paths {
|
||||||
let storage = self.storage.clone();
|
let storage = self.storage.clone();
|
||||||
@@ -76,6 +92,54 @@ impl Core {
|
|||||||
for task in tasks {
|
for task in tasks {
|
||||||
task.await.failed("Failed to wait for task");
|
task.await.failed("Failed to wait for task");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
imported
|
||||||
|
}
|
||||||
|
|
||||||
|
/// inbuxa: an export never carries the full-text index. It is built by
|
||||||
|
/// and for one search backend (the SQL stores index into their own
|
||||||
|
/// tables, the key-value stores into a subspace, external engines keep it
|
||||||
|
/// themselves), so it would be wrong or unreadable after a move to
|
||||||
|
/// another one. Instead, an import queues the same reindex tasks an
|
||||||
|
/// administrator can queue by hand (`reindexAccounts` and
|
||||||
|
/// `reindexTelemetry` store maintenance), and the server rebuilds the
|
||||||
|
/// index for whatever search store it is configured with once it starts.
|
||||||
|
pub async fn queue_reindex(&self, imported: &[u8]) -> Vec<TaskStoreMaintenanceType> {
|
||||||
|
let mut queued = Vec::new();
|
||||||
|
if imported.contains(&SUBSPACE_PROPERTY) {
|
||||||
|
queued.push(TaskStoreMaintenanceType::ReindexAccounts);
|
||||||
|
}
|
||||||
|
if imported.contains(&SUBSPACE_TELEMETRY_SPAN) {
|
||||||
|
queued.push(TaskStoreMaintenanceType::ReindexTelemetry);
|
||||||
|
}
|
||||||
|
if queued.is_empty() {
|
||||||
|
return queued;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut batch = BatchBuilder::new();
|
||||||
|
for maintenance_type in &queued {
|
||||||
|
batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance {
|
||||||
|
maintenance_type: *maintenance_type,
|
||||||
|
status: TaskStatus::now(),
|
||||||
|
shard_index: None,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
self.storage
|
||||||
|
.data
|
||||||
|
.write(batch.build_all())
|
||||||
|
.await
|
||||||
|
.failed("Failed to queue the reindex tasks");
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"Queued {} to rebuild the search index; it runs when the server starts.",
|
||||||
|
queued
|
||||||
|
.iter()
|
||||||
|
.map(|t| t.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" and ")
|
||||||
|
);
|
||||||
|
|
||||||
|
queued
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,17 +189,22 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) {
|
|||||||
}
|
}
|
||||||
SUBSPACE_COUNTER | SUBSPACE_QUOTA => {
|
SUBSPACE_COUNTER | SUBSPACE_QUOTA => {
|
||||||
while let Some((key, value)) = reader.next() {
|
while let Some((key, value)) = reader.next() {
|
||||||
batch.add(
|
let class = ValueClass::Any(AnyClass {
|
||||||
ValueClass::Any(AnyClass {
|
subspace: reader.subspace,
|
||||||
subspace: reader.subspace,
|
key,
|
||||||
key,
|
});
|
||||||
}),
|
let value = u64::from_le_bytes(
|
||||||
u64::from_le_bytes(
|
value
|
||||||
value
|
.try_into()
|
||||||
.try_into()
|
.expect("Failed to deserialize counter/quota"),
|
||||||
.expect("Failed to deserialize counter/quota"),
|
) as i64;
|
||||||
) as i64,
|
// inbuxa: the SQL stores add a negative amount with an UPDATE,
|
||||||
);
|
// which does nothing to a row that isn't there yet, so a
|
||||||
|
// negative counter vanished on import. Create the row first.
|
||||||
|
if value < 0 {
|
||||||
|
batch.add(class.clone(), 0);
|
||||||
|
}
|
||||||
|
batch.add(class, value);
|
||||||
if batch.is_large_batch() {
|
if batch.is_large_batch() {
|
||||||
store
|
store
|
||||||
.write(batch.build_all())
|
.write(batch.build_all())
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
/*
|
||||||
|
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||||
|
*
|
||||||
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
//! inbuxa: the spam filter rules that ship with the server.
|
||||||
|
//!
|
||||||
|
//! Upstream fetches its latest published rules from GitHub at run time, so
|
||||||
|
//! scoring changes with a release nobody here tested and depends on reaching
|
||||||
|
//! it. The fork embeds a pinned copy (resources/spam-filter/, with its version
|
||||||
|
//! and license) and uses it whenever no other source is configured. The rules
|
||||||
|
//! URL remains an operator override (`https://` or `file://`).
|
||||||
|
//!
|
||||||
|
//! Loading rules only ever adds what's missing, never changes an existing rule
|
||||||
|
//! or score. They load on first boot, and again whenever the bundled version
|
||||||
|
//! differs from the one last applied, so an upgrade brings new tags (the AI
|
||||||
|
//! classifier's `LLM_*` scores, say) to an install that already had rules.
|
||||||
|
|
||||||
|
use std::io::Read;
|
||||||
|
use store::{
|
||||||
|
SUBSPACE_INBUXA, Store, ValueKey,
|
||||||
|
write::{AnyClass, BatchBuilder, ValueClass},
|
||||||
|
};
|
||||||
|
use trc::AddContext;
|
||||||
|
|
||||||
|
/// The version of spam-filter the embedded rules come from.
|
||||||
|
pub const BUNDLED_SPAM_RULES_VERSION: &str = "3.0.2";
|
||||||
|
|
||||||
|
static BUNDLED_SPAM_RULES: &[u8] =
|
||||||
|
include_bytes!("../../../../resources/spam-filter/spam-filter-rules.json.gz");
|
||||||
|
|
||||||
|
/// Upstream's default rules source, the value every install created before
|
||||||
|
/// the rules were bundled has saved. Read only to treat it as unset.
|
||||||
|
const LEGACY_DEFAULT_URL: &str =
|
||||||
|
"https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz";
|
||||||
|
|
||||||
|
/// The URL to fetch rules from, or `None` for the bundled rules. An empty
|
||||||
|
/// setting and upstream's old default both mean the bundled rules.
|
||||||
|
pub fn rules_url(configured: Option<String>) -> Option<String> {
|
||||||
|
configured.filter(|url| !url.trim().is_empty() && url != LEGACY_DEFAULT_URL)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bundled rules, uncompressed: the same JSON the rules URL serves.
|
||||||
|
pub fn bundled_rules() -> Result<Vec<u8>, String> {
|
||||||
|
let mut json = Vec::new();
|
||||||
|
mail_auth::flate2::read::GzDecoder::new(BUNDLED_SPAM_RULES)
|
||||||
|
.read_to_end(&mut json)
|
||||||
|
.map_err(|err| format!("Failed to decompress the bundled spam rules: {err}"))?;
|
||||||
|
Ok(json)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn applied_key() -> ValueClass {
|
||||||
|
ValueClass::Any(AnyClass {
|
||||||
|
subspace: SUBSPACE_INBUXA,
|
||||||
|
key: b"Sr".to_vec(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bundled version last loaded into the registry, if any.
|
||||||
|
pub async fn applied_version(data: &Store) -> trc::Result<Option<String>> {
|
||||||
|
data.get_value::<String>(ValueKey::from(applied_key()))
|
||||||
|
.await
|
||||||
|
.caused_by(trc::location!())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Records that the bundled rules of this version have been loaded.
|
||||||
|
pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()> {
|
||||||
|
let mut batch = BatchBuilder::new();
|
||||||
|
batch.set(applied_key(), version.as_bytes().to_vec());
|
||||||
|
data.write(batch.build_all())
|
||||||
|
.await
|
||||||
|
.caused_by(trc::location!())
|
||||||
|
.map(|_| ())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn upstream_default_and_empty_mean_bundled() {
|
||||||
|
assert_eq!(rules_url(None), None);
|
||||||
|
assert_eq!(rules_url(Some(String::new())), None);
|
||||||
|
assert_eq!(rules_url(Some(" ".into())), None);
|
||||||
|
assert_eq!(rules_url(Some(LEGACY_DEFAULT_URL.into())), None);
|
||||||
|
assert_eq!(
|
||||||
|
rules_url(Some("file:///srv/rules.json.gz".into())).as_deref(),
|
||||||
|
Some("file:///srv/rules.json.gz")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bundled_rules_parse_and_score_the_ai_tags() {
|
||||||
|
let rules: serde_json::Value = serde_json::from_slice(&bundled_rules().unwrap()).unwrap();
|
||||||
|
let tags = rules["SpamTag"].as_array().unwrap();
|
||||||
|
for (tag, score) in [("LLM_UNSOLICITED_HIGH", 3.0), ("LLM_LEGITIMATE_HIGH", -3.0)] {
|
||||||
|
let found = tags.iter().find(|t| t["tag"] == tag).unwrap();
|
||||||
|
assert_eq!(found["score"].as_f64(), Some(score), "{tag}");
|
||||||
|
}
|
||||||
|
assert!(!rules["SpamRule"].as_array().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40215,7 +40215,7 @@ impl Default for SpamSettings {
|
|||||||
score_reject: Float::new(0.0f64),
|
score_reject: Float::new(0.0f64),
|
||||||
score_spam: Float::new(5.0f64),
|
score_spam: Float::new(5.0f64),
|
||||||
trust_replies: true,
|
trust_replies: true,
|
||||||
spam_filter_rules_url: Some("https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz".to_string()),
|
spam_filter_rules_url: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,13 +2,15 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
|
*
|
||||||
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::task_manager::{TaskFailureType, TaskResult};
|
use crate::task_manager::{TaskFailureType, TaskResult};
|
||||||
use common::{
|
use common::{
|
||||||
Server,
|
Server,
|
||||||
ipc::{BroadcastEvent, RegistryChange},
|
ipc::{BroadcastEvent, RegistryChange},
|
||||||
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource},
|
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource, spam_rules},
|
||||||
};
|
};
|
||||||
use registry::{
|
use registry::{
|
||||||
schema::{
|
schema::{
|
||||||
@@ -106,6 +108,7 @@ struct RuleUpdateResult {
|
|||||||
|
|
||||||
async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
|
async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
|
let bundled = server.core.spam.spam_rules_url.is_none();
|
||||||
let rules = match fetch_spam_rules(server).await {
|
let rules = match fetch_spam_rules(server).await {
|
||||||
Ok(rules) => rules,
|
Ok(rules) => rules,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
@@ -289,29 +292,36 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
|
|||||||
Elapsed = started.elapsed(),
|
Elapsed = started.elapsed(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// inbuxa: so the next start knows these bundled rules are in
|
||||||
|
if bundled {
|
||||||
|
spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_VERSION)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(TaskResult::Success(vec![]))
|
Ok(TaskResult::Success(vec![]))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_spam_rules(server: &Server) -> Result<Rules, RuleUpdateError> {
|
async fn fetch_spam_rules(server: &Server) -> Result<Rules, RuleUpdateError> {
|
||||||
let Some(rules_url) = server.core.spam.spam_rules_url.as_ref() else {
|
// inbuxa: no URL means the rules bundled with the server
|
||||||
return Err(RuleUpdateError {
|
let bytes = match server.core.spam.spam_rules_url.as_ref() {
|
||||||
typ: TaskFailureType::Permanent,
|
Some(rules_url) => fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500)
|
||||||
reason: "Spam rules resource URL not configured".to_string(),
|
|
||||||
});
|
|
||||||
};
|
|
||||||
let rules_json: AHashMap<String, Vec<serde_json::Value>> =
|
|
||||||
fetch_resource(rules_url, None, Duration::from_secs(60), 1024 * 500)
|
|
||||||
.await
|
.await
|
||||||
.map_err(|reason| RuleUpdateError {
|
.map_err(|reason| RuleUpdateError {
|
||||||
typ: TaskFailureType::Temporary,
|
typ: TaskFailureType::Temporary,
|
||||||
reason,
|
reason,
|
||||||
|
}),
|
||||||
|
None => spam_rules::bundled_rules().map_err(|reason| RuleUpdateError {
|
||||||
|
typ: TaskFailureType::Permanent,
|
||||||
|
reason,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let rules_json: AHashMap<String, Vec<serde_json::Value>> =
|
||||||
|
bytes.and_then(|bytes| {
|
||||||
|
serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError {
|
||||||
|
typ: TaskFailureType::Permanent,
|
||||||
|
reason: format!("Failed to parse spam rules JSON: {err}"),
|
||||||
})
|
})
|
||||||
.and_then(|bytes| {
|
})?;
|
||||||
serde_json::from_slice(&bytes).map_err(|err| RuleUpdateError {
|
|
||||||
typ: TaskFailureType::Permanent,
|
|
||||||
reason: format!("Failed to parse spam rules JSON: {err}"),
|
|
||||||
})
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let mut rules = Rules::default();
|
let mut rules = Rules::default();
|
||||||
for (object_type, values) in rules_json {
|
for (object_type, values) in rules_json {
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
|
|||||||
#[macro_export]
|
#[macro_export]
|
||||||
macro_rules! brand_version {
|
macro_rules! brand_version {
|
||||||
() => {
|
() => {
|
||||||
"2026.9.24"
|
"2026.9.24.3"
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -102,7 +102,10 @@ Permissions: `sysSpamLlmGet`, `sysSpamLlmUpdate`.
|
|||||||
- **Tags and scores.** The classifier's tags are ordinary spam tags, scored
|
- **Tags and scores.** The classifier's tags are ordinary spam tags, scored
|
||||||
by `x:SpamTag` entries like every other tag: `Score` (a number), `Discard`
|
by `x:SpamTag` entries like every other tag: `Score` (a number), `Discard`
|
||||||
or `Reject`. The documented defaults are `LLM_UNSOLICITED_HIGH` 3.0 and
|
or `Reject`. The documented defaults are `LLM_UNSOLICITED_HIGH` 3.0 and
|
||||||
`LLM_LEGITIMATE_HIGH` −3.0. A tag with no entry scores 0.
|
`LLM_LEGITIMATE_HIGH` −3.0. A tag with no entry scores 0. The server ships
|
||||||
|
those entries in its bundled spam rules (`resources/spam-filter/`), loaded
|
||||||
|
on first boot and again when the bundled version changes, so an install
|
||||||
|
that predates them gains them on upgrade (added 2026-09-23).
|
||||||
- **`interactAi`** permission ("Interact with AI models"): lets an account's
|
- **`interactAi`** permission ("Interact with AI models"): lets an account's
|
||||||
own Sieve scripts call `llm_prompt`. This repository's default roles give it
|
own Sieve scripts call `llm_prompt`. This repository's default roles give it
|
||||||
to users, tenant administrators and superusers
|
to users, tenant administrators and superusers
|
||||||
|
|||||||
Binary file not shown.
@@ -1 +1 @@
|
|||||||
rWqJwNJkqgKsbC1eqcEmmIAMtJPmnlDlThR2ZtW_N1c
|
VbnFuwCOTBh0s2T-NuRhb2JaJr8Jl5s3LgXv4Pv2sTg
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
# Bundled spam filter rules
|
||||||
|
|
||||||
|
`spam-filter-rules.json.gz` is the published rules file of
|
||||||
|
[spam-filter](https://github.com/stalwartlabs/spam-filter) **v3.0.2**,
|
||||||
|
unmodified. The server embeds it (`crates/common/src/manager/spam_rules.rs`)
|
||||||
|
and loads it whenever no other rules source is configured, so a release
|
||||||
|
scores mail with the rules it was tested with, offline and with nothing to
|
||||||
|
fetch. The rules URL setting stays an operator override.
|
||||||
|
|
||||||
|
The rules are dual-licensed MIT or Apache-2.0, Copyright (C) 2024, Stalwart
|
||||||
|
Labs LLC; the fork takes them under MIT, with the notice in `THIRD-PARTY.md`.
|
||||||
|
|
||||||
|
They include the scores for the AI classifier's tags (`LLM_*`, 3.0 for the
|
||||||
|
high-confidence spam categories, −3.0 for legitimate), which match
|
||||||
|
`docs/spec/features/ai-spam-classification.md`.
|
||||||
|
|
||||||
|
## Updating
|
||||||
|
|
||||||
|
The `upstream-watch` workflow opens an issue when spam-filter publishes a
|
||||||
|
newer release. To take it:
|
||||||
|
|
||||||
|
1. Download `spam-filter-rules.json.gz` from that release, pinned by tag
|
||||||
|
(`releases/download/vX.Y.Z/…`, not `latest`), over this file.
|
||||||
|
2. Set `BUNDLED_SPAM_RULES_VERSION` in `spam_rules.rs` and the version in
|
||||||
|
this README and in `THIRD-PARTY.md`.
|
||||||
|
3. Run the antispam test (`STORE=RocksDb RUST_MIN_STACK=16777216 cargo test
|
||||||
|
-p tests --lib -- smtp::inbound::antispam::antispam --exact`) and fix
|
||||||
|
expectations the new rules change, knowingly.
|
||||||
|
|
||||||
|
On the next start each server loads the new version once. Loading only adds
|
||||||
|
rules and tags that are missing; it never changes an existing one, so an
|
||||||
|
operator's own adjustments survive.
|
||||||
@@ -86,19 +86,10 @@ async fn antispam() {
|
|||||||
.registry_create_object(SpamSettings {
|
.registry_create_object(SpamSettings {
|
||||||
score_spam: Float::new(5.0),
|
score_spam: Float::new(5.0),
|
||||||
// inbuxa: the rules carry the scores the expectations are written
|
// inbuxa: the rules carry the scores the expectations are written
|
||||||
// against, so they're pinned (spam-filter v3.0.2, beside the test
|
// against. Unset, the server uses the rules bundled with it
|
||||||
// cases) rather than read from a developer's own checkout, which
|
// (resources/spam-filter/), the path production takes;
|
||||||
// left every score at zero. SPAM_RULES_URL still overrides.
|
// SPAM_RULES_URL tests another set.
|
||||||
spam_filter_rules_url: std::env::var("SPAM_RULES_URL")
|
spam_filter_rules_url: std::env::var("SPAM_RULES_URL").ok(),
|
||||||
.unwrap_or_else(|_| {
|
|
||||||
concat!(
|
|
||||||
"file://",
|
|
||||||
env!("CARGO_MANIFEST_DIR"),
|
|
||||||
"/resources/smtp/antispam/spam-filter-rules.json.gz"
|
|
||||||
)
|
|
||||||
.to_string()
|
|
||||||
})
|
|
||||||
.into(),
|
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <[email protected]>
|
||||||
*
|
*
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||||
|
*
|
||||||
|
* Modified by Coffey Labs in 2026 for INBUXA.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
use crate::utils::{
|
use crate::utils::{
|
||||||
@@ -9,14 +11,22 @@ use crate::utils::{
|
|||||||
server::TestServer,
|
server::TestServer,
|
||||||
temp_dir::TempDir,
|
temp_dir::TempDir,
|
||||||
};
|
};
|
||||||
use ::registry::schema::enums::CompressionAlgo;
|
use ::registry::schema::{
|
||||||
|
enums::{CompressionAlgo, TaskStoreMaintenanceType},
|
||||||
|
prelude::ObjectType,
|
||||||
|
structs::Task,
|
||||||
|
};
|
||||||
use ahash::AHashSet;
|
use ahash::AHashSet;
|
||||||
use common::{DATABASE_SCHEMA_VERSION, manager::backup::BackupParams};
|
use common::{
|
||||||
|
DATABASE_SCHEMA_VERSION,
|
||||||
|
manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, backup::BackupParams},
|
||||||
|
};
|
||||||
use store::{
|
use store::{
|
||||||
rand,
|
rand,
|
||||||
write::{
|
write::{
|
||||||
AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, Operation, QueueClass, QueueEvent,
|
AnyClass, AnyKey, BatchBuilder, BlobLink, BlobOp, Operation, QueueClass, QueueEvent,
|
||||||
RegistryClass, ValueClass, key::KeySerializer,
|
RegistryClass, TaskQueueClass, ValueClass,
|
||||||
|
key::{DeserializeBigEndian, KeySerializer},
|
||||||
},
|
},
|
||||||
*,
|
*,
|
||||||
};
|
};
|
||||||
@@ -167,6 +177,50 @@ pub async fn test(test: &TestServer) {
|
|||||||
}
|
}
|
||||||
db.write(batch.build_all()).await.unwrap();
|
db.write(batch.build_all()).await.unwrap();
|
||||||
|
|
||||||
|
// inbuxa: registry objects kept outside the registry subspace (archived
|
||||||
|
// items for undelete, spam training samples, directory entries) and the
|
||||||
|
// fork's own subspace. Exports used to leave the first two behind.
|
||||||
|
println!("Creating archived items, spam samples and fork data...");
|
||||||
|
let mut batch = BatchBuilder::new();
|
||||||
|
for item_id in [1u64, 2, 3] {
|
||||||
|
for object in [
|
||||||
|
ObjectType::ArchivedItem,
|
||||||
|
ObjectType::SpamTrainingSample,
|
||||||
|
ObjectType::Account,
|
||||||
|
] {
|
||||||
|
batch.set(
|
||||||
|
ValueClass::Registry(RegistryClass::Item {
|
||||||
|
object_id: object as u16,
|
||||||
|
item_id,
|
||||||
|
}),
|
||||||
|
random_bytes(item_id as usize * 64),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
batch.set(
|
||||||
|
ValueClass::Any(AnyClass {
|
||||||
|
subspace: SUBSPACE_INBUXA,
|
||||||
|
key: [b'U', b'x']
|
||||||
|
.into_iter()
|
||||||
|
.chain(item_id.to_be_bytes())
|
||||||
|
.collect(),
|
||||||
|
}),
|
||||||
|
random_bytes(32),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
db.write(batch.build_all()).await.unwrap();
|
||||||
|
|
||||||
|
// inbuxa: the trained spam classifier lives in blobs with fixed names
|
||||||
|
let mut named_blobs = Vec::new();
|
||||||
|
for key in [SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY] {
|
||||||
|
let data = random_bytes(4096);
|
||||||
|
test.server
|
||||||
|
.blob_store()
|
||||||
|
.put_blob(key, &data, CompressionAlgo::Lz4)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
named_blobs.push((key, data));
|
||||||
|
}
|
||||||
|
|
||||||
// Create directory data
|
// Create directory data
|
||||||
println!("Creating directory data...");
|
println!("Creating directory data...");
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
@@ -185,6 +239,17 @@ pub async fn test(test: &TestServer) {
|
|||||||
println!("Calculating store hash...");
|
println!("Calculating store hash...");
|
||||||
let snapshot = Snapshot::new(&db).await;
|
let snapshot = Snapshot::new(&db).await;
|
||||||
assert!(!snapshot.keys.is_empty(), "Store hash counts are empty",);
|
assert!(!snapshot.keys.is_empty(), "Store hash counts are empty",);
|
||||||
|
for subspace in [
|
||||||
|
SUBSPACE_DELETED_ITEMS,
|
||||||
|
SUBSPACE_SPAM_SAMPLES,
|
||||||
|
SUBSPACE_INBUXA,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
snapshot.keys.iter().any(|k| k.subspace == subspace),
|
||||||
|
"No test data in subspace {}",
|
||||||
|
char::from(subspace)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Export store
|
// Export store
|
||||||
println!("Exporting store...");
|
println!("Exporting store...");
|
||||||
@@ -210,22 +275,188 @@ pub async fn test(test: &TestServer) {
|
|||||||
.finalize(),
|
.finalize(),
|
||||||
);
|
);
|
||||||
db.write(batch.build_all()).await.unwrap();
|
db.write(batch.build_all()).await.unwrap();
|
||||||
test.server.core.restore(temp_dir.path.clone()).await;
|
for (key, _) in &named_blobs {
|
||||||
|
test.server.blob_store().delete_blob(key).await.unwrap();
|
||||||
|
}
|
||||||
|
let imported = test.server.core.restore(temp_dir.path.clone()).await;
|
||||||
let mut batch = BatchBuilder::new();
|
let mut batch = BatchBuilder::new();
|
||||||
batch.clear(ValueClass::NodeId(0));
|
batch.clear(ValueClass::NodeId(0));
|
||||||
db.write(batch.build_all()).await.unwrap();
|
db.write(batch.build_all()).await.unwrap();
|
||||||
|
for subspace in [
|
||||||
|
SUBSPACE_DELETED_ITEMS,
|
||||||
|
SUBSPACE_SPAM_SAMPLES,
|
||||||
|
SUBSPACE_INBUXA,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
imported.contains(&subspace),
|
||||||
|
"Subspace {} was not exported",
|
||||||
|
char::from(subspace)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Verify hash
|
// Verify hash
|
||||||
print!("Verifying store hash...");
|
print!("Verifying store hash...");
|
||||||
snapshot.assert_is_eq(&Snapshot::new(&db).await);
|
snapshot.assert_is_eq(&Snapshot::new(&db).await);
|
||||||
|
assert_named_blobs(test.server.blob_store(), &named_blobs).await;
|
||||||
println!(" GREAT SUCCESS!");
|
println!(" GREAT SUCCESS!");
|
||||||
|
|
||||||
|
// inbuxa: import the same export into a fresh store of another backend,
|
||||||
|
// the way a move from one database to another does it
|
||||||
|
#[cfg(all(feature = "rocks", feature = "sqlite"))]
|
||||||
|
cross_backend(test, &db, &temp_dir, &named_blobs).await;
|
||||||
|
|
||||||
// Destroy store
|
// Destroy store
|
||||||
|
for (key, _) in &named_blobs {
|
||||||
|
test.server.blob_store().delete_blob(key).await.unwrap();
|
||||||
|
}
|
||||||
store_destroy(&db).await;
|
store_destroy(&db).await;
|
||||||
store_assert_is_empty(&db, db.clone().into(), true).await;
|
store_assert_is_empty(&db, db.clone().into(), true).await;
|
||||||
temp_dir.delete();
|
temp_dir.delete();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "rocks", feature = "sqlite"))]
|
||||||
|
async fn cross_backend(
|
||||||
|
test: &TestServer,
|
||||||
|
source: &Store,
|
||||||
|
export: &TempDir,
|
||||||
|
named_blobs: &[(&[u8], Vec<u8>)],
|
||||||
|
) {
|
||||||
|
let source_type = std::env::var("STORE").unwrap();
|
||||||
|
let target_type = if source_type.eq_ignore_ascii_case("sqlite") {
|
||||||
|
"RocksDb"
|
||||||
|
} else {
|
||||||
|
"Sqlite"
|
||||||
|
};
|
||||||
|
println!("Importing the export into a fresh {target_type} store...");
|
||||||
|
|
||||||
|
let target_dir = TempDir::new("art_vandelay_cross_backend", true);
|
||||||
|
let target = Store::build(
|
||||||
|
crate::utils::storage::build_data_store(target_type, &target_dir.path.to_string_lossy())
|
||||||
|
.await,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
target.create_tables().await.unwrap();
|
||||||
|
store_destroy(&target).await;
|
||||||
|
|
||||||
|
let mut core = test.server.core.as_ref().clone();
|
||||||
|
core.storage.data = target.clone();
|
||||||
|
core.storage.blob = target.clone().into();
|
||||||
|
let imported = core.restore(export.path.clone()).await;
|
||||||
|
|
||||||
|
// Counters are stored differently by the SQL and key-value backends, so
|
||||||
|
// compare their keys here and their values through the counter API.
|
||||||
|
print!("Verifying {target_type} store hash...");
|
||||||
|
Snapshot::new_portable(source)
|
||||||
|
.await
|
||||||
|
.assert_is_eq(&Snapshot::new_portable(&target).await);
|
||||||
|
for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA] {
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
source
|
||||||
|
.iterate(
|
||||||
|
IterateParams::new(
|
||||||
|
AnyKey {
|
||||||
|
subspace,
|
||||||
|
key: vec![0u8],
|
||||||
|
},
|
||||||
|
AnyKey {
|
||||||
|
subspace,
|
||||||
|
key: vec![u8::MAX; 10],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.no_values(),
|
||||||
|
|key, _| {
|
||||||
|
keys.push(key.to_vec());
|
||||||
|
Ok(true)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
for key in keys {
|
||||||
|
let class = || {
|
||||||
|
ValueClass::Any(AnyClass {
|
||||||
|
subspace,
|
||||||
|
key: key.clone(),
|
||||||
|
})
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
source.get_counter(class()).await.unwrap(),
|
||||||
|
target.get_counter(class()).await.unwrap(),
|
||||||
|
"Counter mismatch in {} for {key:?}",
|
||||||
|
char::from(subspace)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_named_blobs(&core.storage.blob, named_blobs).await;
|
||||||
|
println!(" GREAT SUCCESS!");
|
||||||
|
|
||||||
|
// The search index isn't exported; the import queues its rebuild
|
||||||
|
let queued = core.queue_reindex(&imported).await;
|
||||||
|
let expected = [
|
||||||
|
TaskStoreMaintenanceType::ReindexAccounts,
|
||||||
|
TaskStoreMaintenanceType::ReindexTelemetry,
|
||||||
|
];
|
||||||
|
assert_eq!(queued, expected);
|
||||||
|
let mut task_ids = Vec::new();
|
||||||
|
target
|
||||||
|
.iterate(
|
||||||
|
IterateParams::new(
|
||||||
|
AnyKey {
|
||||||
|
subspace: SUBSPACE_TASK_QUEUE,
|
||||||
|
key: vec![0u8],
|
||||||
|
},
|
||||||
|
AnyKey {
|
||||||
|
subspace: SUBSPACE_TASK_QUEUE,
|
||||||
|
key: vec![u8::MAX; 20],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.no_values(),
|
||||||
|
|key, _| {
|
||||||
|
if key.deserialize_be_u64(0)? == 0 {
|
||||||
|
task_ids.push(key.deserialize_be_u64(U64_LEN)?);
|
||||||
|
}
|
||||||
|
Ok(true)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut found = Vec::new();
|
||||||
|
for id in task_ids {
|
||||||
|
match target
|
||||||
|
.get_value::<Task>(ValueKey::from(ValueClass::TaskQueue(
|
||||||
|
TaskQueueClass::Task { id },
|
||||||
|
)))
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
{
|
||||||
|
Some(Task::StoreMaintenance(task)) => found.push(task.maintenance_type),
|
||||||
|
other => panic!("Unexpected task {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
found.sort_by_key(|t| *t as u16);
|
||||||
|
assert_eq!(found, expected, "Queued tasks don't match");
|
||||||
|
|
||||||
|
store_destroy(&target).await;
|
||||||
|
drop(core);
|
||||||
|
drop(target);
|
||||||
|
target_dir.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_named_blobs(blob_store: &BlobStore, named_blobs: &[(&[u8], Vec<u8>)]) {
|
||||||
|
for (key, data) in named_blobs {
|
||||||
|
assert_eq!(
|
||||||
|
blob_store
|
||||||
|
.get_blob(key, 0..usize::MAX)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.as_ref(),
|
||||||
|
Some(data),
|
||||||
|
"Blob {} was not restored",
|
||||||
|
String::from_utf8_lossy(key)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Eq)]
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
struct Snapshot {
|
struct Snapshot {
|
||||||
keys: AHashSet<KeyValue>,
|
keys: AHashSet<KeyValue>,
|
||||||
@@ -240,7 +471,19 @@ struct KeyValue {
|
|||||||
|
|
||||||
impl Snapshot {
|
impl Snapshot {
|
||||||
async fn new(db: &Store) -> Self {
|
async fn new(db: &Store) -> Self {
|
||||||
let is_sql = db.is_sql();
|
Self::build(db, !db.is_sql(), true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Comparable across backends: no counter values, which the SQL and
|
||||||
|
/// key-value stores encode differently, and no blobs, which only live in
|
||||||
|
/// the data store when it doubles as the blob store.
|
||||||
|
#[cfg(all(feature = "rocks", feature = "sqlite"))]
|
||||||
|
async fn new_portable(db: &Store) -> Self {
|
||||||
|
Self::build(db, false, false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn build(db: &Store, counter_values: bool, with_blobs: bool) -> Self {
|
||||||
|
let is_sql = !counter_values;
|
||||||
|
|
||||||
let mut keys = AHashSet::new();
|
let mut keys = AHashSet::new();
|
||||||
|
|
||||||
@@ -265,7 +508,12 @@ impl Snapshot {
|
|||||||
(SUBSPACE_QUOTA, !is_sql),
|
(SUBSPACE_QUOTA, !is_sql),
|
||||||
(SUBSPACE_REPORT_OUT, true),
|
(SUBSPACE_REPORT_OUT, true),
|
||||||
(SUBSPACE_REPORT_IN, true),
|
(SUBSPACE_REPORT_IN, true),
|
||||||
|
(SUBSPACE_DIRECTORY, true),
|
||||||
|
(SUBSPACE_INBUXA, true),
|
||||||
] {
|
] {
|
||||||
|
if subspace == SUBSPACE_BLOBS && !with_blobs {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let from_key = AnyKey {
|
let from_key = AnyKey {
|
||||||
subspace,
|
subspace,
|
||||||
key: vec![0u8],
|
key: vec![0u8],
|
||||||
|
|||||||
Executable
+120
@@ -0,0 +1,120 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
"""Every path Cargo patches has to be in the image's build context.
|
||||||
|
|
||||||
|
Cargo.toml's [patch.crates-io] can point at a directory in this repository,
|
||||||
|
and the Dockerfile builds from a context that .dockerignore prunes to almost
|
||||||
|
nothing. Those two facts met on 2026-09-23: a vendored, patched sieve-rs
|
||||||
|
landed, CI stayed green -- it builds from a checkout, where the directory is
|
||||||
|
simply there -- and the release build failed on
|
||||||
|
|
||||||
|
failed to load source for dependency `sieve-rs`
|
||||||
|
failed to read /build/vendor/sieve-rs/Cargo.toml
|
||||||
|
|
||||||
|
after a tag had already been pushed. This is seconds, and it runs beside the
|
||||||
|
other fork checks rather than waiting for a release to find out.
|
||||||
|
|
||||||
|
Being in the context isn't enough on its own: the Dockerfile cooks the
|
||||||
|
dependencies (`cargo chef cook`) before it copies the tree in, from a recipe
|
||||||
|
that carries only the workspace's manifests. So each patched path must also be
|
||||||
|
copied into that stage before the cook step, or the same error comes back
|
||||||
|
there -- as it did for 2026.9.24.2, the first tag after the context fix.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
root = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def patched_paths(manifest: Path) -> list[str]:
|
||||||
|
"""Directories named by a [patch...] section's `path = "..."` entries."""
|
||||||
|
out, in_patch = [], False
|
||||||
|
for line in manifest.read_text().splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith("["):
|
||||||
|
in_patch = stripped.startswith("[patch")
|
||||||
|
continue
|
||||||
|
if not in_patch:
|
||||||
|
continue
|
||||||
|
m = re.search(r'path\s*=\s*"([^"]+)"', stripped)
|
||||||
|
if m:
|
||||||
|
out.append(m.group(1))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def allowed(dockerignore: Path) -> set[str]:
|
||||||
|
"""The first path segment of every re-inclusion rule."""
|
||||||
|
keep = set()
|
||||||
|
for line in dockerignore.read_text().splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if stripped.startswith("!"):
|
||||||
|
keep.add(stripped[1:].strip("/").split("/")[0])
|
||||||
|
return keep
|
||||||
|
|
||||||
|
|
||||||
|
def copied_before_cook(dockerfile: Path) -> list[str] | None:
|
||||||
|
"""Sources COPY'd into the stage that runs `cargo chef cook`, before it.
|
||||||
|
|
||||||
|
None when no stage cooks. A `COPY . .` covers everything.
|
||||||
|
"""
|
||||||
|
stage: list[str] = []
|
||||||
|
for line in dockerfile.read_text().splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if re.match(r"(?i)^FROM\s", stripped):
|
||||||
|
stage = []
|
||||||
|
continue
|
||||||
|
if "cargo chef cook" in stripped:
|
||||||
|
return stage
|
||||||
|
m = re.match(r"(?i)^COPY\s+(?!--from)(.+)$", stripped)
|
||||||
|
if m:
|
||||||
|
parts = m.group(1).split()
|
||||||
|
stage.extend(p.strip("./").split("/")[0] or "." for p in parts[:-1])
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
paths = patched_paths(root / "Cargo.toml")
|
||||||
|
if not paths:
|
||||||
|
print("no patched paths to check")
|
||||||
|
return 0
|
||||||
|
keep = allowed(root / ".dockerignore")
|
||||||
|
bad = []
|
||||||
|
for p in paths:
|
||||||
|
top = p.strip("/").split("/")[0]
|
||||||
|
if top not in keep:
|
||||||
|
bad.append((p, top))
|
||||||
|
elif not (root / p).is_dir():
|
||||||
|
bad.append((p, None))
|
||||||
|
for path, top in bad:
|
||||||
|
if top is None:
|
||||||
|
print(f"Cargo.toml patches {path}, which does not exist", file=sys.stderr)
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"Cargo.toml patches {path}, but .dockerignore does not re-include {top!r}:\n"
|
||||||
|
f" the image build would not see it, and cargo would fail on it.\n"
|
||||||
|
f" Add `!{top}` to .dockerignore.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
copied = copied_before_cook(root / "Dockerfile")
|
||||||
|
if copied is not None and "." not in copied:
|
||||||
|
for p in paths:
|
||||||
|
top = p.strip("/").split("/")[0]
|
||||||
|
if top not in copied:
|
||||||
|
print(
|
||||||
|
f"Cargo.toml patches {p}, but the Dockerfile doesn't copy {top!r} into the\n"
|
||||||
|
f" stage that runs `cargo chef cook` before that step, so cooking the\n"
|
||||||
|
f" dependencies fails on it. Add `COPY {top}/ {top}/` before the cook.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
bad.append((p, top))
|
||||||
|
if bad:
|
||||||
|
return 1
|
||||||
|
print(f"build context and cook stage include every patched path: {', '.join(paths)}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -23,6 +23,6 @@ crates/migration/src/lib.rs "STALWART_SPAM_CLASSIFIER_MODEL.lz4"
|
|||||||
crates/migration/src/lib.rs "STALWART_SPAM_TRAIN_DATA.lz4"
|
crates/migration/src/lib.rs "STALWART_SPAM_TRAIN_DATA.lz4"
|
||||||
crates/types/src/branding.rs "STALWART"
|
crates/types/src/branding.rs "STALWART"
|
||||||
|
|
||||||
# OPEN, not yet decided (2026-09-22): upstream's published spam-filter rules,
|
# Upstream's old default rules source, read only to treat it as unset: the
|
||||||
# which the server downloads at runtime from this address.
|
# server uses the rules bundled with it (resources/spam-filter/).
|
||||||
crates/registry/src/schema/structs_impl.rs "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz"
|
crates/common/src/manager/spam_rules.rs "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz"
|
||||||
|
|||||||
@@ -46,6 +46,10 @@ TEXT_RENAMES = [
|
|||||||
# that must match their containers and identity provider (database users,
|
# that must match their containers and identity provider (database users,
|
||||||
# passwords, an OIDC audience), and name their databases explicitly.
|
# passwords, an OIDC audience), and name their databases explicitly.
|
||||||
('"stalwart".to_string()', '"inbuxa".to_string()', ('crates',)),
|
('"stalwart".to_string()', '"inbuxa".to_string()', ('crates',)),
|
||||||
|
# The spam filter rules ship with the server (common::manager::spam_rules);
|
||||||
|
# upstream's default of fetching its latest from GitHub becomes unset.
|
||||||
|
('spam_filter_rules_url: Some("https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz".to_string()),',
|
||||||
|
'spam_filter_rules_url: None,', ('crates',)),
|
||||||
]
|
]
|
||||||
ROOTS = ('crates', 'tests', 'resources')
|
ROOTS = ('crates', 'tests', 'resources')
|
||||||
SKIP_SUFFIXES = {'.md', '.txt'}
|
SKIP_SUFFIXES = {'.md', '.txt'}
|
||||||
@@ -58,6 +62,10 @@ SCHEMA_HASH = Path('resources/schema/schema.json.sha256')
|
|||||||
SCHEMA_RENAMES = [
|
SCHEMA_RENAMES = [
|
||||||
('"stalwart"', '"inbuxa"'),
|
('"stalwart"', '"inbuxa"'),
|
||||||
('vnd.stalwart', 'vnd.inbuxa'),
|
('vnd.stalwart', 'vnd.inbuxa'),
|
||||||
|
# The bundled spam rules: no default URL, and say what empty means.
|
||||||
|
('"spamFilterRulesUrl":"https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter-rules.json.gz",', ''),
|
||||||
|
('"URL to download spam filter rules from"',
|
||||||
|
'"URL to download spam filter rules from. Empty uses the rules bundled with the server."'),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user