inbuxa/inbuxa-server
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 27 Wiki Activity Sponsor

Import upstream v0.16.25, stripped

Browse Source 
Upstream commit: 3f657330c0f49a015a3a372fb59669b5cccbca6d
Enterprise-only files removed or emptied: 63
Enterprise-only snippets removed: 118 in 50 files
Dangling module declarations removed: 5
Edits turning enterprise off: 25
Third-party code: 14 files, 0 not in THIRD-PARTY.md
Renamed identifiers: 67 in 19 files
Verification: clean

The same Enterprise footprint as v0.16.24. The build check is clean
apart from the expected errors in the rebuilt-feature tests. A
bug-fix release: DKIM rotation, IMAP failed-login answers, DNSBL
multi-code scoring and negative TTLs, Pyzor on short messages, queue
quotas with an empty match, RocksDB info-log rotation, and
Autodiscover schema handling.
This commit is contained in:
jcoffey-dev committed 2026-10-05 21:15:02 -07:00
1 parent f59b084ce5
commit 72f8ddd5e7
67 files changed
+1643 -492
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/spam-filter/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "spam-filter"
version = "0.16.24"
version = "0.16.25"
edition = "2024"
[dependencies]
crates/spam-filter/src/modules/classifier.rs
+91 -65
View File
@@ -35,7 +35,7 @@ use std::{
hash::{Hash, RandomState},
sync::Arc,
};
use store::ahash::AHashSet;
use store::ahash::AHashMap;
use store::rand::seq::SliceRandom;
use store::write::{BlobLink, RegistryClass, now};
use store::{
@@ -90,7 +90,14 @@ struct TrainingTask {
sample: TrainingSample,
is_spam: bool,
is_replay: bool,
remove: Option<u64>,
remove: SampleRemoval,
}
#[derive(Debug, Clone, Copy)]
enum SampleRemoval {
Keep,
Item,
ItemAndLink { until: u64 },
}
#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)]
@@ -197,7 +204,7 @@ impl SpamClassifier for Server {
object_id,
item_id: u64::MAX,
}));
let mut seen_samples = AHashSet::new();
let mut seen_samples = AHashMap::new();
let mut spam_count = 0;
let mut ham_count = 0;
self.store()
@@ -218,43 +225,57 @@ impl SpamClassifier for Server {
.unwrap_or(u32::MAX),
};
if seen_samples.insert(sample.clone()) {
// Add to reservoir
if !do_remove {
trainer.reservoir.update_reservoir(
&sample,
match seen_samples.entry(sample.clone()) {
Entry::Vacant(entry) => {
entry.insert((!do_remove).then_some(until));
// Add to reservoir
if !do_remove {
trainer.reservoir.update_reservoir(
&sample,
is_spam,
config.reservoir_capacity,
);
} else {
trainer.reservoir.update_counts(is_spam);
}
samples.push(TrainingTask {
id,
sample,
is_spam,
config.reservoir_capacity,
);
} else {
trainer.reservoir.update_counts(is_spam);
is_replay: false,
remove: if do_remove {
SampleRemoval::ItemAndLink { until }
} else {
SampleRemoval::Keep
},
});
remove_entries |= do_remove;
// Update trainer stats
if is_spam {
spam_count += 1;
} else {
ham_count += 1;
}
}
samples.push(TrainingTask {
id,
sample,
is_spam,
is_replay: false,
remove: do_remove.then_some(until),
});
remove_entries |= do_remove;
// Update trainer stats
if is_spam {
spam_count += 1;
} else {
ham_count += 1;
Entry::Occupied(entry) => {
let remove = if *entry.get() == Some(until) {
SampleRemoval::Item
} else {
SampleRemoval::ItemAndLink { until }
};
duplicate_samples.push(TrainingTask {
id,
sample,
is_spam,
is_replay: false,
remove,
});
remove_entries = true;
}
} else {
duplicate_samples.push(TrainingTask {
id,
sample,
is_spam,
is_replay: false,
remove: Some(until),
});
remove_entries = true;
}
trainer.last_id = trainer.last_id.max(id);
@@ -313,7 +334,7 @@ impl SpamClassifier for Server {
sample: sample.clone(),
is_spam: false,
is_replay: true,
remove: None,
remove: SampleRemoval::Keep,
}),
);
} else if ham_count > spam_count {
@@ -327,7 +348,7 @@ impl SpamClassifier for Server {
sample: sample.clone(),
is_spam: true,
is_replay: true,
remove: None,
remove: SampleRemoval::Keep,
}),
);
}
@@ -882,33 +903,38 @@ async fn delete_samples(
let object_id = ObjectType::SpamTrainingSample.to_id();
let mut batch = BatchBuilder::new();
for sample in samples.into_iter().chain(duplicate_samples) {
if let Some(until) = sample.remove {
batch
.with_account_id(sample.sample.account_id)
.clear(BlobOp::Link {
hash: sample.sample.hash,
to: BlobLink::Temporary { until },
})
.clear(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: sample.id,
}))
.clear(ValueClass::Registry(RegistryClass::Index {
index_id: Property::AccountId.to_id(),
object_id,
item_id: sample.id,
key: (sample.sample.account_id as u64).serialize(),
}));
let until = match sample.remove {
SampleRemoval::Keep => continue,
SampleRemoval::Item => None,
SampleRemoval::ItemAndLink { until } => Some(until),
};
batch.with_account_id(sample.sample.account_id);
if let Some(until) = until {
batch.clear(BlobOp::Link {
hash: sample.sample.hash,
to: BlobLink::Temporary { until },
});
}
batch
.clear(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: sample.id,
}))
.clear(ValueClass::Registry(RegistryClass::Index {
index_id: Property::AccountId.to_id(),
object_id,
item_id: sample.id,
key: (sample.sample.account_id as u64).serialize(),
}));
if batch.is_large_batch() {
server
.store()
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
batch = BatchBuilder::new();
batch.with_account_id(sample.sample.account_id);
}
if batch.is_large_batch() {
server
.store()
.write(batch.build_all())
.await
.caused_by(trc::location!())?;
batch = BatchBuilder::new();
batch.with_account_id(sample.sample.account_id);
}
}
if !batch.is_empty() {
crates/spam-filter/src/modules/dnsbl.rs
+143 -95
View File
@@ -11,7 +11,14 @@ use common::{
config::mailstore::spamfilter::{DnsBlServer, Element, IpResolver, Location},
expr::functions::ResolveVariable,
};
use mail_auth::{Error, common::resolver::ToFqdn};
use mail_auth::common::resolver::ToFqdn;
#[cfg(not(feature = "test_mode"))]
use mail_auth::hickory_resolver::{
net::{DnsError, NetError},
proto::rr::{Name, RData},
};
#[cfg(feature = "test_mode")]
use mail_auth::{DnsError, Error};
use std::{
net::Ipv4Addr,
sync::Arc,
@@ -19,6 +26,18 @@ use std::{
};
use trc::SpamEvent;
const MAX_NEGATIVE_TTL: u32 = 3600;
enum DnsblAnswer {
Listed {
ips: Vec<Ipv4Addr>,
expires: Instant,
},
NotListed {
expires: Option<Instant>,
},
}
pub(crate) async fn check_dnsbl(
server: &Server,
ctx: &mut SpamFilterContext<'_>,
@@ -49,7 +68,7 @@ pub(crate) async fn check_dnsbl(
for dnsbl in &server.core.spam.dnsbl.servers {
if dnsbl.scope == scope
&& checks < max_checks
&& let Some(tag) = is_dnsbl(
&& let Some(codes) = dnsbl_codes(
server,
dnsbl,
SpamFilterResolver::new(ctx, resolver, location),
@@ -58,7 +77,19 @@ pub(crate) async fn check_dnsbl(
)
.await
{
ctx.result.add_tag(tag);
for code in codes.iter() {
let tag = server
.eval_if::<String, _>(
&dnsbl.tags,
&SpamFilterResolver::new(ctx, code, location),
ctx.input.span_id,
)
.await;
if let Some(tag) = tag {
ctx.result.add_tag(tag);
}
}
}
}
@@ -71,13 +102,13 @@ pub(crate) async fn check_dnsbl(
}
}
async fn is_dnsbl(
async fn dnsbl_codes(
server: &Server,
config: &DnsBlServer,
resolver: SpamFilterResolver<'_, impl ResolveVariable>,
element: Element,
checks: &mut usize,
) -> Option<String> {
) -> Option<Arc<[IpResolver]>> {
let time = Instant::now();
let zone = server
.eval_if::<String, _>(&config.zone, &resolver, resolver.ctx.input.span_id)
@@ -92,105 +123,122 @@ async fn is_dnsbl(
{
None
} else {
server
.eval_if(
&config.tags,
&SpamFilterResolver::new(
resolver.ctx,
&IpResolver::new(
format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(),
),
resolver.location,
),
resolver.ctx.input.span_id,
)
.await
Some(Arc::from([IpResolver::new(
format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(),
)]))
};
}
}
let result = match server.inner.cache.dns_rbl.get(zone.as_str()) {
Some(Some(result)) => result,
Some(None) => return None,
None => {
*checks += 1;
if let Some(codes) = server.inner.cache.dns_rbl.get(zone.as_str()) {
return codes;
}
match server
.core
.smtp
.resolvers
.dns
.ipv4_lookup_raw(zone.to_fqdn().as_ref())
.await
{
Ok(result) => {
trc::event!(
Spam(SpamEvent::Dnsbl),
Hostname = zone.clone(),
Result = result
.entry
.iter()
.map(|ip| trc::Value::from(ip.to_string()))
.collect::<Vec<_>>(),
Details = element.as_str(),
Elapsed = time.elapsed()
);
*checks += 1;
let entry = Arc::new(IpResolver::new(
result
.entry
.iter()
.copied()
.next()
.unwrap_or(Ipv4Addr::BROADCAST)
.into(),
));
match resolve_zone(server, zone.to_fqdn().as_ref()).await {
Ok(DnsblAnswer::Listed { ips, expires }) => {
trc::event!(
Spam(SpamEvent::Dnsbl),
Hostname = zone.clone(),
Result = ips
.iter()
.map(|ip| trc::Value::from(ip.to_string()))
.collect::<Vec<_>>(),
Details = element.as_str(),
Elapsed = time.elapsed()
);
server.inner.cache.dns_rbl.insert_with_expiry(
zone.into(),
Some(entry.clone()),
result.expires,
);
let codes: Arc<[IpResolver]> = ips
.into_iter()
.map(|ip| IpResolver::new(ip.into()))
.collect();
entry
}
Err(Error::Dns(mail_auth::DnsError::RecordNotFound(_))) => {
trc::event!(
Spam(SpamEvent::Dnsbl),
Hostname = zone.clone(),
Result = trc::Value::None,
Details = element.as_str(),
Elapsed = time.elapsed()
);
server.inner.cache.dns_rbl.insert_with_expiry(
zone.into(),
Some(codes.clone()),
expires,
);
server.inner.cache.dns_rbl.insert(
zone.into(),
None,
Duration::from_secs(86400),
);
return None;
}
Err(err) => {
trc::event!(
Spam(SpamEvent::DnsblError),
Hostname = zone,
Elapsed = time.elapsed(),
Details = element.as_str(),
CausedBy = err.to_string()
);
return None;
}
}
Some(codes)
}
};
Ok(DnsblAnswer::NotListed { expires }) => {
trc::event!(
Spam(SpamEvent::Dnsbl),
Hostname = zone.clone(),
Result = trc::Value::None,
Details = element.as_str(),
Elapsed = time.elapsed()
);
server
.eval_if(
&config.tags,
&SpamFilterResolver::new(resolver.ctx, result.as_ref(), resolver.location),
resolver.ctx.input.span_id,
)
.await
if let Some(expires) = expires {
server
.inner
.cache
.dns_rbl
.insert_with_expiry(zone.into(), None, expires);
}
None
}
Err(err) => {
trc::event!(
Spam(SpamEvent::DnsblError),
Hostname = zone,
Elapsed = time.elapsed(),
Details = element.as_str(),
CausedBy = err
);
None
}
}
}
#[cfg(not(feature = "test_mode"))]
async fn resolve_zone(server: &Server, zone: &str) -> Result<DnsblAnswer, String> {
let name = Name::from_str_relaxed(zone).map_err(|err| err.to_string())?;
match server.core.smtp.resolvers.dns.0.ipv4_lookup(name).await {
Ok(lookup) => {
let expires = lookup.valid_until();
let ips = lookup
.answers()
.iter()
.filter_map(|record| match &record.data {
RData::A(a) => Some(a.0),
_ => None,
})
.collect::<Vec<_>>();
Ok(if !ips.is_empty() {
DnsblAnswer::Listed { ips, expires }
} else {
DnsblAnswer::NotListed {
expires: Some(expires),
}
})
}
Err(NetError::Dns(DnsError::NoRecordsFound(no_records))) => Ok(DnsblAnswer::NotListed {
expires: no_records
.negative_ttl
.filter(|ttl| *ttl > 0)
.map(|ttl| Instant::now() + Duration::from_secs(ttl.min(MAX_NEGATIVE_TTL).into())),
}),
Err(err) => Err(err.to_string()),
}
}
#[cfg(feature = "test_mode")]
async fn resolve_zone(server: &Server, zone: &str) -> Result<DnsblAnswer, String> {
match server.core.smtp.resolvers.dns.ipv4_lookup_raw(zone).await {
Ok(result) => Ok(DnsblAnswer::Listed {
ips: result.entry.to_vec(),
expires: result.expires,
}),
Err(Error::Dns(DnsError::RecordNotFound(_))) => Ok(DnsblAnswer::NotListed {
expires: Some(Instant::now() + Duration::from_secs(MAX_NEGATIVE_TTL.into())),
}),
Err(err) => Err(err.to_string()),
}
}
crates/spam-filter/src/modules/pyzor.rs
+39 -35
View File
@@ -31,17 +31,9 @@ pub(crate) async fn pyzor_check(
message: &Message<'_>,
config: &PyzorConfig,
) -> trc::Result<Option<PyzorResponse>> {
// Make sure there is at least one text part
if !message
.parts
.iter()
.any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_)))
{
let Some(request) = message.pyzor_check_message() else {
return Ok(None);
}
// Hash message
let request = message.pyzor_check_message();
};
#[cfg(feature = "test_mode")]
{
@@ -155,15 +147,15 @@ impl PyzorWrite for Sha1 {
}
trait PyzorDigest<W: PyzorWrite> {
fn pyzor_digest(&self, writer: W) -> W;
fn pyzor_digest(&self, writer: W) -> Option<W>;
}
pub trait PyzorCheck {
fn pyzor_check_message(&self) -> String;
fn pyzor_check_message(&self) -> Option<String>;
}
impl<W: PyzorWrite> PyzorDigest<W> for Message<'_> {
fn pyzor_digest(&self, writer: W) -> W {
fn pyzor_digest(&self, writer: W) -> Option<W> {
let parts = self
.parts
.iter()
@@ -179,7 +171,7 @@ impl<W: PyzorWrite> PyzorDigest<W> for Message<'_> {
}
impl PyzorCheck for Message<'_> {
fn pyzor_check_message(&self) -> String {
fn pyzor_check_message(&self) -> Option<String> {
let time = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
@@ -192,9 +184,9 @@ impl PyzorCheck for Message<'_> {
}
}
fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String {
fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> Option<String> {
// Hash message
let hash = message.pyzor_digest(Sha1::new()).finalize().hex_encode();
let hash = message.pyzor_digest(Sha1::new())?.finalize().hex_encode();
// Hash key
let mut hash_key = Sha1::new();
hash_key.update("anonymous:".as_bytes());
@@ -214,10 +206,10 @@ fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String
sig.update(format!(":{time}:{hash_key}"));
let sig = sig.finalize().hex_encode();
format!("{message}\nSig: {sig}\n")
Some(format!("{message}\nSig: {sig}\n"))
}
fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> W
fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> Option<W>
where
I: Iterator<Item = &'x str>,
W: PyzorWrite,
@@ -279,6 +271,10 @@ where
}
}
if result.is_empty() {
return None;
}
if result.len() > ATOMIC_NUM_LINES {
for (offset, length) in DIGEST_SPEC {
for i in 0..*length {
@@ -293,7 +289,7 @@ where
}
}
writer
Some(writer)
}
fn html_to_text(input: &str) -> String {
@@ -477,7 +473,8 @@ mod test {
&MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(),
1697468672,
49005,
);
)
.unwrap();
assert_eq!(
message,
@@ -510,10 +507,9 @@ mod test {
"http://spammer.com/special-offers?buy=now",
] {
assert_eq!(
String::from_utf8(pyzor_digest(
Vec::new(),
format!("Test {strip_me} Test2").lines(),
))
String::from_utf8(
pyzor_digest(Vec::new(), format!("Test {strip_me} Test2").lines()).unwrap()
)
.unwrap(),
"TestTest2"
);
@@ -521,20 +517,26 @@ mod test {
// Test short lines
assert_eq!(
String::from_utf8(pyzor_digest(
Vec::new(),
concat!("This line is included\n", "not this\n", "This also").lines(),
))
String::from_utf8(
pyzor_digest(
Vec::new(),
concat!("This line is included\n", "not this\n", "This also").lines(),
)
.unwrap()
)
.unwrap(),
"ThislineisincludedThisalso"
);
// Test atomic
assert_eq!(
String::from_utf8(pyzor_digest(
Vec::new(),
"All this message\nShould be included\nIn the digest".lines(),
))
String::from_utf8(
pyzor_digest(
Vec::new(),
"All this message\nShould be included\nIn the digest".lines(),
)
.unwrap()
)
.unwrap(),
"AllthismessageShouldbeincludedInthedigest"
);
@@ -549,7 +551,7 @@ mod test {
expected += format!("Line{i}testtesttest").as_str();
}
assert_eq!(
String::from_utf8(pyzor_digest(Vec::new(), text.lines(),)).unwrap(),
String::from_utf8(pyzor_digest(Vec::new(), text.lines()).unwrap()).unwrap(),
expected
);
@@ -581,7 +583,8 @@ mod test {
MessageParser::new()
.parse(input)
.unwrap()
.pyzor_digest(Vec::new(),)
.pyzor_digest(Vec::new())
.unwrap()
)
.unwrap(),
expected,
@@ -594,7 +597,8 @@ mod test {
MessageParser::new()
.parse(HTML_TEXT_STYLE_SCRIPT)
.unwrap()
.pyzor_digest(Sha1::new(),)
.pyzor_digest(Sha1::new())
.unwrap()
.finalize()
.hex_encode(),
"b2c27325a034c581df0c9ef37e4a0d63208a3e7e",