New installs start with the hashed-address blocklist off, and DNSBL zones read right #90

Merged
jcoffey-dev merged 1 commits from feature/d5-msbl-off into main 2026-09-28 18:32:41 +00:00
6 changed files with 164 additions and 8 deletions
Showing only changes of commit b20b09f81a - Show all commits
+4
View File
@@ -409,6 +409,10 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
bp.registry.write(RegistryWrite::insert(&object)).await?; bp.registry.write(RegistryWrite::insert(&object)).await?;
} }
// D5: the blocklist sent hashed email addresses starts off; the
// rules load later, from a task, which acts on this note
super::spam_rules::mark_new_install(&bp.data_store).await?;
// D1: rotated log files are kept 30 days (a fork-owned setting, // D1: rotated log files are kept 30 days (a fork-owned setting,
// since x:TracerLog is also stored inside x:Bootstrap) // since x:TracerLog is also stored inside x:Bootstrap)
use inbuxa_features::security::log_files; use inbuxa_features::security::log_files;
+72
View File
@@ -118,6 +118,78 @@ pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()>
.map(|_| ()) .map(|_| ())
} }
/// The blocklists a new install starts with switched off (personal-data
/// catalog spec, default D5, settled 2026-09-28): the one that is sent a
/// hash of every email address it's asked about.
pub const NEW_INSTALL_OFF: &[&str] = &["STWT_MSBL_EBL_EMAIL"];
fn new_install_key() -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: b"Sn".to_vec(),
})
}
/// Notes, on a new install's first boot, that [`NEW_INSTALL_OFF`] is to be
/// switched off once the rules are in: they load later, from a task.
pub async fn mark_new_install(data: &Store) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
batch.set(new_install_key(), b"D5".to_vec());
data.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
/// After rules load: on a new install, switches [`NEW_INSTALL_OFF`] off and
/// forgets the note, so it happens once. Returns whether anything changed.
/// An existing server has no note, and keeps every blocklist as it is.
pub async fn apply_new_install(
registry: &store::RegistryStore,
data: &Store,
) -> trc::Result<bool> {
use registry::schema::{prelude::Object, structs::SpamDnsblServer};
use store::registry::write::RegistryWrite;
if data
.get_value::<String>(ValueKey::from(new_install_key()))
.await
.caused_by(trc::location!())?
.is_none()
{
return Ok(false);
}
let mut changed = false;
for server in registry.list::<SpamDnsblServer>().await? {
let mut updated = server.object.clone();
let SpamDnsblServer::Email(email) = &mut updated else {
continue;
};
if !NEW_INSTALL_OFF.contains(&email.name.as_str()) || !email.enable {
continue;
}
email.enable = false;
let old = Object {
inner: server.object.into(),
revision: server.revision,
};
let new = Object {
inner: updated.into(),
revision: server.revision,
};
registry
.write(RegistryWrite::update(types::id::Id::from(server.id.id()), &new, &old))
.await?;
changed = true;
}
let mut batch = BatchBuilder::new();
batch.clear(new_install_key());
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
Ok(changed)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
+34 -6
View File
@@ -88,13 +88,31 @@ fn days(duration: Option<&Duration>) -> Days {
} }
} }
/// An expression's text, if it is a plain constant (a zone, a switch). /// The zones a DNSBL's zone expression can query: each quoted literal that
fn expression_text(value: &Value) -> Option<String> { /// starts with a dot, in any branch (`ip_reverse + '.zen.spamhaus.org'`).
fn zone_hosts(value: &Value) -> Vec<String> {
let mut hosts = Vec::new();
let mut texts = Vec::new();
fn collect<'a>(value: &'a Value, texts: &mut Vec<&'a str>) {
match value { match value {
Value::String(s) => Some(s.clone()), Value::String(s) => texts.push(s),
Value::Object(o) => o.get("else").and_then(|v| v.as_str()).map(str::to_string), Value::Array(items) => items.iter().for_each(|v| collect(v, texts)),
_ => None, Value::Object(map) => map.values().for_each(|v| collect(v, texts)),
_ => {}
} }
}
collect(value, &mut texts);
for text in texts {
for literal in text.split('\'').skip(1).step_by(2) {
if let Some(zone) = literal.strip_prefix('.')
&& zone.contains('.')
&& !hosts.iter().any(|h| h == zone)
{
hosts.push(zone.to_string());
}
}
}
hosts
} }
impl Server { impl Server {
@@ -263,7 +281,7 @@ impl Server {
let value = serde_json::to_value(&server.object).unwrap_or_default(); let value = serde_json::to_value(&server.object).unwrap_or_default();
if value.get("enable").and_then(Value::as_bool).unwrap_or(false) { if value.get("enable").and_then(Value::as_bool).unwrap_or(false) {
dnsbl_on = true; dnsbl_on = true;
if let Some(zone) = value.get("zone").and_then(expression_text) { for zone in value.get("zone").map(zone_hosts).unwrap_or_default() {
endpoint(&mut facts, "spam-dnsbl", zone); endpoint(&mut facts, "spam-dnsbl", zone);
} }
} }
@@ -397,6 +415,16 @@ mod tests {
assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into())); assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into()));
} }
#[test]
fn zones_come_from_every_branch() {
let zone = json!({"else": "false", "match": {"0": {"if": "location == 'tcp'",
"then": "ip_reverse + '.rep.mailspike.net'"}}});
assert_eq!(zone_hosts(&zone), vec!["rep.mailspike.net"]);
let zone = json!({"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}});
assert_eq!(zone_hosts(&zone), vec!["ebl.msbl.org"], "not 'sha1'");
assert!(zone_hosts(&json!({"else": "false"})).is_empty());
}
#[test] #[test]
fn days_round_up() { fn days_round_up() {
assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1)); assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1));
@@ -316,6 +316,15 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED) spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED)
.await?; .await?;
} }
// inbuxa: personal-data catalog, D5: a new install's first rules
// leave the hashed-address blocklist off
if spam_rules::apply_new_install(server.registry(), server.store()).await?
&& let Err(err) = reload_and_broadcast(server, ObjectType::SpamDnsblServer).await
{
return Ok(TaskResult::permanent(format!(
"Spam rules were stored but not activated ({err}); run Reload settings"
)));
}
Ok(TaskResult::Success(vec![])) Ok(TaskResult::Success(vec![]))
} }
} }
+5 -1
View File
@@ -407,7 +407,11 @@ retention is (not a field on `x:TracerLog`, which is also stored inside
`x:Bootstrap` with fields after it, so a new field would change that `x:Bootstrap` with fields after it, so a new field would change that
object's stored format); new installs 30 days, existing servers keep every object's stored format); new installs 30 days, existing servers keep every
file as today. D5 is built after the v0.16.24 import lands, on its reworked file as today. D5 is built after the v0.16.24 import lands, on its reworked
spam-rules loader, which keeps each blocklist's on/off state. spam-rules loader, which keeps each blocklist's on/off state. D5 built after the import: a new install's first boot leaves a note
(`S` `n`), and the rules update, once the bundled rules are in, switches
`STWT_MSBL_EBL_EMAIL` off and forgets the note; the loader keeps that
switch through later updates. An existing server has no note and keeps
every blocklist as it is.
| # | Change | Trade-off | | # | Change | Trade-off |
|---|---|---| |---|---|---|
+39
View File
@@ -274,6 +274,45 @@ pub async fn test(test: &mut TestServer) {
.unwrap(); .unwrap();
assert_eq!(trace["retention"]["days"], json!(7), "{trace}"); assert_eq!(trace["retention"]["days"], json!(7), "{trace}");
// D5: a new install's first rules leave the hashed-address blocklist
// off, once; an existing server (no note) keeps it as it is
let (_, response) = call(
&admin,
"x:SpamDnsblServer/set",
json!({"create": {"m": {"@type": "Email", "name": "STWT_MSBL_EBL_EMAIL", "enable": true,
"zone": {"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}},
"tag": {"else": "'MSBL_EBL'", "match": {}}}}}),
)
.await;
let msbl = response["created"]["m"]["id"]
.as_str()
.unwrap_or_else(|| panic!("{response}"))
.to_string();
let registry = test.server.registry();
let store = test.server.store();
assert!(
!common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
"no note, no change"
);
let enabled = |response: &Value| response["list"][0]["enable"].clone();
let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
assert_eq!(enabled(&response), json!(true));
let (_, response) = call(&officer, "inbuxa:DataInventory/get", json!({"ids": null})).await;
assert!(
response["list"][0]["processors"]
.as_array()
.is_some_and(|p| p.iter().any(|p| p["host"] == "ebl.msbl.org")),
"the zone, not the hash: {response}"
);
common::manager::spam_rules::mark_new_install(store).await.unwrap();
assert!(common::manager::spam_rules::apply_new_install(registry, store).await.unwrap());
let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
assert_eq!(enabled(&response), json!(false), "{response}");
assert!(
!common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
"the note works once"
);
// A tenant can still be deleted: its unused role goes with it // A tenant can still be deleted: its unused role goes with it
let spare = admin let spare = admin
.registry_create_object(Tenant { .registry_create_object(Tenant {