From b20b09f81a4dbb6b99c171688d3789cd363f5225 Mon Sep 17 00:00:00 2001 From: John Coffey Date: Mon, 28 Sep 2026 10:21:15 -0700 Subject: [PATCH] New installs start with the hashed-address blocklist off, and DNSBL zones read right Personal-data catalog spec, default D5 (settled 2026-09-28; built after the v0.16.24 import's spam-rules loader landed). msbl.org's EBL is sent a SHA-1 of every email address it's asked about. A new install's first boot now leaves a note, and the rules update, once the bundled rules are in, switches STWT_MSBL_EBL_EMAIL off and forgets the note, so it happens once; the loader keeps that switch through later updates. An existing server has no note and keeps every blocklist as it is. Also fixes the data inventory's DNSBL endpoints: a zone is an expression (`ip_reverse + '.zen.spamhaus.org'`, conditional branches, `hash(email, 'sha1') + '.ebl.msbl.org'`), and the zone names are now the quoted literals that start with a dot, from every branch, rather than the expression's text. Tested: unit test for the zone rule; the compliance system test (no note, no change; the inventory lists ebl.msbl.org, not a hash; with the note the blocklist goes off; the note works once); the system suite; fork checks. --- crates/common/src/manager/defaults.rs | 4 ++ crates/common/src/manager/spam_rules.rs | 72 +++++++++++++++++++ crates/common/src/privacy.rs | 42 +++++++++-- .../src/task_manager/spam_classifier.rs | 9 +++ docs/spec/features/personal-data-catalog.md | 6 +- tests/src/system/compliance.rs | 39 ++++++++++ 6 files changed, 164 insertions(+), 8 deletions(-) diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index a98d087..2203dde 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -409,6 +409,10 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { bp.registry.write(RegistryWrite::insert(&object)).await?; } + // D5: the blocklist sent hashed email addresses starts off; the + // rules load later, from a task, which acts on this note + super::spam_rules::mark_new_install(&bp.data_store).await?; + // D1: rotated log files are kept 30 days (a fork-owned setting, // since x:TracerLog is also stored inside x:Bootstrap) use inbuxa_features::security::log_files; diff --git a/crates/common/src/manager/spam_rules.rs b/crates/common/src/manager/spam_rules.rs index 050540c..5c89b06 100644 --- a/crates/common/src/manager/spam_rules.rs +++ b/crates/common/src/manager/spam_rules.rs @@ -118,6 +118,78 @@ pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()> .map(|_| ()) } +/// The blocklists a new install starts with switched off (personal-data +/// catalog spec, default D5, settled 2026-09-28): the one that is sent a +/// hash of every email address it's asked about. +pub const NEW_INSTALL_OFF: &[&str] = &["STWT_MSBL_EBL_EMAIL"]; + +fn new_install_key() -> ValueClass { + ValueClass::Any(AnyClass { + subspace: SUBSPACE_INBUXA, + key: b"Sn".to_vec(), + }) +} + +/// Notes, on a new install's first boot, that [`NEW_INSTALL_OFF`] is to be +/// switched off once the rules are in: they load later, from a task. +pub async fn mark_new_install(data: &Store) -> trc::Result<()> { + let mut batch = BatchBuilder::new(); + batch.set(new_install_key(), b"D5".to_vec()); + data.write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + +/// After rules load: on a new install, switches [`NEW_INSTALL_OFF`] off and +/// forgets the note, so it happens once. Returns whether anything changed. +/// An existing server has no note, and keeps every blocklist as it is. +pub async fn apply_new_install( + registry: &store::RegistryStore, + data: &Store, +) -> trc::Result { + use registry::schema::{prelude::Object, structs::SpamDnsblServer}; + use store::registry::write::RegistryWrite; + + if data + .get_value::(ValueKey::from(new_install_key())) + .await + .caused_by(trc::location!())? + .is_none() + { + return Ok(false); + } + let mut changed = false; + for server in registry.list::().await? { + let mut updated = server.object.clone(); + let SpamDnsblServer::Email(email) = &mut updated else { + continue; + }; + if !NEW_INSTALL_OFF.contains(&email.name.as_str()) || !email.enable { + continue; + } + email.enable = false; + let old = Object { + inner: server.object.into(), + revision: server.revision, + }; + let new = Object { + inner: updated.into(), + revision: server.revision, + }; + registry + .write(RegistryWrite::update(types::id::Id::from(server.id.id()), &new, &old)) + .await?; + changed = true; + } + let mut batch = BatchBuilder::new(); + batch.clear(new_install_key()); + data.write(batch.build_all()) + .await + .caused_by(trc::location!())?; + Ok(changed) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/common/src/privacy.rs b/crates/common/src/privacy.rs index 152c1ce..828e52b 100644 --- a/crates/common/src/privacy.rs +++ b/crates/common/src/privacy.rs @@ -88,13 +88,31 @@ fn days(duration: Option<&Duration>) -> Days { } } -/// An expression's text, if it is a plain constant (a zone, a switch). -fn expression_text(value: &Value) -> Option { - match value { - Value::String(s) => Some(s.clone()), - Value::Object(o) => o.get("else").and_then(|v| v.as_str()).map(str::to_string), - _ => None, +/// The zones a DNSBL's zone expression can query: each quoted literal that +/// starts with a dot, in any branch (`ip_reverse + '.zen.spamhaus.org'`). +fn zone_hosts(value: &Value) -> Vec { + let mut hosts = Vec::new(); + let mut texts = Vec::new(); + fn collect<'a>(value: &'a Value, texts: &mut Vec<&'a str>) { + match value { + Value::String(s) => texts.push(s), + Value::Array(items) => items.iter().for_each(|v| collect(v, texts)), + Value::Object(map) => map.values().for_each(|v| collect(v, texts)), + _ => {} + } } + collect(value, &mut texts); + for text in texts { + for literal in text.split('\'').skip(1).step_by(2) { + if let Some(zone) = literal.strip_prefix('.') + && zone.contains('.') + && !hosts.iter().any(|h| h == zone) + { + hosts.push(zone.to_string()); + } + } + } + hosts } impl Server { @@ -263,7 +281,7 @@ impl Server { let value = serde_json::to_value(&server.object).unwrap_or_default(); if value.get("enable").and_then(Value::as_bool).unwrap_or(false) { dnsbl_on = true; - if let Some(zone) = value.get("zone").and_then(expression_text) { + for zone in value.get("zone").map(zone_hosts).unwrap_or_default() { endpoint(&mut facts, "spam-dnsbl", zone); } } @@ -397,6 +415,16 @@ mod tests { assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into())); } + #[test] + fn zones_come_from_every_branch() { + let zone = json!({"else": "false", "match": {"0": {"if": "location == 'tcp'", + "then": "ip_reverse + '.rep.mailspike.net'"}}}); + assert_eq!(zone_hosts(&zone), vec!["rep.mailspike.net"]); + let zone = json!({"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}}); + assert_eq!(zone_hosts(&zone), vec!["ebl.msbl.org"], "not 'sha1'"); + assert!(zone_hosts(&json!({"else": "false"})).is_empty()); + } + #[test] fn days_round_up() { assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1)); diff --git a/crates/services/src/task_manager/spam_classifier.rs b/crates/services/src/task_manager/spam_classifier.rs index f4068f5..0ab8944 100644 --- a/crates/services/src/task_manager/spam_classifier.rs +++ b/crates/services/src/task_manager/spam_classifier.rs @@ -316,6 +316,15 @@ async fn update_spam_rules(server: &Server) -> trc::Result { spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED) .await?; } + // inbuxa: personal-data catalog, D5: a new install's first rules + // leave the hashed-address blocklist off + if spam_rules::apply_new_install(server.registry(), server.store()).await? + && let Err(err) = reload_and_broadcast(server, ObjectType::SpamDnsblServer).await + { + return Ok(TaskResult::permanent(format!( + "Spam rules were stored but not activated ({err}); run Reload settings" + ))); + } Ok(TaskResult::Success(vec![])) } } diff --git a/docs/spec/features/personal-data-catalog.md b/docs/spec/features/personal-data-catalog.md index 6d073ce..fc0224e 100644 --- a/docs/spec/features/personal-data-catalog.md +++ b/docs/spec/features/personal-data-catalog.md @@ -407,7 +407,11 @@ retention is (not a field on `x:TracerLog`, which is also stored inside `x:Bootstrap` with fields after it, so a new field would change that object's stored format); new installs 30 days, existing servers keep every file as today. D5 is built after the v0.16.24 import lands, on its reworked -spam-rules loader, which keeps each blocklist's on/off state. +spam-rules loader, which keeps each blocklist's on/off state. D5 built after the import: a new install's first boot leaves a note +(`S` `n`), and the rules update, once the bundled rules are in, switches +`STWT_MSBL_EBL_EMAIL` off and forgets the note; the loader keeps that +switch through later updates. An existing server has no note and keeps +every blocklist as it is. | # | Change | Trade-off | |---|---|---| diff --git a/tests/src/system/compliance.rs b/tests/src/system/compliance.rs index fd1a3c5..eb31284 100644 --- a/tests/src/system/compliance.rs +++ b/tests/src/system/compliance.rs @@ -274,6 +274,45 @@ pub async fn test(test: &mut TestServer) { .unwrap(); assert_eq!(trace["retention"]["days"], json!(7), "{trace}"); + // D5: a new install's first rules leave the hashed-address blocklist + // off, once; an existing server (no note) keeps it as it is + let (_, response) = call( + &admin, + "x:SpamDnsblServer/set", + json!({"create": {"m": {"@type": "Email", "name": "STWT_MSBL_EBL_EMAIL", "enable": true, + "zone": {"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}}, + "tag": {"else": "'MSBL_EBL'", "match": {}}}}}), + ) + .await; + let msbl = response["created"]["m"]["id"] + .as_str() + .unwrap_or_else(|| panic!("{response}")) + .to_string(); + let registry = test.server.registry(); + let store = test.server.store(); + assert!( + !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(), + "no note, no change" + ); + let enabled = |response: &Value| response["list"][0]["enable"].clone(); + let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await; + assert_eq!(enabled(&response), json!(true)); + let (_, response) = call(&officer, "inbuxa:DataInventory/get", json!({"ids": null})).await; + assert!( + response["list"][0]["processors"] + .as_array() + .is_some_and(|p| p.iter().any(|p| p["host"] == "ebl.msbl.org")), + "the zone, not the hash: {response}" + ); + common::manager::spam_rules::mark_new_install(store).await.unwrap(); + assert!(common::manager::spam_rules::apply_new_install(registry, store).await.unwrap()); + let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await; + assert_eq!(enabled(&response), json!(false), "{response}"); + assert!( + !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(), + "the note works once" + ); + // A tenant can still be deleted: its unused role goes with it let spare = admin .registry_create_object(Tenant { -- 2.54.0