Merge pull request 'New installs start with the hashed-address blocklist off, and DNSBL zones read right' (#90) from feature/d5-msbl-off into main

crates/common/src/manager/defaults.rs

@@ -409,6 +409,10 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
             bp.registry.write(RegistryWrite::insert(&object)).await?;
         }
 
+        // D5: the blocklist sent hashed email addresses starts off; the
+        // rules load later, from a task, which acts on this note
+        super::spam_rules::mark_new_install(&bp.data_store).await?;
+
         // D1: rotated log files are kept 30 days (a fork-owned setting,
         // since x:TracerLog is also stored inside x:Bootstrap)
         use inbuxa_features::security::log_files;

crates/common/src/manager/spam_rules.rs

@@ -118,6 +118,78 @@ pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()>
         .map(|_| ())
 }
 
+/// The blocklists a new install starts with switched off (personal-data
+/// catalog spec, default D5, settled 2026-09-28): the one that is sent a
+/// hash of every email address it's asked about.
+pub const NEW_INSTALL_OFF: &[&str] = &["STWT_MSBL_EBL_EMAIL"];
+
+fn new_install_key() -> ValueClass {
+    ValueClass::Any(AnyClass {
+        subspace: SUBSPACE_INBUXA,
+        key: b"Sn".to_vec(),
+    })
+}
+
+/// Notes, on a new install's first boot, that [`NEW_INSTALL_OFF`] is to be
+/// switched off once the rules are in: they load later, from a task.
+pub async fn mark_new_install(data: &Store) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    batch.set(new_install_key(), b"D5".to_vec());
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())
+        .map(|_| ())
+}
+
+/// After rules load: on a new install, switches [`NEW_INSTALL_OFF`] off and
+/// forgets the note, so it happens once. Returns whether anything changed.
+/// An existing server has no note, and keeps every blocklist as it is.
+pub async fn apply_new_install(
+    registry: &store::RegistryStore,
+    data: &Store,
+) -> trc::Result<bool> {
+    use registry::schema::{prelude::Object, structs::SpamDnsblServer};
+    use store::registry::write::RegistryWrite;
+
+    if data
+        .get_value::<String>(ValueKey::from(new_install_key()))
+        .await
+        .caused_by(trc::location!())?
+        .is_none()
+    {
+        return Ok(false);
+    }
+    let mut changed = false;
+    for server in registry.list::<SpamDnsblServer>().await? {
+        let mut updated = server.object.clone();
+        let SpamDnsblServer::Email(email) = &mut updated else {
+            continue;
+        };
+        if !NEW_INSTALL_OFF.contains(&email.name.as_str()) || !email.enable {
+            continue;
+        }
+        email.enable = false;
+        let old = Object {
+            inner: server.object.into(),
+            revision: server.revision,
+        };
+        let new = Object {
+            inner: updated.into(),
+            revision: server.revision,
+        };
+        registry
+            .write(RegistryWrite::update(types::id::Id::from(server.id.id()), &new, &old))
+            .await?;
+        changed = true;
+    }
+    let mut batch = BatchBuilder::new();
+    batch.clear(new_install_key());
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+    Ok(changed)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/common/src/privacy.rs

@@ -88,13 +88,31 @@ fn days(duration: Option<&Duration>) -> Days {
     }
 }
 
-/// An expression's text, if it is a plain constant (a zone, a switch).
+/// The zones a DNSBL's zone expression can query: each quoted literal that
-fn expression_text(value: &Value) -> Option<String> {
+/// starts with a dot, in any branch (`ip_reverse + '.zen.spamhaus.org'`).
-    match value {
+fn zone_hosts(value: &Value) -> Vec<String> {
-        Value::String(s) => Some(s.clone()),
+    let mut hosts = Vec::new();
-        Value::Object(o) => o.get("else").and_then(|v| v.as_str()).map(str::to_string),
+    let mut texts = Vec::new();
-        _ => None,
+    fn collect<'a>(value: &'a Value, texts: &mut Vec<&'a str>) {
+        match value {
+            Value::String(s) => texts.push(s),
+            Value::Array(items) => items.iter().for_each(|v| collect(v, texts)),
+            Value::Object(map) => map.values().for_each(|v| collect(v, texts)),
+            _ => {}
+        }
     }
+    collect(value, &mut texts);
+    for text in texts {
+        for literal in text.split('\'').skip(1).step_by(2) {
+            if let Some(zone) = literal.strip_prefix('.')
+                && zone.contains('.')
+                && !hosts.iter().any(|h| h == zone)
+            {
+                hosts.push(zone.to_string());
+            }
+        }
+    }
+    hosts
 }
 
 impl Server {
@@ -263,7 +281,7 @@ impl Server {
             let value = serde_json::to_value(&server.object).unwrap_or_default();
             if value.get("enable").and_then(Value::as_bool).unwrap_or(false) {
                 dnsbl_on = true;
-                if let Some(zone) = value.get("zone").and_then(expression_text) {
+                for zone in value.get("zone").map(zone_hosts).unwrap_or_default() {
                     endpoint(&mut facts, "spam-dnsbl", zone);
                 }
             }
@@ -397,6 +415,16 @@ mod tests {
         assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into()));
     }
 
+    #[test]
+    fn zones_come_from_every_branch() {
+        let zone = json!({"else": "false", "match": {"0": {"if": "location == 'tcp'",
+            "then": "ip_reverse + '.rep.mailspike.net'"}}});
+        assert_eq!(zone_hosts(&zone), vec!["rep.mailspike.net"]);
+        let zone = json!({"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}});
+        assert_eq!(zone_hosts(&zone), vec!["ebl.msbl.org"], "not 'sha1'");
+        assert!(zone_hosts(&json!({"else": "false"})).is_empty());
+    }
+
     #[test]
     fn days_round_up() {
         assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1));

crates/services/src/task_manager/spam_classifier.rs

@@ -316,6 +316,15 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
             spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED)
                 .await?;
         }
+        // inbuxa: personal-data catalog, D5: a new install's first rules
+        // leave the hashed-address blocklist off
+        if spam_rules::apply_new_install(server.registry(), server.store()).await?
+            && let Err(err) = reload_and_broadcast(server, ObjectType::SpamDnsblServer).await
+        {
+            return Ok(TaskResult::permanent(format!(
+                "Spam rules were stored but not activated ({err}); run Reload settings"
+            )));
+        }
         Ok(TaskResult::Success(vec![]))
     }
 }

docs/spec/features/personal-data-catalog.md

@@ -407,7 +407,11 @@ retention is (not a field on `x:TracerLog`, which is also stored inside
 `x:Bootstrap` with fields after it, so a new field would change that
 object's stored format); new installs 30 days, existing servers keep every
 file as today. D5 is built after the v0.16.24 import lands, on its reworked
-spam-rules loader, which keeps each blocklist's on/off state.
+spam-rules loader, which keeps each blocklist's on/off state. D5 built after the import: a new install's first boot leaves a note
+(`S` `n`), and the rules update, once the bundled rules are in, switches
+`STWT_MSBL_EBL_EMAIL` off and forgets the note; the loader keeps that
+switch through later updates. An existing server has no note and keeps
+every blocklist as it is.
 
 | # | Change | Trade-off |
 |---|---|---|

tests/src/system/compliance.rs

@@ -274,6 +274,45 @@ pub async fn test(test: &mut TestServer) {
         .unwrap();
     assert_eq!(trace["retention"]["days"], json!(7), "{trace}");
 
+    // D5: a new install's first rules leave the hashed-address blocklist
+    // off, once; an existing server (no note) keeps it as it is
+    let (_, response) = call(
+        &admin,
+        "x:SpamDnsblServer/set",
+        json!({"create": {"m": {"@type": "Email", "name": "STWT_MSBL_EBL_EMAIL", "enable": true,
+            "zone": {"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}},
+            "tag": {"else": "'MSBL_EBL'", "match": {}}}}}),
+    )
+    .await;
+    let msbl = response["created"]["m"]["id"]
+        .as_str()
+        .unwrap_or_else(|| panic!("{response}"))
+        .to_string();
+    let registry = test.server.registry();
+    let store = test.server.store();
+    assert!(
+        !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
+        "no note, no change"
+    );
+    let enabled = |response: &Value| response["list"][0]["enable"].clone();
+    let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
+    assert_eq!(enabled(&response), json!(true));
+    let (_, response) = call(&officer, "inbuxa:DataInventory/get", json!({"ids": null})).await;
+    assert!(
+        response["list"][0]["processors"]
+            .as_array()
+            .is_some_and(|p| p.iter().any(|p| p["host"] == "ebl.msbl.org")),
+        "the zone, not the hash: {response}"
+    );
+    common::manager::spam_rules::mark_new_install(store).await.unwrap();
+    assert!(common::manager::spam_rules::apply_new_install(registry, store).await.unwrap());
+    let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
+    assert_eq!(enabled(&response), json!(false), "{response}");
+    assert!(
+        !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
+        "the note works once"
+    );
+
     // A tenant can still be deleted: its unused role goes with it
     let spare = admin
         .registry_create_object(Tenant {