Merge pull request 'Release 2026.9.28.4' (#94) from release/2026.9.28.4-pr into main

The personal-data catalog and what it feeds: the data inventory and its
snapshots (#83, #89), the Compliance Officer roles (#88), the Compliance
Overview and Data Inventory menu entries (#92). Log file retention
(#87), privacy defaults for new installs (#85, #90), webhooks that send
only the events they name (#82), and upstream v0.16.24 (#84), whose
spam rules updates keep what an admin edited.

Explain: 12 settings asked about (upstream's new createdAt fields, the
certificate dates and the webhook events policy), with the release's
recommended model built locally; 705 answers carry over.

.gitea/workflows/ci.yml

@@ -44,10 +44,6 @@ jobs:
       # schema has, and name nothing that is gone.
       - if: always()
         run: python3 tools/fork/privacy-check.py
-      # The admin reads each expression field's allowed values and variables
-      # from the schema; they're generated from the registry and must match it.
-      - if: always()
-        run: python3 tools/fork/expr-schema.py --check
       - if: always()
         run: python3 -m unittest discover -s tools/fork/tests
 

crates/common/src/manager/defaults.rs

@@ -409,6 +409,10 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
             bp.registry.write(RegistryWrite::insert(&object)).await?;
         }
 
+        // D5: the blocklist sent hashed email addresses starts off; the
+        // rules load later, from a task, which acts on this note
+        super::spam_rules::mark_new_install(&bp.data_store).await?;
+
         // D1: rotated log files are kept 30 days (a fork-owned setting,
         // since x:TracerLog is also stored inside x:Bootstrap)
         use inbuxa_features::security::log_files;

crates/common/src/manager/spam_rules.rs

@@ -118,6 +118,78 @@ pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()>
         .map(|_| ())
 }
 
+/// The blocklists a new install starts with switched off (personal-data
+/// catalog spec, default D5, settled 2026-09-28): the one that is sent a
+/// hash of every email address it's asked about.
+pub const NEW_INSTALL_OFF: &[&str] = &["STWT_MSBL_EBL_EMAIL"];
+
+fn new_install_key() -> ValueClass {
+    ValueClass::Any(AnyClass {
+        subspace: SUBSPACE_INBUXA,
+        key: b"Sn".to_vec(),
+    })
+}
+
+/// Notes, on a new install's first boot, that [`NEW_INSTALL_OFF`] is to be
+/// switched off once the rules are in: they load later, from a task.
+pub async fn mark_new_install(data: &Store) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    batch.set(new_install_key(), b"D5".to_vec());
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())
+        .map(|_| ())
+}
+
+/// After rules load: on a new install, switches [`NEW_INSTALL_OFF`] off and
+/// forgets the note, so it happens once. Returns whether anything changed.
+/// An existing server has no note, and keeps every blocklist as it is.
+pub async fn apply_new_install(
+    registry: &store::RegistryStore,
+    data: &Store,
+) -> trc::Result<bool> {
+    use registry::schema::{prelude::Object, structs::SpamDnsblServer};
+    use store::registry::write::RegistryWrite;
+
+    if data
+        .get_value::<String>(ValueKey::from(new_install_key()))
+        .await
+        .caused_by(trc::location!())?
+        .is_none()
+    {
+        return Ok(false);
+    }
+    let mut changed = false;
+    for server in registry.list::<SpamDnsblServer>().await? {
+        let mut updated = server.object.clone();
+        let SpamDnsblServer::Email(email) = &mut updated else {
+            continue;
+        };
+        if !NEW_INSTALL_OFF.contains(&email.name.as_str()) || !email.enable {
+            continue;
+        }
+        email.enable = false;
+        let old = Object {
+            inner: server.object.into(),
+            revision: server.revision,
+        };
+        let new = Object {
+            inner: updated.into(),
+            revision: server.revision,
+        };
+        registry
+            .write(RegistryWrite::update(types::id::Id::from(server.id.id()), &new, &old))
+            .await?;
+        changed = true;
+    }
+    let mut batch = BatchBuilder::new();
+    batch.clear(new_install_key());
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+    Ok(changed)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/common/src/privacy.rs

@@ -88,14 +88,32 @@ fn days(duration: Option<&Duration>) -> Days {
     }
 }
 
-/// An expression's text, if it is a plain constant (a zone, a switch).
-fn expression_text(value: &Value) -> Option<String> {
+/// The zones a DNSBL's zone expression can query: each quoted literal that
+/// starts with a dot, in any branch (`ip_reverse + '.zen.spamhaus.org'`).
+fn zone_hosts(value: &Value) -> Vec<String> {
+    let mut hosts = Vec::new();
+    let mut texts = Vec::new();
+    fn collect<'a>(value: &'a Value, texts: &mut Vec<&'a str>) {
         match value {
-        Value::String(s) => Some(s.clone()),
-        Value::Object(o) => o.get("else").and_then(|v| v.as_str()).map(str::to_string),
-        _ => None,
+            Value::String(s) => texts.push(s),
+            Value::Array(items) => items.iter().for_each(|v| collect(v, texts)),
+            Value::Object(map) => map.values().for_each(|v| collect(v, texts)),
+            _ => {}
         }
     }
+    collect(value, &mut texts);
+    for text in texts {
+        for literal in text.split('\'').skip(1).step_by(2) {
+            if let Some(zone) = literal.strip_prefix('.')
+                && zone.contains('.')
+                && !hosts.iter().any(|h| h == zone)
+            {
+                hosts.push(zone.to_string());
+            }
+        }
+    }
+    hosts
+}
 
 impl Server {
     async fn singleton<T: registry::types::ObjectImpl + From<Object> + Default>(&self) -> trc::Result<T> {
@@ -263,7 +281,7 @@ impl Server {
             let value = serde_json::to_value(&server.object).unwrap_or_default();
             if value.get("enable").and_then(Value::as_bool).unwrap_or(false) {
                 dnsbl_on = true;
-                if let Some(zone) = value.get("zone").and_then(expression_text) {
+                for zone in value.get("zone").map(zone_hosts).unwrap_or_default() {
                     endpoint(&mut facts, "spam-dnsbl", zone);
                 }
             }
@@ -397,6 +415,16 @@ mod tests {
         assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into()));
     }
 
+    #[test]
+    fn zones_come_from_every_branch() {
+        let zone = json!({"else": "false", "match": {"0": {"if": "location == 'tcp'",
+            "then": "ip_reverse + '.rep.mailspike.net'"}}});
+        assert_eq!(zone_hosts(&zone), vec!["rep.mailspike.net"]);
+        let zone = json!({"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}});
+        assert_eq!(zone_hosts(&zone), vec!["ebl.msbl.org"], "not 'sha1'");
+        assert!(zone_hosts(&json!({"else": "false"})).is_empty());
+    }
+
     #[test]
     fn days_round_up() {
         assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1));

crates/services/src/task_manager/spam_classifier.rs

@@ -316,6 +316,15 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
             spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED)
                 .await?;
         }
+        // inbuxa: personal-data catalog, D5: a new install's first rules
+        // leave the hashed-address blocklist off
+        if spam_rules::apply_new_install(server.registry(), server.store()).await?
+            && let Err(err) = reload_and_broadcast(server, ObjectType::SpamDnsblServer).await
+        {
+            return Ok(TaskResult::permanent(format!(
+                "Spam rules were stored but not activated ({err}); run Reload settings"
+            )));
+        }
         Ok(TaskResult::Success(vec![]))
     }
 }

crates/types/src/branding.rs

@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
 #[macro_export]
 macro_rules! brand_version {
     () => {
-        "2026.9.28.3"
+        "2026.9.28.4"
     };
 }
 

docs/spec/features/personal-data-catalog.md

@@ -407,7 +407,11 @@ retention is (not a field on `x:TracerLog`, which is also stored inside
 `x:Bootstrap` with fields after it, so a new field would change that
 object's stored format); new installs 30 days, existing servers keep every
 file as today. D5 is built after the v0.16.24 import lands, on its reworked
-spam-rules loader, which keeps each blocklist's on/off state.
+spam-rules loader, which keeps each blocklist's on/off state. D5 built after the import: a new install's first boot leaves a note
+(`S` `n`), and the rules update, once the bundled rules are in, switches
+`STWT_MSBL_EBL_EMAIL` off and forgets the note; the loader keeps that
+switch through later updates. An existing server has no note and keeps
+every blocklist as it is.
 
 | # | Change | Trade-off |
 |---|---|---|

resources/privacy/catalog.toml

@@ -847,7 +847,7 @@ default = "none"
 whose = ["correspondent"]
 where = ["memory"]
 scope = "server"
-retention = "unbounded"
+retention = "object-life"
 [object."x:DmarcTroubleshoot".properties]
 ehloDomain = ["network"]
 ipRevPtr = ["network"]
@@ -1266,7 +1266,7 @@ default = "none"
 whose = ["correspondent", "holder", "administrator"]
 where = ["log-file"]
 scope = "server"
-retention = "unbounded"
+retention = { setting = "inbuxa:LogSettings.keepForDays" }
 [object."x:Log".properties]
 details = ["network", "identifier", "metadata"]
 timestamp = ["metadata"]
@@ -1689,7 +1689,7 @@ default = "none"
 whose = ["correspondent"]
 where = ["memory"]
 scope = "server"
-retention = "unbounded"
+retention = "object-life"
 [object."x:SpamClassify".properties]
 authenticatedAs = ["identifier"]
 ehloDomain = ["network"]
@@ -1810,7 +1810,7 @@ default = "none"
 whose = ["holder", "correspondent"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskCalendarItipContents".properties]
 from = ["identifier"]
 iCalendarData = ["content"]
@@ -1825,7 +1825,7 @@ default = "none"
 whose = ["holder"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskDestroyAccount".properties]
 accountName = ["identifier"]
 
@@ -1852,7 +1852,7 @@ default = "none"
 whose = ["holder"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskMergeThreads".properties]
 messageIds = ["metadata"]
 threadName = ["content"]
@@ -1886,7 +1886,7 @@ default = "none"
 whose = ["holder"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskStatusRetry".properties]
 failureReason = ["content"]
 
@@ -2040,30 +2040,23 @@ default = "none"
 
 [object."x:TracerLog"]
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["log-file"]
-scope = "server"
-retention = "unbounded"
+
 [object."x:TracerLog".properties]
 path = ["metadata"]
 
 [object."x:TracerOtelGrpc"]
+# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["external"]
-scope = "server"
-retention = "receiver"
+
 [object."x:TracerOtelGrpc".properties]
 endpoint = ["network"]
 httpAuth = ["credential"]
 httpHeaders = ["credential"]
 
 [object."x:TracerOtelHttp"]
+# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["external"]
-scope = "server"
-retention = "receiver"
+
 [object."x:TracerOtelHttp".properties]
 endpoint = ["network"]
 httpAuth = ["credential"]
@@ -2095,11 +2088,9 @@ usedDiskQuota = ["metadata"]
 default = "none"
 
 [object."x:WebHook"]
+# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["external"]
-scope = "server"
-retention = "receiver"
+
 [object."x:WebHook".properties]
 httpAuth = ["credential"]
 httpHeaders = ["credential"]

resources/schema/schema.json.sha256

@@ -1 +1 @@
-r0pqQlntxFWW4X3bTLq57ObxRipXJXdzjsL9cyzz2Bo
+wDJZ1KdKs21tjHD-UBI9R_XwPEET7Iul8XEXbPlgBPE

tests/src/system/compliance.rs

@@ -274,6 +274,45 @@ pub async fn test(test: &mut TestServer) {
         .unwrap();
     assert_eq!(trace["retention"]["days"], json!(7), "{trace}");
 
+    // D5: a new install's first rules leave the hashed-address blocklist
+    // off, once; an existing server (no note) keeps it as it is
+    let (_, response) = call(
+        &admin,
+        "x:SpamDnsblServer/set",
+        json!({"create": {"m": {"@type": "Email", "name": "STWT_MSBL_EBL_EMAIL", "enable": true,
+            "zone": {"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}},
+            "tag": {"else": "'MSBL_EBL'", "match": {}}}}}),
+    )
+    .await;
+    let msbl = response["created"]["m"]["id"]
+        .as_str()
+        .unwrap_or_else(|| panic!("{response}"))
+        .to_string();
+    let registry = test.server.registry();
+    let store = test.server.store();
+    assert!(
+        !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
+        "no note, no change"
+    );
+    let enabled = |response: &Value| response["list"][0]["enable"].clone();
+    let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
+    assert_eq!(enabled(&response), json!(true));
+    let (_, response) = call(&officer, "inbuxa:DataInventory/get", json!({"ids": null})).await;
+    assert!(
+        response["list"][0]["processors"]
+            .as_array()
+            .is_some_and(|p| p.iter().any(|p| p["host"] == "ebl.msbl.org")),
+        "the zone, not the hash: {response}"
+    );
+    common::manager::spam_rules::mark_new_install(store).await.unwrap();
+    assert!(common::manager::spam_rules::apply_new_install(registry, store).await.unwrap());
+    let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
+    assert_eq!(enabled(&response), json!(false), "{response}");
+    assert!(
+        !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
+        "the note works once"
+    );
+
     // A tenant can still be deleted: its unused role goes with it
     let spare = admin
         .registry_create_object(Tenant {

tools/fork/README.md

@@ -112,18 +112,3 @@ a fresh copy for each one. See `docs/spec/compat-tests.md`.
 tools/fork/run-compat.sh --store /srv/inbuxa-copy/rocks.db \
     --admin '[email protected]:PASSWORD' --recordings ~/compat
 ```
-
-## expr-schema.py
-
-Writes each expression field's allowed constants and variables, read from the
-generated registry code, into the schema the server serves INBUXA Admin
-(`resources/schema/schema.json.gz` and its checksum). The admin uses them to
-offer plain choices instead of a free-text box.
-
-```bash
-tools/fork/expr-schema.py          # update the schema
-tools/fork/expr-schema.py --check  # exit 1 if it's out of date (CI)
-```
-
-Re-run it after anything that regenerates the registry, an upstream import
-included.

tools/fork/expr-schema.py

@@ -1,144 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2026 Coffey Labs
-# SPDX-License-Identifier: AGPL-3.0-or-later
-"""Tell INBUXA Admin what each expression field accepts.
-
-Every expression field in the registry has a context: the constants it may
-evaluate to (DKIM verification: relaxed, strict or disable) and the variables
-its conditions may read (sender_domain, local_port...). The server enforces
-both, but the schema it serves the admin describes every expression field as
-only an `x:Expression` object, so the admin can offer nothing better than a
-free-text box.
-
-This reads those contexts from the generated registry code and writes them
-into the served schema, on each expression field's type:
-
-    "type": {"type": "object", "objectName": "x:Expression",
-             "expression": {"constants": ["relaxed", "strict", "disable"],
-                            "variables": ["sender", "sender_domain", ...]}}
-
-The registry code is the source, so re-run this after anything that
-regenerates it (an upstream import, a new expression field). `--check` exits 1
-when the schema is out of date; CI runs it.
-"""
-
-import argparse
-import base64
-import gzip
-import hashlib
-import json
-import re
-import sys
-from pathlib import Path
-
-root = Path(__file__).resolve().parents[2]
-REGISTRY = root / 'crates' / 'registry' / 'src' / 'schema'
-SCHEMA = root / 'resources' / 'schema' / 'schema.json.gz'
-SCHEMA_HASH = root / 'resources' / 'schema' / 'schema.json.sha256'
-
-
-def names(enum, text):
-    """Variant → wire name, from the `Enum::Variant => "name"` arms."""
-    return dict(re.findall(rf'{enum}::(\w+) => "([^"]+)"', text))
-
-
-def lists(text):
-    """Every `pub static NAME: &[ExpressionConstant|Variable] = &[...]`."""
-    out = {}
-    for name, kind, body in re.findall(
-        r'pub static (\w+): &\[(ExpressionConstant|ExpressionVariable)\] = &\[(.*?)\];', text, re.S
-    ):
-        out[name] = (kind, re.findall(rf'{kind}::(\w+)', body))
-    return out
-
-
-def contexts(text):
-    """(struct, Property variant, variables list name, constants list name) per context."""
-    out = []
-    for block in re.finditer(r'(?m)^impl (\w+) \{(.*?)^\}', text, re.S):
-        struct, body = block.group(1), block.group(2)
-        for ctx in re.finditer(r'ExpressionContext \{(.*?)\n\s*\}\n', body, re.S):
-            fields = ctx.group(1)
-            prop = re.search(r'property: Property::(\w+),', fields)
-            var = re.search(r'allowed_variables: (&\[\]|\w+),', fields)
-            const = re.search(r'allowed_constants: (&\[\]|\w+),', fields)
-            if prop and var and const:
-                out.append((struct, prop.group(1), var.group(1), const.group(1)))
-    return out
-
-
-def build():
-    enums = (REGISTRY / 'enums.rs').read_text(encoding='utf-8')
-    enums_impl = (REGISTRY / 'enums_impl.rs').read_text(encoding='utf-8')
-    props = names('Property', (REGISTRY / 'properties_impl.rs').read_text(encoding='utf-8'))
-    const_names = names('ExpressionConstant', enums_impl)
-    var_names = names('ExpressionVariable', enums_impl)
-    known = lists(enums)
-
-    def resolve(ref, kind, wire):
-        if ref == '&[]':
-            return []
-        found = known.get(ref)
-        if not found or found[0] != kind:
-            raise SystemExit(f'expr-schema: no {kind} list named {ref}')
-        return [wire[v] for v in found[1]]
-
-    table = {}
-    for struct, prop, var, const in contexts((REGISTRY / 'structs_impl.rs').read_text(encoding='utf-8')):
-        table[(f'x:{struct}', props[prop])] = {
-            'constants': resolve(const, 'ExpressionConstant', const_names),
-            'variables': resolve(var, 'ExpressionVariable', var_names),
-        }
-    return table
-
-
-def apply(schema, table):
-    """Write the table into the schema; returns the (object, field) pairs it couldn't place."""
-    missing = []
-    for (obj, field), expr in sorted(table.items()):
-        target = schema['fields'].get(obj, {}).get('properties', {}).get(field)
-        if target is None or target['type'].get('objectName') != 'x:Expression':
-            missing.append(f'{obj}.{field}')
-            continue
-        target['type']['expression'] = expr
-    return missing
-
-
-def encode(schema):
-    text = json.dumps(schema, ensure_ascii=False, separators=(',', ':'))
-    out = gzip.compress(text.encode('utf-8'), compresslevel=9, mtime=0)
-    digest = base64.urlsafe_b64encode(hashlib.sha256(out).digest()).decode().rstrip('=')
-    return out, digest
-
-
-def main():
-    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
-    parser.add_argument('--check', action='store_true', help='exit 1 if the schema is out of date')
-    args = parser.parse_args()
-
-    before = SCHEMA.read_bytes()
-    schema = json.loads(gzip.decompress(before))
-    table = build()
-    missing = apply(schema, table)
-    if missing:
-        print('expr-schema: expression contexts with no matching schema field:', file=sys.stderr)
-        for m in missing:
-            print(f'  {m}', file=sys.stderr)
-        return 1
-
-    current = json.loads(gzip.decompress(before))
-    if current == schema:
-        print(f'expr-schema: {len(table)} expression fields, schema up to date')
-        return 0
-    if args.check:
-        print('expr-schema: schema is out of date; run tools/fork/expr-schema.py', file=sys.stderr)
-        return 1
-    out, digest = encode(schema)
-    SCHEMA.write_bytes(out)
-    SCHEMA_HASH.write_text(digest, encoding='utf-8')
-    print(f'expr-schema: wrote {len(table)} expression fields into {SCHEMA.relative_to(root)}')
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main())