Schema: each expression field says which values and variables it accepts

The registry knows, for every expression field, the constants it may
evaluate to and the variables its conditions may read, and enforces both.
The schema served to INBUXA Admin described every one as a bare
x:Expression, so the console could offer nothing better than free text.

tools/fork/expr-schema.py reads those contexts from the generated registry
code and writes them onto each field's type as
expression: {constants, variables}. All 124 expression fields are covered.
CI runs it with --check so the schema can't drift from the registry.

.gitea/workflows/ci.yml

@@ -44,6 +44,10 @@ jobs:
       # schema has, and name nothing that is gone.
       - if: always()
         run: python3 tools/fork/privacy-check.py
+      # The admin reads each expression field's allowed values and variables
+      # from the schema; they're generated from the registry and must match it.
+      - if: always()
+        run: python3 tools/fork/expr-schema.py --check
       - if: always()
         run: python3 -m unittest discover -s tools/fork/tests
 

crates/common/src/manager/defaults.rs

@@ -409,6 +409,10 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
             bp.registry.write(RegistryWrite::insert(&object)).await?;
         }
 
+        // D5: the blocklist sent hashed email addresses starts off; the
+        // rules load later, from a task, which acts on this note
+        super::spam_rules::mark_new_install(&bp.data_store).await?;
+
         // D1: rotated log files are kept 30 days (a fork-owned setting,
         // since x:TracerLog is also stored inside x:Bootstrap)
         use inbuxa_features::security::log_files;

crates/common/src/manager/spam_rules.rs

@@ -118,6 +118,78 @@ pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()>
         .map(|_| ())
 }
 
+/// The blocklists a new install starts with switched off (personal-data
+/// catalog spec, default D5, settled 2026-09-28): the one that is sent a
+/// hash of every email address it's asked about.
+pub const NEW_INSTALL_OFF: &[&str] = &["STWT_MSBL_EBL_EMAIL"];
+
+fn new_install_key() -> ValueClass {
+    ValueClass::Any(AnyClass {
+        subspace: SUBSPACE_INBUXA,
+        key: b"Sn".to_vec(),
+    })
+}
+
+/// Notes, on a new install's first boot, that [`NEW_INSTALL_OFF`] is to be
+/// switched off once the rules are in: they load later, from a task.
+pub async fn mark_new_install(data: &Store) -> trc::Result<()> {
+    let mut batch = BatchBuilder::new();
+    batch.set(new_install_key(), b"D5".to_vec());
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())
+        .map(|_| ())
+}
+
+/// After rules load: on a new install, switches [`NEW_INSTALL_OFF`] off and
+/// forgets the note, so it happens once. Returns whether anything changed.
+/// An existing server has no note, and keeps every blocklist as it is.
+pub async fn apply_new_install(
+    registry: &store::RegistryStore,
+    data: &Store,
+) -> trc::Result<bool> {
+    use registry::schema::{prelude::Object, structs::SpamDnsblServer};
+    use store::registry::write::RegistryWrite;
+
+    if data
+        .get_value::<String>(ValueKey::from(new_install_key()))
+        .await
+        .caused_by(trc::location!())?
+        .is_none()
+    {
+        return Ok(false);
+    }
+    let mut changed = false;
+    for server in registry.list::<SpamDnsblServer>().await? {
+        let mut updated = server.object.clone();
+        let SpamDnsblServer::Email(email) = &mut updated else {
+            continue;
+        };
+        if !NEW_INSTALL_OFF.contains(&email.name.as_str()) || !email.enable {
+            continue;
+        }
+        email.enable = false;
+        let old = Object {
+            inner: server.object.into(),
+            revision: server.revision,
+        };
+        let new = Object {
+            inner: updated.into(),
+            revision: server.revision,
+        };
+        registry
+            .write(RegistryWrite::update(types::id::Id::from(server.id.id()), &new, &old))
+            .await?;
+        changed = true;
+    }
+    let mut batch = BatchBuilder::new();
+    batch.clear(new_install_key());
+    data.write(batch.build_all())
+        .await
+        .caused_by(trc::location!())?;
+    Ok(changed)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/common/src/privacy.rs

@@ -88,14 +88,32 @@ fn days(duration: Option<&Duration>) -> Days {
     }
 }
 
-/// An expression's text, if it is a plain constant (a zone, a switch).
-fn expression_text(value: &Value) -> Option<String> {
+/// The zones a DNSBL's zone expression can query: each quoted literal that
+/// starts with a dot, in any branch (`ip_reverse + '.zen.spamhaus.org'`).
+fn zone_hosts(value: &Value) -> Vec<String> {
+    let mut hosts = Vec::new();
+    let mut texts = Vec::new();
+    fn collect<'a>(value: &'a Value, texts: &mut Vec<&'a str>) {
         match value {
-        Value::String(s) => Some(s.clone()),
-        Value::Object(o) => o.get("else").and_then(|v| v.as_str()).map(str::to_string),
-        _ => None,
+            Value::String(s) => texts.push(s),
+            Value::Array(items) => items.iter().for_each(|v| collect(v, texts)),
+            Value::Object(map) => map.values().for_each(|v| collect(v, texts)),
+            _ => {}
         }
     }
+    collect(value, &mut texts);
+    for text in texts {
+        for literal in text.split('\'').skip(1).step_by(2) {
+            if let Some(zone) = literal.strip_prefix('.')
+                && zone.contains('.')
+                && !hosts.iter().any(|h| h == zone)
+            {
+                hosts.push(zone.to_string());
+            }
+        }
+    }
+    hosts
+}
 
 impl Server {
     async fn singleton<T: registry::types::ObjectImpl + From<Object> + Default>(&self) -> trc::Result<T> {
@@ -263,7 +281,7 @@ impl Server {
             let value = serde_json::to_value(&server.object).unwrap_or_default();
             if value.get("enable").and_then(Value::as_bool).unwrap_or(false) {
                 dnsbl_on = true;
-                if let Some(zone) = value.get("zone").and_then(expression_text) {
+                for zone in value.get("zone").map(zone_hosts).unwrap_or_default() {
                     endpoint(&mut facts, "spam-dnsbl", zone);
                 }
             }
@@ -397,6 +415,16 @@ mod tests {
         assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into()));
     }
 
+    #[test]
+    fn zones_come_from_every_branch() {
+        let zone = json!({"else": "false", "match": {"0": {"if": "location == 'tcp'",
+            "then": "ip_reverse + '.rep.mailspike.net'"}}});
+        assert_eq!(zone_hosts(&zone), vec!["rep.mailspike.net"]);
+        let zone = json!({"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}});
+        assert_eq!(zone_hosts(&zone), vec!["ebl.msbl.org"], "not 'sha1'");
+        assert!(zone_hosts(&json!({"else": "false"})).is_empty());
+    }
+
     #[test]
     fn days_round_up() {
         assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1));

crates/services/src/task_manager/spam_classifier.rs

@@ -316,6 +316,15 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
             spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED)
                 .await?;
         }
+        // inbuxa: personal-data catalog, D5: a new install's first rules
+        // leave the hashed-address blocklist off
+        if spam_rules::apply_new_install(server.registry(), server.store()).await?
+            && let Err(err) = reload_and_broadcast(server, ObjectType::SpamDnsblServer).await
+        {
+            return Ok(TaskResult::permanent(format!(
+                "Spam rules were stored but not activated ({err}); run Reload settings"
+            )));
+        }
         Ok(TaskResult::Success(vec![]))
     }
 }

docs/spec/features/personal-data-catalog.md

@@ -407,7 +407,11 @@ retention is (not a field on `x:TracerLog`, which is also stored inside
 `x:Bootstrap` with fields after it, so a new field would change that
 object's stored format); new installs 30 days, existing servers keep every
 file as today. D5 is built after the v0.16.24 import lands, on its reworked
-spam-rules loader, which keeps each blocklist's on/off state.
+spam-rules loader, which keeps each blocklist's on/off state. D5 built after the import: a new install's first boot leaves a note
+(`S` `n`), and the rules update, once the bundled rules are in, switches
+`STWT_MSBL_EBL_EMAIL` off and forgets the note; the loader keeps that
+switch through later updates. An existing server has no note and keeps
+every blocklist as it is.
 
 | # | Change | Trade-off |
 |---|---|---|

resources/privacy/catalog.toml

@@ -847,7 +847,7 @@ default = "none"
 whose = ["correspondent"]
 where = ["memory"]
 scope = "server"
-retention = "unbounded"
+retention = "object-life"
 [object."x:DmarcTroubleshoot".properties]
 ehloDomain = ["network"]
 ipRevPtr = ["network"]
@@ -1266,7 +1266,7 @@ default = "none"
 whose = ["correspondent", "holder", "administrator"]
 where = ["log-file"]
 scope = "server"
-retention = "unbounded"
+retention = { setting = "inbuxa:LogSettings.keepForDays" }
 [object."x:Log".properties]
 details = ["network", "identifier", "metadata"]
 timestamp = ["metadata"]
@@ -1689,7 +1689,7 @@ default = "none"
 whose = ["correspondent"]
 where = ["memory"]
 scope = "server"
-retention = "unbounded"
+retention = "object-life"
 [object."x:SpamClassify".properties]
 authenticatedAs = ["identifier"]
 ehloDomain = ["network"]
@@ -1810,7 +1810,7 @@ default = "none"
 whose = ["holder", "correspondent"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskCalendarItipContents".properties]
 from = ["identifier"]
 iCalendarData = ["content"]
@@ -1825,7 +1825,7 @@ default = "none"
 whose = ["holder"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskDestroyAccount".properties]
 accountName = ["identifier"]
 
@@ -1852,7 +1852,7 @@ default = "none"
 whose = ["holder"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskMergeThreads".properties]
 messageIds = ["metadata"]
 threadName = ["content"]
@@ -1886,7 +1886,7 @@ default = "none"
 whose = ["holder"]
 where = ["data-store"]
 scope = "tenant"
-retention = "unbounded"
+retention = "object-life"
 [object."x:TaskStatusRetry".properties]
 failureReason = ["content"]
 
@@ -2040,30 +2040,23 @@ default = "none"
 
 [object."x:TracerLog"]
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["log-file"]
-scope = "server"
-retention = "unbounded"
+
 [object."x:TracerLog".properties]
 path = ["metadata"]
 
 [object."x:TracerOtelGrpc"]
+# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["external"]
-scope = "server"
-retention = "receiver"
+
 [object."x:TracerOtelGrpc".properties]
 endpoint = ["network"]
 httpAuth = ["credential"]
 httpHeaders = ["credential"]
 
 [object."x:TracerOtelHttp"]
+# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["external"]
-scope = "server"
-retention = "receiver"
+
 [object."x:TracerOtelHttp".properties]
 endpoint = ["network"]
 httpAuth = ["credential"]
@@ -2095,11 +2088,9 @@ usedDiskQuota = ["metadata"]
 default = "none"
 
 [object."x:WebHook"]
+# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
 default = "none"
-whose = ["correspondent", "holder", "administrator"]
-where = ["external"]
-scope = "server"
-retention = "receiver"
+
 [object."x:WebHook".properties]
 httpAuth = ["credential"]
 httpHeaders = ["credential"]

resources/schema/schema.json.sha256

@@ -1 +1 @@
-8Wyl9buv_eSbbGGhV-UZcjGHG_H77eorpuoI2u6wgnY
+k496pjVWlQ2p4bkZCh8agzaCKMLD4c3Z9WtoxpckYDU

tests/src/system/compliance.rs

@@ -274,6 +274,45 @@ pub async fn test(test: &mut TestServer) {
         .unwrap();
     assert_eq!(trace["retention"]["days"], json!(7), "{trace}");
 
+    // D5: a new install's first rules leave the hashed-address blocklist
+    // off, once; an existing server (no note) keeps it as it is
+    let (_, response) = call(
+        &admin,
+        "x:SpamDnsblServer/set",
+        json!({"create": {"m": {"@type": "Email", "name": "STWT_MSBL_EBL_EMAIL", "enable": true,
+            "zone": {"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}},
+            "tag": {"else": "'MSBL_EBL'", "match": {}}}}}),
+    )
+    .await;
+    let msbl = response["created"]["m"]["id"]
+        .as_str()
+        .unwrap_or_else(|| panic!("{response}"))
+        .to_string();
+    let registry = test.server.registry();
+    let store = test.server.store();
+    assert!(
+        !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
+        "no note, no change"
+    );
+    let enabled = |response: &Value| response["list"][0]["enable"].clone();
+    let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
+    assert_eq!(enabled(&response), json!(true));
+    let (_, response) = call(&officer, "inbuxa:DataInventory/get", json!({"ids": null})).await;
+    assert!(
+        response["list"][0]["processors"]
+            .as_array()
+            .is_some_and(|p| p.iter().any(|p| p["host"] == "ebl.msbl.org")),
+        "the zone, not the hash: {response}"
+    );
+    common::manager::spam_rules::mark_new_install(store).await.unwrap();
+    assert!(common::manager::spam_rules::apply_new_install(registry, store).await.unwrap());
+    let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
+    assert_eq!(enabled(&response), json!(false), "{response}");
+    assert!(
+        !common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
+        "the note works once"
+    );
+
     // A tenant can still be deleted: its unused role goes with it
     let spare = admin
         .registry_create_object(Tenant {

tools/fork/README.md

@@ -112,3 +112,18 @@ a fresh copy for each one. See `docs/spec/compat-tests.md`.
 tools/fork/run-compat.sh --store /srv/inbuxa-copy/rocks.db \
     --admin '[email protected]:PASSWORD' --recordings ~/compat
 ```
+
+## expr-schema.py
+
+Writes each expression field's allowed constants and variables, read from the
+generated registry code, into the schema the server serves INBUXA Admin
+(`resources/schema/schema.json.gz` and its checksum). The admin uses them to
+offer plain choices instead of a free-text box.
+
+```bash
+tools/fork/expr-schema.py          # update the schema
+tools/fork/expr-schema.py --check  # exit 1 if it's out of date (CI)
+```
+
+Re-run it after anything that regenerates the registry, an upstream import
+included.

tools/fork/expr-schema.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2026 Coffey Labs
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""Tell INBUXA Admin what each expression field accepts.
+
+Every expression field in the registry has a context: the constants it may
+evaluate to (DKIM verification: relaxed, strict or disable) and the variables
+its conditions may read (sender_domain, local_port...). The server enforces
+both, but the schema it serves the admin describes every expression field as
+only an `x:Expression` object, so the admin can offer nothing better than a
+free-text box.
+
+This reads those contexts from the generated registry code and writes them
+into the served schema, on each expression field's type:
+
+    "type": {"type": "object", "objectName": "x:Expression",
+             "expression": {"constants": ["relaxed", "strict", "disable"],
+                            "variables": ["sender", "sender_domain", ...]}}
+
+The registry code is the source, so re-run this after anything that
+regenerates it (an upstream import, a new expression field). `--check` exits 1
+when the schema is out of date; CI runs it.
+"""
+
+import argparse
+import base64
+import gzip
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+
+root = Path(__file__).resolve().parents[2]
+REGISTRY = root / 'crates' / 'registry' / 'src' / 'schema'
+SCHEMA = root / 'resources' / 'schema' / 'schema.json.gz'
+SCHEMA_HASH = root / 'resources' / 'schema' / 'schema.json.sha256'
+
+
+def names(enum, text):
+    """Variant → wire name, from the `Enum::Variant => "name"` arms."""
+    return dict(re.findall(rf'{enum}::(\w+) => "([^"]+)"', text))
+
+
+def lists(text):
+    """Every `pub static NAME: &[ExpressionConstant|Variable] = &[...]`."""
+    out = {}
+    for name, kind, body in re.findall(
+        r'pub static (\w+): &\[(ExpressionConstant|ExpressionVariable)\] = &\[(.*?)\];', text, re.S
+    ):
+        out[name] = (kind, re.findall(rf'{kind}::(\w+)', body))
+    return out
+
+
+def contexts(text):
+    """(struct, Property variant, variables list name, constants list name) per context."""
+    out = []
+    for block in re.finditer(r'(?m)^impl (\w+) \{(.*?)^\}', text, re.S):
+        struct, body = block.group(1), block.group(2)
+        for ctx in re.finditer(r'ExpressionContext \{(.*?)\n\s*\}\n', body, re.S):
+            fields = ctx.group(1)
+            prop = re.search(r'property: Property::(\w+),', fields)
+            var = re.search(r'allowed_variables: (&\[\]|\w+),', fields)
+            const = re.search(r'allowed_constants: (&\[\]|\w+),', fields)
+            if prop and var and const:
+                out.append((struct, prop.group(1), var.group(1), const.group(1)))
+    return out
+
+
+def build():
+    enums = (REGISTRY / 'enums.rs').read_text(encoding='utf-8')
+    enums_impl = (REGISTRY / 'enums_impl.rs').read_text(encoding='utf-8')
+    props = names('Property', (REGISTRY / 'properties_impl.rs').read_text(encoding='utf-8'))
+    const_names = names('ExpressionConstant', enums_impl)
+    var_names = names('ExpressionVariable', enums_impl)
+    known = lists(enums)
+
+    def resolve(ref, kind, wire):
+        if ref == '&[]':
+            return []
+        found = known.get(ref)
+        if not found or found[0] != kind:
+            raise SystemExit(f'expr-schema: no {kind} list named {ref}')
+        return [wire[v] for v in found[1]]
+
+    table = {}
+    for struct, prop, var, const in contexts((REGISTRY / 'structs_impl.rs').read_text(encoding='utf-8')):
+        table[(f'x:{struct}', props[prop])] = {
+            'constants': resolve(const, 'ExpressionConstant', const_names),
+            'variables': resolve(var, 'ExpressionVariable', var_names),
+        }
+    return table
+
+
+def apply(schema, table):
+    """Write the table into the schema; returns the (object, field) pairs it couldn't place."""
+    missing = []
+    for (obj, field), expr in sorted(table.items()):
+        target = schema['fields'].get(obj, {}).get('properties', {}).get(field)
+        if target is None or target['type'].get('objectName') != 'x:Expression':
+            missing.append(f'{obj}.{field}')
+            continue
+        target['type']['expression'] = expr
+    return missing
+
+
+def encode(schema):
+    text = json.dumps(schema, ensure_ascii=False, separators=(',', ':'))
+    out = gzip.compress(text.encode('utf-8'), compresslevel=9, mtime=0)
+    digest = base64.urlsafe_b64encode(hashlib.sha256(out).digest()).decode().rstrip('=')
+    return out, digest
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument('--check', action='store_true', help='exit 1 if the schema is out of date')
+    args = parser.parse_args()
+
+    before = SCHEMA.read_bytes()
+    schema = json.loads(gzip.decompress(before))
+    table = build()
+    missing = apply(schema, table)
+    if missing:
+        print('expr-schema: expression contexts with no matching schema field:', file=sys.stderr)
+        for m in missing:
+            print(f'  {m}', file=sys.stderr)
+        return 1
+
+    current = json.loads(gzip.decompress(before))
+    if current == schema:
+        print(f'expr-schema: {len(table)} expression fields, schema up to date')
+        return 0
+    if args.check:
+        print('expr-schema: schema is out of date; run tools/fork/expr-schema.py', file=sys.stderr)
+        return 1
+    out, digest = encode(schema)
+    SCHEMA.write_bytes(out)
+    SCHEMA_HASH.write_text(digest, encoding='utf-8')
+    print(f'expr-schema: wrote {len(table)} expression fields into {SCHEMA.relative_to(root)}')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())