Merge main into merge/upstream-v0.16.24

The schema, which both sides changed, merged as JSON with no conflicts.
The personal-data catalog (#83) gains upstream's new x:DnsServerPowerDns:
nothing personal but its API key, like the other DNS providers.

.gitea/workflows/ci.yml

@@ -40,6 +40,12 @@ jobs:
       # nothing. CI never sees the difference; a release does.
       - if: always()
         run: python3 tools/fork/context-check.py
+      # The personal-data catalog must classify every object and field the
+      # schema has, and name nothing that is gone.
+      - if: always()
+        run: python3 tools/fork/privacy-check.py
+      - if: always()
+        run: python3 -m unittest discover -s tools/fork/tests
 
   build:
     # Either runner (host1 or host2): the build needs no docker socket.

crates/common/src/config/telemetry.rs

@@ -483,8 +483,16 @@ impl Tracers {
                 };
 
                 // Parse webhook events
+                // inbuxa: personal-data catalog, finding 1: an include list is
+                // sent as named; otherwise a webhook honors its level as a
+                // tracer does, and never sends a protocol's raw input or
+                // output (whole messages)
+                let level = Level::from(hook.level);
+                let named = (hook.events_policy == EventPolicy::Include)
+                    .then(|| hook.events.iter().copied().collect::<AHashSet<_>>())
+                    .unwrap_or_default();
                 apply_events(hook.events, hook.events_policy, |event_type| {
-                    if event_type != EventType::Telemetry(TelemetryEvent::WebhookError) {
+                    if webhook_wants(event_type, level, &custom_levels, &named) {
                         tracer.interests.set(event_type);
                         global_interests.set(event_type);
                     }
@@ -743,6 +751,31 @@ fn tracer_settings(tracer: &Tracer) -> u64 {
     settings_hash(&tracer)
 }
 
+/// inbuxa: whether a webhook at `level` receives this event type. Its own
+/// error event never, or a failing webhook would report itself to itself.
+/// An event `named` in an include list always: naming it is the choice.
+/// Otherwise (the exclude policy, the default) only events at or above its
+/// level, as for a tracer, and never a protocol's raw input or output, which
+/// carries whole messages and credentials.
+fn webhook_wants(
+    event_type: EventType,
+    level: Level,
+    custom_levels: &AHashMap<EventType, Level>,
+    named: &AHashSet<EventType>,
+) -> bool {
+    if event_type == EventType::Telemetry(TelemetryEvent::WebhookError) {
+        return false;
+    }
+    if named.contains(&event_type) {
+        return true;
+    }
+    let event_level = custom_levels
+        .get(&event_type)
+        .copied()
+        .unwrap_or(event_type.level());
+    level.is_contained(event_level) && !event_type.is_raw_io()
+}
+
 fn webhook_settings(hook: &WebHook) -> u64 {
     let mut hook = hook.clone();
     in_place_reset!(hook);
@@ -804,3 +837,61 @@ impl std::fmt::Debug for OtelMetrics {
             .finish()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use trc::{AuthEvent, SmtpEvent};
+
+    fn wants(event: EventType, level: Level, named: &[EventType]) -> bool {
+        webhook_wants(
+            event,
+            level,
+            &AHashMap::new(),
+            &named.iter().copied().collect(),
+        )
+    }
+
+    #[test]
+    fn a_webhook_honors_its_level() {
+        let success = EventType::Auth(AuthEvent::Success);
+        assert!(wants(success, Level::Info, &[]));
+        assert!(!wants(success, Level::Error, &[]), "info is below error");
+    }
+
+    #[test]
+    fn raw_io_goes_out_only_when_named() {
+        let raw = EventType::Smtp(SmtpEvent::RawInput);
+        assert!(raw.is_raw_io());
+        // Not with the exclude policy, even at trace
+        assert!(!wants(raw, Level::Info, &[]));
+        assert!(!wants(raw, Level::Trace, &[]));
+        // Named in an include list, whatever the level
+        assert!(wants(raw, Level::Info, &[raw]));
+    }
+
+    #[test]
+    fn a_named_event_is_sent_whatever_its_level() {
+        let start = EventType::Smtp(SmtpEvent::ConnectionStart);
+        assert!(!Level::Info.is_contained(start.level()), "below info");
+        assert!(!wants(start, Level::Info, &[]));
+        assert!(wants(start, Level::Info, &[start]));
+    }
+
+    #[test]
+    fn a_custom_level_counts() {
+        let start = EventType::Smtp(SmtpEvent::ConnectionStart);
+        let custom = [(start, Level::Info)].into_iter().collect::<AHashMap<_, _>>();
+        assert!(webhook_wants(start, Level::Info, &custom, &AHashSet::new()));
+        // Raw I/O raised to info still needs naming
+        let raw = EventType::Smtp(SmtpEvent::RawInput);
+        let custom = [(raw, Level::Info)].into_iter().collect::<AHashMap<_, _>>();
+        assert!(!webhook_wants(raw, Level::Info, &custom, &AHashSet::new()));
+    }
+
+    #[test]
+    fn a_webhook_never_hears_its_own_errors() {
+        let own = EventType::Telemetry(TelemetryEvent::WebhookError);
+        assert!(!wants(own, Level::Trace, &[own]));
+    }
+}

crates/common/src/manager/defaults.rs

@@ -14,7 +14,7 @@ use aws_lc_rs::{
 use registry::{
     schema::{
         enums::*,
-        prelude::{ObjectType, SocketAddr},
+        prelude::{Object, ObjectType, SocketAddr},
         structs::*,
     },
     types::{duration::Duration, error::Error, list::List, map::Map},
@@ -388,6 +388,28 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
         }
     }
 
+    // inbuxa: personal-data catalog, defaults D2, D3, D4 and D6 (settled
+    // 2026-09-28): privacy-leaning values, for new installs only. A server
+    // with roles is not new, and keeps its settings whether saved or left at
+    // the default. Each singleton is read, changed and written back whole, so
+    // anything already in it stays.
+    #[cfg(not(feature = "test_mode"))]
+    if bp.registry.count_object(ObjectType::Role).await? == 0 {
+        let mut security = bp.setting_infallible::<Security>().await;
+        let mut classifier = bp.setting_infallible::<SpamClassifier>().await;
+        let mut pyzor = bp.setting_infallible::<SpamPyzor>().await;
+        let mut retention = bp.setting_infallible::<DataRetention>().await;
+        new_install_privacy_defaults(&mut security, &mut classifier, &mut pyzor, &mut retention);
+        for object in [
+            Object::from(security),
+            classifier.into(),
+            pyzor.into(),
+            retention.into(),
+        ] {
+            bp.registry.write(RegistryWrite::insert(&object)).await?;
+        }
+    }
+
     if bp.registry.count_object(ObjectType::Role).await? == 0 {
         let permissions = DefaultPermissions::default();
         let mut role_ids = Vec::with_capacity(4);
@@ -560,3 +582,81 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
 
     Ok(())
 }
+
+/// inbuxa: the new-install values of defaults D2, D3, D4 and D6 from the
+/// personal-data catalog spec. Automatic IP bans expire after 30 days instead
+/// of never; spam training samples are kept 90 days instead of 180; Pyzor,
+/// which sends a digest of each message's text to a public server, is off;
+/// delivery history is kept 14 days instead of 30.
+fn new_install_privacy_defaults(
+    security: &mut Security,
+    classifier: &mut SpamClassifier,
+    pyzor: &mut SpamPyzor,
+    retention: &mut DataRetention,
+) {
+    const DAY: u64 = 24 * 60 * 60 * 1000;
+    let ban_period = Some(Duration::from_millis(30 * DAY));
+    security.auth_ban_period = ban_period;
+    security.abuse_ban_period = ban_period;
+    security.loiter_ban_period = ban_period;
+    security.scan_ban_period = ban_period;
+    classifier.hold_samples_for = Duration::from_millis(90 * DAY);
+    pyzor.enable = false;
+    retention.hold_traces_for = Some(Duration::from_millis(14 * DAY));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const DAY: u64 = 24 * 60 * 60 * 1000;
+
+    #[test]
+    fn new_installs_get_the_privacy_defaults() {
+        let (mut security, mut classifier, mut pyzor, mut retention) = (
+            Security::default(),
+            SpamClassifier::default(),
+            SpamPyzor::default(),
+            DataRetention::default(),
+        );
+        // What an install gets without them: bans that never lift, 180-day
+        // samples, Pyzor on, 30-day traces.
+        assert_eq!(security.auth_ban_period, None);
+        assert!(pyzor.enable);
+
+        new_install_privacy_defaults(&mut security, &mut classifier, &mut pyzor, &mut retention);
+
+        for period in [
+            security.auth_ban_period,
+            security.abuse_ban_period,
+            security.loiter_ban_period,
+            security.scan_ban_period,
+        ] {
+            assert_eq!(period.map(|p| p.into_inner().as_millis() as u64), Some(30 * DAY));
+        }
+        assert_eq!(classifier.hold_samples_for.into_inner().as_millis() as u64, 90 * DAY);
+        assert!(!pyzor.enable);
+        assert_eq!(
+            retention.hold_traces_for.map(|p| p.into_inner().as_millis() as u64),
+            Some(14 * DAY)
+        );
+    }
+
+    #[test]
+    fn everything_else_in_the_settings_stays() {
+        let mut retention = DataRetention {
+            archive_deleted_items_for: Some(Duration::from_millis(7 * DAY)),
+            ..Default::default()
+        };
+        let before = retention.clone();
+        new_install_privacy_defaults(
+            &mut Security::default(),
+            &mut SpamClassifier::default(),
+            &mut SpamPyzor::default(),
+            &mut retention,
+        );
+        assert_eq!(retention.archive_deleted_items_for, before.archive_deleted_items_for);
+        assert_eq!(retention.hold_metrics_for, before.hold_metrics_for);
+        assert_eq!(retention.expunge_trash_after, before.expunge_trash_after);
+    }
+}

crates/common/src/network/security.rs

@@ -426,6 +426,37 @@ impl Server {
     }
 }
 
+impl Server {
+    /// inbuxa: personal-data catalog, D2: removes bans whose period is over.
+    /// They already stop blocking when they expire, and go when settings are
+    /// next loaded; the daily clean-up makes sure a server that seldom
+    /// reloads doesn't keep them.
+    pub async fn purge_expired_blocked_ips(&self) -> trc::Result<()> {
+        let now = now() as i64;
+        let mut expired = Vec::new();
+        for ip in self.registry().list::<BlockedIp>().await? {
+            if ip.object.expires_at.as_ref().is_some_and(|at| at.timestamp() <= now) {
+                let address = ip.object.address.clone();
+                let object = Object {
+                    inner: ip.object.into(),
+                    revision: ip.revision,
+                };
+                self.registry()
+                    .write(RegistryWrite::delete_object(ip.id, &object))
+                    .await?;
+                expired.push(trc::Value::from(address.into_inner().0));
+            }
+        }
+        if !expired.is_empty() {
+            trc::event!(
+                Security(trc::SecurityEvent::IpBlockExpired),
+                Details = expired
+            );
+        }
+        Ok(())
+    }
+}
+
 impl BlockedIps {
     pub async fn parse(bp: &mut Bootstrap) -> Self {
         let mut ips = Self::default();

crates/registry/src/schema/structs_impl.rs

@@ -47532,7 +47532,9 @@ impl Default for WebHook {
             level: TracingLevel::Info,
             lossy: false,
             events: Default::default(),
-            events_policy: EventPolicy::Exclude,
+            // inbuxa: personal-data catalog, D7: a new webhook sends nothing
+            // until its events are chosen
+            events_policy: EventPolicy::Include,
         }
     }
 }

crates/services/src/task_manager/maintenance.rs

@@ -269,6 +269,11 @@ async fn store_maintenance(
                 trc::error!(err.details("Failed to re-apply account locks"));
             }
 
+            // inbuxa: personal-data catalog, D2: bans past their period go
+            if let Err(err) = server.purge_expired_blocked_ips().await {
+                trc::error!(err.details("Failed to purge expired IP bans"));
+            }
+
             // inbuxa: AU-7: audit records past their retention go; a
             // failure leaves them for the next run
             if let Err(err) = server.audit_purge().await {

docs/spec/features/personal-data-catalog.md

@@ -0,0 +1,647 @@
+# Feature spec: personal-data catalog, compliance role, and the Compliance section's first pages
+
+Status: approved 2026-09-28, with the answers under [Settled](#settled).
+Phase 1 of the GDPR auditor foundation: the investigation and the design. Not a
+rebuild of an upstream feature, so it has no line in SPEC.md §4's table.
+
+## Provenance
+
+Written for the record SPEC.md §3 rule 3 asks for. Sources, and nothing else:
+
+| Source | License | Used for |
+|---|---|---|
+| This repository at `de275ba` (2026-09-28): the code, `resources/schema/schema.json.gz`, `tools/fork/`, `.gitea/workflows/` | AGPL-3.0-only | Every claim in the source map: each row names the code that writes the data |
+| inbuxa-admin (the console) at `8e281e5` | AGPL-3.0-only | How navigation, hand-built pages and visibility work |
+| docs.inbuxa.org (the inbuxa.org repository at `6b12467`) | Own | What the documentation says, for "Contradictions" |
+| `inbuxa-drafts/specs/audit-hold-lock.md`, `legacy-protocols.md` | Own | The Compliance pages that already exist, and the permission-rollout notes |
+| Regulation (EU) 2016/679 (GDPR), Articles 5, 25, 30 and 32 | Public law | The shape of a record of processing: categories, whose data, where, how long, who receives it |
+
+No Enterprise-only file or snippet was used. Claims were checked by reading the
+code; where a search found nothing, the row says so rather than asserting a
+negative.
+
+## What it is
+
+Three things, the foundation for a GDPR auditor:
+
+1. **A personal-data catalog.** Every place the server can store or send
+   personal data, with its categories, whose data it is, where it lives, what
+   bounds its retention, and the settings that control it. What a particular
+   server holds depends on how its operator configured it, so the catalog
+   describes *possibilities*; the server evaluates it against live settings to
+   say what *this* server does (Phase 3).
+2. **A compliance role** that can see and act on compliance data without being
+   able to change server settings.
+3. **Two console pages** under Compliance: **Overview** and **Data
+   inventory**.
+
+The catalog records facts and the auditor reports findings. Neither judges:
+nothing in code, docs, UI text or output claims the server meets a legal
+standard, or that one is guaranteed. Nothing here changes a default; proposed
+changes are listed under [Default profile](#default-profile) for John to
+decide.
+
+**Out of scope** (their own specs): retention policies, data subject requests,
+the guided questionnaire, jurisdiction packs, mobile and ihasmail screens.
+Legal hold and the audit log are built already (audit-hold-lock spec, phases
+1–4); this spec places them in the Compliance navigation and gives the
+compliance role access to them, nothing more.
+
+## 1. Vocabulary
+
+**Categories** (fixed; a source may have several):
+
+| Category | Means |
+|---|---|
+| `identifier` | Names, email addresses, account and principal ids, login names |
+| `contact` | Phone numbers, postal addresses, vCard details, contact addresses given for an object (an OAuth client's contacts) |
+| `network` | IP addresses and ports, client hostnames (EHLO, PTR), user agents, device push URLs |
+| `content` | Message bodies, subjects, attachments, events, contacts' cards, files, scripts, free-text descriptions and reasons |
+| `metadata` | Timestamps, sizes, flags, envelopes, delivery status, message-ids, counts |
+| `credential` | Passwords and their hashes, tokens, keys, secrets |
+
+**Whose** (data subjects): `holder` (the account holder), `correspondent`
+(people who send to or receive from the server, and people in contact cards
+and events), `administrator` (people acting on the server).
+
+**Where** (location): `data-store`, `blob-store`, `search-store`,
+`in-memory-store` (the configured one; the data store by default),
+`memory` (process RAM, per node), `log-file`, `external` (sent to an
+endpoint). **Leaves the host** is a separate column: a store is local unless
+the operator points it at a remote backend.
+
+**Scope**: `tenant` (attributable to a tenant through the account or domain),
+`server` (server-wide only).
+
+## 2. Source map
+
+Defaults are those of a new install: the struct defaults, plus what first boot
+inserts (`crates/common/src/manager/defaults.rs`). "Unbounded" means no setting
+or process removes the data on a schedule.
+
+### 2.1 Mail, groupware and user content
+
+| Source | Categories | Whose | Controlled by | Retention bound | Where / leaves host | Scope | Written by | Default |
+|---|---|---|---|---|---|---|---|---|
+| Emails, mailboxes, threads | content, identifier, metadata | holder, correspondent | always on; `x:Email.encryptAtRest`, `x:Email.maxMessages` | unbounded until deleted; Trash emptied after `x:DataRetention.expungeTrashAfter` | data + blob store; leaves when sent | tenant | `email/src/message/ingest.rs` `email_ingest`; `delivery.rs` `deliver_message`; `message/delete.rs` `emails_delete`, `emails_auto_expunge` | collected; Trash 30 d |
+| Email submissions | metadata (envelopes) | holder, correspondent | always on | `x:DataRetention.expungeSubmissionsAfter` | data store | tenant | `message/delete.rs` `purge_email_submissions` (purge) | 3 d |
+| Full-text index: mail | content, identifier, metadata | holder, correspondent | `x:Search.indexEmail`, `indexEmailFields` | follows the item | search store (external if Elasticsearch/Meilisearch) | tenant | `services/src/task_manager/index.rs` `build_email_document` | indexed, all fields |
+| Calendars, events, iTIP | content, identifier, metadata | holder, correspondent | `x:CalendarScheduling.enable`, `httpRsvpEnable`; `x:CalendarAlarm.enable` | unbounded; scheduling inbox `expungeSchedulingInboxAfter`, share notices `expungeShareNotifyAfter` | data + blob store; **iMIP mails event details to external attendees** | tenant | `dav/src/calendar/update.rs`; `jmap/src/calendar_event/set.rs`; `groupware/src/calendar/storage.rs`; `services/src/task_manager/imip.rs` | collected; 30 d / 30 d |
+| Contacts, address books | contact, identifier | correspondent, holder | always on | unbounded | data store | tenant | `dav/src/card/update.rs`; `jmap/src/contact/set.rs`; `groupware/src/contact/storage.rs` | collected |
+| Full-text index: calendars, contacts | content, contact, identifier | correspondent, holder | `x:Search.indexCalendar`/`Fields`, `indexContacts`/`Fields` | follows the item | search store | tenant | `index.rs` `build_calendar_document`, `build_contact_document` | indexed |
+| Files (FileNode, WebDAV) | content, metadata | holder | `x:FileStorage.maxSize`, `maxFiles` | unbounded | data + blob store | tenant | `dav/src/file/update.rs`; `jmap/src/file/set.rs`; `groupware/src/file/storage.rs` | collected |
+| Sieve scripts, vacation responses | content | holder (scripts may name correspondents) | user-set | unbounded | data + blob store | tenant | `jmap/src/sieve/set.rs`; `managesieve/src/op/putscript.rs`; `jmap/src/vacation/set.rs` | when set |
+| Sieve "seen" ids (vacation, duplicate) | identifier (blake3 hash) | correspondent | used only by scripts that call vacation/duplicate | the script's TTL | in-memory store | tenant | `email/src/sieve/ingest.rs` | when used |
+| Identities | identifier, content (signatures) | holder | `x:Email.maxIdentities` | unbounded | data store | tenant | `jmap/src/identity/set.rs` | collected |
+| Push subscriptions | network (push URL), credential (keys); **EmailPush can carry email properties to the push URL** | holder | client opt-in; `x:Jmap.maxSubscriptions` | expire within 7 days | data store → **external push service** | tenant | `jmap/src/push/set.rs`; `services/src/state_manager/http.rs` | client-driven |
+
+### 2.2 Records kept by inbuxa's own features
+
+These live in inbuxa's own key space (`SUBSPACE_INBUXA`) unless noted.
+`Store::danger_destroy_account` doesn't clear that space, so a row's
+"survives account deletion" note matters.
+
+| Source | Categories | Whose | Controlled by | Retention bound | Where | Scope | Written by | Default |
+|---|---|---|---|---|---|---|---|---|
+| Archived items (undelete) | content, identifier, metadata | holder, correspondent | `x:DataRetention.archiveDeletedItemsFor`; a legal hold forces keeping | each item's `archivedUntil`; **held: year 9999** | registry (data store) + blob store | tenant | `features/src/undelete/records.rs` `insert`, `set_deadline`, `remove_expired`; `undelete/email.rs` `note`; `groupware/src/inbuxa.rs`; `services/.../index.rs` `archive_noted` | **off** |
+| Deleted accounts kept | identifier (name, addresses), **credential (the whole account record, password hashes included)**, metadata | holder | `x:DataRetention.archiveDeletedAccountsFor`; a hold forces keeping | `kept_until`; held: year 9999 | data store | tenant | `jmap/src/inbuxa/deleted_account.rs` `keep`; `features/src/undelete/data.rs` `set_kept_account` | **off** |
+| Legal holds | identifier, content (reference, description, reasons) | administrator, holder | `sysLegalHold*` | **unbounded, by design** | data store | server (scope may name tenants) | `features/src/hold/mod.rs` `create`, `update`, `pin_account`, `keep_moved` | none |
+| Hold exports: records | identifier, content (reason), metadata | administrator | `sysLegalHoldExport` | **unbounded, by design** | data store | server | `hold/mod.rs` `create_export`, `update_export` | none |
+| Hold exports: ZIP | content (everything the hold covers) | holder, correspondent | as above | `x:Jmap.uploadTtl`, then the next blob purge (`blobCleanupSchedule`) | blob store; **leaves when downloaded** | server | `jmap/src/inbuxa/hold_export.rs` `build` → `put_jmap_blob` | 1 h + up to a day |
+| Account locks, delegations | identifier, content (reason), metadata | holder, administrator | `sysAccountLock*` | until the lock is removed; **no removal on account deletion found** | data store | tenant | `features/src/lock/mod.rs` `set`, `remove` | none |
+| Audit log | identifier, **network (sign-in IPs)**, content (setting diffs, secrets redacted; reasons) | administrator, holder (as target), correspondent (e.g. banned IPs) | always on; `inbuxa:AuditSettings.keepForDays` (not a registry setting) | `keepForDays`, minimum 90; records about held accounts kept while held; **kept after account deletion** | data store (hash-chained) | tenant-filterable | `common/src/audit.rs` `audit_append`, `audit_sign_in`, `audit_sign_in_failed`, `audit_foreign_access`, `audit_delegate`; `jmap/src/inbuxa/audit.rs` `recorded`; purge `audit_purge` | **collected, 730 d** |
+| Audit exports | as the audit log | as above | `sysAuditExport` | `x:Jmap.uploadTtl`, then the next blob purge | blob store; leaves when downloaded | server or tenant | `jmap/src/inbuxa/audit_log.rs` `build_export` | 1 h + up to a day |
+| Masked addresses | identifier, content (description, site), metadata (last message) | holder | `x:Email.maxMaskedAddresses` | the registry object goes with the account; **the address tombstone is kept forever by design**; its `Mm`/`Mc` records aren't cleared on deletion (search found no clearing) | registry + data store | tenant | `features/src/masked_email/ops.rs`; `data.rs` `set_address`, `set_record`, `log_change` | when used (5 max) |
+| Legacy-protocol last use | metadata (one timestamp per account and protocol; no IP) | holder | none | **unbounded; not cleared on account deletion** | data store | tenant | `features/src/security/legacy_use.rs` `record` (from `common/src/network/legacy.rs` `admit_legacy_session`) | **collected** |
+
+### 2.3 Mail transport and reports
+
+| Source | Categories | Whose | Controlled by | Retention bound | Where / leaves host | Scope | Written by | Default |
+|---|---|---|---|---|---|---|---|---|
+| Queue (`x:QueuedMessage`) | content, metadata, network (`receivedFromIp`) | holder, correspondent | always on | delivered, or `x:MtaDeliveryExpirationTtl.expire` | data + blob store; delivered out | server | `smtp/src/queue/spool.rs` `queue`, `save_changes`, `remove` | 3 d max |
+| DSNs | content (**original headers**), metadata | correspondent, holder | `x:DsnReportSettings` | sent through the queue | **leaves** | server | `smtp/src/queue/dsn.rs` `send_dsn` | on |
+| Received DMARC, TLS-RPT, ARF reports | network (source IPs), identifier (envelope/header from); ARF: **message and headers**, original recipient, user agent | correspondent | `x:ReportSettings.inboundReportAddresses`, `inboundReportForwarding` | `x:DataRetention.holdMtaReportsFor` | data store; also forwarded to the mailbox | tenant (by domain) | `smtp/src/reporting/analysis.rs` | collected, 30 d |
+| DMARC aggregate reports we send | network (source IPs), identifier (domains) | correspondent | `x:DmarcReportSettings.aggregateSendFrequency`; only to domains publishing `rua` | kept until sent | data store → **external `rua`** | server | `smtp/src/reporting/dmarc.rs` | daily |
+| DMARC/SPF/DKIM failure reports we send | content (**raw headers**), network, identifier | correspondent | `x:DmarcReportSettings.failureSendFrequency`, `x:SpfReportSettings`, `x:DkimReportSettings`; only when requested | through the queue | **external `ruf`** | server | `reporting/dmarc.rs` `send_dmarc_report`; `spf.rs`; `dkim.rs` | on, rate-limited |
+| TLS-RPT we send | network (remote MX hosts) | operators of other servers | `x:TlsReportSettings.sendFrequency` | until sent (deletion path not confirmed) | **external** | server | `smtp/src/reporting/tls.rs` | daily |
+| Relay (smart host) | everything in the message | holder, correspondent | `x:MtaRoute` (Relay) | receiver's | **external** | server | `smtp/src/outbound` | none |
+| Milters, MTA hooks | network (client IP, PTR, HELO), identifier (SASL login), **full message** | holder, correspondent | `x:MtaMilter`, `x:MtaHook` | receiver's | **external** | server | `smtp/src/inbound/milter`, `inbound/hooks/mod.rs` | none |
+
+### 2.4 Telemetry and logs
+
+| Source | Categories | Whose | Controlled by | Retention bound | Where / leaves host | Scope | Written by | Default |
+|---|---|---|---|---|---|---|---|---|
+| Stored traces (`x:Trace`) | network (IP, EHLO), identifier (envelope addresses, account names), metadata (message-ids, results) | correspondent, holder | `x:TracingStore`; `x:EventTracingLevel` | `x:DataRetention.holdTracesFor` (unset = unbounded) | data store, or a remote PostgreSQL/MySQL/FDB | server (records name accounts) | `common/src/telemetry/tracers/store.rs` `spawn_store_tracer`; purge `purge_spans` | **collected** (first boot inserts `TracingStore::Default` although the struct default is off); 30 d |
+| Trace search index | same keywords, lowercased | as above | `x:Search.indexTelemetry`, `indexTracingFields` | purged with the traces | search store | server | `index.rs` `trace_search_document` | indexed |
+| Stored metrics | none personal (aggregates) | — | `x:MetricsStore`, `x:Metrics` | `holdMetricsFor` | data store | server | `telemetry/metrics/store.rs` | collected, 90 d |
+| Log file | network (**client IP on every in-session line**, from the span), identifier (MAIL FROM, RCPT TO, account names), metadata (message-ids). No subject key exists; bodies only in trace-level raw-I/O events | correspondent, holder, administrator | `x:TracerLog` (`level`, `events`, `path`, `rotate`) | **unbounded: rotated daily, never deleted** | log file (`/var/log/inbuxa`); read back as `x:Log` and fed to Explain | server | `common/src/telemetry/tracers/log.rs` | **collected, info level** |
+| Console / journald tracers | as the log file | as above | `x:Tracer` Stdout/Journal | the host's journal policy | stdout / journald (leaves only if the host forwards it) | server | `tracers/stdout.rs`, `journald.rs` | none (recovery mode adds a console tracer) |
+| OpenTelemetry tracer | as the log file, every event and span key | as above | `x:Tracer` OtelHttp/OtelGrpc (`level`, `events`, `endpoint`) | receiver's | **external** | server | `tracers/otel.rs` | none |
+| Webhooks | as the log file; **can include raw SMTP input (message content) and model replies** — see [Findings](#findings) 1 | as above | `x:WebHook` (`events`, `eventsPolicy`, `url`) | receiver's; retried up to `discardAfter` | **external** | server | `telemetry/webhooks/mod.rs` `post_webhook_events` | none |
+| Alerts | contact (the addresses an administrator sets) | administrator | `x:Alert` | as mail | mail; may leave | server | `telemetry/alerts.rs` | none |
+| Failed tasks | identifier (account names, e.g. a failed account destroy), content (iTIP messages) | holder, correspondent | none | **not confirmed: failed tasks are rescheduled at `u64::MAX` and no purge was found** | data store | server | `services/src/task_manager/manager.rs` | when a task fails |
+
+### 2.5 Sign-in and network defence
+
+| Source | Categories | Whose | Controlled by | Retention bound | Where | Scope | Written by | Default |
+|---|---|---|---|---|---|---|---|---|
+| Accounts, groups, lists, domains, tenants (registry) | identifier, contact, metadata | holder | registry | the object's life | data store | tenant | registry writes (`jmap/src/registry/`) | — |
+| Passwords, app passwords, API keys | credential (argon2id hashes), network (`allowedIps`), metadata | holder | `x:Authentication.passwordHashAlgorithm`; `maxAppPasswords`, `maxApiKeys`; `expiresAt` | the account's life or `expiresAt` | data store | tenant | `jmap/src/registry/mapping/account.rs` | — |
+| OAuth clients | identifier, contact (`contacts`), credential (secret hash) | administrator, third-party developer | `x:OidcProvider.requireClientRegistration` | `expiresAt` or unbounded | data store | tenant | `http/src/auth/oauth/registration.rs` | none |
+| OAuth codes and tokens | credential | holder | `x:OidcProvider.*Expiry` | tokens are sealed and stateless (not stored); codes in the in-memory store with TTL | in-memory store | tenant | `http/src/auth/oauth/auth.rs`, `token.rs` | codes 10 min |
+| Rate-limit state | network, identifier (login names) | holder, correspondent | `x:Http.rateLimit*`, `x:Imap.maxRequestRate`, `x:Security.*BanRate` | the rate's period | in-memory store | server | `common/src/auth/rate_limit.rs`, `network/security.rs` | on |
+| Greylist | identifier (sender/recipient pairs, plain) | correspondent, holder | `x:SpamSettings.greylistFor` | that period | in-memory store | server | `smtp/src/inbound/rcpt.rs` | **off** |
+| Automatic IP bans (`x:BlockedIp`) | network | correspondent, holder | `x:Security.authBanRate`, `abuseBanRate`, `loiterBanRate`, `scanBanRate` (on); `*BanPeriod` (**no default**) | **unbounded: a ban with no period never expires**; an expired ban's record goes when settings next load; each ban is also an audit record | data store (registry) | server | `common/src/network/security.rs` `block_ip` | **collected, permanent** |
+| Allowed IPs | network | administrator's choice | manual; `expiresAt` | optional | data store | server | registry | none |
+
+### 2.6 Spam filter and AI
+
+| Source | Categories | Whose | Controlled by | Retention bound | Where / leaves host | Scope | Written by | Default |
+|---|---|---|---|---|---|---|---|---|
+| Training samples (`x:SpamTrainingSample`) | content (**the whole message, pinned**), identifier (`from`), content (`subject`) | correspondent, holder | `x:SpamClassifier.model`, `learnSpamFromTraps`, `learnSpamFromRblHits`, `learnHamFromCard`, `learnHamFromReply`; users' Junk moves | `x:SpamClassifier.holdSamplesFor`; **outlives the user deleting the mail** | data + blob store | tenant for users' samples; **server, unattributed, for SMTP autolearn** | `email/src/message/ingest.rs` `add_spam_sample`; `smtp/src/queue/spool.rs` | **collected, 180 d** |
+| Trainer state (`INBUXA_SPAM_TRAIN_DATA`) | metadata (blob hashes with account ids; hashed tokens that include addresses and domains) | correspondent, holder | as above | **unbounded (overwritten on training, never expired)** | blob store | server | `spam-filter/src/modules/classifier.rs` `spam_train` | after 100 + 100 samples |
+| Trained model | hashed weights, not directly personal | — | as above | unbounded | blob store | server | same | same |
+| AI spam classification | content (subject + up to `inbuxa:AiLimits.maxContentBytes` of text) | correspondent | `x:SpamLlm`, `x:AiModel` (`url`), `inbuxa:AiLimits` | the endpoint's; the verdict is written into the stored message | **external endpoint** (local or hosted; a remote URL only warns) | server | `spam-filter/src/analysis/llm.rs`; `common/src/enterprise/llm.rs` | **off** |
+| Sieve `llm_prompt` | whatever a script sends | any | `x:AiModel`; `interactAi` | endpoint's | **external** | per account | `enterprise/llm.rs` `sieve_prompt` | no model |
+| Explain | identifier (return path, recipients), network (trace IPs), metadata (remote replies, log details); never contents or raw I/O | correspondent, holder | `inbuxa:AiLimits.explainEnabled`, `explainModelId` | answer cache: memory, per node, 24 h, 1000 answers | **external endpoint** + memory | server | `jmap/src/inbuxa/explanation.rs`; `features/src/ai/explain/memory.rs` | no model |
+| DNSBL lookups | network (client IPs), identifier (domains; **SHA-1 of addresses** to msbl.org), metadata (MD5 of URLs) | correspondent | bundled `x:SpamDnsblServer` rules; `x:SpamDnsblSettings` | receiver's | **external (DNS)** | server | spam-filter DNSBL module | **on: 17 of 18 bundled lists** |
+| Pyzor | content (SHA-1 digest of normalized body) | correspondent | `x:SpamPyzor` | receiver's | **external: `public.pyzor.org`** | server | `spam-filter/src/modules/pyzor.rs` | **on** |
+| URL-shortener expansion | network (fetches links, which can carry per-recipient tracking tokens) | correspondent, holder | bundled redirector list | — | **external (HTTP)** | server | `spam-filter/src/analysis/url.rs` | **on** |
+
+### 2.7 Stores and directories an operator can point elsewhere
+
+| Registry object | Receives | Default |
+|---|---|---|
+| `x:DataStore` | everything above that says data store | RocksDB, local |
+| `x:BlobStore` (S3, Azure, FS, SQL, FDB) | message bodies, attachments, files, samples, exports | the data store |
+| `x:SearchStore` (Elasticsearch, Meilisearch, SQL, FDB) | full text of mail, contacts, calendars; trace keywords | the data store |
+| `x:InMemoryStore` (Redis) | rate limits, greylist, OAuth codes, Sieve seen ids | the data store |
+| `x:TracingStore`, `x:MetricsStore` | traces, metrics | the data store |
+| `x:Coordinator` (Kafka, NATS, Zenoh, Redis) | cluster broadcasts (whether personal: not confirmed) | off |
+| `x:Directory` (LDAP, SQL, OIDC) | login names, passwords to verify, address lookups | internal |
+| `x:AcmeProvider`, `x:DnsServer` | hostnames, a role address; no personal data found | Let's Encrypt if chosen; manual DNS |
+
+## Findings
+
+Facts from the investigation that matter beyond the catalog. Each is a finding,
+not a judgment; what to do about each is John's call.
+
+1. **A webhook ignores levels and, left at its defaults, receives every
+   event, message content included.** `config/telemetry.rs` builds a webhook's
+   interests with `apply_events` alone: no level check. With the default
+   `eventsPolicy` (exclude) and no events listed, that is every event type,
+   including trace-level `smtp.raw-input` (the raw SMTP bytes, `DATA`
+   included) and `ai.llm-response`. The audit-log docs suggest a webhook to
+   pass records to a SIEM. Probably upstream behavior; not yet checked
+   against upstream.
+2. **Log files are never deleted.** Daily rotation opens a new file; nothing
+   removes old ones, and no logrotate configuration ships. At the default
+   level every in-session line carries the client IP.
+3. **Automatic IP bans are permanent.** No `*BanPeriod` has a default, so a
+   ban never expires. (Corrected 2026-09-28: a ban that does expire stops
+   blocking, and its record is deleted when settings are next loaded, in
+   `BlockedIps::parse`; the investigation missed that path.)
+4. **Some records outlive the account.** Deleting an account doesn't clear
+   inbuxa's own key space: legacy-protocol last use, account locks, masked
+   address records and audit records stay (audit records by design).
+5. **A kept deleted account keeps its password hashes** for
+   `archiveDeletedAccountsFor`, or indefinitely under a legal hold.
+6. **Spam training keeps whole messages for 180 days**, after the user
+   deleted them, and SMTP autolearn keeps them with no account attribution, so
+   they can't be found by person. The trainer state never expires.
+7. **Traces are on in a new install** although the struct default says off:
+   first boot inserts the tracing and metrics stores.
+8. **Data leaves the host by default** through the spam filter (DNSBL queries
+   with IPs, domains and hashed addresses to 17 lists; Pyzor body digests;
+   shortener URL fetches) and through reports (DMARC aggregate daily; failure
+   reports with headers where requested). No AI, webhook, OpenTelemetry,
+   milter, hook or relay endpoint is configured by default.
+9. **Export files outlive their stated lifetime by up to a day.** The link
+   expires with `uploadTtl`; the bytes stay until the next blob purge
+   (04:00 daily).
+10. **Failed tasks may stay forever** (rescheduled at `u64::MAX`; no purge
+    found), holding account names or iTIP messages. Not confirmed by a test.
+
+## 3. How the schema is produced, and what the catalog works from
+
+The prompt says not to hand-edit generated files and to work from the source
+of truth. What the investigation found:
+
+- **`resources/schema/schema.json.gz` and the registry Rust code
+  (`crates/registry/src/schema/*.rs`, headed "auto-generated") come from
+  upstream's generator, which isn't in this repository or upstream's public
+  tree.** Nothing here regenerates them.
+- **The fork edits both by hand.** Since the v0.16.22 import the gz has
+  changed in 18 commits: the Compliance layout (Audit Log, Legal Holds, Locked
+  Accounts), the Local AI and Legacy Protocols pages, the fork's permissions
+  (`sysAudit*`, `sysAccountLock*`, `sysLegalHold*`, `sysAiExplain`), always
+  paired with `enums.rs`/`enums_impl.rs` edits marked `// inbuxa:`. Scripted
+  edits go through `tools/fork/renames.py` (`schema_bytes`), which writes the gz
+  with `gzip.compress(compresslevel=9, mtime=0)` and the `.sha256` sidecar as
+  unpadded URL-safe base64 of the gz's SHA-256.
+- **`/api/schema`** (`crates/http/src/api/mod.rs`, `"schema"` arm) serves the
+  embedded gz as-is to any authenticated caller, unfiltered; the console
+  hides what a person can't use from the permission list `/api/account`
+  returns.
+- **The schema carries 150 objects and 2,102 properties** (316 field
+  schemas). About 34 objects hold data about people, roughly 150–180
+  properties counting nested structs; about 20 configuration objects carry
+  about 100 credential or contact properties. inbuxa's own objects
+  (`inbuxa:AuditEvent`, `LegalHold`, `HoldExport`, `AccountLock`,
+  `DeletedAccount`, `ProtocolPolicy`, `TenantProtocolPolicy`,
+  `Explanation`, `AiLimits`, Fastmail `MaskedEmail`) aren't in the schema
+  at all: they are defined in `crates/jmap-proto/src/object/inbuxa_*.rs`.
+
+So the source of truth for *what exists* is the schema (for registry objects)
+and `jmap-proto` (for inbuxa's own). Neither should carry the catalog:
+**Decision (Settled 1):** the catalog is a sidecar, and the schema and generated
+code are read, never annotated, by it. The fork's existing hand edits to the
+schema (layout entries, permissions) continue as today for the Compliance
+pages and the new permissions, because the console's navigation comes from
+the schema and there is no other way in; each is marked in the commit and
+listed in the strip report as now.
+
+## 4. The catalog: format and location
+
+**Location:** `resources/privacy/catalog.toml`, a fork-owned file. The server
+embeds it (`include_str!`) and parses it at start, as it does the schema.
+
+**Why a sidecar** rather than annotating in place:
+
+- The schema and registry code are upstream's generated output; annotations
+  there are overwritten or conflict on every import (SPEC.md §2.2, §2.3).
+- Many sources aren't registry objects: log files, telemetry exporters,
+  inbuxa's key-space records, DNSBL and Pyzor, the in-memory cache. An
+  in-place annotation has nowhere to put them.
+- One file is one diff to review when an import brings new fields, and it is
+  what the CI check and the strip report read.
+
+**Shape.** Three kinds of entry:
+
+```toml
+# A registry object: a default for its properties, and the ones that differ.
+[object."x:Account"]
+default = "none"                  # properties not listed hold no personal data
+whose = ["holder"]
+where = ["data-store"]
+scope = "tenant"
+retention = "object-life"
+[object."x:Account".properties]
+name = ["identifier"]
+emailAddress = ["identifier"]
+aliases = ["identifier"]
+description = ["content"]
+credentials = ["credential", "network"]   # secrets, allowedIps
+locale = ["metadata"]
+
+# One of inbuxa's own JMAP objects (not in the schema).
+[object."inbuxa:AuditEvent"]
+default = "none"
+whose = ["administrator", "holder", "correspondent"]
+where = ["data-store"]
+scope = "tenant"
+retention = { setting = "inbuxa:AuditSettings.keepForDays" }
+[object."inbuxa:AuditEvent".properties]
+actor = ["identifier"]
+remoteIp = ["network"]
+changes = ["content"]
+reason = ["content"]
+
+# A source that is no object: files, exporters, lookups, caches.
+[source."log-file"]
+categories = ["network", "identifier", "metadata"]
+whose = ["correspondent", "holder", "administrator"]
+where = ["log-file"]
+scope = "server"
+enabled_by = ["x:TracerLog.enable"]
+captures = ["x:TracerLog.level", "x:TracerLog.events", "x:EventTracingLevel"]
+retention = "unbounded"
+leaves_host = false
+written_by = ["crates/common/src/telemetry/tracers/log.rs"]
+
+[source."spam-dnsbl"]
+categories = ["network", "identifier", "metadata"]
+whose = ["correspondent"]
+where = ["external"]
+scope = "server"
+enabled_by = ["x:SpamDnsblServer.enable"]
+retention = "receiver"
+leaves_host = true
+written_by = ["crates/spam-filter/src/modules/dnsbl.rs"]
+```
+
+Rules: `retention` is `unbounded`, `object-life`, `receiver` (it left the
+host), or `{ setting = "<object>.<property>" }`; every setting named is a real
+`x:` or `inbuxa:` property. Every object in the schema and every `inbuxa:`
+object has an entry, even if only `default = "none"` (configuration objects
+with nothing personal). The file holds facts only: no wording about legal
+status.
+
+**Evaluation (Phase 3).** Each entry's `enabled_by`, `retention` and
+`leaves_host` are evaluated against the live registry: a store pointed at a
+remote backend makes everything in it `leaves_host = true`; an unset
+retention setting reads `unbounded`; an external endpoint lists its URL's
+host as a candidate processor.
+
+## 5. The CI check
+
+`tools/fork/privacy-check.py`, modeled on `name-check.py` and
+`notice-check.py` (plain Python, no dependencies, exit 1 on findings). Wired
+into the `fork-checks` job in `.gitea/workflows/ci.yml`, beside them. It fails
+when:
+
+1. an object in `schema.json.gz` `fields`, or an `inbuxa:` object in
+   `crates/jmap-proto/src/object/`, has no catalog entry;
+2. a property whose schema format is `emailAddress`, `ipAddress`,
+   `ipNetwork`, `secret`/`secretText`, or whose type is `x:SecretKey*` or
+   `x:HttpAuth`, is covered only by an object's `default = "none"` — those
+   must be classified explicitly, so a new personal field can't hide behind a
+   default;
+3. an entry names an object, property or setting that no longer exists
+   (stale);
+4. an entry uses a category, subject or location outside the vocabulary.
+
+`tools/fork/strip.py` gains `privacy_flags(tree)`, beside `schema_flags`:
+objects and properties new in an import and not in the catalog, reported in
+`STRIP-REPORT.md` under "Unclassified in the privacy catalog", the way
+Enterprise flags are reported. Informational, as the Enterprise list is; CI
+is what fails.
+
+Tests (Phase 2): the check passes on main; fails on a synthetic unclassified
+field; fails on a synthetic `emailAddress` property covered only by a
+default; fails on a stale entry.
+
+## Default profile
+
+**Today's defaults in a new install** that collect or send personal data:
+
+| Source | Collected | Retention |
+|---|---|---|
+| Traces + trace index | yes | 30 days |
+| Log file | yes, info level | **unbounded** |
+| Audit log | yes | 730 days |
+| Spam training samples (whole messages) | yes | 180 days |
+| Spam trainer state | yes, after 200 samples | **unbounded** |
+| Automatic IP bans | yes | **unbounded** |
+| Legacy-protocol last use | yes | **unbounded** |
+| Received DMARC/TLS/ARF reports | yes | 30 days |
+| DNSBL lookups (17 lists), Pyzor, URL-shortener fetches | yes | leave the host |
+| Outbound DMARC aggregate and failure reports | yes, where the domain asks | leave the host |
+| Trash | yes | 30 days |
+| Undelete, deleted-account keeping | **off** | — |
+| AI classification, Explain | **off** (no model) | — |
+| Greylisting | **off** | — |
+
+**Changes for new installs only.** Settled (5): all seven are built, in
+Phase 3; existing servers keep their settings. D7 is also covered by the bug
+fix for finding 1 (Settled 6).
+
+**As built (2026-09-28).** D2, D3, D4, D6 are written on first boot of a new
+install only (no roles yet), each singleton read and written back whole
+(`manager/defaults.rs`, `new_install_privacy_defaults`); expired bans are
+also purged daily (`purge_expired_blocked_ips`). D7 changes the default for
+webhooks created from now on; stored webhooks keep theirs (the registry
+stores every field). **Settled (John, 2026-09-28):** D1 becomes a
+fork-owned log-retention setting, kept in inbuxa's own storage as audit
+retention is (not a field on `x:TracerLog`, which is also stored inside
+`x:Bootstrap` with fields after it, so a new field would change that
+object's stored format); new installs 30 days, existing servers keep every
+file as today. D5 is built after the v0.16.24 import lands, on its reworked
+spam-rules loader, which keeps each blocklist's on/off state.
+
+| # | Change | Trade-off |
+|---|---|---|
+| D1 | A **log retention** setting on `x:TracerLog` (delete rotated files older than N days), default 30 days for new installs | Needs code (a new field, so a schema edit); older logs gone for troubleshooting; operators wanting longer set it |
+| D2 | A default **ban period**, e.g. 30 days, for the four ban rates, and a purge of expired `x:BlockedIp` | A persistent attacker is re-banned after expiry rather than kept out; permanent bans of shared/NAT addresses stop being permanent |
+| D3 | **Spam sample retention** 180 → 90 days | Fewer samples to retrain from; the classifier is retrained regularly, so the effect on accuracy is likely small (not measured) |
+| D4 | **Pyzor off** by default | One fewer spam signal; no body digests leave the host |
+| D5 | **msbl.org EBL off** by default (the list that receives hashed addresses) | One fewer signal on address-based spam; no hashed addresses leave the host |
+| D6 | **Trace retention** 30 → 14 days | Shorter delivery history in **Emails › History** and for support |
+| D7 | Webhooks default to `eventsPolicy = include` with no events, and never receive raw-I/O events unless listed by name | A webhook does nothing until events are chosen; closes finding 1 |
+
+Separately, **not defaults but gaps** a later spec should close (listed so
+they aren't lost): clearing inbuxa's own records on account deletion
+(finding 4); an expiry for the spam trainer state; a purge for failed tasks
+(finding 10, once confirmed); removing export blobs when their link expires
+(finding 9).
+
+## 6. The inventory, over JMAP (Phase 3)
+
+Under `urn:inbuxa:jmap`, read-only.
+
+### `inbuxa:DataInventory/get`
+
+A singleton, evaluated on request.
+
+Request: `{ accountId, ids: null | ["singleton"], properties? }`.
+
+Response `list[0]`:
+
+```json
+{
+  "id": "singleton",
+  "evaluatedAt": "2026-09-28T10:00:00Z",
+  "catalogVersion": "2026.9.28.3",
+  "sources": [
+    {
+      "id": "log-file",
+      "kind": "source",
+      "categories": ["network", "identifier", "metadata"],
+      "whose": ["correspondent", "holder", "administrator"],
+      "where": ["log-file"],
+      "collected": true,
+      "retention": { "unbounded": true },
+      "leavesHost": false,
+      "scope": "server",
+      "controlledBy": ["x:TracerLog.enable", "x:TracerLog.level"],
+      "endpoints": []
+    },
+    {
+      "id": "x:Trace",
+      "kind": "object",
+      "categories": ["network", "identifier", "metadata"],
+      "collected": true,
+      "retention": { "days": 30, "setting": "x:DataRetention.holdTracesFor" },
+      "leavesHost": false,
+      "scope": "server",
+      "endpoints": []
+    }
+  ],
+  "processors": [
+    { "host": "public.pyzor.org", "receives": ["content"], "sources": ["spam-pyzor"] }
+  ]
+}
+```
+
+`collected` is false when the source's switch is off; `retention` reads the
+live setting; `leavesHost` is true when the source's location is a remote
+store or an external endpoint; `processors` lists each external host once,
+with what it receives — candidates, since whether a host is a processor in
+law is the operator's determination.
+
+**Permission:** `sysComplianceGet`. **Tenant scoping:** inside a tenant,
+`sources` holds only `scope = "tenant"` entries, evaluated with the tenant's
+own settings where it has them (its protocol switches, its domains' stores
+are the server's); server-scope sources and `processors` are left out, because
+they describe the whole server. A tenant principal without the permission gets
+`forbidden`.
+
+### `inbuxa:InventorySnapshot/get`, `/query`
+
+A dated copy of the evaluated inventory, taken when a setting the catalog
+references changes (hooked where the audit log already sees registry writes)
+and at most once a day otherwise. `query` filters by `after`/`before`,
+newest first. Fields: `id`, `takenAt`, `trigger` (`setting-changed` with the
+setting, or `daily`), `summary` (counts: sources collected, unbounded,
+leaving the host, processors), and on `get` the full inventory as above.
+Kept for `inbuxa:AuditSettings.keepForDays`, so history is as long as the
+audit log's. Same permission and tenant scoping as the inventory.
+
+## 7. The compliance role
+
+### What it holds
+
+A new built-in role, **Compliance Officer**, at server level:
+
+| Permission | New? | Gives |
+|---|---|---|
+| `sysComplianceGet` | new | Overview, Data inventory, snapshots |
+| `sysAuditGet`, `sysAuditExport` | existing | read and export the audit log, verify the chain |
+| `sysLegalHoldGet`, `sysLegalHoldCreate`, `sysLegalHoldUpdate`, `sysLegalHoldExport` | existing | place, widen, release and export holds |
+| `sysAccountLockGet` | existing | see locks and delegations |
+| read (`*Get`, `*Query`) on accounts, groups, lists, domains, tenants, roles | existing | know who and what the records refer to |
+
+It holds **no** `*Create`/`*Update`/`*Destroy` on registry objects, no
+`sysAuditSettingsUpdate`, no `impersonate`, no `fetchAnyBlob`. It can't
+change a server setting.
+
+**Settled (2):** the officer *places and releases* holds, because that is
+the job; every placing, widening and release is already recorded with its
+reason, under the officer's own identity (AU-12), and a release can't be
+undone silently (LH-10).
+
+A **Tenant Compliance Officer** (inside a tenant: `sysComplianceGet`,
+`sysAuditGet`, `sysAuditExport`, `sysAccountLockGet`, reads) is built with
+it (Settled 3). It has no hold permissions, since legal holds are
+server-only by the tenant ceiling (LH-13), and it sees the tenant's slice of
+the inventory only (§6).
+
+### Reaching existing servers
+
+New permissions get ids 673 onward and `COUNT` grows (`enums.rs`,
+`enums_impl.rs`, and the schema's `Permission` enum, a marked hand edit as
+before). `DefaultPermissions` places `sysComplianceGet` with superusers;
+`granted_permissions.rs` `ADMIN_GRANTS` adds it once to stored administrator
+roles, as it did for the audit and hold permissions. The **role object**
+itself is seeded only on a fresh install (`defaults.rs`, `count == 0`); for
+existing servers a one-time step modeled on `granted_permissions.rs` creates
+it once, recorded so that deleting it sticks. That step is new code.
+
+### Keeping administrators reviewable
+
+What already holds: every administrator's change to the registry and to
+inbuxa's own objects is recorded before it is allowed, under their identity,
+with a reason where required; sign-ins by anyone holding `sys*` permissions
+are recorded; records can't be deleted over JMAP; the chain is verifiable;
+changing retention is itself recorded.
+
+What this adds: the compliance officer reads all of it, server administrators'
+actions included, without being an administrator. The Overview (§8) puts on
+top anything that weakens review: audit retention shortened, a tracer or the
+audit export webhook removed, a change to who holds the compliance or
+administrator roles, a hold released — each with who and when.
+
+**Settled (4):** a server administrator can still shorten audit retention to
+90 days; that is recorded and surfaced on the Overview, not gated on a second
+person, since a server may have only one. Anyone with shell access can edit
+the store directly; that is outside what the server can review.
+
+## 8. The Compliance section in the console
+
+The navigation comes from the schema's `layouts` (`/api/schema`); a
+hand-built page is a `CustomComponent/<Name>` link there, a render branch in
+`MainContent.tsx`, and a visibility rule in `layout.ts` `checkSpecialLink`.
+The section shows when at least one of its pages is visible to the person
+(Phase 4 confirms the container behaves so with every child hidden).
+
+Planned navigation, in order:
+
+| Page | This task | Built as | Visible with |
+|---|---|---|---|
+| **Overview** | **builds** | hand-built (`CustomComponent/ComplianceOverview`) | `sysComplianceGet` |
+| **Data inventory** | **builds** | hand-built (`CustomComponent/DataInventory`) | `sysComplianceGet` |
+| Retention | later spec | likely hand-built over `x:DataRetention`, `x:SpamClassifier`, `x:TracerLog` and `inbuxa:AuditSettings`, since they're spread across objects | — |
+| Legal holds | exists | hand-built | `sysLegalHoldGet` |
+| Audit log | exists | hand-built | `sysAuditGet` |
+| Locked accounts | exists | hand-built | `sysAccountLockGet` |
+| Data subject requests | later spec | hand-built | — |
+| Records and documents | later spec | could be schema-driven if it becomes a registry object | — |
+| Jurisdiction packs | later spec | hand-built | — |
+
+**Decision:** later pages are left out of the navigation until
+built, not shown disabled: a disabled entry reads as a feature that exists.
+
+**Overview** shows the latest snapshot's findings as facts ("Log files are
+kept with no limit", "Sent to a host outside this network: public.pyzor.org"),
+the review items from §7, and the snapshot history (when and why the inventory
+changed). **Data inventory** lists the evaluated sources, filterable by
+source, category and location, with each external endpoint listed as a
+candidate processor. UI text states facts and never claims a legal standard is met.
+
+## Contradictions with the docs and SPEC.md
+
+1. **docs, Security › "What the AI features do and do not send":** "message
+   content is never written to the logs". True at the default level. The
+   model's reply to the spam classifier is logged as `ai.llm-response` at
+   trace level (up to 1,024 characters, which can restate the message), so a
+   trace-level tracer writes it, and any webhook at its defaults receives it,
+   along with raw SMTP input (finding 1).
+2. **docs, Audit log:** suggests a webhook to pass records to a SIEM, without
+   saying a webhook's `level` is ignored and its default policy sends every
+   event.
+3. **docs, Monitoring:** "rotated daily" is right, but doesn't say old files
+   are never removed (finding 2). Not strictly a contradiction; an omission
+   that matters here.
+4. **SPEC.md §2.3** calls `schema.json.gz` upstream's published schema and
+   the checklist for rebuilt features. It doesn't say the fork now hand-edits
+   it (layout, permissions); §2.2's strip step would overwrite those edits on
+   an import unless they're re-applied. Worth a sentence in SPEC.md.
+5. **The schema's `TracingStore`/`MetricsStore` defaults** say disabled; first
+   boot turns both on. The docs describe the real behavior; the schema default
+   (what the console shows as the default) doesn't.
+
+The rest checked out: tracing 30 days, metrics 90 days, the Explain cache (in
+memory, a day), audit retention (two years, minimum 90), undelete off by
+default.
+
+## Settled
+
+John's answers, 2026-09-28, to the questions this spec asked:
+
+1. **The catalog is a sidecar**, `resources/privacy/catalog.toml`; the
+   Compliance pages and the new permission are hand-added to the schema as
+   the audit and hold work did.
+2. **The Compliance Officer places and releases legal holds.** It's their
+   role.
+3. **The Tenant Compliance Officer is built now**, with the server role
+   (Phase 3), not after data subject requests.
+4. **Shortening audit retention is recorded and surfaced**, not gated on a
+   second person: a server may have only one.
+5. **All seven defaults, D1–D7, for new installs.** Built in Phase 3.
+6. **Finding 1 is fixed now**, as a bug, separately from this work.
+7. **Snapshots are kept as long as the audit log** (`keepForDays`).
+
+## Phases
+
+1. **This spec.** Stop for approval.
+2. **The catalog and its check:** `resources/privacy/catalog.toml`,
+   `tools/fork/privacy-check.py` in `fork-checks`, the strip report section,
+   and the tests in §5.
+3. **Server:** `sysComplianceGet`, the Compliance Officer and Tenant
+   Compliance Officer roles (with the existing-server step),
+   `inbuxa:DataInventory`, `inbuxa:InventorySnapshot`, and defaults D1–D7
+   for new installs; tests with several configurations (defaults, a remote
+   store, a hosted AI endpoint, telemetry off), tenant scoping, refusal
+   without the permission.
+4. **Console:** Overview and Data inventory, the navigation entries, a PR
+   linking this spec.

resources/schema/schema.json.sha256

@@ -1 +1 @@
-6SH8D-bnOr8ynvSjjkUDkpWWy-QxJQdfZc3KaDlE-hk
+ZjiCQWnwHbujcF7hNolPZQa-VtrNTNfc_HbSuZZ_TiY

tests/src/system/security.rs

@@ -221,6 +221,21 @@ pub async fn test(test: &mut TestServer) {
     )
     .await;
 
+    // inbuxa: personal-data catalog, D2: the daily clean-up removes the
+    // expired ban's record, without waiting for settings to reload
+    test.server.purge_expired_blocked_ips().await.unwrap();
+    assert_eq!(
+        admin
+            .registry_query_ids(
+                ObjectType::BlockedIp,
+                [(Property::Address, "10.0.0.2")],
+                Vec::<&str>::new(),
+            )
+            .await,
+        Vec::<Id>::new(),
+        "the expired ban's record is gone"
+    );
+
     // Make sure the IP remains unblocked after reload
     admin.registry_create_object(Action::ReloadBlockedIps).await;
     validate_password_with_ip(

tools/fork/README.md

@@ -72,6 +72,25 @@ tools/fork/notice-check.py --fix    # add it where it's missing
 Run `--fix` after resolving an upstream merge: a conflict resolved by taking
 upstream's side can drop a notice the file had.
 
+## privacy-check.py
+
+Fails when the personal-data catalog (`resources/privacy/catalog.toml`) and
+the code disagree: an object in the schema or one of inbuxa's own JMAP
+objects with no entry, a property the schema types as an address, an IP or
+a secret left to its object's default, or an entry naming an object,
+property, setting or code path that no longer exists. CI runs it beside the
+name check, with its tests (`python3 -m unittest discover -s tools/fork/tests`).
+See `docs/spec/features/personal-data-catalog.md`.
+
+```bash
+tools/fork/privacy-check.py              # exit 1 on any finding
+tools/fork/privacy-check.py --unlisted   # starting entries for what's missing
+```
+
+After an upstream import, the strip report lists what is new and unclassified
+under "Unclassified in the privacy catalog". `--unlisted` types each from the
+schema alone; read the field's description before trusting it.
+
 ## record-compat.py
 
 Records what the `*_compat` tests compare against, from the Enterprise

tools/fork/privacy-check.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2026 Coffey Labs
+# SPDX-License-Identifier: AGPL-3.0-only
+"""
+Fail when the personal-data catalog and the code disagree.
+
+    tools/fork/privacy-check.py              # check; exit 1 on any finding
+    tools/fork/privacy-check.py --unlisted   # print catalog entries for what's missing
+
+The catalog (`resources/privacy/catalog.toml`, spec
+`docs/spec/features/personal-data-catalog.md`) says, for every object and
+source, what personal data it can hold. An upstream import can bring objects
+and fields nobody has classified, and a refactor can leave the catalog naming
+things that are gone; either way the catalog stops being true without anyone
+noticing, so this runs in CI on every push and pull request. It fails when:
+
+  1. an object in the schema's `fields`, or one of inbuxa's own JMAP objects,
+     has no catalog entry;
+  2. a property the schema types as an email address, an IP address or
+     network, or a secret is covered only by its object's `default` -- it
+     must be listed, so a new personal field can't hide behind a default;
+  3. an entry names an object, property, setting or code path that doesn't
+     exist (stale);
+  4. an entry uses a word outside the catalog's own vocabulary.
+
+When it fails on a new object or field, classify it: `--unlisted` prints a
+starting entry for each, typed from the schema alone. Read the property's
+description before trusting it.
+"""
+import argparse
+import gzip
+import json
+import os
+import re
+import sys
+import tomllib
+
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+SCHEMA = 'resources/schema/schema.json.gz'
+CATALOG = 'resources/privacy/catalog.toml'
+OBJECTS_DIR = 'crates/jmap-proto/src/object'
+METHODS = 'crates/jmap-proto/src/request/method.rs'
+
+SENSITIVE_FORMATS = {
+    'emailAddress': 'identifier',
+    'ipAddress': 'network',
+    'ipNetwork': 'network',
+    'secret': 'credential',
+    'secretText': 'credential',
+}
+INBUXA_OBJECT = re.compile(r'"(inbuxa:[A-Z][A-Za-z]*)"')
+PROPERTY_NAME = re.compile(r'=> "([a-z][A-Za-z0-9]*)"')
+
+
+def sensitive(type_):
+    """The category a property's schema type alone implies, or None."""
+    fmt = type_.get('format') or (type_.get('class') or {}).get('format')
+    if fmt in SENSITIVE_FORMATS:
+        return SENSITIVE_FORMATS[fmt]
+    name = type_.get('objectName') or (type_.get('class') or {}).get('objectName') or ''
+    if name.startswith(('x:SecretKey', 'x:SecretText')) or name == 'x:HttpAuth':
+        return 'credential'
+    return None
+
+
+def load_schema(root):
+    with gzip.open(os.path.join(root, SCHEMA)) as f:
+        return json.load(f)['fields']
+
+
+def load_inbuxa_objects(root):
+    """inbuxa's own objects, from the method names jmap-proto parses."""
+    with open(os.path.join(root, METHODS), encoding='utf-8') as f:
+        return set(INBUXA_OBJECT.findall(f.read()))
+
+
+def inbuxa_properties(root, file_name):
+    """The property names a jmap-proto object file maps, or None if it's gone."""
+    path = os.path.join(root, OBJECTS_DIR, file_name)
+    if not os.path.isfile(path):
+        return None
+    with open(path, encoding='utf-8') as f:
+        return set(PROPERTY_NAME.findall(f.read()))
+
+
+def findings(root, catalog):
+    """Everything wrong with `catalog` against the tree at `root`."""
+    fields = load_schema(root)
+    inbuxa = load_inbuxa_objects(root)
+    vocab = catalog.get('vocabulary', {})
+    objects = catalog.get('object', {})
+    sources = catalog.get('source', {})
+    out = []
+    inbuxa_props = {}
+
+    def vocabulary(where, key, values):
+        allowed = set(vocab.get(key, []))
+        for value in values if isinstance(values, list) else [values]:
+            if value not in allowed:
+                out.append(f'{where}: "{value}" is not in vocabulary.{key}')
+
+    def setting_exists(where, setting):
+        obj, _, prop = setting.partition('.')
+        if obj in fields:
+            if prop not in fields[obj].get('properties', {}):
+                out.append(f'{where}: setting {setting} names no property of {obj}')
+        elif obj in inbuxa:
+            props = inbuxa_props.get(obj)
+            if props is not None and prop not in props:
+                out.append(f'{where}: setting {setting} names no property of {obj}')
+        else:
+            out.append(f'{where}: setting {setting} names no object')
+
+    def common(where, entry):
+        for key in ('whose', 'where', 'scope'):
+            if key in entry:
+                vocabulary(where, key, entry[key])
+        retention = entry.get('retention')
+        if isinstance(retention, dict):
+            setting_exists(where, retention.get('setting', ''))
+        elif retention is not None:
+            vocabulary(where, 'retention', retention)
+        for key in ('enabled_by', 'captures'):
+            for setting in entry.get(key, []):
+                setting_exists(where, setting)
+
+    # inbuxa objects' properties first, so settings can name them
+    for name, entry in objects.items():
+        if name.startswith('inbuxa:'):
+            inbuxa_props[name] = inbuxa_properties(root, entry.get('file', ''))
+
+    # 1: nothing unlisted
+    for name in sorted(fields):
+        if name not in objects:
+            out.append(f'{name}: schema object has no catalog entry')
+    for name in sorted(inbuxa):
+        if name not in objects:
+            out.append(f'{name}: inbuxa object has no catalog entry')
+
+    for name, entry in sorted(objects.items()):
+        props = entry.get('properties', {})
+        if name.startswith('inbuxa:'):
+            known = inbuxa_props.get(name)
+            if name not in inbuxa:
+                out.append(f'{name}: no such inbuxa object (stale)')
+            if known is None:
+                out.append(f'{name}: file "{entry.get("file", "")}" not found in {OBJECTS_DIR}')
+                known = set()
+        elif name in fields:
+            known = set(fields[name].get('properties', {}))
+        else:
+            out.append(f'{name}: no such schema object (stale)')
+            continue
+        if entry.get('default') != 'none':
+            out.append(f'{name}: default must be "none"')
+        for prop, categories in props.items():
+            if prop not in known:
+                out.append(f'{name}.{prop}: no such property (stale)')
+            vocabulary(f'{name}.{prop}', 'categories', categories)
+        common(name, entry)
+        # 2: typed-sensitive properties are listed
+        if name in fields:
+            for prop, spec in fields[name].get('properties', {}).items():
+                if prop not in props and sensitive(spec['type']):
+                    out.append(f'{name}.{prop}: typed as {sensitive(spec["type"])} but not listed')
+
+    for name, entry in sorted(sources.items()):
+        where = f'source "{name}"'
+        vocabulary(where, 'categories', entry.get('categories', []))
+        common(where, entry)
+        if 'leaves_host' not in entry:
+            out.append(f'{where}: leaves_host is missing')
+        for path in entry.get('written_by', []):
+            if not os.path.exists(os.path.join(root, path)):
+                out.append(f'{where}: written_by {path} doesn\'t exist (stale)')
+    return out
+
+
+def unlisted(root, catalog):
+    """Starting catalog entries for unlisted objects and properties."""
+    fields = load_schema(root)
+    objects = catalog.get('object', {})
+    lines = []
+    for name in sorted(fields):
+        entry = objects.get(name)
+        listed = (entry or {}).get('properties', {})
+        missing = {
+            p: sensitive(spec['type'])
+            for p, spec in fields[name].get('properties', {}).items()
+            if p not in listed and sensitive(spec['type'])
+        }
+        if entry is None or missing:
+            lines.append(f'[object.{json.dumps(name)}]' if entry is None else f'# add to {name}:')
+            if entry is None:
+                lines.append('default = "none"')
+            if missing:
+                if entry is None:
+                    lines.append(f'[object.{json.dumps(name)}.properties]')
+                for prop, category in sorted(missing.items()):
+                    lines.append(f'{prop} = ["{category}"]')
+            lines.append('')
+    return lines
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.split('\n')[1])
+    parser.add_argument('--unlisted', action='store_true', help='print entries for what is missing')
+    parser.add_argument('--root', default=ROOT, help=argparse.SUPPRESS)
+    args = parser.parse_args(argv)
+    with open(os.path.join(args.root, CATALOG), 'rb') as f:
+        catalog = tomllib.load(f)
+    if args.unlisted:
+        print('\n'.join(unlisted(args.root, catalog)))
+        return 0
+    found = findings(args.root, catalog)
+    if found:
+        print(f'privacy check: {len(found)} finding(s) in {CATALOG}:')
+        for line in found:
+            print(f'  {line}')
+        print('Classify new objects and fields (--unlisted helps); remove what no longer exists.')
+        return 1
+    print(f'privacy check: clean ({len(catalog.get("object", {}))} objects, '
+          f'{len(catalog.get("source", {}))} sources).')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/fork/strip.py

@@ -373,6 +373,39 @@ def schema_flags(tree):
     return {'objects': objects, 'fields': fields}
 
 
+def privacy_flags(tree):
+    """Objects and properties new in this import that the personal-data
+    catalog doesn't classify (docs/spec/features/personal-data-catalog.md).
+    Informational, like the Enterprise flags: privacy-check.py is what fails
+    CI once the import is merged."""
+    import importlib.util
+    import tomllib
+    here = Path(__file__).resolve().parent
+    catalog_path = here.parents[1] / 'resources' / 'privacy' / 'catalog.toml'
+    current_path = here.parents[1] / 'resources' / 'schema' / 'schema.json.gz'
+    path = tree / 'resources' / 'schema' / 'schema.json.gz'
+    if not (path.is_file() and catalog_path.is_file() and current_path.is_file()):
+        return None
+    spec = importlib.util.spec_from_file_location('privacy_check', here / 'privacy-check.py')
+    check = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(check)
+    catalog = tomllib.loads(catalog_path.read_text(encoding='utf-8')).get('object', {})
+    current = json.loads(gzip.decompress(current_path.read_bytes())).get('fields', {})
+    upstream = json.loads(gzip.decompress(path.read_bytes())).get('fields', {})
+    objects = sorted(name for name in upstream if name not in catalog)
+    fields = []
+    for name, spec_ in upstream.items():
+        if name not in catalog:
+            continue
+        listed = catalog[name].get('properties', {})
+        known = current.get(name, {}).get('properties', {})
+        for prop, p in spec_.get('properties', {}).items():
+            if prop not in known and prop not in listed:
+                typed = check.sensitive(p.get('type', {}))
+                fields.append(f'{name}.{prop}' + (f' ({typed})' if typed else ''))
+    return {'objects': objects, 'fields': sorted(fields)}
+
+
 def remaining_hooks(tree):
     gates, checks = {}, {}
     for path in tree.rglob('*.rs'):
@@ -456,6 +489,8 @@ def write_report(out_dir, report):
     ]
     if r['schema']:
         md.append(f'- Upstream schema flags {len(r["schema"]["objects"])} objects and {len(r["schema"]["fields"])} fields as Enterprise')
+    if r.get('privacy'):
+        md.append(f'- Unclassified in the privacy catalog: {len(r["privacy"]["objects"])} objects and {len(r["privacy"]["fields"])} new fields')
     md += ['', '## Removed files', ''] + [f'- `{f}`' for f in r['removed_files']]
     md += ['', '## Removed snippets', ''] + [f'- `{f}`: {n}' for f, n in r['removed_snippets'].items()]
     md += ['', '## Dangling module declarations removed', ''] + [f'- `{d["file"]}:{d["line"]}`: `mod {d["module"]}` ({" / ".join(d["lines"])})' for d in r['dangling_mods']]
@@ -463,6 +498,13 @@ def write_report(out_dir, report):
     if r['schema']:
         md += ['', '## Flagged Enterprise in upstream\'s schema', '', '**Objects:** ' + ', '.join(f'`{o}`' for o in r['schema']['objects']),
                '', '**Fields:** ' + ', '.join(f'`{f}`' for f in r['schema']['fields'])]
+    if r.get('privacy'):
+        md += ['', '## Unclassified in the privacy catalog', '',
+               'New in this import and not in `resources/privacy/catalog.toml`. `tools/fork/privacy-check.py` '
+               'fails CI on these once merged; `--unlisted` prints a starting entry. A type in brackets is what '
+               'the schema alone says the field holds.', '',
+               '**Objects:** ' + (', '.join(f'`{o}`' for o in r['privacy']['objects']) or 'none'),
+               '', '**Fields:** ' + (', '.join(f'`{f}`' for f in r['privacy']['fields']) or 'none')]
     md += ['', '## Third-party code', '',
            'Comments in the stripped tree that name another copyright holder, another license, or a source the '
            'code came from. Files marked **new** aren\'t in THIRD-PARTY.md yet.', '']
@@ -537,7 +579,7 @@ def main():
         'removed_files': removed_files, 'removed_snippets': removed_snippets,
         'cargo_edits': edits, 'dangling_mods': dangling, 'problems': problems,
         'feature_gates': gates, 'edition_checks': checks,
-        'schema': schema_flags(tree), 'third_party': others, 'third_party_unlisted': new_others,
+        'schema': schema_flags(tree), 'privacy': privacy_flags(tree), 'third_party': others, 'third_party_unlisted': new_others,
         'renames': renames, 'build': build, 'ossify_log': log,
     }
     write_report(args.out, report)

tools/fork/tests/test_privacy_check.py

@@ -0,0 +1,177 @@
+# SPDX-FileCopyrightText: 2026 Coffey Labs
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Tests for tools/fork/privacy-check.py: python3 -m unittest discover tools/fork/tests"""
+import gzip
+import importlib.util
+import json
+import os
+import tempfile
+import tomllib
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+spec = importlib.util.spec_from_file_location('privacy_check', os.path.join(HERE, '..', 'privacy-check.py'))
+check = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(check)
+
+VOCAB = """
+[vocabulary]
+categories = ["identifier", "contact", "network", "content", "metadata", "credential"]
+whose = ["holder", "correspondent", "administrator"]
+where = ["data-store", "blob-store", "search-store", "in-memory-store", "memory", "log-file", "external"]
+scope = ["tenant", "server"]
+retention = ["unbounded", "object-life", "receiver"]
+"""
+
+GOOD = VOCAB + """
+[object."x:Widget"]
+default = "none"
+whose = ["holder"]
+where = ["data-store"]
+scope = "tenant"
+retention = { setting = "x:Widget.keepFor" }
+[object."x:Widget".properties]
+owner = ["identifier"]
+
+[object."inbuxa:Gadget"]
+file = "inbuxa_gadget.rs"
+default = "none"
+[object."inbuxa:Gadget".properties]
+remoteIp = ["network"]
+
+[source."widget-log"]
+categories = ["network"]
+where = ["log-file"]
+scope = "server"
+retention = "unbounded"
+enabled_by = ["x:Widget.enable", "inbuxa:Gadget.remoteIp"]
+leaves_host = false
+written_by = ["crates/widget.rs"]
+"""
+
+
+def make_tree(fields):
+    root = tempfile.mkdtemp()
+    os.makedirs(os.path.join(root, 'resources/schema'))
+    with gzip.open(os.path.join(root, check.SCHEMA), 'wt') as f:
+        json.dump({'fields': fields}, f)
+    os.makedirs(os.path.join(root, check.OBJECTS_DIR))
+    with open(os.path.join(root, check.OBJECTS_DIR, 'inbuxa_gadget.rs'), 'w') as f:
+        f.write('Property::RemoteIp => "remoteIp",\nProperty::Id => "id",\n')
+    os.makedirs(os.path.join(root, 'crates/jmap-proto/src/request'), exist_ok=True)
+    with open(os.path.join(root, check.METHODS), 'w') as f:
+        f.write('"inbuxa:Gadget" => MethodObject::Gadget,\n')
+    with open(os.path.join(root, 'crates/widget.rs'), 'w') as f:
+        f.write('')
+    return root
+
+
+def prop(type_, fmt=None):
+    t = {'type': type_}
+    if fmt:
+        t['format'] = fmt
+    return {'type': t}
+
+
+FIELDS = {
+    'x:Widget': {'properties': {
+        'owner': prop('string', 'emailAddress'),
+        'enable': prop('boolean'),
+        'keepFor': prop('number', 'duration'),
+        'label': prop('string', 'string'),
+    }},
+}
+
+
+class PrivacyCheck(unittest.TestCase):
+    def run_check(self, catalog, fields=FIELDS):
+        return check.findings(make_tree(fields), tomllib.loads(catalog))
+
+    def test_the_repository_passes(self):
+        with open(os.path.join(check.ROOT, check.CATALOG), 'rb') as f:
+            catalog = tomllib.load(f)
+        self.assertEqual(check.findings(check.ROOT, catalog), [])
+
+    def test_a_consistent_catalog_passes(self):
+        self.assertEqual(self.run_check(GOOD), [])
+
+    def test_an_unclassified_object_fails(self):
+        fields = dict(FIELDS, **{'x:Sprocket': {'properties': {'size': prop('number', 'size')}}})
+        found = self.run_check(GOOD, fields)
+        self.assertEqual(found, ['x:Sprocket: schema object has no catalog entry'])
+
+    def test_an_address_hidden_behind_the_default_fails(self):
+        fields = {'x:Widget': {'properties': dict(FIELDS['x:Widget']['properties'],
+                                                  contactEmail=prop('string', 'emailAddress'))}}
+        found = self.run_check(GOOD, fields)
+        self.assertEqual(found, ['x:Widget.contactEmail: typed as identifier but not listed'])
+
+    def test_a_secret_in_a_set_or_object_counts(self):
+        fields = {'x:Widget': {'properties': dict(
+            FIELDS['x:Widget']['properties'],
+            ips={'type': {'type': 'set', 'class': {'type': 'string', 'format': 'ipNetwork'}}},
+            key={'type': {'type': 'object', 'objectName': 'x:SecretKey'}},
+        )}}
+        found = self.run_check(GOOD, fields)
+        self.assertIn('x:Widget.ips: typed as network but not listed', found)
+        self.assertIn('x:Widget.key: typed as credential but not listed', found)
+
+    def test_a_stale_property_fails(self):
+        found = self.run_check(GOOD.replace('owner = ["identifier"]', 'owner = ["identifier"]\ngone = ["content"]'))
+        self.assertEqual(found, ['x:Widget.gone: no such property (stale)'])
+
+    def test_a_stale_object_fails(self):
+        found = self.run_check(GOOD + '\n[object."x:Removed"]\ndefault = "none"\n')
+        self.assertEqual(found, ['x:Removed: no such schema object (stale)'])
+
+    def test_a_stale_setting_fails(self):
+        found = self.run_check(GOOD.replace('x:Widget.keepFor', 'x:Widget.keepForever'))
+        self.assertEqual(found, ['x:Widget: setting x:Widget.keepForever names no property of x:Widget'])
+
+    def test_a_stale_code_path_fails(self):
+        found = self.run_check(GOOD.replace('crates/widget.rs', 'crates/gone.rs'))
+        self.assertEqual(found, ['source "widget-log": written_by crates/gone.rs doesn\'t exist (stale)'])
+
+    def test_an_unlisted_inbuxa_object_fails(self):
+        catalog = VOCAB + """
+[object."x:Widget"]
+default = "none"
+[object."x:Widget".properties]
+owner = ["identifier"]
+"""
+        found = self.run_check(catalog)
+        self.assertEqual(found, ['inbuxa:Gadget: inbuxa object has no catalog entry'])
+
+    def test_words_outside_the_vocabulary_fail(self):
+        found = self.run_check(GOOD.replace('owner = ["identifier"]', 'owner = ["personal"]'))
+        self.assertEqual(found, ['x:Widget.owner: "personal" is not in vocabulary.categories'])
+
+    def test_unlisted_prints_a_starting_entry(self):
+        fields = dict(FIELDS, **{'x:Sprocket': {'properties': {'mail': prop('string', 'emailAddress')}}})
+        lines = check.unlisted(make_tree(fields), tomllib.loads(GOOD))
+        self.assertIn('[object."x:Sprocket"]', lines)
+        self.assertIn('mail = ["identifier"]', lines)
+
+
+class StripReport(unittest.TestCase):
+    def test_an_import_reports_what_is_new_and_unclassified(self):
+        import pathlib
+        strip_spec = importlib.util.spec_from_file_location('strip', os.path.join(HERE, '..', 'strip.py'))
+        strip = importlib.util.module_from_spec(strip_spec)
+        strip_spec.loader.exec_module(strip)
+        with gzip.open(os.path.join(check.ROOT, check.SCHEMA)) as f:
+            upstream = json.load(f)
+        upstream['fields']['x:NewThing'] = {'properties': {}}
+        upstream['fields']['x:UserAccount']['properties']['backupEmail'] = prop('string', 'emailAddress')
+        tree = pathlib.Path(tempfile.mkdtemp())
+        (tree / 'resources/schema').mkdir(parents=True)
+        with gzip.open(tree / 'resources/schema/schema.json.gz', 'wt') as f:
+            json.dump(upstream, f)
+        self.assertEqual(strip.privacy_flags(tree), {
+            'objects': ['x:NewThing'],
+            'fields': ['x:UserAccount.backupEmail (identifier)'],
+        })
+
+
+if __name__ == '__main__':
+    unittest.main()