Merge main into merge/upstream-v0.16.24

The schema, which both sides changed, merged as JSON with no conflicts.
The personal-data catalog (#83) gains upstream's new x:DnsServerPowerDns:
nothing personal but its API key, like the other DNS providers.

crates/common/src/config/telemetry.rs

@@ -483,8 +483,16 @@ impl Tracers {
                 };
 
                 // Parse webhook events
+                // inbuxa: personal-data catalog, finding 1: an include list is
+                // sent as named; otherwise a webhook honors its level as a
+                // tracer does, and never sends a protocol's raw input or
+                // output (whole messages)
+                let level = Level::from(hook.level);
+                let named = (hook.events_policy == EventPolicy::Include)
+                    .then(|| hook.events.iter().copied().collect::<AHashSet<_>>())
+                    .unwrap_or_default();
                 apply_events(hook.events, hook.events_policy, |event_type| {
-                    if event_type != EventType::Telemetry(TelemetryEvent::WebhookError) {
+                    if webhook_wants(event_type, level, &custom_levels, &named) {
                         tracer.interests.set(event_type);
                         global_interests.set(event_type);
                     }
@@ -743,6 +751,31 @@ fn tracer_settings(tracer: &Tracer) -> u64 {
     settings_hash(&tracer)
 }
 
+/// inbuxa: whether a webhook at `level` receives this event type. Its own
+/// error event never, or a failing webhook would report itself to itself.
+/// An event `named` in an include list always: naming it is the choice.
+/// Otherwise (the exclude policy, the default) only events at or above its
+/// level, as for a tracer, and never a protocol's raw input or output, which
+/// carries whole messages and credentials.
+fn webhook_wants(
+    event_type: EventType,
+    level: Level,
+    custom_levels: &AHashMap<EventType, Level>,
+    named: &AHashSet<EventType>,
+) -> bool {
+    if event_type == EventType::Telemetry(TelemetryEvent::WebhookError) {
+        return false;
+    }
+    if named.contains(&event_type) {
+        return true;
+    }
+    let event_level = custom_levels
+        .get(&event_type)
+        .copied()
+        .unwrap_or(event_type.level());
+    level.is_contained(event_level) && !event_type.is_raw_io()
+}
+
 fn webhook_settings(hook: &WebHook) -> u64 {
     let mut hook = hook.clone();
     in_place_reset!(hook);
@@ -804,3 +837,61 @@ impl std::fmt::Debug for OtelMetrics {
             .finish()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use trc::{AuthEvent, SmtpEvent};
+
+    fn wants(event: EventType, level: Level, named: &[EventType]) -> bool {
+        webhook_wants(
+            event,
+            level,
+            &AHashMap::new(),
+            &named.iter().copied().collect(),
+        )
+    }
+
+    #[test]
+    fn a_webhook_honors_its_level() {
+        let success = EventType::Auth(AuthEvent::Success);
+        assert!(wants(success, Level::Info, &[]));
+        assert!(!wants(success, Level::Error, &[]), "info is below error");
+    }
+
+    #[test]
+    fn raw_io_goes_out_only_when_named() {
+        let raw = EventType::Smtp(SmtpEvent::RawInput);
+        assert!(raw.is_raw_io());
+        // Not with the exclude policy, even at trace
+        assert!(!wants(raw, Level::Info, &[]));
+        assert!(!wants(raw, Level::Trace, &[]));
+        // Named in an include list, whatever the level
+        assert!(wants(raw, Level::Info, &[raw]));
+    }
+
+    #[test]
+    fn a_named_event_is_sent_whatever_its_level() {
+        let start = EventType::Smtp(SmtpEvent::ConnectionStart);
+        assert!(!Level::Info.is_contained(start.level()), "below info");
+        assert!(!wants(start, Level::Info, &[]));
+        assert!(wants(start, Level::Info, &[start]));
+    }
+
+    #[test]
+    fn a_custom_level_counts() {
+        let start = EventType::Smtp(SmtpEvent::ConnectionStart);
+        let custom = [(start, Level::Info)].into_iter().collect::<AHashMap<_, _>>();
+        assert!(webhook_wants(start, Level::Info, &custom, &AHashSet::new()));
+        // Raw I/O raised to info still needs naming
+        let raw = EventType::Smtp(SmtpEvent::RawInput);
+        let custom = [(raw, Level::Info)].into_iter().collect::<AHashMap<_, _>>();
+        assert!(!webhook_wants(raw, Level::Info, &custom, &AHashSet::new()));
+    }
+
+    #[test]
+    fn a_webhook_never_hears_its_own_errors() {
+        let own = EventType::Telemetry(TelemetryEvent::WebhookError);
+        assert!(!wants(own, Level::Trace, &[own]));
+    }
+}

crates/common/src/manager/defaults.rs

@@ -14,7 +14,7 @@ use aws_lc_rs::{
 use registry::{
     schema::{
         enums::*,
-        prelude::{ObjectType, SocketAddr},
+        prelude::{Object, ObjectType, SocketAddr},
         structs::*,
     },
     types::{duration::Duration, error::Error, list::List, map::Map},
@@ -388,6 +388,28 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
         }
     }
 
+    // inbuxa: personal-data catalog, defaults D2, D3, D4 and D6 (settled
+    // 2026-09-28): privacy-leaning values, for new installs only. A server
+    // with roles is not new, and keeps its settings whether saved or left at
+    // the default. Each singleton is read, changed and written back whole, so
+    // anything already in it stays.
+    #[cfg(not(feature = "test_mode"))]
+    if bp.registry.count_object(ObjectType::Role).await? == 0 {
+        let mut security = bp.setting_infallible::<Security>().await;
+        let mut classifier = bp.setting_infallible::<SpamClassifier>().await;
+        let mut pyzor = bp.setting_infallible::<SpamPyzor>().await;
+        let mut retention = bp.setting_infallible::<DataRetention>().await;
+        new_install_privacy_defaults(&mut security, &mut classifier, &mut pyzor, &mut retention);
+        for object in [
+            Object::from(security),
+            classifier.into(),
+            pyzor.into(),
+            retention.into(),
+        ] {
+            bp.registry.write(RegistryWrite::insert(&object)).await?;
+        }
+    }
+
     if bp.registry.count_object(ObjectType::Role).await? == 0 {
         let permissions = DefaultPermissions::default();
         let mut role_ids = Vec::with_capacity(4);
@@ -560,3 +582,81 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
 
     Ok(())
 }
+
+/// inbuxa: the new-install values of defaults D2, D3, D4 and D6 from the
+/// personal-data catalog spec. Automatic IP bans expire after 30 days instead
+/// of never; spam training samples are kept 90 days instead of 180; Pyzor,
+/// which sends a digest of each message's text to a public server, is off;
+/// delivery history is kept 14 days instead of 30.
+fn new_install_privacy_defaults(
+    security: &mut Security,
+    classifier: &mut SpamClassifier,
+    pyzor: &mut SpamPyzor,
+    retention: &mut DataRetention,
+) {
+    const DAY: u64 = 24 * 60 * 60 * 1000;
+    let ban_period = Some(Duration::from_millis(30 * DAY));
+    security.auth_ban_period = ban_period;
+    security.abuse_ban_period = ban_period;
+    security.loiter_ban_period = ban_period;
+    security.scan_ban_period = ban_period;
+    classifier.hold_samples_for = Duration::from_millis(90 * DAY);
+    pyzor.enable = false;
+    retention.hold_traces_for = Some(Duration::from_millis(14 * DAY));
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const DAY: u64 = 24 * 60 * 60 * 1000;
+
+    #[test]
+    fn new_installs_get_the_privacy_defaults() {
+        let (mut security, mut classifier, mut pyzor, mut retention) = (
+            Security::default(),
+            SpamClassifier::default(),
+            SpamPyzor::default(),
+            DataRetention::default(),
+        );
+        // What an install gets without them: bans that never lift, 180-day
+        // samples, Pyzor on, 30-day traces.
+        assert_eq!(security.auth_ban_period, None);
+        assert!(pyzor.enable);
+
+        new_install_privacy_defaults(&mut security, &mut classifier, &mut pyzor, &mut retention);
+
+        for period in [
+            security.auth_ban_period,
+            security.abuse_ban_period,
+            security.loiter_ban_period,
+            security.scan_ban_period,
+        ] {
+            assert_eq!(period.map(|p| p.into_inner().as_millis() as u64), Some(30 * DAY));
+        }
+        assert_eq!(classifier.hold_samples_for.into_inner().as_millis() as u64, 90 * DAY);
+        assert!(!pyzor.enable);
+        assert_eq!(
+            retention.hold_traces_for.map(|p| p.into_inner().as_millis() as u64),
+            Some(14 * DAY)
+        );
+    }
+
+    #[test]
+    fn everything_else_in_the_settings_stays() {
+        let mut retention = DataRetention {
+            archive_deleted_items_for: Some(Duration::from_millis(7 * DAY)),
+            ..Default::default()
+        };
+        let before = retention.clone();
+        new_install_privacy_defaults(
+            &mut Security::default(),
+            &mut SpamClassifier::default(),
+            &mut SpamPyzor::default(),
+            &mut retention,
+        );
+        assert_eq!(retention.archive_deleted_items_for, before.archive_deleted_items_for);
+        assert_eq!(retention.hold_metrics_for, before.hold_metrics_for);
+        assert_eq!(retention.expunge_trash_after, before.expunge_trash_after);
+    }
+}

crates/common/src/network/security.rs

@@ -426,6 +426,37 @@ impl Server {
     }
 }
 
+impl Server {
+    /// inbuxa: personal-data catalog, D2: removes bans whose period is over.
+    /// They already stop blocking when they expire, and go when settings are
+    /// next loaded; the daily clean-up makes sure a server that seldom
+    /// reloads doesn't keep them.
+    pub async fn purge_expired_blocked_ips(&self) -> trc::Result<()> {
+        let now = now() as i64;
+        let mut expired = Vec::new();
+        for ip in self.registry().list::<BlockedIp>().await? {
+            if ip.object.expires_at.as_ref().is_some_and(|at| at.timestamp() <= now) {
+                let address = ip.object.address.clone();
+                let object = Object {
+                    inner: ip.object.into(),
+                    revision: ip.revision,
+                };
+                self.registry()
+                    .write(RegistryWrite::delete_object(ip.id, &object))
+                    .await?;
+                expired.push(trc::Value::from(address.into_inner().0));
+            }
+        }
+        if !expired.is_empty() {
+            trc::event!(
+                Security(trc::SecurityEvent::IpBlockExpired),
+                Details = expired
+            );
+        }
+        Ok(())
+    }
+}
+
 impl BlockedIps {
     pub async fn parse(bp: &mut Bootstrap) -> Self {
         let mut ips = Self::default();

crates/registry/src/schema/structs_impl.rs

@@ -47532,7 +47532,9 @@ impl Default for WebHook {
             level: TracingLevel::Info,
             lossy: false,
             events: Default::default(),
-            events_policy: EventPolicy::Exclude,
+            // inbuxa: personal-data catalog, D7: a new webhook sends nothing
+            // until its events are chosen
+            events_policy: EventPolicy::Include,
         }
     }
 }

crates/services/src/task_manager/maintenance.rs

@@ -269,6 +269,11 @@ async fn store_maintenance(
                 trc::error!(err.details("Failed to re-apply account locks"));
             }
 
+            // inbuxa: personal-data catalog, D2: bans past their period go
+            if let Err(err) = server.purge_expired_blocked_ips().await {
+                trc::error!(err.details("Failed to purge expired IP bans"));
+            }
+
             // inbuxa: AU-7: audit records past their retention go; a
             // failure leaves them for the next run
             if let Err(err) = server.audit_purge().await {