Compare commits

..
Author SHA1 Message Date
jcoffey-dev b183d4fc8a Compliance menu: Overview and Data Inventory first
ci / fork-checks (pull_request) Canceled after 14s
ci / build (pull_request) Canceled after 12s
Personal-data catalog spec, §8: the Compliance section in the console
reads Overview, Data Inventory, Legal Holds, Audit Log, Locked
Accounts. The two new entries are hand-built console pages
(CustomComponent/ComplianceOverview, CustomComponent/DataInventory),
shown to people with sysComplianceGet; the console shows each only
where it knows the page, so an older console skips them. Schema edited
as the fork's earlier Compliance entries were, hash updated.
2026-09-28 10:54:17 -07:00
9 changed files with 32 additions and 179 deletions
-4
View File
@@ -409,10 +409,6 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> {
bp.registry.write(RegistryWrite::insert(&object)).await?; bp.registry.write(RegistryWrite::insert(&object)).await?;
} }
// D5: the blocklist sent hashed email addresses starts off; the
// rules load later, from a task, which acts on this note
super::spam_rules::mark_new_install(&bp.data_store).await?;
// D1: rotated log files are kept 30 days (a fork-owned setting, // D1: rotated log files are kept 30 days (a fork-owned setting,
// since x:TracerLog is also stored inside x:Bootstrap) // since x:TracerLog is also stored inside x:Bootstrap)
use inbuxa_features::security::log_files; use inbuxa_features::security::log_files;
-72
View File
@@ -118,78 +118,6 @@ pub async fn set_applied_version(data: &Store, version: &str) -> trc::Result<()>
.map(|_| ()) .map(|_| ())
} }
/// The blocklists a new install starts with switched off (personal-data
/// catalog spec, default D5, settled 2026-09-28): the one that is sent a
/// hash of every email address it's asked about.
pub const NEW_INSTALL_OFF: &[&str] = &["STWT_MSBL_EBL_EMAIL"];
fn new_install_key() -> ValueClass {
ValueClass::Any(AnyClass {
subspace: SUBSPACE_INBUXA,
key: b"Sn".to_vec(),
})
}
/// Notes, on a new install's first boot, that [`NEW_INSTALL_OFF`] is to be
/// switched off once the rules are in: they load later, from a task.
pub async fn mark_new_install(data: &Store) -> trc::Result<()> {
let mut batch = BatchBuilder::new();
batch.set(new_install_key(), b"D5".to_vec());
data.write(batch.build_all())
.await
.caused_by(trc::location!())
.map(|_| ())
}
/// After rules load: on a new install, switches [`NEW_INSTALL_OFF`] off and
/// forgets the note, so it happens once. Returns whether anything changed.
/// An existing server has no note, and keeps every blocklist as it is.
pub async fn apply_new_install(
registry: &store::RegistryStore,
data: &Store,
) -> trc::Result<bool> {
use registry::schema::{prelude::Object, structs::SpamDnsblServer};
use store::registry::write::RegistryWrite;
if data
.get_value::<String>(ValueKey::from(new_install_key()))
.await
.caused_by(trc::location!())?
.is_none()
{
return Ok(false);
}
let mut changed = false;
for server in registry.list::<SpamDnsblServer>().await? {
let mut updated = server.object.clone();
let SpamDnsblServer::Email(email) = &mut updated else {
continue;
};
if !NEW_INSTALL_OFF.contains(&email.name.as_str()) || !email.enable {
continue;
}
email.enable = false;
let old = Object {
inner: server.object.into(),
revision: server.revision,
};
let new = Object {
inner: updated.into(),
revision: server.revision,
};
registry
.write(RegistryWrite::update(types::id::Id::from(server.id.id()), &new, &old))
.await?;
changed = true;
}
let mut batch = BatchBuilder::new();
batch.clear(new_install_key());
data.write(batch.build_all())
.await
.caused_by(trc::location!())?;
Ok(changed)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
+6 -34
View File
@@ -88,31 +88,13 @@ fn days(duration: Option<&Duration>) -> Days {
} }
} }
/// The zones a DNSBL's zone expression can query: each quoted literal that /// An expression's text, if it is a plain constant (a zone, a switch).
/// starts with a dot, in any branch (`ip_reverse + '.zen.spamhaus.org'`). fn expression_text(value: &Value) -> Option<String> {
fn zone_hosts(value: &Value) -> Vec<String> {
let mut hosts = Vec::new();
let mut texts = Vec::new();
fn collect<'a>(value: &'a Value, texts: &mut Vec<&'a str>) {
match value { match value {
Value::String(s) => texts.push(s), Value::String(s) => Some(s.clone()),
Value::Array(items) => items.iter().for_each(|v| collect(v, texts)), Value::Object(o) => o.get("else").and_then(|v| v.as_str()).map(str::to_string),
Value::Object(map) => map.values().for_each(|v| collect(v, texts)), _ => None,
_ => {}
} }
}
collect(value, &mut texts);
for text in texts {
for literal in text.split('\'').skip(1).step_by(2) {
if let Some(zone) = literal.strip_prefix('.')
&& zone.contains('.')
&& !hosts.iter().any(|h| h == zone)
{
hosts.push(zone.to_string());
}
}
}
hosts
} }
impl Server { impl Server {
@@ -281,7 +263,7 @@ impl Server {
let value = serde_json::to_value(&server.object).unwrap_or_default(); let value = serde_json::to_value(&server.object).unwrap_or_default();
if value.get("enable").and_then(Value::as_bool).unwrap_or(false) { if value.get("enable").and_then(Value::as_bool).unwrap_or(false) {
dnsbl_on = true; dnsbl_on = true;
for zone in value.get("zone").map(zone_hosts).unwrap_or_default() { if let Some(zone) = value.get("zone").and_then(expression_text) {
endpoint(&mut facts, "spam-dnsbl", zone); endpoint(&mut facts, "spam-dnsbl", zone);
} }
} }
@@ -415,16 +397,6 @@ mod tests {
assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into())); assert_eq!(remote_host(&json!({"@type": "S3", "bucket": "mail"})), Some("S3".into()));
} }
#[test]
fn zones_come_from_every_branch() {
let zone = json!({"else": "false", "match": {"0": {"if": "location == 'tcp'",
"then": "ip_reverse + '.rep.mailspike.net'"}}});
assert_eq!(zone_hosts(&zone), vec!["rep.mailspike.net"]);
let zone = json!({"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}});
assert_eq!(zone_hosts(&zone), vec!["ebl.msbl.org"], "not 'sha1'");
assert!(zone_hosts(&json!({"else": "false"})).is_empty());
}
#[test] #[test]
fn days_round_up() { fn days_round_up() {
assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1)); assert_eq!(days(Some(&Duration::from_millis(86_400_000))), Days::Days(1));
@@ -316,15 +316,6 @@ async fn update_spam_rules(server: &Server) -> trc::Result<TaskResult> {
spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED) spam_rules::set_applied_version(server.store(), spam_rules::BUNDLED_SPAM_RULES_APPLIED)
.await?; .await?;
} }
// inbuxa: personal-data catalog, D5: a new install's first rules
// leave the hashed-address blocklist off
if spam_rules::apply_new_install(server.registry(), server.store()).await?
&& let Err(err) = reload_and_broadcast(server, ObjectType::SpamDnsblServer).await
{
return Ok(TaskResult::permanent(format!(
"Spam rules were stored but not activated ({err}); run Reload settings"
)));
}
Ok(TaskResult::Success(vec![])) Ok(TaskResult::Success(vec![]))
} }
} }
+1 -1
View File
@@ -81,7 +81,7 @@ fn legacy_setting(name: &str, is_set: impl Fn(&str) -> bool) -> Option<String> {
#[macro_export] #[macro_export]
macro_rules! brand_version { macro_rules! brand_version {
() => { () => {
"2026.9.28.4" "2026.9.28.3"
}; };
} }
+1 -5
View File
@@ -407,11 +407,7 @@ retention is (not a field on `x:TracerLog`, which is also stored inside
`x:Bootstrap` with fields after it, so a new field would change that `x:Bootstrap` with fields after it, so a new field would change that
object's stored format); new installs 30 days, existing servers keep every object's stored format); new installs 30 days, existing servers keep every
file as today. D5 is built after the v0.16.24 import lands, on its reworked file as today. D5 is built after the v0.16.24 import lands, on its reworked
spam-rules loader, which keeps each blocklist's on/off state. D5 built after the import: a new install's first boot leaves a note spam-rules loader, which keeps each blocklist's on/off state.
(`S` `n`), and the rules update, once the bundled rules are in, switches
`STWT_MSBL_EBL_EMAIL` off and forgets the note; the loader keeps that
switch through later updates. An existing server has no note and keeps
every blocklist as it is.
| # | Change | Trade-off | | # | Change | Trade-off |
|---|---|---| |---|---|---|
Binary file not shown.
+23 -14
View File
@@ -847,7 +847,7 @@ default = "none"
whose = ["correspondent"] whose = ["correspondent"]
where = ["memory"] where = ["memory"]
scope = "server" scope = "server"
retention = "object-life" retention = "unbounded"
[object."x:DmarcTroubleshoot".properties] [object."x:DmarcTroubleshoot".properties]
ehloDomain = ["network"] ehloDomain = ["network"]
ipRevPtr = ["network"] ipRevPtr = ["network"]
@@ -1266,7 +1266,7 @@ default = "none"
whose = ["correspondent", "holder", "administrator"] whose = ["correspondent", "holder", "administrator"]
where = ["log-file"] where = ["log-file"]
scope = "server" scope = "server"
retention = { setting = "inbuxa:LogSettings.keepForDays" } retention = "unbounded"
[object."x:Log".properties] [object."x:Log".properties]
details = ["network", "identifier", "metadata"] details = ["network", "identifier", "metadata"]
timestamp = ["metadata"] timestamp = ["metadata"]
@@ -1689,7 +1689,7 @@ default = "none"
whose = ["correspondent"] whose = ["correspondent"]
where = ["memory"] where = ["memory"]
scope = "server" scope = "server"
retention = "object-life" retention = "unbounded"
[object."x:SpamClassify".properties] [object."x:SpamClassify".properties]
authenticatedAs = ["identifier"] authenticatedAs = ["identifier"]
ehloDomain = ["network"] ehloDomain = ["network"]
@@ -1810,7 +1810,7 @@ default = "none"
whose = ["holder", "correspondent"] whose = ["holder", "correspondent"]
where = ["data-store"] where = ["data-store"]
scope = "tenant" scope = "tenant"
retention = "object-life" retention = "unbounded"
[object."x:TaskCalendarItipContents".properties] [object."x:TaskCalendarItipContents".properties]
from = ["identifier"] from = ["identifier"]
iCalendarData = ["content"] iCalendarData = ["content"]
@@ -1825,7 +1825,7 @@ default = "none"
whose = ["holder"] whose = ["holder"]
where = ["data-store"] where = ["data-store"]
scope = "tenant" scope = "tenant"
retention = "object-life" retention = "unbounded"
[object."x:TaskDestroyAccount".properties] [object."x:TaskDestroyAccount".properties]
accountName = ["identifier"] accountName = ["identifier"]
@@ -1852,7 +1852,7 @@ default = "none"
whose = ["holder"] whose = ["holder"]
where = ["data-store"] where = ["data-store"]
scope = "tenant" scope = "tenant"
retention = "object-life" retention = "unbounded"
[object."x:TaskMergeThreads".properties] [object."x:TaskMergeThreads".properties]
messageIds = ["metadata"] messageIds = ["metadata"]
threadName = ["content"] threadName = ["content"]
@@ -1886,7 +1886,7 @@ default = "none"
whose = ["holder"] whose = ["holder"]
where = ["data-store"] where = ["data-store"]
scope = "tenant" scope = "tenant"
retention = "object-life" retention = "unbounded"
[object."x:TaskStatusRetry".properties] [object."x:TaskStatusRetry".properties]
failureReason = ["content"] failureReason = ["content"]
@@ -2040,23 +2040,30 @@ default = "none"
[object."x:TracerLog"] [object."x:TracerLog"]
default = "none" default = "none"
whose = ["correspondent", "holder", "administrator"]
where = ["log-file"]
scope = "server"
retention = "unbounded"
[object."x:TracerLog".properties] [object."x:TracerLog".properties]
path = ["metadata"] path = ["metadata"]
[object."x:TracerOtelGrpc"] [object."x:TracerOtelGrpc"]
# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
default = "none" default = "none"
whose = ["correspondent", "holder", "administrator"]
where = ["external"]
scope = "server"
retention = "receiver"
[object."x:TracerOtelGrpc".properties] [object."x:TracerOtelGrpc".properties]
endpoint = ["network"] endpoint = ["network"]
httpAuth = ["credential"] httpAuth = ["credential"]
httpHeaders = ["credential"] httpHeaders = ["credential"]
[object."x:TracerOtelHttp"] [object."x:TracerOtelHttp"]
# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
default = "none" default = "none"
whose = ["correspondent", "holder", "administrator"]
where = ["external"]
scope = "server"
retention = "receiver"
[object."x:TracerOtelHttp".properties] [object."x:TracerOtelHttp".properties]
endpoint = ["network"] endpoint = ["network"]
httpAuth = ["credential"] httpAuth = ["credential"]
@@ -2088,9 +2095,11 @@ usedDiskQuota = ["metadata"]
default = "none" default = "none"
[object."x:WebHook"] [object."x:WebHook"]
# Configuration: the "webhooks" and "otel-tracer" sources carry what it sends
default = "none" default = "none"
whose = ["correspondent", "holder", "administrator"]
where = ["external"]
scope = "server"
retention = "receiver"
[object."x:WebHook".properties] [object."x:WebHook".properties]
httpAuth = ["credential"] httpAuth = ["credential"]
httpHeaders = ["credential"] httpHeaders = ["credential"]
-39
View File
@@ -274,45 +274,6 @@ pub async fn test(test: &mut TestServer) {
.unwrap(); .unwrap();
assert_eq!(trace["retention"]["days"], json!(7), "{trace}"); assert_eq!(trace["retention"]["days"], json!(7), "{trace}");
// D5: a new install's first rules leave the hashed-address blocklist
// off, once; an existing server (no note) keeps it as it is
let (_, response) = call(
&admin,
"x:SpamDnsblServer/set",
json!({"create": {"m": {"@type": "Email", "name": "STWT_MSBL_EBL_EMAIL", "enable": true,
"zone": {"else": "hash(email, 'sha1') + '.ebl.msbl.org'", "match": {}},
"tag": {"else": "'MSBL_EBL'", "match": {}}}}}),
)
.await;
let msbl = response["created"]["m"]["id"]
.as_str()
.unwrap_or_else(|| panic!("{response}"))
.to_string();
let registry = test.server.registry();
let store = test.server.store();
assert!(
!common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
"no note, no change"
);
let enabled = |response: &Value| response["list"][0]["enable"].clone();
let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
assert_eq!(enabled(&response), json!(true));
let (_, response) = call(&officer, "inbuxa:DataInventory/get", json!({"ids": null})).await;
assert!(
response["list"][0]["processors"]
.as_array()
.is_some_and(|p| p.iter().any(|p| p["host"] == "ebl.msbl.org")),
"the zone, not the hash: {response}"
);
common::manager::spam_rules::mark_new_install(store).await.unwrap();
assert!(common::manager::spam_rules::apply_new_install(registry, store).await.unwrap());
let (_, response) = call(&admin, "x:SpamDnsblServer/get", json!({"ids": [msbl]})).await;
assert_eq!(enabled(&response), json!(false), "{response}");
assert!(
!common::manager::spam_rules::apply_new_install(registry, store).await.unwrap(),
"the note works once"
);
// A tenant can still be deleted: its unused role goes with it // A tenant can still be deleted: its unused role goes with it
let spare = admin let spare = admin
.registry_create_object(Tenant { .registry_create_object(Tenant {