Evaluate the personal-data catalog: the data inventory and its history
Personal-data catalog spec, §6 (Phase 3c). inbuxa:DataInventory/get evaluates the catalog against the server's live settings and says what this server holds: for each source and each object that can hold personal data, its categories and whose data it is, whether it is collected here at all, what bounds its retention (the live value of the setting that does, or unbounded), whether it leaves the host and to which endpoints, and a summary. Every host that receives something is listed once as a candidate processor with what it receives. Inside a tenant it answers with the tenant's slice and none of the server's processors. Read-only, with sysComplianceGet. inbuxa:InventorySnapshot/get is the history: a dated copy of the evaluated inventory, recorded when it changes -- after a registry write to an object the inventory reads, after inbuxa's log, audit or AI settings change, and on the daily clean-up -- and kept as long as the audit log's records. ids: null lists every snapshot, newest first; the full inventory only when asked for. The catalog is embedded and parsed at start (new dependency: toml, MIT/Apache); the evaluation is a pure function of it and the live facts, so each configuration is tested without a server. Loopback endpoints stay on the host; any other configured endpoint leaves it. Tested: unit tests for the evaluation (a new install's defaults, an external blob store, a hosted AI endpoint, telemetry off, a tenant's slice, hosts from URLs, loopback), snapshots, and the fact gathering's store and duration rules; the compliance system test, extended (the officer reads the inventory, a plain user is refused, a tenant's officer sees its slice and no processors, a webhook to another host becomes a processor and a snapshot names x:WebHook, a retention change reads through); the system, audit, legal hold and account lock suites; fork checks. The system suite failed once of three runs with an email import's blob not found, in antispam.rs; the same happened once in purge.rs on the previous branch. Nothing here touches uploads; noted for a separate look.
This commit is contained in:
@@ -92,6 +92,10 @@ impl JmapAuthorization for AccessToken {
|
||||
GetRequestMethod::AiLimits(_) => Permission::SysSpamLlmGet,
|
||||
// inbuxa: log file retention, with the tracers' permissions
|
||||
GetRequestMethod::LogSettings(_) => Permission::SysTracerGet,
|
||||
// inbuxa: personal-data catalog, the inventory and its history
|
||||
GetRequestMethod::DataInventory(_) | GetRequestMethod::InventorySnapshot(_) => {
|
||||
Permission::SysComplianceGet
|
||||
}
|
||||
// inbuxa: the audit log (AU-9)
|
||||
GetRequestMethod::AuditEvent(_) | GetRequestMethod::AuditSettings(_) => {
|
||||
Permission::SysAuditGet
|
||||
@@ -393,6 +397,8 @@ impl JmapAuthorization for AccessToken {
|
||||
| MethodObject::DeletedAccount
|
||||
| MethodObject::AiLimits
|
||||
| MethodObject::LogSettings
|
||||
| MethodObject::DataInventory
|
||||
| MethodObject::InventorySnapshot
|
||||
| MethodObject::Explanation
|
||||
| MethodObject::AuditEvent
|
||||
| MethodObject::AuditSettings
|
||||
|
||||
@@ -455,6 +455,20 @@ impl RequestHandler for Server {
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: inbuxa:DataInventory/get
|
||||
GetRequestMethod::DataInventory(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::data_inventory::inventory_get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: inbuxa:InventorySnapshot/get
|
||||
GetRequestMethod::InventorySnapshot(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
crate::inbuxa::data_inventory::snapshot_get(self, access_token, *req)
|
||||
.await?
|
||||
.into()
|
||||
}
|
||||
// inbuxa: account lock with delegation (AL-1)
|
||||
GetRequestMethod::AccountLock(mut req) => {
|
||||
resolve_account_id(&mut req.account_id, method_name.obj, access_token)?;
|
||||
|
||||
@@ -419,6 +419,8 @@ impl IntermediateChangesResponse {
|
||||
| MethodObject::DeletedAccount
|
||||
| MethodObject::AiLimits
|
||||
| MethodObject::LogSettings
|
||||
| MethodObject::DataInventory
|
||||
| MethodObject::InventorySnapshot
|
||||
| MethodObject::Explanation
|
||||
| MethodObject::AuditEvent
|
||||
| MethodObject::AuditSettings
|
||||
|
||||
@@ -206,6 +206,8 @@ pub async fn set(
|
||||
Some(error) => response.not_updated.append(id, error),
|
||||
None => {
|
||||
limits::set(data, &limits).await?;
|
||||
// inbuxa: personal-data catalog: the inventory's history
|
||||
server.inventory_snapshot_after("inbuxa:AiLimits").await;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -392,6 +392,8 @@ pub async fn settings_set(
|
||||
Some(error) => response.not_updated.append(id, error),
|
||||
None => {
|
||||
log::set_settings(server.store(), &settings).await?;
|
||||
// Audit retention is also the inventory's (personal-data catalog)
|
||||
server.inventory_snapshot_after("inbuxa:AuditSettings").await;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2026 Coffey Labs
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
//! `inbuxa:DataInventory/get` and `inbuxa:InventorySnapshot/get`: the
|
||||
//! personal-data catalog evaluated against this server, and its history
|
||||
//! (personal-data catalog spec, §6). Read-only, with `sysComplianceGet`.
|
||||
//!
|
||||
//! Inside a tenant both answer with the tenant's slice: tenant-scoped
|
||||
//! sources only, and none of the server's processors, which describe the
|
||||
//! whole server. The same holds for snapshots, which are taken of the whole
|
||||
//! server and cut to the slice when read.
|
||||
|
||||
use common::{Server, auth::AccessToken};
|
||||
use inbuxa_features::privacy::{Inventory, snapshot};
|
||||
use jmap_proto::{
|
||||
method::get::{GetRequest, GetResponse},
|
||||
object::{
|
||||
inbuxa_data_inventory::{DataInventory, DataInventoryProperty as P, DataInventoryValue},
|
||||
inbuxa_inventory_snapshot::{
|
||||
InventorySnapshot, InventorySnapshotProperty as S, InventorySnapshotValue,
|
||||
},
|
||||
},
|
||||
};
|
||||
use jmap_tools::{Element, Key, Map, Property, Value};
|
||||
use std::borrow::Cow;
|
||||
use types::{brand_version, id::Id};
|
||||
|
||||
fn json_to_value<Pr: Property, E: Element<Property = Pr>>(
|
||||
json: serde_json::Value,
|
||||
) -> Value<'static, Pr, E> {
|
||||
match json {
|
||||
serde_json::Value::Null => Value::Null,
|
||||
serde_json::Value::Bool(b) => Value::Bool(b),
|
||||
serde_json::Value::Number(n) => {
|
||||
if let Some(n) = n.as_u64() {
|
||||
Value::Number(n.into())
|
||||
} else if let Some(n) = n.as_i64() {
|
||||
Value::Number(n.into())
|
||||
} else {
|
||||
Value::Number(n.as_f64().unwrap_or_default().into())
|
||||
}
|
||||
}
|
||||
serde_json::Value::String(s) => Value::Str(Cow::Owned(s)),
|
||||
serde_json::Value::Array(items) => {
|
||||
Value::Array(items.into_iter().map(json_to_value).collect())
|
||||
}
|
||||
serde_json::Value::Object(map) => {
|
||||
let mut out = Map::with_capacity(map.len());
|
||||
for (key, value) in map {
|
||||
out.insert_unchecked(Key::Owned(key), json_to_value(value));
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn to_json<T: serde::Serialize>(value: &T) -> serde_json::Value {
|
||||
serde_json::to_value(value).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn utc(seconds: u64) -> String {
|
||||
jmap_proto::types::date::UTCDate::from_timestamp(seconds as i64).to_string()
|
||||
}
|
||||
|
||||
/// A snapshot's inventory, cut to a tenant's slice when asked from one.
|
||||
fn slice(mut inventory: Inventory, tenant_only: bool) -> Inventory {
|
||||
if tenant_only {
|
||||
inventory.items.retain(|item| item.scope == "tenant");
|
||||
inventory.processors.clear();
|
||||
}
|
||||
inventory
|
||||
}
|
||||
|
||||
/// `inbuxa:DataInventory/get`.
|
||||
pub async fn inventory_get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<DataInventory>,
|
||||
) -> trc::Result<GetResponse<DataInventory>> {
|
||||
let properties = request.unwrap_properties(&[
|
||||
P::Id,
|
||||
P::EvaluatedAt,
|
||||
P::CatalogVersion,
|
||||
P::Summary,
|
||||
P::Items,
|
||||
P::Processors,
|
||||
]);
|
||||
let (ids, not_found) = request.unwrap_ids(1)?;
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found,
|
||||
};
|
||||
let wanted = match ids {
|
||||
None => true,
|
||||
Some(ids) => {
|
||||
let mut wanted = false;
|
||||
for id in ids {
|
||||
if id.is_singleton() {
|
||||
wanted = true;
|
||||
} else {
|
||||
response.push_not_found(id);
|
||||
}
|
||||
}
|
||||
wanted
|
||||
}
|
||||
};
|
||||
if wanted {
|
||||
let inventory = server
|
||||
.data_inventory(access_token.tenant_id().is_some())
|
||||
.await?;
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in &properties {
|
||||
let value = match property {
|
||||
P::Id => Value::Element(DataInventoryValue::Id(Id::singleton())),
|
||||
P::EvaluatedAt => Value::Str(utc(store::write::now()).into()),
|
||||
P::CatalogVersion => Value::Str(brand_version!().into()),
|
||||
P::Summary => json_to_value(to_json(&inventory.summary())),
|
||||
P::Items => json_to_value(to_json(&inventory.items)),
|
||||
P::Processors => json_to_value(to_json(&inventory.processors)),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
response.list.push(Value::Object(out));
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// `inbuxa:InventorySnapshot/get`: by id (the time taken, as an id), or
|
||||
/// `ids: null` for every snapshot kept, newest first. `inventory` is the
|
||||
/// whole evaluated inventory; leave it out of `properties` for the list.
|
||||
pub async fn snapshot_get(
|
||||
server: &Server,
|
||||
access_token: &AccessToken,
|
||||
mut request: GetRequest<InventorySnapshot>,
|
||||
) -> trc::Result<GetResponse<InventorySnapshot>> {
|
||||
let tenant_only = access_token.tenant_id().is_some();
|
||||
let data = &server.core.storage.data;
|
||||
let properties =
|
||||
request.unwrap_properties(&[S::Id, S::TakenAt, S::Trigger, S::Summary, S::Inventory]);
|
||||
let times: Vec<u64> = match request.ids.take() {
|
||||
None => snapshot::list(data, 0, u64::MAX).await?,
|
||||
Some(_) => {
|
||||
let (ids, _) = request.unwrap_ids(server.core.jmap.get_max_objects)?;
|
||||
ids.unwrap_or_default().into_iter().map(|id| id.id()).collect()
|
||||
}
|
||||
};
|
||||
let mut response = GetResponse {
|
||||
account_id: request.account_id.into(),
|
||||
state: None,
|
||||
list: Vec::new(),
|
||||
not_found: vec![],
|
||||
};
|
||||
for taken_at in times {
|
||||
let Some(found) = snapshot::get(data, taken_at).await? else {
|
||||
response.push_not_found(Id::from(taken_at));
|
||||
continue;
|
||||
};
|
||||
let inventory = slice(found.inventory, tenant_only);
|
||||
let summary = if tenant_only { inventory.summary() } else { found.summary };
|
||||
let mut out = Map::with_capacity(properties.len());
|
||||
for property in &properties {
|
||||
let value = match property {
|
||||
S::Id => Value::Element(InventorySnapshotValue::Id(Id::from(taken_at))),
|
||||
S::TakenAt => Value::Str(utc(found.taken_at).into()),
|
||||
S::Trigger => json_to_value(to_json(&found.trigger)),
|
||||
S::Summary => json_to_value(to_json(&summary)),
|
||||
S::Inventory => json_to_value(to_json(&inventory)),
|
||||
};
|
||||
out.insert_unchecked(Key::Property(property.clone()), value);
|
||||
}
|
||||
response.list.push(Value::Object(out));
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -154,6 +154,7 @@ pub async fn set(
|
||||
log_files::set(data, &settings).await?;
|
||||
// This node purges now; the others within the hour
|
||||
log_files::CHANGED.notify_one();
|
||||
server.inventory_snapshot_after("inbuxa:LogSettings").await;
|
||||
response.updated.append(id, None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod audit;
|
||||
pub mod audit_log;
|
||||
pub mod ai_limits;
|
||||
pub mod log_settings;
|
||||
pub mod data_inventory;
|
||||
pub mod explanation;
|
||||
pub mod protocol_policy;
|
||||
pub mod tenant_protocol_policy;
|
||||
|
||||
@@ -902,7 +902,20 @@ impl RegistrySet for Server {
|
||||
// Finalize cache invalidation
|
||||
self.invalidate_caches(cache_invalidator).await?;
|
||||
|
||||
Ok(set.into_response())
|
||||
// inbuxa: personal-data catalog: what the server holds may
|
||||
// have changed, so the inventory's history is brought up to date
|
||||
let response = set.into_response();
|
||||
if !response.created.is_empty()
|
||||
|| !response.updated.is_empty()
|
||||
|| !response.destroyed.is_empty()
|
||||
{
|
||||
self.inventory_snapshot_after(&format!(
|
||||
"x:{}",
|
||||
registry::types::EnumImpl::as_str(&object_type)
|
||||
))
|
||||
.await;
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
ObjectType::ArfExternalReport
|
||||
| ObjectType::DmarcExternalReport
|
||||
|
||||
Reference in New Issue
Block a user